1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 #include <sys/ddi.h> 24 #include <sys/sunddi.h> 25 #include <sys/sunndi.h> 26 #include <sys/atomic.h> 27 #include <sys/dlpi.h> 28 #include <sys/pattr.h> 29 #include <sys/strsubr.h> 30 #include <sys/stream.h> 31 #include <sys/strsun.h> 32 #include <inet/ip.h> 33 #include <inet/tcp.h> 34 35 #include "version.h" 36 #include "common/common.h" 37 #include "common/t4_msg.h" 38 #include "common/t4_regs.h" 39 #include "common/t4_regs_values.h" 40 41 /* TODO: Tune. */ 42 int rx_buf_size = 8192; 43 int tx_copy_threshold = 256; 44 uint16_t rx_copy_threshold = 256; 45 46 /* Used to track coalesced tx work request */ 47 struct txpkts { 48 mblk_t *tail; /* head is in the software descriptor */ 49 uint64_t *flitp; /* ptr to flit where next pkt should start */ 50 uint8_t npkt; /* # of packets in this work request */ 51 uint8_t nflits; /* # of flits used by this work request */ 52 uint16_t plen; /* total payload (sum of all packets) */ 53 }; 54 55 /* All information needed to tx a frame */ 56 struct txinfo { 57 uint32_t len; /* Total length of frame */ 58 uint32_t flags; /* Checksum and LSO flags */ 59 uint32_t mss; /* MSS for LSO */ 60 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */ 61 uint8_t nflits; /* # of flits needed for the SGL */ 62 uint8_t hdls_used; /* # of DMA handles used */ 63 uint32_t txb_used; /* txb_space used */ 64 struct ulptx_sgl sgl __attribute__((aligned(8))); 65 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2]; 66 }; 67 68 static int service_iq(struct sge_iq *iq, int budget); 69 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, 70 int8_t pktc_idx, int qsize, uint8_t esize); 71 static inline void init_fl(struct sge_fl *fl, uint16_t qsize); 72 static inline void init_eq(struct adapter *sc, struct sge_eq *eq, 73 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid); 74 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, 75 struct sge_fl *fl, int intr_idx, int cong); 76 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq, 77 struct sge_fl *fl); 78 static int alloc_fwq(struct adapter *sc); 79 static int free_fwq(struct adapter *sc); 80 #ifdef TCP_OFFLOAD_ENABLE 81 static int alloc_mgmtq(struct adapter *sc); 82 #endif 83 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, 84 int i); 85 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq); 86 #ifdef TCP_OFFLOAD_ENABLE 87 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 88 int intr_idx); 89 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq); 90 #endif 91 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq); 92 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi, 93 struct sge_eq *eq); 94 #ifdef TCP_OFFLOAD_ENABLE 95 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi, 96 struct sge_eq *eq); 97 #endif 98 static int alloc_eq(struct adapter *sc, struct port_info *pi, 99 struct sge_eq *eq); 100 static int free_eq(struct adapter *sc, struct sge_eq *eq); 101 #ifdef TCP_OFFLOAD_ENABLE 102 static int alloc_wrq(struct adapter *sc, struct port_info *pi, 103 struct sge_wrq *wrq, int idx); 104 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq); 105 #endif 106 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx); 107 static int free_txq(struct port_info *pi, struct sge_txq *txq); 108 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags, 109 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 110 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 111 caddr_t *pva); 112 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 113 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw, 114 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 115 caddr_t *pva); 116 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl); 117 static int alloc_tx_copybuffer(struct adapter *sc, size_t len, 118 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba, 119 caddr_t *pva); 120 static inline bool is_new_response(const struct sge_iq *iq, 121 struct rsp_ctrl **ctrl); 122 static inline void iq_next(struct sge_iq *iq); 123 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs); 124 static void refill_sfl(void *arg); 125 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl); 126 static void free_fl_bufs(struct sge_fl *fl); 127 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl, 128 uint32_t len_newbuf, int *fl_bufs_used); 129 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, 130 struct txinfo *txinfo, int sgl_only); 131 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste); 132 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, 133 struct txinfo *txinfo); 134 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len); 135 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo, 136 mblk_t *m, int len); 137 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo); 138 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 139 struct txinfo *txinfo); 140 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts); 141 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 142 struct txinfo *txinfo); 143 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 144 struct txpkts *txpkts, struct txinfo *txinfo); 145 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, 146 int len); 147 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq); 148 static int reclaim_tx_descs(struct sge_txq *txq, int howmany); 149 static void write_txqflush_wr(struct sge_txq *txq); 150 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, 151 mblk_t *m); 152 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl); 153 static kstat_t *setup_port_config_kstats(struct port_info *pi); 154 static kstat_t *setup_port_info_kstats(struct port_info *pi); 155 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, 156 int idx); 157 static int update_rxq_kstats(kstat_t *ksp, int rw); 158 static int update_port_info_kstats(kstat_t *ksp, int rw); 159 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, 160 int idx); 161 static int update_txq_kstats(kstat_t *ksp, int rw); 162 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *, 163 mblk_t *); 164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, 165 mblk_t *m); 166 167 static inline int 168 reclaimable(struct sge_eq *eq) 169 { 170 unsigned int cidx; 171 172 cidx = eq->spg->cidx; /* stable snapshot */ 173 cidx = be16_to_cpu(cidx); 174 175 if (cidx >= eq->cidx) 176 return (cidx - eq->cidx); 177 else 178 return (cidx + eq->cap - eq->cidx); 179 } 180 181 void 182 t4_sge_init(struct adapter *sc) 183 { 184 struct driver_properties *p = &sc->props; 185 ddi_dma_attr_t *dma_attr; 186 ddi_device_acc_attr_t *acc_attr; 187 uint32_t sge_control, sge_conm_ctrl; 188 int egress_threshold; 189 190 /* 191 * Device access and DMA attributes for descriptor rings 192 */ 193 acc_attr = &sc->sge.acc_attr_desc; 194 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 195 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 196 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC; 197 198 dma_attr = &sc->sge.dma_attr_desc; 199 dma_attr->dma_attr_version = DMA_ATTR_V0; 200 dma_attr->dma_attr_addr_lo = 0; 201 dma_attr->dma_attr_addr_hi = UINT64_MAX; 202 dma_attr->dma_attr_count_max = UINT64_MAX; 203 dma_attr->dma_attr_align = 512; 204 dma_attr->dma_attr_burstsizes = 0xfff; 205 dma_attr->dma_attr_minxfer = 1; 206 dma_attr->dma_attr_maxxfer = UINT64_MAX; 207 dma_attr->dma_attr_seg = UINT64_MAX; 208 dma_attr->dma_attr_sgllen = 1; 209 dma_attr->dma_attr_granular = 1; 210 dma_attr->dma_attr_flags = 0; 211 212 /* 213 * Device access and DMA attributes for tx buffers 214 */ 215 acc_attr = &sc->sge.acc_attr_tx; 216 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 217 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 218 219 dma_attr = &sc->sge.dma_attr_tx; 220 dma_attr->dma_attr_version = DMA_ATTR_V0; 221 dma_attr->dma_attr_addr_lo = 0; 222 dma_attr->dma_attr_addr_hi = UINT64_MAX; 223 dma_attr->dma_attr_count_max = UINT64_MAX; 224 dma_attr->dma_attr_align = 1; 225 dma_attr->dma_attr_burstsizes = 0xfff; 226 dma_attr->dma_attr_minxfer = 1; 227 dma_attr->dma_attr_maxxfer = UINT64_MAX; 228 dma_attr->dma_attr_seg = UINT64_MAX; 229 dma_attr->dma_attr_sgllen = TX_SGL_SEGS; 230 dma_attr->dma_attr_granular = 1; 231 dma_attr->dma_attr_flags = 0; 232 233 /* 234 * Ingress Padding Boundary and Egress Status Page Size are set up by 235 * t4_fixup_host_params(). 236 */ 237 sge_control = t4_read_reg(sc, A_SGE_CONTROL); 238 sc->sge.pktshift = G_PKTSHIFT(sge_control); 239 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; 240 241 /* t4_nex uses FLM packed mode */ 242 sc->sge.fl_align = t4_fl_pkt_align(sc, true); 243 244 /* 245 * Device access and DMA attributes for rx buffers 246 */ 247 sc->sge.rxb_params.dip = sc->dip; 248 sc->sge.rxb_params.buf_size = rx_buf_size; 249 250 acc_attr = &sc->sge.rxb_params.acc_attr_rx; 251 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0; 252 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; 253 254 dma_attr = &sc->sge.rxb_params.dma_attr_rx; 255 dma_attr->dma_attr_version = DMA_ATTR_V0; 256 dma_attr->dma_attr_addr_lo = 0; 257 dma_attr->dma_attr_addr_hi = UINT64_MAX; 258 dma_attr->dma_attr_count_max = UINT64_MAX; 259 /* 260 * Low 4 bits of an rx buffer address have a special meaning to the SGE 261 * and an rx buf cannot have an address with any of these bits set. 262 * FL_ALIGN is >= 32 so we're sure things are ok. 263 */ 264 dma_attr->dma_attr_align = sc->sge.fl_align; 265 dma_attr->dma_attr_burstsizes = 0xfff; 266 dma_attr->dma_attr_minxfer = 1; 267 dma_attr->dma_attr_maxxfer = UINT64_MAX; 268 dma_attr->dma_attr_seg = UINT64_MAX; 269 dma_attr->dma_attr_sgllen = 1; 270 dma_attr->dma_attr_granular = 1; 271 dma_attr->dma_attr_flags = 0; 272 273 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params); 274 275 /* 276 * A FL with <= fl_starve_thres buffers is starving and a periodic 277 * timer will attempt to refill it. This needs to be larger than the 278 * SGE's Egress Congestion Threshold. If it isn't, then we can get 279 * stuck waiting for new packets while the SGE is waiting for us to 280 * give it more Free List entries. (Note that the SGE's Egress 281 * Congestion Threshold is in units of 2 Free List pointers.) For T4, 282 * there was only a single field to control this. For T5 there's the 283 * original field which now only applies to Unpacked Mode Free List 284 * buffers and a new field which only applies to Packed Mode Free List 285 * buffers. 286 */ 287 288 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL); 289 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 290 case CHELSIO_T4: 291 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl); 292 break; 293 case CHELSIO_T5: 294 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl); 295 break; 296 case CHELSIO_T6: 297 default: 298 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl); 299 } 300 sc->sge.fl_starve_threshold = 2*egress_threshold + 1; 301 302 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size); 303 304 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD, 305 V_THRESHOLD_0(p->counter_val[0]) | 306 V_THRESHOLD_1(p->counter_val[1]) | 307 V_THRESHOLD_2(p->counter_val[2]) | 308 V_THRESHOLD_3(p->counter_val[3])); 309 310 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1, 311 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) | 312 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1]))); 313 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3, 314 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) | 315 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3]))); 316 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, 317 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) | 318 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5]))); 319 320 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl); 321 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl); 322 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update); 323 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx); 324 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL, 325 t4_handle_fw_rpl); 326 } 327 328 /* 329 * Allocate and initialize the firmware event queue and the forwarded interrupt 330 * queues, if any. The adapter owns all these queues as they are not associated 331 * with any particular port. 332 * 333 * Returns errno on failure. Resources allocated up to that point may still be 334 * allocated. Caller is responsible for cleanup in case this function fails. 335 */ 336 int 337 t4_setup_adapter_queues(struct adapter *sc) 338 { 339 int rc; 340 341 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 342 343 /* 344 * Firmware event queue 345 */ 346 rc = alloc_fwq(sc); 347 if (rc != 0) 348 return (rc); 349 350 #ifdef TCP_OFFLOAD_ENABLE 351 /* 352 * Management queue. This is just a control queue that uses the fwq as 353 * its associated iq. 354 */ 355 rc = alloc_mgmtq(sc); 356 #endif 357 358 return (rc); 359 } 360 361 /* 362 * Idempotent 363 */ 364 int 365 t4_teardown_adapter_queues(struct adapter *sc) 366 { 367 368 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 369 370 (void) free_fwq(sc); 371 372 return (0); 373 } 374 375 static inline int 376 first_vector(struct port_info *pi) 377 { 378 struct adapter *sc = pi->adapter; 379 int rc = T4_EXTRA_INTR, i; 380 381 if (sc->intr_count == 1) 382 return (0); 383 384 for_each_port(sc, i) { 385 struct port_info *p = sc->port[i]; 386 387 if (i == pi->port_id) 388 break; 389 390 #ifdef TCP_OFFLOAD_ENABLE 391 if (!(sc->flags & INTR_FWD)) 392 rc += p->nrxq + p->nofldrxq; 393 else 394 rc += max(p->nrxq, p->nofldrxq); 395 #else 396 /* 397 * Not compiled with offload support and intr_count > 1. Only 398 * NIC queues exist and they'd better be taking direct 399 * interrupts. 400 */ 401 ASSERT(!(sc->flags & INTR_FWD)); 402 rc += p->nrxq; 403 #endif 404 } 405 return (rc); 406 } 407 408 /* 409 * Given an arbitrary "index," come up with an iq that can be used by other 410 * queues (of this port) for interrupt forwarding, SGE egress updates, etc. 411 * The iq returned is guaranteed to be something that takes direct interrupts. 412 */ 413 static struct sge_iq * 414 port_intr_iq(struct port_info *pi, int idx) 415 { 416 struct adapter *sc = pi->adapter; 417 struct sge *s = &sc->sge; 418 struct sge_iq *iq = NULL; 419 420 if (sc->intr_count == 1) 421 return (&sc->sge.fwq); 422 423 #ifdef TCP_OFFLOAD_ENABLE 424 if (!(sc->flags & INTR_FWD)) { 425 idx %= pi->nrxq + pi->nofldrxq; 426 427 if (idx >= pi->nrxq) { 428 idx -= pi->nrxq; 429 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 430 } else 431 iq = &s->rxq[pi->first_rxq + idx].iq; 432 433 } else { 434 idx %= max(pi->nrxq, pi->nofldrxq); 435 436 if (pi->nrxq >= pi->nofldrxq) 437 iq = &s->rxq[pi->first_rxq + idx].iq; 438 else 439 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq; 440 } 441 #else 442 /* 443 * Not compiled with offload support and intr_count > 1. Only NIC 444 * queues exist and they'd better be taking direct interrupts. 445 */ 446 ASSERT(!(sc->flags & INTR_FWD)); 447 448 idx %= pi->nrxq; 449 iq = &s->rxq[pi->first_rxq + idx].iq; 450 #endif 451 452 return (iq); 453 } 454 455 int 456 t4_setup_port_queues(struct port_info *pi) 457 { 458 int rc = 0, i, intr_idx, j; 459 struct sge_rxq *rxq; 460 struct sge_txq *txq; 461 #ifdef TCP_OFFLOAD_ENABLE 462 int iqid; 463 struct sge_wrq *ctrlq; 464 struct sge_ofld_rxq *ofld_rxq; 465 struct sge_wrq *ofld_txq; 466 #endif 467 struct adapter *sc = pi->adapter; 468 struct driver_properties *p = &sc->props; 469 470 pi->ksp_config = setup_port_config_kstats(pi); 471 pi->ksp_info = setup_port_info_kstats(pi); 472 473 /* Interrupt vector to start from (when using multiple vectors) */ 474 intr_idx = first_vector(pi); 475 476 /* 477 * First pass over all rx queues (NIC and TOE): 478 * a) initialize iq and fl 479 * b) allocate queue iff it will take direct interrupts. 480 */ 481 482 for_each_rxq(pi, i, rxq) { 483 484 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq, 485 RX_IQ_ESIZE); 486 487 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */ 488 489 if ((!(sc->flags & INTR_FWD)) 490 #ifdef TCP_OFFLOAD_ENABLE 491 || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq) 492 #else 493 || (sc->intr_count > 1 && pi->nrxq) 494 #endif 495 ) { 496 rxq->iq.flags |= IQ_INTR; 497 rc = alloc_rxq(pi, rxq, intr_idx, i); 498 if (rc != 0) 499 goto done; 500 intr_idx++; 501 } 502 503 } 504 505 #ifdef TCP_OFFLOAD_ENABLE 506 for_each_ofld_rxq(pi, i, ofld_rxq) { 507 508 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, 509 p->qsize_rxq, RX_IQ_ESIZE); 510 511 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8); 512 513 if (!(sc->flags & INTR_FWD) || 514 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) { 515 ofld_rxq->iq.flags = IQ_INTR; 516 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 517 if (rc != 0) 518 goto done; 519 520 intr_idx++; 521 } 522 } 523 #endif 524 525 /* 526 * Second pass over all rx queues (NIC and TOE). The queues forwarding 527 * their interrupts are allocated now. 528 */ 529 j = 0; 530 for_each_rxq(pi, i, rxq) { 531 if (rxq->iq.flags & IQ_INTR) 532 continue; 533 534 intr_idx = port_intr_iq(pi, j)->abs_id; 535 536 rc = alloc_rxq(pi, rxq, intr_idx, i); 537 if (rc != 0) 538 goto done; 539 j++; 540 } 541 542 #ifdef TCP_OFFLOAD_ENABLE 543 for_each_ofld_rxq(pi, i, ofld_rxq) { 544 if (ofld_rxq->iq.flags & IQ_INTR) 545 continue; 546 547 intr_idx = port_intr_iq(pi, j)->abs_id; 548 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx); 549 if (rc != 0) 550 goto done; 551 j++; 552 } 553 #endif 554 /* 555 * Now the tx queues. Only one pass needed. 556 */ 557 j = 0; 558 for_each_txq(pi, i, txq) { 559 uint16_t iqid; 560 561 iqid = port_intr_iq(pi, j)->cntxt_id; 562 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid); 563 rc = alloc_txq(pi, txq, i); 564 if (rc != 0) 565 goto done; 566 } 567 568 #ifdef TCP_OFFLOAD_ENABLE 569 for_each_ofld_txq(pi, i, ofld_txq) { 570 uint16_t iqid; 571 572 iqid = port_intr_iq(pi, j)->cntxt_id; 573 init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan, 574 iqid); 575 rc = alloc_wrq(sc, pi, ofld_txq, i); 576 if (rc != 0) 577 goto done; 578 } 579 580 /* 581 * Finally, the control queue. 582 */ 583 ctrlq = &sc->sge.ctrlq[pi->port_id]; 584 iqid = port_intr_iq(pi, 0)->cntxt_id; 585 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid); 586 rc = alloc_wrq(sc, pi, ctrlq, 0); 587 #endif 588 589 done: 590 if (rc != 0) 591 (void) t4_teardown_port_queues(pi); 592 593 return (rc); 594 } 595 596 /* 597 * Idempotent 598 */ 599 int 600 t4_teardown_port_queues(struct port_info *pi) 601 { 602 int i; 603 struct sge_rxq *rxq; 604 struct sge_txq *txq; 605 #ifdef TCP_OFFLOAD_ENABLE 606 struct adapter *sc = pi->adapter; 607 struct sge_ofld_rxq *ofld_rxq; 608 struct sge_wrq *ofld_txq; 609 #endif 610 611 if (pi->ksp_config != NULL) { 612 kstat_delete(pi->ksp_config); 613 pi->ksp_config = NULL; 614 } 615 if (pi->ksp_info != NULL) { 616 kstat_delete(pi->ksp_info); 617 pi->ksp_info = NULL; 618 } 619 620 #ifdef TCP_OFFLOAD_ENABLE 621 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 622 #endif 623 624 for_each_txq(pi, i, txq) { 625 (void) free_txq(pi, txq); 626 } 627 628 #ifdef TCP_OFFLOAD_ENABLE 629 for_each_ofld_txq(pi, i, ofld_txq) { 630 (void) free_wrq(sc, ofld_txq); 631 } 632 633 for_each_ofld_rxq(pi, i, ofld_rxq) { 634 if ((ofld_rxq->iq.flags & IQ_INTR) == 0) 635 (void) free_ofld_rxq(pi, ofld_rxq); 636 } 637 #endif 638 639 for_each_rxq(pi, i, rxq) { 640 if ((rxq->iq.flags & IQ_INTR) == 0) 641 (void) free_rxq(pi, rxq); 642 } 643 644 /* 645 * Then take down the rx queues that take direct interrupts. 646 */ 647 648 for_each_rxq(pi, i, rxq) { 649 if (rxq->iq.flags & IQ_INTR) 650 (void) free_rxq(pi, rxq); 651 } 652 653 #ifdef TCP_OFFLOAD_ENABLE 654 for_each_ofld_rxq(pi, i, ofld_rxq) { 655 if (ofld_rxq->iq.flags & IQ_INTR) 656 (void) free_ofld_rxq(pi, ofld_rxq); 657 } 658 #endif 659 660 return (0); 661 } 662 663 /* Deals with errors and forwarded interrupts */ 664 uint_t 665 t4_intr_all(caddr_t arg1, caddr_t arg2) 666 { 667 668 (void) t4_intr_err(arg1, arg2); 669 (void) t4_intr(arg1, arg2); 670 671 return (DDI_INTR_CLAIMED); 672 } 673 674 static void 675 t4_intr_rx_work(struct sge_iq *iq) 676 { 677 mblk_t *mp = NULL; 678 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 679 RXQ_LOCK(rxq); 680 if (!iq->polling) { 681 mp = t4_ring_rx(rxq, iq->qsize/8); 682 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS), 683 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 684 } 685 RXQ_UNLOCK(rxq); 686 if (mp != NULL) 687 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp, 688 rxq->ring_gen_num); 689 } 690 691 /* Deals with interrupts on the given ingress queue */ 692 /* ARGSUSED */ 693 uint_t 694 t4_intr(caddr_t arg1, caddr_t arg2) 695 { 696 struct sge_iq *iq = (struct sge_iq *)arg2; 697 int state; 698 699 /* Right now receive polling is only enabled for MSI-X and 700 * when we have enough msi-x vectors i.e no interrupt forwarding. 701 */ 702 if (iq->adapter->props.multi_rings) { 703 t4_intr_rx_work(iq); 704 } else { 705 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY); 706 if (state == IQS_IDLE) { 707 (void) service_iq(iq, 0); 708 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE); 709 } 710 } 711 return (DDI_INTR_CLAIMED); 712 } 713 714 /* Deals with error interrupts */ 715 /* ARGSUSED */ 716 uint_t 717 t4_intr_err(caddr_t arg1, caddr_t arg2) 718 { 719 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 720 struct adapter *sc = (struct adapter *)arg1; 721 722 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0); 723 (void) t4_slow_intr_handler(sc); 724 725 return (DDI_INTR_CLAIMED); 726 } 727 728 /* 729 * t4_ring_rx - Process responses from an SGE response queue. 730 * 731 * This function processes responses from an SGE response queue up to the supplied budget. 732 * Responses include received packets as well as control messages from FW 733 * or HW. 734 * It returns a chain of mblks containing the received data, to be 735 * passed up to mac_ring_rx(). 736 */ 737 mblk_t * 738 t4_ring_rx(struct sge_rxq *rxq, int budget) 739 { 740 struct sge_iq *iq = &rxq->iq; 741 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 742 struct adapter *sc = iq->adapter; 743 struct rsp_ctrl *ctrl; 744 const struct rss_header *rss; 745 int ndescs = 0, fl_bufs_used = 0; 746 int rsp_type; 747 uint32_t lq; 748 mblk_t *mblk_head = NULL, **mblk_tail, *m; 749 struct cpl_rx_pkt *cpl; 750 uint32_t received_bytes = 0, pkt_len = 0; 751 bool csum_ok; 752 uint16_t err_vec; 753 754 mblk_tail = &mblk_head; 755 756 while (is_new_response(iq, &ctrl)) { 757 758 membar_consumer(); 759 760 m = NULL; 761 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 762 lq = be32_to_cpu(ctrl->pldbuflen_qid); 763 rss = (const void *)iq->cdesc; 764 765 switch (rsp_type) { 766 case X_RSPD_TYPE_FLBUF: 767 768 ASSERT(iq->flags & IQ_HAS_FL); 769 770 if (CPL_RX_PKT == rss->opcode) { 771 cpl = (void *)(rss + 1); 772 pkt_len = be16_to_cpu(cpl->len); 773 774 if (iq->polling && ((received_bytes + pkt_len) > budget)) 775 goto done; 776 777 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 778 if (m == NULL) 779 goto done; 780 781 iq->intr_next = iq->intr_params; 782 m->b_rptr += sc->sge.pktshift; 783 if (sc->params.tp.rx_pkt_encap) 784 /* It is enabled only in T6 config file */ 785 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 786 else 787 err_vec = ntohs(cpl->err_vec); 788 789 csum_ok = cpl->csum_calc && !err_vec; 790 791 /* TODO: what about cpl->ip_frag? */ 792 if (csum_ok && !cpl->ip_frag) { 793 mac_hcksum_set(m, 0, 0, 0, 0xffff, 794 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 795 HCK_IPV4_HDRCKSUM_OK); 796 rxq->rxcsum++; 797 } 798 rxq->rxpkts++; 799 rxq->rxbytes += pkt_len; 800 received_bytes += pkt_len; 801 802 *mblk_tail = m; 803 mblk_tail = &m->b_next; 804 805 break; 806 } 807 808 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 809 if (m == NULL) 810 goto done; 811 /* FALLTHROUGH */ 812 813 case X_RSPD_TYPE_CPL: 814 ASSERT(rss->opcode < NUM_CPL_CMDS); 815 sc->cpl_handler[rss->opcode](iq, rss, m); 816 break; 817 818 default: 819 break; 820 } 821 iq_next(iq); 822 ++ndescs; 823 if (!iq->polling && (ndescs == budget)) 824 break; 825 } 826 827 done: 828 829 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 830 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) | 831 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX))); 832 833 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) { 834 int starved; 835 FL_LOCK(fl); 836 fl->needed += fl_bufs_used; 837 starved = refill_fl(sc, fl, fl->cap / 8); 838 FL_UNLOCK(fl); 839 if (starved) 840 add_fl_to_sfl(sc, fl); 841 } 842 return (mblk_head); 843 } 844 845 /* 846 * Deals with anything and everything on the given ingress queue. 847 */ 848 static int 849 service_iq(struct sge_iq *iq, int budget) 850 { 851 struct sge_iq *q; 852 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */ 853 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */ 854 struct adapter *sc = iq->adapter; 855 struct rsp_ctrl *ctrl; 856 const struct rss_header *rss; 857 int ndescs = 0, limit, fl_bufs_used = 0; 858 int rsp_type; 859 uint32_t lq; 860 int starved; 861 mblk_t *m; 862 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql); 863 864 limit = budget ? budget : iq->qsize / 8; 865 866 /* 867 * We always come back and check the descriptor ring for new indirect 868 * interrupts and other responses after running a single handler. 869 */ 870 for (;;) { 871 while (is_new_response(iq, &ctrl)) { 872 873 membar_consumer(); 874 875 m = NULL; 876 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen); 877 lq = be32_to_cpu(ctrl->pldbuflen_qid); 878 rss = (const void *)iq->cdesc; 879 880 switch (rsp_type) { 881 case X_RSPD_TYPE_FLBUF: 882 883 ASSERT(iq->flags & IQ_HAS_FL); 884 885 m = get_fl_payload(sc, fl, lq, &fl_bufs_used); 886 if (m == NULL) { 887 /* 888 * Rearm the iq with a 889 * longer-than-default timer 890 */ 891 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 892 V_INGRESSQID((u32)iq->cntxt_id) | 893 V_SEINTARM(V_QINTR_TIMER_IDX(SGE_NTIMERS-1))); 894 if (fl_bufs_used > 0) { 895 ASSERT(iq->flags & IQ_HAS_FL); 896 FL_LOCK(fl); 897 fl->needed += fl_bufs_used; 898 starved = refill_fl(sc, fl, fl->cap / 8); 899 FL_UNLOCK(fl); 900 if (starved) 901 add_fl_to_sfl(sc, fl); 902 } 903 return (0); 904 } 905 906 /* FALLTHRU */ 907 case X_RSPD_TYPE_CPL: 908 909 ASSERT(rss->opcode < NUM_CPL_CMDS); 910 sc->cpl_handler[rss->opcode](iq, rss, m); 911 break; 912 913 case X_RSPD_TYPE_INTR: 914 915 /* 916 * Interrupts should be forwarded only to queues 917 * that are not forwarding their interrupts. 918 * This means service_iq can recurse but only 1 919 * level deep. 920 */ 921 ASSERT(budget == 0); 922 923 q = sc->sge.iqmap[lq - sc->sge.iq_start]; 924 if (atomic_cas_uint(&q->state, IQS_IDLE, 925 IQS_BUSY) == IQS_IDLE) { 926 if (service_iq(q, q->qsize / 8) == 0) { 927 (void) atomic_cas_uint( 928 &q->state, IQS_BUSY, 929 IQS_IDLE); 930 } else { 931 STAILQ_INSERT_TAIL(&iql, q, 932 link); 933 } 934 } 935 break; 936 937 default: 938 break; 939 } 940 941 iq_next(iq); 942 if (++ndescs == limit) { 943 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 944 V_CIDXINC(ndescs) | 945 V_INGRESSQID(iq->cntxt_id) | 946 V_SEINTARM(V_QINTR_TIMER_IDX( 947 X_TIMERREG_UPDATE_CIDX))); 948 ndescs = 0; 949 950 if (fl_bufs_used > 0) { 951 ASSERT(iq->flags & IQ_HAS_FL); 952 FL_LOCK(fl); 953 fl->needed += fl_bufs_used; 954 (void) refill_fl(sc, fl, fl->cap / 8); 955 FL_UNLOCK(fl); 956 fl_bufs_used = 0; 957 } 958 959 if (budget != 0) 960 return (EINPROGRESS); 961 } 962 } 963 964 if (STAILQ_EMPTY(&iql) != 0) 965 break; 966 967 /* 968 * Process the head only, and send it to the back of the list if 969 * it's still not done. 970 */ 971 q = STAILQ_FIRST(&iql); 972 STAILQ_REMOVE_HEAD(&iql, link); 973 if (service_iq(q, q->qsize / 8) == 0) 974 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE); 975 else 976 STAILQ_INSERT_TAIL(&iql, q, link); 977 } 978 979 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) | 980 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next)); 981 982 if (iq->flags & IQ_HAS_FL) { 983 984 FL_LOCK(fl); 985 fl->needed += fl_bufs_used; 986 starved = refill_fl(sc, fl, fl->cap / 4); 987 FL_UNLOCK(fl); 988 if (starved != 0) 989 add_fl_to_sfl(sc, fl); 990 } 991 992 return (0); 993 } 994 995 #ifdef TCP_OFFLOAD_ENABLE 996 int 997 t4_mgmt_tx(struct adapter *sc, mblk_t *m) 998 { 999 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m)); 1000 } 1001 1002 /* 1003 * Doesn't fail. Holds on to work requests it can't send right away. 1004 */ 1005 int 1006 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0) 1007 { 1008 struct sge_eq *eq = &wrq->eq; 1009 struct mblk_pair *wr_list = &wrq->wr_list; 1010 int can_reclaim; 1011 caddr_t dst; 1012 mblk_t *wr, *next; 1013 1014 TXQ_LOCK_ASSERT_OWNED(wrq); 1015 #ifdef TCP_OFFLOAD_ENABLE 1016 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD || 1017 (eq->flags & EQ_TYPEMASK) == EQ_CTRL); 1018 #else 1019 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL); 1020 #endif 1021 1022 if (m0 != NULL) { 1023 if (wr_list->head != NULL) 1024 wr_list->tail->b_next = m0; 1025 else 1026 wr_list->head = m0; 1027 while (m0->b_next) 1028 m0 = m0->b_next; 1029 wr_list->tail = m0; 1030 } 1031 1032 can_reclaim = reclaimable(eq); 1033 eq->cidx += can_reclaim; 1034 eq->avail += can_reclaim; 1035 if (eq->cidx >= eq->cap) 1036 eq->cidx -= eq->cap; 1037 1038 for (wr = wr_list->head; wr; wr = next) { 1039 int ndesc, len = 0; 1040 mblk_t *m; 1041 1042 next = wr->b_next; 1043 wr->b_next = NULL; 1044 1045 for (m = wr; m; m = m->b_cont) 1046 len += MBLKL(m); 1047 1048 ASSERT(len > 0 && (len & 0x7) == 0); 1049 ASSERT(len <= SGE_MAX_WR_LEN); 1050 1051 ndesc = howmany(len, EQ_ESIZE); 1052 if (eq->avail < ndesc) { 1053 wr->b_next = next; 1054 wrq->no_desc++; 1055 break; 1056 } 1057 1058 dst = (void *)&eq->desc[eq->pidx]; 1059 for (m = wr; m; m = m->b_cont) 1060 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 1061 1062 eq->pidx += ndesc; 1063 eq->avail -= ndesc; 1064 if (eq->pidx >= eq->cap) 1065 eq->pidx -= eq->cap; 1066 1067 eq->pending += ndesc; 1068 if (eq->pending > 16) 1069 ring_tx_db(sc, eq); 1070 1071 wrq->tx_wrs++; 1072 freemsg(wr); 1073 1074 if (eq->avail < 8) { 1075 can_reclaim = reclaimable(eq); 1076 eq->cidx += can_reclaim; 1077 eq->avail += can_reclaim; 1078 if (eq->cidx >= eq->cap) 1079 eq->cidx -= eq->cap; 1080 } 1081 } 1082 1083 if (eq->pending != 0) 1084 ring_tx_db(sc, eq); 1085 1086 if (wr == NULL) 1087 wr_list->head = wr_list->tail = NULL; 1088 else { 1089 wr_list->head = wr; 1090 1091 ASSERT(wr_list->tail->b_next == NULL); 1092 } 1093 1094 return (0); 1095 } 1096 #endif 1097 1098 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */ 1099 #define TXPKTS_PKT_HDR ((\ 1100 sizeof (struct ulp_txpkt) + \ 1101 sizeof (struct ulptx_idata) + \ 1102 sizeof (struct cpl_tx_pkt_core)) / 8) 1103 1104 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */ 1105 #define TXPKTS_WR_HDR (\ 1106 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \ 1107 TXPKTS_PKT_HDR) 1108 1109 /* Header of a tx WR, before SGL of first packet (in flits) */ 1110 #define TXPKT_WR_HDR ((\ 1111 sizeof (struct fw_eth_tx_pkt_wr) + \ 1112 sizeof (struct cpl_tx_pkt_core)) / 8) 1113 1114 /* Header of a tx LSO WR, before SGL of first packet (in flits) */ 1115 #define TXPKT_LSO_WR_HDR ((\ 1116 sizeof (struct fw_eth_tx_pkt_wr) + \ 1117 sizeof(struct cpl_tx_pkt_lso_core) + \ 1118 sizeof (struct cpl_tx_pkt_core)) / 8) 1119 1120 mblk_t * 1121 t4_eth_tx(void *arg, mblk_t *frame) 1122 { 1123 struct sge_txq *txq = (struct sge_txq *) arg; 1124 struct port_info *pi = txq->port; 1125 struct adapter *sc = pi->adapter; 1126 struct sge_eq *eq = &txq->eq; 1127 mblk_t *next_frame; 1128 int rc, coalescing; 1129 struct txpkts txpkts; 1130 struct txinfo txinfo; 1131 1132 txpkts.npkt = 0; /* indicates there's nothing in txpkts */ 1133 coalescing = 0; 1134 1135 TXQ_LOCK(txq); 1136 if (eq->avail < 8) 1137 (void) reclaim_tx_descs(txq, 8); 1138 for (; frame; frame = next_frame) { 1139 1140 if (eq->avail < 8) 1141 break; 1142 1143 next_frame = frame->b_next; 1144 frame->b_next = NULL; 1145 1146 if (next_frame != NULL) 1147 coalescing = 1; 1148 1149 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing); 1150 if (rc != 0) { 1151 if (rc == ENOMEM) { 1152 1153 /* Short of resources, suspend tx */ 1154 1155 frame->b_next = next_frame; 1156 break; 1157 } 1158 1159 /* 1160 * Unrecoverable error for this frame, throw it 1161 * away and move on to the next. 1162 */ 1163 1164 freemsg(frame); 1165 continue; 1166 } 1167 1168 if (coalescing != 0 && 1169 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) { 1170 1171 /* Successfully absorbed into txpkts */ 1172 1173 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo); 1174 goto doorbell; 1175 } 1176 1177 /* 1178 * We weren't coalescing to begin with, or current frame could 1179 * not be coalesced (add_to_txpkts flushes txpkts if a frame 1180 * given to it can't be coalesced). Either way there should be 1181 * nothing in txpkts. 1182 */ 1183 ASSERT(txpkts.npkt == 0); 1184 1185 /* We're sending out individual frames now */ 1186 coalescing = 0; 1187 1188 if (eq->avail < 8) 1189 (void) reclaim_tx_descs(txq, 8); 1190 rc = write_txpkt_wr(pi, txq, frame, &txinfo); 1191 if (rc != 0) { 1192 1193 /* Short of hardware descriptors, suspend tx */ 1194 1195 /* 1196 * This is an unlikely but expensive failure. We've 1197 * done all the hard work (DMA bindings etc.) and now we 1198 * can't send out the frame. What's worse, we have to 1199 * spend even more time freeing up everything in txinfo. 1200 */ 1201 txq->qfull++; 1202 free_txinfo_resources(txq, &txinfo); 1203 1204 frame->b_next = next_frame; 1205 break; 1206 } 1207 1208 doorbell: 1209 /* Fewer and fewer doorbells as the queue fills up */ 1210 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) { 1211 txq->txbytes += txinfo.len; 1212 txq->txpkts++; 1213 ring_tx_db(sc, eq); 1214 } 1215 (void) reclaim_tx_descs(txq, 32); 1216 } 1217 1218 if (txpkts.npkt > 0) 1219 write_txpkts_wr(txq, &txpkts); 1220 1221 /* 1222 * frame not NULL means there was an error but we haven't thrown it 1223 * away. This can happen when we're short of tx descriptors (qfull) or 1224 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush 1225 * and reclaim will get things going again. 1226 * 1227 * If eq->avail is already 0 we know a credit flush was requested in the 1228 * WR that reduced it to 0 so we don't need another flush (we don't have 1229 * any descriptor for a flush WR anyway, duh). 1230 */ 1231 if (frame && eq->avail > 0) 1232 write_txqflush_wr(txq); 1233 1234 if (eq->pending != 0) 1235 ring_tx_db(sc, eq); 1236 1237 (void) reclaim_tx_descs(txq, eq->qsize); 1238 TXQ_UNLOCK(txq); 1239 1240 return (frame); 1241 } 1242 1243 static inline void 1244 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx, 1245 int qsize, uint8_t esize) 1246 { 1247 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS); 1248 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */ 1249 1250 iq->flags = 0; 1251 iq->adapter = sc; 1252 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx); 1253 iq->intr_pktc_idx = SGE_NCOUNTERS - 1; 1254 if (pktc_idx >= 0) { 1255 iq->intr_params |= F_QINTR_CNT_EN; 1256 iq->intr_pktc_idx = pktc_idx; 1257 } 1258 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */ 1259 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */ 1260 } 1261 1262 static inline void 1263 init_fl(struct sge_fl *fl, uint16_t qsize) 1264 { 1265 1266 fl->qsize = qsize; 1267 fl->allocb_fail = 0; 1268 } 1269 1270 static inline void 1271 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize, 1272 uint8_t tx_chan, uint16_t iqid) 1273 { 1274 struct sge *s = &sc->sge; 1275 uint32_t r; 1276 1277 ASSERT(tx_chan < NCHAN); 1278 ASSERT(eqtype <= EQ_TYPEMASK); 1279 1280 if (is_t5(sc->params.chip)) { 1281 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF); 1282 r >>= S_QUEUESPERPAGEPF0 + 1283 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf; 1284 s->s_qpp = r & M_QUEUESPERPAGEPF0; 1285 } 1286 1287 eq->flags = eqtype & EQ_TYPEMASK; 1288 eq->tx_chan = tx_chan; 1289 eq->iqid = iqid; 1290 eq->qsize = qsize; 1291 } 1292 1293 /* 1294 * Allocates the ring for an ingress queue and an optional freelist. If the 1295 * freelist is specified it will be allocated and then associated with the 1296 * ingress queue. 1297 * 1298 * Returns errno on failure. Resources allocated up to that point may still be 1299 * allocated. Caller is responsible for cleanup in case this function fails. 1300 * 1301 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then 1302 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies 1303 * the index of the queue to which its interrupts will be forwarded. 1304 */ 1305 static int 1306 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl, 1307 int intr_idx, int cong) 1308 { 1309 int rc, i, cntxt_id; 1310 size_t len; 1311 struct fw_iq_cmd c; 1312 struct adapter *sc = iq->adapter; 1313 uint32_t v = 0; 1314 1315 len = iq->qsize * iq->esize; 1316 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl, 1317 &iq->ba, (caddr_t *)&iq->desc); 1318 if (rc != 0) 1319 return (rc); 1320 1321 bzero(&c, sizeof (c)); 1322 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | 1323 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) | 1324 V_FW_IQ_CMD_VFN(0)); 1325 1326 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | 1327 FW_LEN16(c)); 1328 1329 /* Special handling for firmware event queue */ 1330 if (iq == &sc->sge.fwq) 1331 v |= F_FW_IQ_CMD_IQASYNCH; 1332 1333 if (iq->flags & IQ_INTR) 1334 ASSERT(intr_idx < sc->intr_count); 1335 else 1336 v |= F_FW_IQ_CMD_IQANDST; 1337 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx); 1338 1339 c.type_to_iqandstindex = cpu_to_be32(v | 1340 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 1341 V_FW_IQ_CMD_VIID(pi->viid) | 1342 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); 1343 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | 1344 F_FW_IQ_CMD_IQGTSMODE | 1345 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | 1346 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4)); 1347 c.iqsize = cpu_to_be16(iq->qsize); 1348 c.iqaddr = cpu_to_be64(iq->ba); 1349 if (cong >= 0) 1350 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN | 1351 V_FW_IQ_CMD_IQTYPE(cong ? 1352 FW_IQ_IQTYPE_NIC : FW_IQ_IQTYPE_OFLD)); 1353 1354 if (fl != NULL) { 1355 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip); 1356 1357 mutex_init(&fl->lock, NULL, MUTEX_DRIVER, 1358 DDI_INTR_PRI(sc->intr_pri)); 1359 fl->flags |= FL_MTX; 1360 1361 len = fl->qsize * RX_FL_ESIZE; 1362 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl, 1363 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc); 1364 if (rc != 0) 1365 return (rc); 1366 1367 /* Allocate space for one software descriptor per buffer. */ 1368 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8; 1369 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap, 1370 KM_SLEEP); 1371 fl->needed = fl->cap; 1372 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8); 1373 1374 c.iqns_to_fl0congen |= 1375 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) | 1376 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN); 1377 if (cong >= 0) { 1378 c.iqns_to_fl0congen |= 1379 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) | 1380 F_FW_IQ_CMD_FL0CONGCIF | 1381 F_FW_IQ_CMD_FL0CONGEN); 1382 } 1383 1384 /* In T6, for egress queue type FL there is internal overhead 1385 * of 16B for header going into FLM module. Hence the maximum 1386 * allowed burst size is 448 bytes. For T4/T5, the hardware 1387 * doesn't coalesce fetch requests if more than 64 bytes of 1388 * Free List pointers are provided, so we use a 128-byte Fetch 1389 * Burst Minimum there (T6 implements coalescing so we can use 1390 * the smaller 64-byte value there). 1391 */ 1392 1393 c.fl0dcaen_to_fl0cidxfthresh = 1394 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 1395 ? X_FETCHBURSTMIN_128B 1396 : X_FETCHBURSTMIN_64B) | 1397 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 1398 ? X_FETCHBURSTMAX_512B 1399 : X_FETCHBURSTMAX_256B)); 1400 c.fl0size = cpu_to_be16(fl->qsize); 1401 c.fl0addr = cpu_to_be64(fl->ba); 1402 } 1403 1404 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1405 if (rc != 0) { 1406 cxgb_printf(sc->dip, CE_WARN, 1407 "failed to create ingress queue: %d", rc); 1408 return (rc); 1409 } 1410 1411 iq->cdesc = iq->desc; 1412 iq->cidx = 0; 1413 iq->gen = 1; 1414 iq->intr_next = iq->intr_params; 1415 iq->adapter = sc; 1416 iq->cntxt_id = be16_to_cpu(c.iqid); 1417 iq->abs_id = be16_to_cpu(c.physiqid); 1418 iq->flags |= IQ_ALLOCATED; 1419 mutex_init(&iq->lock, NULL, 1420 MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri))); 1421 iq->polling = 0; 1422 1423 cntxt_id = iq->cntxt_id - sc->sge.iq_start; 1424 if (cntxt_id >= sc->sge.niq) { 1425 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__, 1426 cntxt_id, sc->sge.niq - 1); 1427 } 1428 sc->sge.iqmap[cntxt_id] = iq; 1429 1430 if (fl != NULL) { 1431 fl->cntxt_id = be16_to_cpu(c.fl0id); 1432 fl->pidx = fl->cidx = 0; 1433 fl->copy_threshold = rx_copy_threshold; 1434 1435 cntxt_id = fl->cntxt_id - sc->sge.eq_start; 1436 if (cntxt_id >= sc->sge.neq) { 1437 panic("%s: fl->cntxt_id (%d) more than the max (%d)", 1438 __func__, cntxt_id, sc->sge.neq - 1); 1439 } 1440 sc->sge.eqmap[cntxt_id] = (void *)fl; 1441 1442 FL_LOCK(fl); 1443 (void) refill_fl(sc, fl, fl->lowat); 1444 FL_UNLOCK(fl); 1445 1446 iq->flags |= IQ_HAS_FL; 1447 } 1448 1449 if (is_t5(sc->params.chip) && cong >= 0) { 1450 uint32_t param, val; 1451 1452 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 1453 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) | 1454 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id); 1455 if (cong == 0) 1456 val = 1 << 19; 1457 else { 1458 val = 2 << 19; 1459 for (i = 0; i < 4; i++) { 1460 if (cong & (1 << i)) 1461 val |= 1 << (i << 2); 1462 } 1463 } 1464 1465 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1466 if (rc != 0) { 1467 /* report error but carry on */ 1468 cxgb_printf(sc->dip, CE_WARN, 1469 "failed to set congestion manager context for " 1470 "ingress queue %d: %d", iq->cntxt_id, rc); 1471 } 1472 } 1473 1474 /* Enable IQ interrupts */ 1475 iq->state = IQS_IDLE; 1476 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) | 1477 V_INGRESSQID(iq->cntxt_id)); 1478 1479 return (0); 1480 } 1481 1482 static int 1483 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl) 1484 { 1485 int rc; 1486 struct adapter *sc = iq->adapter; 1487 dev_info_t *dip; 1488 1489 dip = pi ? pi->dip : sc->dip; 1490 1491 if (iq != NULL) { 1492 if (iq->flags & IQ_ALLOCATED) { 1493 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0, 1494 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id, 1495 fl ? fl->cntxt_id : 0xffff, 0xffff); 1496 if (rc != 0) { 1497 cxgb_printf(dip, CE_WARN, 1498 "failed to free queue %p: %d", iq, rc); 1499 return (rc); 1500 } 1501 mutex_destroy(&iq->lock); 1502 iq->flags &= ~IQ_ALLOCATED; 1503 } 1504 1505 if (iq->desc != NULL) { 1506 (void) free_desc_ring(&iq->dhdl, &iq->ahdl); 1507 iq->desc = NULL; 1508 } 1509 1510 bzero(iq, sizeof (*iq)); 1511 } 1512 1513 if (fl != NULL) { 1514 if (fl->sdesc != NULL) { 1515 FL_LOCK(fl); 1516 free_fl_bufs(fl); 1517 FL_UNLOCK(fl); 1518 1519 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) * 1520 fl->cap); 1521 fl->sdesc = NULL; 1522 } 1523 1524 if (fl->desc != NULL) { 1525 (void) free_desc_ring(&fl->dhdl, &fl->ahdl); 1526 fl->desc = NULL; 1527 } 1528 1529 if (fl->flags & FL_MTX) { 1530 mutex_destroy(&fl->lock); 1531 fl->flags &= ~FL_MTX; 1532 } 1533 1534 bzero(fl, sizeof (struct sge_fl)); 1535 } 1536 1537 return (0); 1538 } 1539 1540 static int 1541 alloc_fwq(struct adapter *sc) 1542 { 1543 int rc, intr_idx; 1544 struct sge_iq *fwq = &sc->sge.fwq; 1545 1546 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE); 1547 fwq->flags |= IQ_INTR; /* always */ 1548 intr_idx = sc->intr_count > 1 ? 1 : 0; 1549 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1); 1550 if (rc != 0) { 1551 cxgb_printf(sc->dip, CE_WARN, 1552 "failed to create firmware event queue: %d.", rc); 1553 return (rc); 1554 } 1555 1556 return (0); 1557 } 1558 1559 static int 1560 free_fwq(struct adapter *sc) 1561 { 1562 1563 return (free_iq_fl(NULL, &sc->sge.fwq, NULL)); 1564 } 1565 1566 #ifdef TCP_OFFLOAD_ENABLE 1567 static int 1568 alloc_mgmtq(struct adapter *sc) 1569 { 1570 int rc; 1571 struct sge_wrq *mgmtq = &sc->sge.mgmtq; 1572 1573 init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan, 1574 sc->sge.fwq.cntxt_id); 1575 rc = alloc_wrq(sc, NULL, mgmtq, 0); 1576 if (rc != 0) { 1577 cxgb_printf(sc->dip, CE_WARN, 1578 "failed to create management queue: %d\n", rc); 1579 return (rc); 1580 } 1581 1582 return (0); 1583 } 1584 #endif 1585 1586 static int 1587 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i) 1588 { 1589 int rc; 1590 1591 rxq->port = pi; 1592 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1593 t4_get_tp_ch_map(pi->adapter, pi->tx_chan)); 1594 if (rc != 0) 1595 return (rc); 1596 1597 rxq->ksp = setup_rxq_kstats(pi, rxq, i); 1598 1599 return (rc); 1600 } 1601 1602 static int 1603 free_rxq(struct port_info *pi, struct sge_rxq *rxq) 1604 { 1605 int rc; 1606 1607 if (rxq->ksp != NULL) { 1608 kstat_delete(rxq->ksp); 1609 rxq->ksp = NULL; 1610 } 1611 1612 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl); 1613 if (rc == 0) 1614 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl)); 1615 1616 return (rc); 1617 } 1618 1619 #ifdef TCP_OFFLOAD_ENABLE 1620 static int 1621 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq, 1622 int intr_idx) 1623 { 1624 int rc; 1625 1626 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx, 1627 t4_get_tp_ch_map(pi->adapter, pi->tx_chan)); 1628 if (rc != 0) 1629 return (rc); 1630 1631 return (rc); 1632 } 1633 1634 static int 1635 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq) 1636 { 1637 int rc; 1638 1639 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl); 1640 if (rc == 0) 1641 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) - 1642 offsetof(struct sge_ofld_rxq, fl)); 1643 1644 return (rc); 1645 } 1646 #endif 1647 1648 static int 1649 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) 1650 { 1651 int rc, cntxt_id; 1652 struct fw_eq_ctrl_cmd c; 1653 1654 bzero(&c, sizeof (c)); 1655 1656 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST | 1657 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) | 1658 V_FW_EQ_CTRL_CMD_VFN(0)); 1659 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC | 1660 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c)); 1661 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */ 1662 c.physeqid_pkd = BE_32(0); 1663 c.fetchszm_to_iqid = 1664 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1665 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | 1666 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); 1667 c.dcaen_to_eqsize = 1668 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1669 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1670 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1671 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize)); 1672 c.eqaddr = BE_64(eq->ba); 1673 1674 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1675 if (rc != 0) { 1676 cxgb_printf(sc->dip, CE_WARN, 1677 "failed to create control queue %d: %d", eq->tx_chan, rc); 1678 return (rc); 1679 } 1680 eq->flags |= EQ_ALLOCATED; 1681 1682 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid)); 1683 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1684 if (cntxt_id >= sc->sge.neq) 1685 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1686 cntxt_id, sc->sge.neq - 1); 1687 sc->sge.eqmap[cntxt_id] = eq; 1688 1689 return (rc); 1690 } 1691 1692 static int 1693 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1694 { 1695 int rc, cntxt_id; 1696 struct fw_eq_eth_cmd c; 1697 1698 bzero(&c, sizeof (c)); 1699 1700 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST | 1701 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) | 1702 V_FW_EQ_ETH_CMD_VFN(0)); 1703 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC | 1704 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c)); 1705 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE | 1706 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid)); 1707 c.fetchszm_to_iqid = 1708 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1709 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | 1710 V_FW_EQ_ETH_CMD_IQID(eq->iqid)); 1711 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1712 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1713 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1714 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize)); 1715 c.eqaddr = BE_64(eq->ba); 1716 1717 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1718 if (rc != 0) { 1719 cxgb_printf(pi->dip, CE_WARN, 1720 "failed to create Ethernet egress queue: %d", rc); 1721 return (rc); 1722 } 1723 eq->flags |= EQ_ALLOCATED; 1724 1725 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd)); 1726 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1727 if (cntxt_id >= sc->sge.neq) 1728 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1729 cntxt_id, sc->sge.neq - 1); 1730 sc->sge.eqmap[cntxt_id] = eq; 1731 1732 return (rc); 1733 } 1734 1735 #ifdef TCP_OFFLOAD_ENABLE 1736 static int 1737 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1738 { 1739 int rc, cntxt_id; 1740 struct fw_eq_ofld_cmd c; 1741 1742 bzero(&c, sizeof (c)); 1743 1744 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST | 1745 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) | 1746 V_FW_EQ_OFLD_CMD_VFN(0)); 1747 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC | 1748 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); 1749 c.fetchszm_to_iqid = 1750 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | 1751 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | 1752 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); 1753 c.dcaen_to_eqsize = 1754 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) | 1755 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) | 1756 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | 1757 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize)); 1758 c.eqaddr = BE_64(eq->ba); 1759 1760 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c); 1761 if (rc != 0) { 1762 cxgb_printf(pi->dip, CE_WARN, 1763 "failed to create egress queue for TCP offload: %d", rc); 1764 return (rc); 1765 } 1766 eq->flags |= EQ_ALLOCATED; 1767 1768 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd)); 1769 cntxt_id = eq->cntxt_id - sc->sge.eq_start; 1770 if (cntxt_id >= sc->sge.neq) 1771 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__, 1772 cntxt_id, sc->sge.neq - 1); 1773 sc->sge.eqmap[cntxt_id] = eq; 1774 1775 return (rc); 1776 } 1777 #endif 1778 1779 static int 1780 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq) 1781 { 1782 int rc; 1783 size_t len; 1784 1785 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri)); 1786 eq->flags |= EQ_MTX; 1787 1788 len = eq->qsize * EQ_ESIZE; 1789 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl, 1790 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc); 1791 if (rc != 0) 1792 return (rc); 1793 1794 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE; 1795 eq->spg = (void *)&eq->desc[eq->cap]; 1796 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */ 1797 eq->pidx = eq->cidx = 0; 1798 eq->doorbells = sc->doorbells; 1799 1800 switch (eq->flags & EQ_TYPEMASK) { 1801 case EQ_CTRL: 1802 rc = ctrl_eq_alloc(sc, eq); 1803 break; 1804 1805 case EQ_ETH: 1806 rc = eth_eq_alloc(sc, pi, eq); 1807 break; 1808 1809 #ifdef TCP_OFFLOAD_ENABLE 1810 case EQ_OFLD: 1811 rc = ofld_eq_alloc(sc, pi, eq); 1812 break; 1813 #endif 1814 1815 default: 1816 panic("%s: invalid eq type %d.", __func__, 1817 eq->flags & EQ_TYPEMASK); 1818 } 1819 1820 if (eq->doorbells & 1821 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) { 1822 uint32_t s_qpp = sc->sge.s_qpp; 1823 uint32_t mask = (1 << s_qpp) - 1; 1824 volatile uint8_t *udb; 1825 1826 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET; 1827 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */ 1828 eq->udb_qid = eq->cntxt_id & mask; /* id in page */ 1829 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE) 1830 eq->doorbells &= ~DOORBELL_WCWR; 1831 else { 1832 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */ 1833 eq->udb_qid = 0; 1834 } 1835 eq->udb = (volatile void *)udb; 1836 } 1837 1838 if (rc != 0) { 1839 cxgb_printf(sc->dip, CE_WARN, 1840 "failed to allocate egress queue(%d): %d", 1841 eq->flags & EQ_TYPEMASK, rc); 1842 } 1843 1844 return (rc); 1845 } 1846 1847 static int 1848 free_eq(struct adapter *sc, struct sge_eq *eq) 1849 { 1850 int rc; 1851 1852 if (eq->flags & EQ_ALLOCATED) { 1853 switch (eq->flags & EQ_TYPEMASK) { 1854 case EQ_CTRL: 1855 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, 1856 eq->cntxt_id); 1857 break; 1858 1859 case EQ_ETH: 1860 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, 1861 eq->cntxt_id); 1862 break; 1863 1864 #ifdef TCP_OFFLOAD_ENABLE 1865 case EQ_OFLD: 1866 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0, 1867 eq->cntxt_id); 1868 break; 1869 #endif 1870 1871 default: 1872 panic("%s: invalid eq type %d.", __func__, 1873 eq->flags & EQ_TYPEMASK); 1874 } 1875 if (rc != 0) { 1876 cxgb_printf(sc->dip, CE_WARN, 1877 "failed to free egress queue (%d): %d", 1878 eq->flags & EQ_TYPEMASK, rc); 1879 return (rc); 1880 } 1881 eq->flags &= ~EQ_ALLOCATED; 1882 } 1883 1884 if (eq->desc != NULL) { 1885 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl); 1886 eq->desc = NULL; 1887 } 1888 1889 if (eq->flags & EQ_MTX) 1890 mutex_destroy(&eq->lock); 1891 1892 bzero(eq, sizeof (*eq)); 1893 return (0); 1894 } 1895 1896 #ifdef TCP_OFFLOAD_ENABLE 1897 /* ARGSUSED */ 1898 static int 1899 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq, 1900 int idx) 1901 { 1902 int rc; 1903 1904 rc = alloc_eq(sc, pi, &wrq->eq); 1905 if (rc != 0) 1906 return (rc); 1907 1908 wrq->adapter = sc; 1909 wrq->wr_list.head = NULL; 1910 wrq->wr_list.tail = NULL; 1911 1912 /* 1913 * TODO: use idx to figure out what kind of wrq this is and install 1914 * useful kstats for it. 1915 */ 1916 1917 return (rc); 1918 } 1919 1920 static int 1921 free_wrq(struct adapter *sc, struct sge_wrq *wrq) 1922 { 1923 int rc; 1924 1925 rc = free_eq(sc, &wrq->eq); 1926 if (rc != 0) 1927 return (rc); 1928 1929 bzero(wrq, sizeof (*wrq)); 1930 return (0); 1931 } 1932 #endif 1933 1934 static int 1935 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx) 1936 { 1937 int rc, i; 1938 struct adapter *sc = pi->adapter; 1939 struct sge_eq *eq = &txq->eq; 1940 1941 rc = alloc_eq(sc, pi, eq); 1942 if (rc != 0) 1943 return (rc); 1944 1945 txq->port = pi; 1946 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP); 1947 txq->txb_size = eq->qsize * tx_copy_threshold; 1948 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl, 1949 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va); 1950 if (rc == 0) 1951 txq->txb_avail = txq->txb_size; 1952 else 1953 txq->txb_avail = txq->txb_size = 0; 1954 1955 /* 1956 * TODO: is this too low? Worst case would need around 4 times qsize 1957 * (all tx descriptors filled to the brim with SGLs, with each entry in 1958 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total 1959 * if you see too many dma_hdl_failed. 1960 */ 1961 txq->tx_dhdl_total = eq->qsize * 2; 1962 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) * 1963 txq->tx_dhdl_total, KM_SLEEP); 1964 for (i = 0; i < txq->tx_dhdl_total; i++) { 1965 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 1966 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]); 1967 if (rc != DDI_SUCCESS) { 1968 cxgb_printf(sc->dip, CE_WARN, 1969 "%s: failed to allocate DMA handle (%d)", 1970 __func__, rc); 1971 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 1972 } 1973 txq->tx_dhdl_avail++; 1974 } 1975 1976 txq->ksp = setup_txq_kstats(pi, txq, idx); 1977 1978 return (rc); 1979 } 1980 1981 static int 1982 free_txq(struct port_info *pi, struct sge_txq *txq) 1983 { 1984 int i; 1985 struct adapter *sc = pi->adapter; 1986 struct sge_eq *eq = &txq->eq; 1987 1988 if (txq->ksp != NULL) { 1989 kstat_delete(txq->ksp); 1990 txq->ksp = NULL; 1991 } 1992 1993 if (txq->txb_va != NULL) { 1994 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl); 1995 txq->txb_va = NULL; 1996 } 1997 1998 if (txq->sdesc != NULL) { 1999 struct tx_sdesc *sd; 2000 ddi_dma_handle_t hdl; 2001 2002 TXQ_LOCK(txq); 2003 while (eq->cidx != eq->pidx) { 2004 sd = &txq->sdesc[eq->cidx]; 2005 2006 for (i = sd->hdls_used; i; i--) { 2007 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx]; 2008 (void) ddi_dma_unbind_handle(hdl); 2009 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 2010 txq->tx_dhdl_cidx = 0; 2011 } 2012 2013 ASSERT(sd->m); 2014 freemsgchain(sd->m); 2015 2016 eq->cidx += sd->desc_used; 2017 if (eq->cidx >= eq->cap) 2018 eq->cidx -= eq->cap; 2019 2020 txq->txb_avail += txq->txb_used; 2021 } 2022 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx); 2023 ASSERT(txq->txb_avail == txq->txb_size); 2024 TXQ_UNLOCK(txq); 2025 2026 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap); 2027 txq->sdesc = NULL; 2028 } 2029 2030 if (txq->tx_dhdl != NULL) { 2031 for (i = 0; i < txq->tx_dhdl_total; i++) { 2032 if (txq->tx_dhdl[i] != NULL) 2033 ddi_dma_free_handle(&txq->tx_dhdl[i]); 2034 } 2035 } 2036 2037 (void) free_eq(sc, &txq->eq); 2038 2039 bzero(txq, sizeof (*txq)); 2040 return (0); 2041 } 2042 2043 /* 2044 * Allocates a block of contiguous memory for DMA. Can be used to allocate 2045 * memory for descriptor rings or for tx/rx copy buffers. 2046 * 2047 * Caller does not have to clean up anything if this function fails, it cleans 2048 * up after itself. 2049 * 2050 * Caller provides the following: 2051 * len length of the block of memory to allocate. 2052 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR) 2053 * acc_attr device access attributes for the allocation. 2054 * dma_attr DMA attributes for the allocation 2055 * 2056 * If the function is successful it fills up this information: 2057 * dma_hdl DMA handle for the allocated memory 2058 * acc_hdl access handle for the allocated memory 2059 * ba bus address of the allocated memory 2060 * va KVA of the allocated memory. 2061 */ 2062 static int 2063 alloc_dma_memory(struct adapter *sc, size_t len, int flags, 2064 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr, 2065 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 2066 uint64_t *pba, caddr_t *pva) 2067 { 2068 int rc; 2069 ddi_dma_handle_t dhdl; 2070 ddi_acc_handle_t ahdl; 2071 ddi_dma_cookie_t cookie; 2072 uint_t ccount; 2073 caddr_t va; 2074 size_t real_len; 2075 2076 *pva = NULL; 2077 2078 /* 2079 * DMA handle. 2080 */ 2081 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl); 2082 if (rc != DDI_SUCCESS) { 2083 cxgb_printf(sc->dip, CE_WARN, 2084 "failed to allocate DMA handle: %d", rc); 2085 2086 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 2087 } 2088 2089 /* 2090 * Memory suitable for DMA. 2091 */ 2092 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr, 2093 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING, 2094 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl); 2095 if (rc != DDI_SUCCESS) { 2096 cxgb_printf(sc->dip, CE_WARN, 2097 "failed to allocate DMA memory: %d", rc); 2098 2099 ddi_dma_free_handle(&dhdl); 2100 return (ENOMEM); 2101 } 2102 2103 if (len != real_len) { 2104 cxgb_printf(sc->dip, CE_WARN, 2105 "%s: len (%u) != real_len (%u)\n", len, real_len); 2106 } 2107 2108 /* 2109 * DMA bindings. 2110 */ 2111 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL, 2112 NULL, &cookie, &ccount); 2113 if (rc != DDI_DMA_MAPPED) { 2114 cxgb_printf(sc->dip, CE_WARN, 2115 "failed to map DMA memory: %d", rc); 2116 2117 ddi_dma_mem_free(&ahdl); 2118 ddi_dma_free_handle(&dhdl); 2119 return (ENOMEM); 2120 } 2121 if (ccount != 1) { 2122 cxgb_printf(sc->dip, CE_WARN, 2123 "unusable DMA mapping (%d segments)", ccount); 2124 (void) free_desc_ring(&dhdl, &ahdl); 2125 } 2126 2127 bzero(va, real_len); 2128 *dma_hdl = dhdl; 2129 *acc_hdl = ahdl; 2130 *pba = cookie.dmac_laddress; 2131 *pva = va; 2132 2133 return (0); 2134 } 2135 2136 static int 2137 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 2138 { 2139 (void) ddi_dma_unbind_handle(*dhdl); 2140 ddi_dma_mem_free(ahdl); 2141 ddi_dma_free_handle(dhdl); 2142 2143 return (0); 2144 } 2145 2146 static int 2147 alloc_desc_ring(struct adapter *sc, size_t len, int rw, 2148 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 2149 uint64_t *pba, caddr_t *pva) 2150 { 2151 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc; 2152 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; 2153 2154 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr, 2155 dma_attr, dma_hdl, acc_hdl, pba, pva)); 2156 } 2157 2158 static int 2159 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl) 2160 { 2161 return (free_dma_memory(dhdl, ahdl)); 2162 } 2163 2164 static int 2165 alloc_tx_copybuffer(struct adapter *sc, size_t len, 2166 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, 2167 uint64_t *pba, caddr_t *pva) 2168 { 2169 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx; 2170 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */ 2171 2172 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE, 2173 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva)); 2174 } 2175 2176 static inline bool 2177 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl) 2178 { 2179 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc - 2180 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL); 2181 2182 *ctrl = (void *)((uintptr_t)iq->cdesc + 2183 (iq->esize - sizeof (struct rsp_ctrl))); 2184 2185 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen)); 2186 } 2187 2188 static inline void 2189 iq_next(struct sge_iq *iq) 2190 { 2191 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize); 2192 if (++iq->cidx == iq->qsize - 1) { 2193 iq->cidx = 0; 2194 iq->gen ^= 1; 2195 iq->cdesc = iq->desc; 2196 } 2197 } 2198 2199 /* 2200 * Fill up the freelist by upto nbufs and maybe ring its doorbell. 2201 * 2202 * Returns non-zero to indicate that it should be added to the list of starving 2203 * freelists. 2204 */ 2205 static int 2206 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs) 2207 { 2208 uint64_t *d = &fl->desc[fl->pidx]; 2209 struct fl_sdesc *sd = &fl->sdesc[fl->pidx]; 2210 2211 FL_LOCK_ASSERT_OWNED(fl); 2212 ASSERT(nbufs >= 0); 2213 2214 if (nbufs > fl->needed) 2215 nbufs = fl->needed; 2216 2217 while (nbufs--) { 2218 if (sd->rxb != NULL) { 2219 if (sd->rxb->ref_cnt == 1) { 2220 /* 2221 * Buffer is available for recycling. Two ways 2222 * this can happen: 2223 * 2224 * a) All the packets DMA'd into it last time 2225 * around were within the rx_copy_threshold 2226 * and no part of the buffer was ever passed 2227 * up (ref_cnt never went over 1). 2228 * 2229 * b) Packets DMA'd into the buffer were passed 2230 * up but have all been freed by the upper 2231 * layers by now (ref_cnt went over 1 but is 2232 * now back to 1). 2233 * 2234 * Either way the bus address in the descriptor 2235 * ring is already valid. 2236 */ 2237 ASSERT(*d == cpu_to_be64(sd->rxb->ba)); 2238 d++; 2239 goto recycled; 2240 } else { 2241 /* 2242 * Buffer still in use and we need a 2243 * replacement. But first release our reference 2244 * on the existing buffer. 2245 */ 2246 rxbuf_free(sd->rxb); 2247 } 2248 } 2249 2250 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1); 2251 if (sd->rxb == NULL) 2252 break; 2253 *d++ = cpu_to_be64(sd->rxb->ba); 2254 2255 recycled: fl->pending++; 2256 sd++; 2257 fl->needed--; 2258 if (++fl->pidx == fl->cap) { 2259 fl->pidx = 0; 2260 sd = fl->sdesc; 2261 d = fl->desc; 2262 } 2263 } 2264 2265 if (fl->pending >= 8) 2266 ring_fl_db(sc, fl); 2267 2268 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING)); 2269 } 2270 2271 #ifndef TAILQ_FOREACH_SAFE 2272 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \ 2273 for ((var) = TAILQ_FIRST((head)); \ 2274 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \ 2275 (var) = (tvar)) 2276 #endif 2277 2278 /* 2279 * Attempt to refill all starving freelists. 2280 */ 2281 static void 2282 refill_sfl(void *arg) 2283 { 2284 struct adapter *sc = arg; 2285 struct sge_fl *fl, *fl_temp; 2286 2287 mutex_enter(&sc->sfl_lock); 2288 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) { 2289 FL_LOCK(fl); 2290 (void) refill_fl(sc, fl, 64); 2291 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) { 2292 TAILQ_REMOVE(&sc->sfl, fl, link); 2293 fl->flags &= ~FL_STARVING; 2294 } 2295 FL_UNLOCK(fl); 2296 } 2297 2298 if (!TAILQ_EMPTY(&sc->sfl) != 0) 2299 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000)); 2300 mutex_exit(&sc->sfl_lock); 2301 } 2302 2303 static void 2304 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl) 2305 { 2306 mutex_enter(&sc->sfl_lock); 2307 FL_LOCK(fl); 2308 if ((fl->flags & FL_DOOMED) == 0) { 2309 if (TAILQ_EMPTY(&sc->sfl) != 0) { 2310 sc->sfl_timer = timeout(refill_sfl, sc, 2311 drv_usectohz(100000)); 2312 } 2313 fl->flags |= FL_STARVING; 2314 TAILQ_INSERT_TAIL(&sc->sfl, fl, link); 2315 } 2316 FL_UNLOCK(fl); 2317 mutex_exit(&sc->sfl_lock); 2318 } 2319 2320 static void 2321 free_fl_bufs(struct sge_fl *fl) 2322 { 2323 struct fl_sdesc *sd; 2324 unsigned int i; 2325 2326 FL_LOCK_ASSERT_OWNED(fl); 2327 2328 for (i = 0; i < fl->cap; i++) { 2329 sd = &fl->sdesc[i]; 2330 2331 if (sd->rxb != NULL) { 2332 rxbuf_free(sd->rxb); 2333 sd->rxb = NULL; 2334 } 2335 } 2336 } 2337 2338 /* 2339 * Note that fl->cidx and fl->offset are left unchanged in case of failure. 2340 */ 2341 static mblk_t * 2342 get_fl_payload(struct adapter *sc, struct sge_fl *fl, 2343 uint32_t len_newbuf, int *fl_bufs_used) 2344 { 2345 struct mblk_pair frame = {0}; 2346 struct rxbuf *rxb; 2347 mblk_t *m = NULL; 2348 uint_t nbuf = 0, len, copy, n; 2349 uint32_t cidx, offset, rcidx, roffset; 2350 2351 /* 2352 * The SGE won't pack a new frame into the current buffer if the entire 2353 * payload doesn't fit in the remaining space. Move on to the next buf 2354 * in that case. 2355 */ 2356 rcidx = fl->cidx; 2357 roffset = fl->offset; 2358 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) { 2359 fl->offset = 0; 2360 if (++fl->cidx == fl->cap) 2361 fl->cidx = 0; 2362 nbuf++; 2363 } 2364 cidx = fl->cidx; 2365 offset = fl->offset; 2366 2367 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */ 2368 copy = (len <= fl->copy_threshold); 2369 if (copy != 0) { 2370 frame.head = m = allocb(len, BPRI_HI); 2371 if (m == NULL) { 2372 fl->allocb_fail++; 2373 cmn_err(CE_WARN,"%s: mbuf allocation failure " 2374 "count = %llu", __func__, 2375 (unsigned long long)fl->allocb_fail); 2376 fl->cidx = rcidx; 2377 fl->offset = roffset; 2378 return (NULL); 2379 } 2380 } 2381 2382 while (len) { 2383 rxb = fl->sdesc[cidx].rxb; 2384 n = min(len, rxb->buf_size - offset); 2385 2386 (void) ddi_dma_sync(rxb->dhdl, offset, n, 2387 DDI_DMA_SYNC_FORKERNEL); 2388 2389 if (copy != 0) 2390 bcopy(rxb->va + offset, m->b_wptr, n); 2391 else { 2392 m = desballoc((unsigned char *)rxb->va + offset, n, 2393 BPRI_HI, &rxb->freefunc); 2394 if (m == NULL) { 2395 fl->allocb_fail++; 2396 cmn_err(CE_WARN, 2397 "%s: mbuf allocation failure " 2398 "count = %llu", __func__, 2399 (unsigned long long)fl->allocb_fail); 2400 if (frame.head) 2401 freemsgchain(frame.head); 2402 fl->cidx = rcidx; 2403 fl->offset = roffset; 2404 return (NULL); 2405 } 2406 atomic_inc_uint(&rxb->ref_cnt); 2407 if (frame.head != NULL) 2408 frame.tail->b_cont = m; 2409 else 2410 frame.head = m; 2411 frame.tail = m; 2412 } 2413 m->b_wptr += n; 2414 len -= n; 2415 offset += roundup(n, sc->sge.fl_align); 2416 ASSERT(offset <= rxb->buf_size); 2417 if (offset == rxb->buf_size) { 2418 offset = 0; 2419 if (++cidx == fl->cap) 2420 cidx = 0; 2421 nbuf++; 2422 } 2423 } 2424 2425 fl->cidx = cidx; 2426 fl->offset = offset; 2427 (*fl_bufs_used) += nbuf; 2428 2429 ASSERT(frame.head != NULL); 2430 return (frame.head); 2431 } 2432 2433 /* 2434 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're 2435 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes 2436 * of immediate data. 2437 */ 2438 #define IMM_LEN ( \ 2439 2 * EQ_ESIZE \ 2440 - sizeof (struct fw_eth_tx_pkt_wr) \ 2441 - sizeof (struct cpl_tx_pkt_core)) 2442 2443 /* 2444 * Returns non-zero on failure, no need to cleanup anything in that case. 2445 * 2446 * Note 1: We always try to pull up the mblk if required and return E2BIG only 2447 * if this fails. 2448 * 2449 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk 2450 * does not have the TCP header in it. 2451 */ 2452 static int 2453 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo, 2454 int sgl_only) 2455 { 2456 uint32_t flags = 0, len, n; 2457 mblk_t *m = *fp; 2458 int rc; 2459 2460 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */ 2461 2462 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags); 2463 txinfo->flags = flags; 2464 2465 mac_lso_get(m, &txinfo->mss, &flags); 2466 txinfo->flags |= flags; 2467 2468 if (flags & HW_LSO) 2469 sgl_only = 1; /* Do not allow immediate data with LSO */ 2470 2471 start: txinfo->nsegs = 0; 2472 txinfo->hdls_used = 0; 2473 txinfo->txb_used = 0; 2474 txinfo->len = 0; 2475 2476 /* total length and a rough estimate of # of segments */ 2477 n = 0; 2478 for (; m; m = m->b_cont) { 2479 len = MBLKL(m); 2480 n += (len / PAGE_SIZE) + 1; 2481 txinfo->len += len; 2482 } 2483 m = *fp; 2484 2485 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) { 2486 txq->pullup_early++; 2487 m = msgpullup(*fp, -1); 2488 if (m == NULL) { 2489 txq->pullup_failed++; 2490 return (E2BIG); /* (*fp) left as it was */ 2491 } 2492 freemsg(*fp); 2493 *fp = m; 2494 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2495 } 2496 2497 if (txinfo->len <= IMM_LEN && !sgl_only) 2498 return (0); /* nsegs = 0 tells caller to use imm. tx */ 2499 2500 if (txinfo->len <= txq->copy_threshold && 2501 copy_into_txb(txq, m, txinfo->len, txinfo) == 0) 2502 goto done; 2503 2504 for (; m; m = m->b_cont) { 2505 2506 len = MBLKL(m); 2507 2508 /* Use tx copy buffer if this mblk is small enough */ 2509 if (len <= txq->copy_threshold && 2510 copy_into_txb(txq, m, len, txinfo) == 0) 2511 continue; 2512 2513 /* Add DMA bindings for this mblk to the SGL */ 2514 rc = add_mblk(txq, txinfo, m, len); 2515 2516 if (rc == E2BIG || 2517 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) { 2518 2519 txq->pullup_late++; 2520 m = msgpullup(*fp, -1); 2521 if (m != NULL) { 2522 free_txinfo_resources(txq, txinfo); 2523 freemsg(*fp); 2524 *fp = m; 2525 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags); 2526 goto start; 2527 } 2528 2529 txq->pullup_failed++; 2530 rc = E2BIG; 2531 } 2532 2533 if (rc != 0) { 2534 free_txinfo_resources(txq, txinfo); 2535 return (rc); 2536 } 2537 } 2538 2539 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS); 2540 2541 done: 2542 2543 /* 2544 * Store the # of flits required to hold this frame's SGL in nflits. An 2545 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by 2546 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used 2547 * then len1 must be set to 0. 2548 */ 2549 n = txinfo->nsegs - 1; 2550 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2; 2551 if (n & 1) 2552 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0); 2553 2554 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) | 2555 V_ULPTX_NSGE(txinfo->nsegs)); 2556 2557 return (0); 2558 } 2559 2560 static inline int 2561 fits_in_txb(struct sge_txq *txq, int len, int *waste) 2562 { 2563 if (txq->txb_avail < len) 2564 return (0); 2565 2566 if (txq->txb_next + len <= txq->txb_size) { 2567 *waste = 0; 2568 return (1); 2569 } 2570 2571 *waste = txq->txb_size - txq->txb_next; 2572 2573 return (txq->txb_avail - *waste < len ? 0 : 1); 2574 } 2575 2576 #define TXB_CHUNK 64 2577 2578 /* 2579 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo 2580 * and txq to indicate resources used. Caller has to make sure that those many 2581 * bytes are available in the mblk chain (b_cont linked). 2582 */ 2583 static inline int 2584 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo) 2585 { 2586 int waste, n; 2587 2588 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */ 2589 2590 if (!fits_in_txb(txq, len, &waste)) { 2591 txq->txb_full++; 2592 return (ENOMEM); 2593 } 2594 2595 if (waste != 0) { 2596 ASSERT((waste & (TXB_CHUNK - 1)) == 0); 2597 txinfo->txb_used += waste; 2598 txq->txb_avail -= waste; 2599 txq->txb_next = 0; 2600 } 2601 2602 for (n = 0; n < len; m = m->b_cont) { 2603 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m)); 2604 n += MBLKL(m); 2605 } 2606 2607 add_seg(txinfo, txq->txb_ba + txq->txb_next, len); 2608 2609 n = roundup(len, TXB_CHUNK); 2610 txinfo->txb_used += n; 2611 txq->txb_avail -= n; 2612 txq->txb_next += n; 2613 ASSERT(txq->txb_next <= txq->txb_size); 2614 if (txq->txb_next == txq->txb_size) 2615 txq->txb_next = 0; 2616 2617 return (0); 2618 } 2619 2620 static inline void 2621 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len) 2622 { 2623 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */ 2624 2625 if (txinfo->nsegs != 0) { 2626 int idx = txinfo->nsegs - 1; 2627 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len); 2628 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba); 2629 } else { 2630 txinfo->sgl.len0 = cpu_to_be32(len); 2631 txinfo->sgl.addr0 = cpu_to_be64(ba); 2632 } 2633 txinfo->nsegs++; 2634 } 2635 2636 /* 2637 * This function cleans up any partially allocated resources when it fails so 2638 * there's nothing for the caller to clean up in that case. 2639 * 2640 * EIO indicates permanent failure. Caller should drop the frame containing 2641 * this mblk and continue. 2642 * 2643 * E2BIG indicates that the SGL length for this mblk exceeds the hardware 2644 * limit. Caller should pull up the frame before trying to send it out. 2645 * (This error means our pullup_early heuristic did not work for this frame) 2646 * 2647 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA 2648 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to 2649 * free up resources. 2650 */ 2651 static inline int 2652 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len) 2653 { 2654 ddi_dma_handle_t dhdl; 2655 ddi_dma_cookie_t cookie; 2656 uint_t ccount = 0; 2657 int rc; 2658 2659 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */ 2660 2661 if (txq->tx_dhdl_avail == 0) { 2662 txq->dma_hdl_failed++; 2663 return (ENOMEM); 2664 } 2665 2666 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx]; 2667 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len, 2668 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie, 2669 &ccount); 2670 if (rc != DDI_DMA_MAPPED) { 2671 txq->dma_map_failed++; 2672 2673 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP); 2674 2675 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO); 2676 } 2677 2678 if (ccount + txinfo->nsegs > TX_SGL_SEGS) { 2679 (void) ddi_dma_unbind_handle(dhdl); 2680 return (E2BIG); 2681 } 2682 2683 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2684 while (--ccount) { 2685 ddi_dma_nextcookie(dhdl, &cookie); 2686 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size); 2687 } 2688 2689 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total) 2690 txq->tx_dhdl_pidx = 0; 2691 txq->tx_dhdl_avail--; 2692 txinfo->hdls_used++; 2693 2694 return (0); 2695 } 2696 2697 /* 2698 * Releases all the txq resources used up in the specified txinfo. 2699 */ 2700 static void 2701 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo) 2702 { 2703 int n; 2704 2705 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */ 2706 2707 n = txinfo->txb_used; 2708 if (n > 0) { 2709 txq->txb_avail += n; 2710 if (n <= txq->txb_next) 2711 txq->txb_next -= n; 2712 else { 2713 n -= txq->txb_next; 2714 txq->txb_next = txq->txb_size - n; 2715 } 2716 } 2717 2718 for (n = txinfo->hdls_used; n > 0; n--) { 2719 if (txq->tx_dhdl_pidx > 0) 2720 txq->tx_dhdl_pidx--; 2721 else 2722 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1; 2723 txq->tx_dhdl_avail++; 2724 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]); 2725 } 2726 } 2727 2728 /* 2729 * Returns 0 to indicate that m has been accepted into a coalesced tx work 2730 * request. It has either been folded into txpkts or txpkts was flushed and m 2731 * has started a new coalesced work request (as the first frame in a fresh 2732 * txpkts). 2733 * 2734 * Returns non-zero to indicate a failure - caller is responsible for 2735 * transmitting m, if there was anything in txpkts it has been flushed. 2736 */ 2737 static int 2738 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m, 2739 struct txinfo *txinfo) 2740 { 2741 struct sge_eq *eq = &txq->eq; 2742 int can_coalesce; 2743 struct tx_sdesc *txsd; 2744 uint8_t flits; 2745 2746 TXQ_LOCK_ASSERT_OWNED(txq); 2747 2748 if (txpkts->npkt > 0) { 2749 flits = TXPKTS_PKT_HDR + txinfo->nflits; 2750 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2751 txpkts->nflits + flits <= TX_WR_FLITS && 2752 txpkts->nflits + flits <= eq->avail * 8 && 2753 txpkts->plen + txinfo->len < 65536; 2754 2755 if (can_coalesce != 0) { 2756 txpkts->tail->b_next = m; 2757 txpkts->tail = m; 2758 txpkts->npkt++; 2759 txpkts->nflits += flits; 2760 txpkts->plen += txinfo->len; 2761 2762 txsd = &txq->sdesc[eq->pidx]; 2763 txsd->txb_used += txinfo->txb_used; 2764 txsd->hdls_used += txinfo->hdls_used; 2765 2766 return (0); 2767 } 2768 2769 /* 2770 * Couldn't coalesce m into txpkts. The first order of business 2771 * is to send txpkts on its way. Then we'll revisit m. 2772 */ 2773 write_txpkts_wr(txq, txpkts); 2774 } 2775 2776 /* 2777 * Check if we can start a new coalesced tx work request with m as 2778 * the first packet in it. 2779 */ 2780 2781 ASSERT(txpkts->npkt == 0); 2782 ASSERT(txinfo->len < 65536); 2783 2784 flits = TXPKTS_WR_HDR + txinfo->nflits; 2785 can_coalesce = (txinfo->flags & HW_LSO) == 0 && 2786 flits <= eq->avail * 8 && flits <= TX_WR_FLITS; 2787 2788 if (can_coalesce == 0) 2789 return (EINVAL); 2790 2791 /* 2792 * Start a fresh coalesced tx WR with m as the first frame in it. 2793 */ 2794 txpkts->tail = m; 2795 txpkts->npkt = 1; 2796 txpkts->nflits = flits; 2797 txpkts->flitp = &eq->desc[eq->pidx].flit[2]; 2798 txpkts->plen = txinfo->len; 2799 2800 txsd = &txq->sdesc[eq->pidx]; 2801 txsd->m = m; 2802 txsd->txb_used = txinfo->txb_used; 2803 txsd->hdls_used = txinfo->hdls_used; 2804 2805 return (0); 2806 } 2807 2808 /* 2809 * Note that write_txpkts_wr can never run out of hardware descriptors (but 2810 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for 2811 * coalescing only if sufficient hardware descriptors are available. 2812 */ 2813 static void 2814 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts) 2815 { 2816 struct sge_eq *eq = &txq->eq; 2817 struct fw_eth_tx_pkts_wr *wr; 2818 struct tx_sdesc *txsd; 2819 uint32_t ctrl; 2820 uint16_t ndesc; 2821 2822 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2823 2824 ndesc = howmany(txpkts->nflits, 8); 2825 2826 wr = (void *)&eq->desc[eq->pidx]; 2827 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) | 2828 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */ 2829 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2)); 2830 if (eq->avail == ndesc) 2831 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2832 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2833 wr->plen = cpu_to_be16(txpkts->plen); 2834 wr->npkt = txpkts->npkt; 2835 wr->r3 = wr->type = 0; 2836 2837 /* Everything else already written */ 2838 2839 txsd = &txq->sdesc[eq->pidx]; 2840 txsd->desc_used = ndesc; 2841 2842 txq->txb_used += txsd->txb_used / TXB_CHUNK; 2843 txq->hdl_used += txsd->hdls_used; 2844 2845 ASSERT(eq->avail >= ndesc); 2846 2847 eq->pending += ndesc; 2848 eq->avail -= ndesc; 2849 eq->pidx += ndesc; 2850 if (eq->pidx >= eq->cap) 2851 eq->pidx -= eq->cap; 2852 2853 txq->txpkts_pkts += txpkts->npkt; 2854 txq->txpkts_wrs++; 2855 txpkts->npkt = 0; /* emptied */ 2856 } 2857 2858 static int 2859 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m, 2860 struct txinfo *txinfo) 2861 { 2862 struct sge_eq *eq = &txq->eq; 2863 struct fw_eth_tx_pkt_wr *wr; 2864 struct cpl_tx_pkt_core *cpl; 2865 uint32_t ctrl; /* used in many unrelated places */ 2866 uint64_t ctrl1; 2867 int nflits, ndesc; 2868 struct tx_sdesc *txsd; 2869 caddr_t dst; 2870 2871 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */ 2872 2873 /* 2874 * Do we have enough flits to send this frame out? 2875 */ 2876 ctrl = sizeof (struct cpl_tx_pkt_core); 2877 if (txinfo->flags & HW_LSO) { 2878 nflits = TXPKT_LSO_WR_HDR; 2879 ctrl += sizeof(struct cpl_tx_pkt_lso_core); 2880 } else 2881 nflits = TXPKT_WR_HDR; 2882 if (txinfo->nsegs > 0) 2883 nflits += txinfo->nflits; 2884 else { 2885 nflits += howmany(txinfo->len, 8); 2886 ctrl += txinfo->len; 2887 } 2888 ndesc = howmany(nflits, 8); 2889 if (ndesc > eq->avail) 2890 return (ENOMEM); 2891 2892 /* Firmware work request header */ 2893 wr = (void *)&eq->desc[eq->pidx]; 2894 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | 2895 V_FW_WR_IMMDLEN(ctrl)); 2896 ctrl = V_FW_WR_LEN16(howmany(nflits, 2)); 2897 if (eq->avail == ndesc) 2898 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ; 2899 wr->equiq_to_len16 = cpu_to_be32(ctrl); 2900 wr->r3 = 0; 2901 2902 if (txinfo->flags & HW_LSO) { 2903 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 2904 char *p = (void *)m->b_rptr; 2905 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE | 2906 F_LSO_LAST_SLICE; 2907 2908 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2909 if (((struct ether_header *)p)->ether_type == 2910 htons(ETHERTYPE_VLAN)) { 2911 ctrl |= V_LSO_ETHHDR_LEN(1); 2912 p += sizeof (struct ether_vlan_header); 2913 } else 2914 p += sizeof (struct ether_header); 2915 2916 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */ 2917 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4); 2918 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */ 2919 p += IPH_HDR_LENGTH(p); 2920 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4); 2921 2922 lso->lso_ctrl = cpu_to_be32(ctrl); 2923 lso->ipid_ofst = cpu_to_be16(0); 2924 lso->mss = cpu_to_be16(txinfo->mss); 2925 lso->seqno_offset = cpu_to_be32(0); 2926 if (is_t4(pi->adapter->params.chip)) 2927 lso->len = cpu_to_be32(txinfo->len); 2928 else 2929 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len)); 2930 2931 cpl = (void *)(lso + 1); 2932 2933 txq->tso_wrs++; 2934 } else 2935 cpl = (void *)(wr + 1); 2936 2937 /* Checksum offload */ 2938 ctrl1 = 0; 2939 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 2940 ctrl1 |= F_TXPKT_IPCSUM_DIS; 2941 if (!(txinfo->flags & HCK_FULLCKSUM)) 2942 ctrl1 |= F_TXPKT_L4CSUM_DIS; 2943 if (ctrl1 == 0) 2944 txq->txcsum++; /* some hardware assistance provided */ 2945 2946 /* CPL header */ 2947 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 2948 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 2949 cpl->pack = 0; 2950 cpl->len = cpu_to_be16(txinfo->len); 2951 cpl->ctrl1 = cpu_to_be64(ctrl1); 2952 2953 /* Software descriptor */ 2954 txsd = &txq->sdesc[eq->pidx]; 2955 txsd->m = m; 2956 txsd->txb_used = txinfo->txb_used; 2957 txsd->hdls_used = txinfo->hdls_used; 2958 /* LINTED: E_ASSIGN_NARROW_CONV */ 2959 txsd->desc_used = ndesc; 2960 2961 txq->txb_used += txinfo->txb_used / TXB_CHUNK; 2962 txq->hdl_used += txinfo->hdls_used; 2963 2964 eq->pending += ndesc; 2965 eq->avail -= ndesc; 2966 eq->pidx += ndesc; 2967 if (eq->pidx >= eq->cap) 2968 eq->pidx -= eq->cap; 2969 2970 /* SGL */ 2971 dst = (void *)(cpl + 1); 2972 if (txinfo->nsegs > 0) { 2973 txq->sgl_wrs++; 2974 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 2975 2976 /* Need to zero-pad to a 16 byte boundary if not on one */ 2977 if ((uintptr_t)dst & 0xf) 2978 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 2979 *(uint64_t *)dst = 0; 2980 2981 } else { 2982 txq->imm_wrs++; 2983 #ifdef DEBUG 2984 ctrl = txinfo->len; 2985 #endif 2986 for (; m; m = m->b_cont) { 2987 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m)); 2988 #ifdef DEBUG 2989 ctrl -= MBLKL(m); 2990 #endif 2991 } 2992 ASSERT(ctrl == 0); 2993 } 2994 2995 txq->txpkt_wrs++; 2996 return (0); 2997 } 2998 2999 static inline void 3000 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq, 3001 struct txpkts *txpkts, struct txinfo *txinfo) 3002 { 3003 struct ulp_txpkt *ulpmc; 3004 struct ulptx_idata *ulpsc; 3005 struct cpl_tx_pkt_core *cpl; 3006 uintptr_t flitp, start, end; 3007 uint64_t ctrl; 3008 caddr_t dst; 3009 3010 ASSERT(txpkts->npkt > 0); 3011 3012 start = (uintptr_t)txq->eq.desc; 3013 end = (uintptr_t)txq->eq.spg; 3014 3015 /* Checksum offload */ 3016 ctrl = 0; 3017 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM)) 3018 ctrl |= F_TXPKT_IPCSUM_DIS; 3019 if (!(txinfo->flags & HCK_FULLCKSUM)) 3020 ctrl |= F_TXPKT_L4CSUM_DIS; 3021 if (ctrl == 0) 3022 txq->txcsum++; /* some hardware assistance provided */ 3023 3024 /* 3025 * The previous packet's SGL must have ended at a 16 byte boundary (this 3026 * is required by the firmware/hardware). It follows that flitp cannot 3027 * wrap around between the ULPTX master command and ULPTX subcommand (8 3028 * bytes each), and that it can not wrap around in the middle of the 3029 * cpl_tx_pkt_core either. 3030 */ 3031 flitp = (uintptr_t)txpkts->flitp; 3032 ASSERT((flitp & 0xf) == 0); 3033 3034 /* ULP master command */ 3035 ulpmc = (void *)flitp; 3036 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0)); 3037 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) + 3038 sizeof (*cpl) + 8 * txinfo->nflits, 16)); 3039 3040 /* ULP subcommand */ 3041 ulpsc = (void *)(ulpmc + 1); 3042 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) | 3043 F_ULP_TX_SC_MORE); 3044 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core)); 3045 3046 flitp += sizeof (*ulpmc) + sizeof (*ulpsc); 3047 if (flitp == end) 3048 flitp = start; 3049 3050 /* CPL_TX_PKT */ 3051 cpl = (void *)flitp; 3052 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) | 3053 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf)); 3054 cpl->pack = 0; 3055 cpl->len = cpu_to_be16(txinfo->len); 3056 cpl->ctrl1 = cpu_to_be64(ctrl); 3057 3058 flitp += sizeof (*cpl); 3059 if (flitp == end) 3060 flitp = start; 3061 3062 /* SGL for this frame */ 3063 dst = (caddr_t)flitp; 3064 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8); 3065 flitp = (uintptr_t)dst; 3066 3067 /* Zero pad and advance to a 16 byte boundary if not already at one. */ 3068 if (flitp & 0xf) { 3069 3070 /* no matter what, flitp should be on an 8 byte boundary */ 3071 ASSERT((flitp & 0x7) == 0); 3072 3073 *(uint64_t *)flitp = 0; 3074 flitp += sizeof (uint64_t); 3075 txpkts->nflits++; 3076 } 3077 3078 if (flitp == end) 3079 flitp = start; 3080 3081 txpkts->flitp = (void *)flitp; 3082 } 3083 3084 static inline void 3085 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len) 3086 { 3087 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) { 3088 bcopy(from, *to, len); 3089 (*to) += len; 3090 } else { 3091 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to); 3092 3093 bcopy(from, *to, portion); 3094 from += portion; 3095 portion = len - portion; /* remaining */ 3096 bcopy(from, (void *)eq->desc, portion); 3097 (*to) = (caddr_t)eq->desc + portion; 3098 } 3099 } 3100 3101 static inline void 3102 ring_tx_db(struct adapter *sc, struct sge_eq *eq) 3103 { 3104 int val, db_mode; 3105 u_int db = eq->doorbells; 3106 3107 if (eq->pending > 1) 3108 db &= ~DOORBELL_WCWR; 3109 3110 if (eq->pending > eq->pidx) { 3111 int offset = eq->cap - (eq->pending - eq->pidx); 3112 3113 /* pidx has wrapped around since last doorbell */ 3114 3115 (void) ddi_dma_sync(eq->desc_dhdl, 3116 offset * sizeof (struct tx_desc), 0, 3117 DDI_DMA_SYNC_FORDEV); 3118 (void) ddi_dma_sync(eq->desc_dhdl, 3119 0, eq->pidx * sizeof (struct tx_desc), 3120 DDI_DMA_SYNC_FORDEV); 3121 } else if (eq->pending > 0) { 3122 (void) ddi_dma_sync(eq->desc_dhdl, 3123 (eq->pidx - eq->pending) * sizeof (struct tx_desc), 3124 eq->pending * sizeof (struct tx_desc), 3125 DDI_DMA_SYNC_FORDEV); 3126 } 3127 3128 membar_producer(); 3129 3130 if (is_t4(sc->params.chip)) 3131 val = V_PIDX(eq->pending); 3132 else 3133 val = V_PIDX_T5(eq->pending); 3134 3135 db_mode = (1 << (ffs(db) - 1)); 3136 switch (db_mode) { 3137 case DOORBELL_UDB: 3138 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 3139 break; 3140 3141 case DOORBELL_WCWR: 3142 { 3143 volatile uint64_t *dst, *src; 3144 int i; 3145 /* 3146 * Queues whose 128B doorbell segment fits in 3147 * the page do not use relative qid 3148 * (udb_qid is always 0). Only queues with 3149 * doorbell segments can do WCWR. 3150 */ 3151 ASSERT(eq->udb_qid == 0 && eq->pending == 1); 3152 3153 dst = (volatile void *)((uintptr_t)eq->udb + 3154 UDBS_WR_OFFSET - UDBS_DB_OFFSET); 3155 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1; 3156 src = (void *)&eq->desc[i]; 3157 while (src != (void *)&eq->desc[i + 1]) 3158 *dst++ = *src++; 3159 membar_producer(); 3160 break; 3161 } 3162 3163 case DOORBELL_UDBWC: 3164 *eq->udb = LE_32(V_QID(eq->udb_qid) | val); 3165 membar_producer(); 3166 break; 3167 3168 case DOORBELL_KDB: 3169 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), 3170 V_QID(eq->cntxt_id) | val); 3171 break; 3172 } 3173 3174 eq->pending = 0; 3175 } 3176 3177 static int 3178 reclaim_tx_descs(struct sge_txq *txq, int howmany) 3179 { 3180 struct tx_sdesc *txsd; 3181 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed; 3182 struct sge_eq *eq = &txq->eq; 3183 3184 EQ_LOCK_ASSERT_OWNED(eq); 3185 3186 cidx = eq->spg->cidx; /* stable snapshot */ 3187 cidx = be16_to_cpu(cidx); 3188 3189 if (cidx >= eq->cidx) 3190 can_reclaim = cidx - eq->cidx; 3191 else 3192 can_reclaim = cidx + eq->cap - eq->cidx; 3193 3194 if (can_reclaim == 0) 3195 return (0); 3196 3197 txb_freed = hdls_freed = reclaimed = 0; 3198 do { 3199 int ndesc; 3200 3201 txsd = &txq->sdesc[eq->cidx]; 3202 ndesc = txsd->desc_used; 3203 3204 /* Firmware doesn't return "partial" credits. */ 3205 ASSERT(can_reclaim >= ndesc); 3206 3207 /* 3208 * We always keep mblk around, even for immediate data. If mblk 3209 * is NULL, this has to be the software descriptor for a credit 3210 * flush work request. 3211 */ 3212 if (txsd->m != NULL) 3213 freemsgchain(txsd->m); 3214 #ifdef DEBUG 3215 else { 3216 ASSERT(txsd->txb_used == 0); 3217 ASSERT(txsd->hdls_used == 0); 3218 ASSERT(ndesc == 1); 3219 } 3220 #endif 3221 3222 txb_freed += txsd->txb_used; 3223 hdls_freed += txsd->hdls_used; 3224 reclaimed += ndesc; 3225 3226 eq->cidx += ndesc; 3227 if (eq->cidx >= eq->cap) 3228 eq->cidx -= eq->cap; 3229 3230 can_reclaim -= ndesc; 3231 3232 } while (can_reclaim && reclaimed < howmany); 3233 3234 eq->avail += reclaimed; 3235 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */ 3236 3237 txq->txb_avail += txb_freed; 3238 3239 txq->tx_dhdl_avail += hdls_freed; 3240 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total); 3241 for (; hdls_freed; hdls_freed--) { 3242 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]); 3243 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total) 3244 txq->tx_dhdl_cidx = 0; 3245 } 3246 3247 return (reclaimed); 3248 } 3249 3250 static void 3251 write_txqflush_wr(struct sge_txq *txq) 3252 { 3253 struct sge_eq *eq = &txq->eq; 3254 struct fw_eq_flush_wr *wr; 3255 struct tx_sdesc *txsd; 3256 3257 EQ_LOCK_ASSERT_OWNED(eq); 3258 ASSERT(eq->avail > 0); 3259 3260 wr = (void *)&eq->desc[eq->pidx]; 3261 bzero(wr, sizeof (*wr)); 3262 wr->opcode = FW_EQ_FLUSH_WR; 3263 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) | 3264 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ); 3265 3266 txsd = &txq->sdesc[eq->pidx]; 3267 txsd->m = NULL; 3268 txsd->txb_used = 0; 3269 txsd->hdls_used = 0; 3270 txsd->desc_used = 1; 3271 3272 eq->pending++; 3273 eq->avail--; 3274 if (++eq->pidx == eq->cap) 3275 eq->pidx = 0; 3276 } 3277 3278 static int 3279 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3280 { 3281 bool csum_ok; 3282 uint16_t err_vec; 3283 struct sge_rxq *rxq = (void *)iq; 3284 struct mblk_pair chain = {0}; 3285 struct adapter *sc = iq->adapter; 3286 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1); 3287 3288 iq->intr_next = iq->intr_params; 3289 3290 m->b_rptr += sc->sge.pktshift; 3291 3292 /* Compressed error vector is enabled for T6 only */ 3293 if (sc->params.tp.rx_pkt_encap) 3294 /* It is enabled only in T6 config file */ 3295 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec)); 3296 else 3297 err_vec = ntohs(cpl->err_vec); 3298 3299 csum_ok = cpl->csum_calc && !err_vec; 3300 /* TODO: what about cpl->ip_frag? */ 3301 if (csum_ok && !cpl->ip_frag) { 3302 mac_hcksum_set(m, 0, 0, 0, 0xffff, 3303 HCK_FULLCKSUM_OK | HCK_FULLCKSUM | 3304 HCK_IPV4_HDRCKSUM_OK); 3305 rxq->rxcsum++; 3306 } 3307 3308 /* Add to the chain that we'll send up */ 3309 if (chain.head != NULL) 3310 chain.tail->b_next = m; 3311 else 3312 chain.head = m; 3313 chain.tail = m; 3314 3315 t4_mac_rx(rxq->port, rxq, chain.head); 3316 3317 rxq->rxpkts++; 3318 rxq->rxbytes += be16_to_cpu(cpl->len); 3319 return (0); 3320 } 3321 3322 #define FL_HW_IDX(idx) ((idx) >> 3) 3323 3324 static inline void 3325 ring_fl_db(struct adapter *sc, struct sge_fl *fl) 3326 { 3327 int desc_start, desc_last, ndesc; 3328 uint32_t v = sc->params.arch.sge_fl_db ; 3329 3330 ndesc = FL_HW_IDX(fl->pending); 3331 3332 /* Hold back one credit if pidx = cidx */ 3333 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx)) 3334 ndesc--; 3335 3336 /* 3337 * There are chances of ndesc modified above (to avoid pidx = cidx). 3338 * If there is nothing to post, return. 3339 */ 3340 if (ndesc <= 0) 3341 return; 3342 3343 desc_last = FL_HW_IDX(fl->pidx); 3344 3345 if (fl->pidx < fl->pending) { 3346 /* There was a wrap */ 3347 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending); 3348 3349 /* From desc_start to the end of list */ 3350 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0, 3351 DDI_DMA_SYNC_FORDEV); 3352 3353 /* From start of list to the desc_last */ 3354 if (desc_last != 0) 3355 (void) ddi_dma_sync(fl->dhdl, 0, desc_last * 3356 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3357 } else { 3358 /* There was no wrap, sync from start_desc to last_desc */ 3359 desc_start = FL_HW_IDX(fl->pidx - fl->pending); 3360 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 3361 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV); 3362 } 3363 3364 if (is_t4(sc->params.chip)) 3365 v |= V_PIDX(ndesc); 3366 else 3367 v |= V_PIDX_T5(ndesc); 3368 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc); 3369 3370 membar_producer(); 3371 3372 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v); 3373 3374 /* 3375 * Update pending count: 3376 * Deduct the number of descriptors posted 3377 */ 3378 fl->pending -= ndesc * 8; 3379 } 3380 3381 static void 3382 tx_reclaim_task(void *arg) 3383 { 3384 struct sge_txq *txq = arg; 3385 3386 TXQ_LOCK(txq); 3387 reclaim_tx_descs(txq, txq->eq.qsize); 3388 TXQ_UNLOCK(txq); 3389 } 3390 3391 /* ARGSUSED */ 3392 static int 3393 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss, 3394 mblk_t *m) 3395 { 3396 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1); 3397 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid)); 3398 struct adapter *sc = iq->adapter; 3399 struct sge *s = &sc->sge; 3400 struct sge_eq *eq; 3401 struct sge_txq *txq; 3402 3403 txq = (void *)s->eqmap[qid - s->eq_start]; 3404 eq = &txq->eq; 3405 txq->qflush++; 3406 t4_mac_tx_update(txq->port, txq); 3407 3408 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task, 3409 (void *)txq, DDI_NOSLEEP); 3410 3411 return (0); 3412 } 3413 3414 static int 3415 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 3416 { 3417 struct adapter *sc = iq->adapter; 3418 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1); 3419 3420 ASSERT(m == NULL); 3421 3422 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) { 3423 const struct rss_header *rss2; 3424 3425 rss2 = (const struct rss_header *)&cpl->data[0]; 3426 return (sc->cpl_handler[rss2->opcode](iq, rss2, m)); 3427 } 3428 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0])); 3429 } 3430 3431 int 3432 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count, 3433 int flags) 3434 { 3435 int i, rc; 3436 3437 txmaps->map_total = count; 3438 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0; 3439 3440 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) * 3441 txmaps->map_total, flags); 3442 3443 for (i = 0; i < count; i++) { 3444 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx, 3445 DDI_DMA_SLEEP, 0, &txmaps->map[i]); 3446 if (rc != DDI_SUCCESS) { 3447 cxgb_printf(sc->dip, CE_WARN, 3448 "%s: failed to allocate DMA handle (%d)", 3449 __func__, rc); 3450 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL); 3451 } 3452 txmaps->map_avail++; 3453 } 3454 3455 return (0); 3456 } 3457 3458 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 3459 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 3460 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 3461 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x 3462 #define KS_C_SET(x, ...) \ 3463 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 3464 3465 /* 3466 * cxgbe:X:config 3467 */ 3468 struct cxgbe_port_config_kstats { 3469 kstat_named_t idx; 3470 kstat_named_t nrxq; 3471 kstat_named_t ntxq; 3472 kstat_named_t first_rxq; 3473 kstat_named_t first_txq; 3474 kstat_named_t controller; 3475 kstat_named_t factory_mac_address; 3476 }; 3477 3478 /* 3479 * cxgbe:X:info 3480 */ 3481 struct cxgbe_port_info_kstats { 3482 kstat_named_t transceiver; 3483 kstat_named_t rx_ovflow0; 3484 kstat_named_t rx_ovflow1; 3485 kstat_named_t rx_ovflow2; 3486 kstat_named_t rx_ovflow3; 3487 kstat_named_t rx_trunc0; 3488 kstat_named_t rx_trunc1; 3489 kstat_named_t rx_trunc2; 3490 kstat_named_t rx_trunc3; 3491 kstat_named_t tx_pause; 3492 kstat_named_t rx_pause; 3493 }; 3494 3495 static kstat_t * 3496 setup_port_config_kstats(struct port_info *pi) 3497 { 3498 kstat_t *ksp; 3499 struct cxgbe_port_config_kstats *kstatp; 3500 int ndata; 3501 dev_info_t *pdip = ddi_get_parent(pi->dip); 3502 uint8_t *ma = &pi->hw_addr[0]; 3503 3504 ndata = sizeof (struct cxgbe_port_config_kstats) / 3505 sizeof (kstat_named_t); 3506 3507 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config", 3508 "net", KSTAT_TYPE_NAMED, ndata, 0); 3509 if (ksp == NULL) { 3510 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3511 return (NULL); 3512 } 3513 3514 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data; 3515 3516 KS_UINIT(idx); 3517 KS_UINIT(nrxq); 3518 KS_UINIT(ntxq); 3519 KS_UINIT(first_rxq); 3520 KS_UINIT(first_txq); 3521 KS_CINIT(controller); 3522 KS_CINIT(factory_mac_address); 3523 3524 KS_U_SET(idx, pi->port_id); 3525 KS_U_SET(nrxq, pi->nrxq); 3526 KS_U_SET(ntxq, pi->ntxq); 3527 KS_U_SET(first_rxq, pi->first_rxq); 3528 KS_U_SET(first_txq, pi->first_txq); 3529 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip), 3530 ddi_get_instance(pdip)); 3531 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X", 3532 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]); 3533 3534 /* Do NOT set ksp->ks_update. These kstats do not change. */ 3535 3536 /* Install the kstat */ 3537 ksp->ks_private = (void *)pi; 3538 kstat_install(ksp); 3539 3540 return (ksp); 3541 } 3542 3543 static kstat_t * 3544 setup_port_info_kstats(struct port_info *pi) 3545 { 3546 kstat_t *ksp; 3547 struct cxgbe_port_info_kstats *kstatp; 3548 int ndata; 3549 3550 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t); 3551 3552 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info", 3553 "net", KSTAT_TYPE_NAMED, ndata, 0); 3554 if (ksp == NULL) { 3555 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats."); 3556 return (NULL); 3557 } 3558 3559 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data; 3560 3561 KS_CINIT(transceiver); 3562 KS_UINIT(rx_ovflow0); 3563 KS_UINIT(rx_ovflow1); 3564 KS_UINIT(rx_ovflow2); 3565 KS_UINIT(rx_ovflow3); 3566 KS_UINIT(rx_trunc0); 3567 KS_UINIT(rx_trunc1); 3568 KS_UINIT(rx_trunc2); 3569 KS_UINIT(rx_trunc3); 3570 KS_UINIT(tx_pause); 3571 KS_UINIT(rx_pause); 3572 3573 /* Install the kstat */ 3574 ksp->ks_update = update_port_info_kstats; 3575 ksp->ks_private = (void *)pi; 3576 kstat_install(ksp); 3577 3578 return (ksp); 3579 } 3580 3581 static int 3582 update_port_info_kstats(kstat_t *ksp, int rw) 3583 { 3584 struct cxgbe_port_info_kstats *kstatp = 3585 (struct cxgbe_port_info_kstats *)ksp->ks_data; 3586 struct port_info *pi = ksp->ks_private; 3587 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX", 3588 "active TWINAX", "LRM" }; 3589 uint32_t bgmap; 3590 3591 if (rw == KSTAT_WRITE) 3592 return (0); 3593 3594 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 3595 KS_C_SET(transceiver, "unplugged"); 3596 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 3597 KS_C_SET(transceiver, "unknown"); 3598 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 3599 KS_C_SET(transceiver, "unsupported"); 3600 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 3601 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]); 3602 else 3603 KS_C_SET(transceiver, "type %d", pi->mod_type); 3604 3605 #define GET_STAT(name) t4_read_reg64(pi->adapter, \ 3606 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L)) 3607 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \ 3608 A_MPS_STAT_##name##_L) 3609 3610 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL)); 3611 if (bgmap == 0) 3612 bgmap = (pi->port_id == 0) ? 0xf : 0; 3613 else if (bgmap == 1) 3614 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0; 3615 else 3616 bgmap = 1; 3617 3618 KS_U_SET(rx_ovflow0, (bgmap & 1) ? 3619 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0); 3620 KS_U_SET(rx_ovflow1, (bgmap & 2) ? 3621 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0); 3622 KS_U_SET(rx_ovflow2, (bgmap & 4) ? 3623 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0); 3624 KS_U_SET(rx_ovflow3, (bgmap & 8) ? 3625 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0); 3626 KS_U_SET(rx_trunc0, (bgmap & 1) ? 3627 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0); 3628 KS_U_SET(rx_trunc1, (bgmap & 2) ? 3629 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0); 3630 KS_U_SET(rx_trunc2, (bgmap & 4) ? 3631 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0); 3632 KS_U_SET(rx_trunc3, (bgmap & 8) ? 3633 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0); 3634 3635 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE)); 3636 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE)); 3637 3638 return (0); 3639 3640 } 3641 3642 /* 3643 * cxgbe:X:rxqY 3644 */ 3645 struct rxq_kstats { 3646 kstat_named_t rxcsum; 3647 kstat_named_t rxpkts; 3648 kstat_named_t rxbytes; 3649 kstat_named_t nomem; 3650 }; 3651 3652 static kstat_t * 3653 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx) 3654 { 3655 struct kstat *ksp; 3656 struct rxq_kstats *kstatp; 3657 int ndata; 3658 char str[16]; 3659 3660 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t); 3661 (void) snprintf(str, sizeof (str), "rxq%u", idx); 3662 3663 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq", 3664 KSTAT_TYPE_NAMED, ndata, 0); 3665 if (ksp == NULL) { 3666 cxgb_printf(pi->dip, CE_WARN, 3667 "%s: failed to initialize rxq kstats for queue %d.", 3668 __func__, idx); 3669 return (NULL); 3670 } 3671 3672 kstatp = (struct rxq_kstats *)ksp->ks_data; 3673 3674 KS_UINIT(rxcsum); 3675 KS_UINIT(rxpkts); 3676 KS_UINIT(rxbytes); 3677 KS_UINIT(nomem); 3678 3679 ksp->ks_update = update_rxq_kstats; 3680 ksp->ks_private = (void *)rxq; 3681 kstat_install(ksp); 3682 3683 return (ksp); 3684 } 3685 3686 static int 3687 update_rxq_kstats(kstat_t *ksp, int rw) 3688 { 3689 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data; 3690 struct sge_rxq *rxq = ksp->ks_private; 3691 3692 if (rw == KSTAT_WRITE) 3693 return (0); 3694 3695 KS_U_FROM(rxcsum, rxq); 3696 KS_U_FROM(rxpkts, rxq); 3697 KS_U_FROM(rxbytes, rxq); 3698 KS_U_FROM(nomem, rxq); 3699 3700 return (0); 3701 } 3702 3703 /* 3704 * cxgbe:X:txqY 3705 */ 3706 struct txq_kstats { 3707 kstat_named_t txcsum; 3708 kstat_named_t tso_wrs; 3709 kstat_named_t imm_wrs; 3710 kstat_named_t sgl_wrs; 3711 kstat_named_t txpkt_wrs; 3712 kstat_named_t txpkts_wrs; 3713 kstat_named_t txpkts_pkts; 3714 kstat_named_t txb_used; 3715 kstat_named_t hdl_used; 3716 kstat_named_t txb_full; 3717 kstat_named_t dma_hdl_failed; 3718 kstat_named_t dma_map_failed; 3719 kstat_named_t qfull; 3720 kstat_named_t qflush; 3721 kstat_named_t pullup_early; 3722 kstat_named_t pullup_late; 3723 kstat_named_t pullup_failed; 3724 }; 3725 3726 static kstat_t * 3727 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx) 3728 { 3729 struct kstat *ksp; 3730 struct txq_kstats *kstatp; 3731 int ndata; 3732 char str[16]; 3733 3734 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t); 3735 (void) snprintf(str, sizeof (str), "txq%u", idx); 3736 3737 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq", 3738 KSTAT_TYPE_NAMED, ndata, 0); 3739 if (ksp == NULL) { 3740 cxgb_printf(pi->dip, CE_WARN, 3741 "%s: failed to initialize txq kstats for queue %d.", 3742 __func__, idx); 3743 return (NULL); 3744 } 3745 3746 kstatp = (struct txq_kstats *)ksp->ks_data; 3747 3748 KS_UINIT(txcsum); 3749 KS_UINIT(tso_wrs); 3750 KS_UINIT(imm_wrs); 3751 KS_UINIT(sgl_wrs); 3752 KS_UINIT(txpkt_wrs); 3753 KS_UINIT(txpkts_wrs); 3754 KS_UINIT(txpkts_pkts); 3755 KS_UINIT(txb_used); 3756 KS_UINIT(hdl_used); 3757 KS_UINIT(txb_full); 3758 KS_UINIT(dma_hdl_failed); 3759 KS_UINIT(dma_map_failed); 3760 KS_UINIT(qfull); 3761 KS_UINIT(qflush); 3762 KS_UINIT(pullup_early); 3763 KS_UINIT(pullup_late); 3764 KS_UINIT(pullup_failed); 3765 3766 ksp->ks_update = update_txq_kstats; 3767 ksp->ks_private = (void *)txq; 3768 kstat_install(ksp); 3769 3770 return (ksp); 3771 } 3772 3773 static int 3774 update_txq_kstats(kstat_t *ksp, int rw) 3775 { 3776 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data; 3777 struct sge_txq *txq = ksp->ks_private; 3778 3779 if (rw == KSTAT_WRITE) 3780 return (0); 3781 3782 KS_U_FROM(txcsum, txq); 3783 KS_U_FROM(tso_wrs, txq); 3784 KS_U_FROM(imm_wrs, txq); 3785 KS_U_FROM(sgl_wrs, txq); 3786 KS_U_FROM(txpkt_wrs, txq); 3787 KS_U_FROM(txpkts_wrs, txq); 3788 KS_U_FROM(txpkts_pkts, txq); 3789 KS_U_FROM(txb_used, txq); 3790 KS_U_FROM(hdl_used, txq); 3791 KS_U_FROM(txb_full, txq); 3792 KS_U_FROM(dma_hdl_failed, txq); 3793 KS_U_FROM(dma_map_failed, txq); 3794 KS_U_FROM(qfull, txq); 3795 KS_U_FROM(qflush, txq); 3796 KS_U_FROM(pullup_early, txq); 3797 KS_U_FROM(pullup_late, txq); 3798 KS_U_FROM(pullup_failed, txq); 3799 3800 return (0); 3801 } 3802