1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 #include <sys/ddi.h> 24 #include <sys/sunddi.h> 25 #include <sys/sunndi.h> 26 #include <sys/modctl.h> 27 #include <sys/conf.h> 28 #include <sys/devops.h> 29 #include <sys/pci.h> 30 #include <sys/atomic.h> 31 #include <sys/types.h> 32 #include <sys/file.h> 33 #include <sys/errno.h> 34 #include <sys/open.h> 35 #include <sys/cred.h> 36 #include <sys/stat.h> 37 #include <sys/mkdev.h> 38 #include <sys/queue.h> 39 40 #include "version.h" 41 #include "common/common.h" 42 #include "common/t4_msg.h" 43 #include "common/t4_regs.h" 44 #include "firmware/t4_fw.h" 45 #include "firmware/t4_cfg.h" 46 #include "firmware/t5_fw.h" 47 #include "firmware/t5_cfg.h" 48 #include "firmware/t6_fw.h" 49 #include "firmware/t6_cfg.h" 50 #include "t4_l2t.h" 51 52 static int t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp); 53 static int t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp); 54 static int t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, 55 int *rp); 56 struct cb_ops t4_cb_ops = { 57 .cb_open = t4_cb_open, 58 .cb_close = t4_cb_close, 59 .cb_strategy = nodev, 60 .cb_print = nodev, 61 .cb_dump = nodev, 62 .cb_read = nodev, 63 .cb_write = nodev, 64 .cb_ioctl = t4_cb_ioctl, 65 .cb_devmap = nodev, 66 .cb_mmap = nodev, 67 .cb_segmap = nodev, 68 .cb_chpoll = nochpoll, 69 .cb_prop_op = ddi_prop_op, 70 .cb_flag = D_MP, 71 .cb_rev = CB_REV, 72 .cb_aread = nodev, 73 .cb_awrite = nodev 74 }; 75 76 static int t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, 77 void *arg, void *result); 78 static int t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, 79 void *arg, dev_info_t **cdipp); 80 static int t4_bus_unconfig(dev_info_t *dip, uint_t flags, 81 ddi_bus_config_op_t op, void *arg); 82 struct bus_ops t4_bus_ops = { 83 .busops_rev = BUSO_REV, 84 .bus_ctl = t4_bus_ctl, 85 .bus_prop_op = ddi_bus_prop_op, 86 .bus_config = t4_bus_config, 87 .bus_unconfig = t4_bus_unconfig, 88 }; 89 90 static int t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, 91 void **rp); 92 static int t4_devo_probe(dev_info_t *dip); 93 static int t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 94 static int t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 95 static int t4_devo_quiesce(dev_info_t *dip); 96 struct dev_ops t4_dev_ops = { 97 .devo_rev = DEVO_REV, 98 .devo_getinfo = t4_devo_getinfo, 99 .devo_identify = nulldev, 100 .devo_probe = t4_devo_probe, 101 .devo_attach = t4_devo_attach, 102 .devo_detach = t4_devo_detach, 103 .devo_reset = nodev, 104 .devo_cb_ops = &t4_cb_ops, 105 .devo_bus_ops = &t4_bus_ops, 106 .devo_quiesce = &t4_devo_quiesce, 107 }; 108 109 static struct modldrv modldrv = { 110 .drv_modops = &mod_driverops, 111 .drv_linkinfo = "Chelsio T4 nexus " DRV_VERSION, 112 .drv_dev_ops = &t4_dev_ops 113 }; 114 115 static struct modlinkage modlinkage = { 116 .ml_rev = MODREV_1, 117 .ml_linkage = {&modldrv, NULL}, 118 }; 119 120 void *t4_list; 121 122 struct intrs_and_queues { 123 int intr_type; /* DDI_INTR_TYPE_* */ 124 int nirq; /* Number of vectors */ 125 int intr_fwd; /* Interrupts forwarded */ 126 int ntxq10g; /* # of NIC txq's for each 10G port */ 127 int nrxq10g; /* # of NIC rxq's for each 10G port */ 128 int ntxq1g; /* # of NIC txq's for each 1G port */ 129 int nrxq1g; /* # of NIC rxq's for each 1G port */ 130 #ifdef TCP_OFFLOAD_ENABLE 131 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 132 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 133 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 134 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 135 #endif 136 }; 137 138 struct fw_info fi[3]; 139 140 static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, 141 mblk_t *m); 142 static int fw_msg_not_handled(struct adapter *, const __be64 *); 143 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h); 144 static unsigned int getpf(struct adapter *sc); 145 static int prep_firmware(struct adapter *sc); 146 static int upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma); 147 static int partition_resources(struct adapter *sc); 148 static int adap__pre_init_tweaks(struct adapter *sc); 149 static int get_params__pre_init(struct adapter *sc); 150 static int get_params__post_init(struct adapter *sc); 151 static int set_params__post_init(struct adapter *); 152 static void setup_memwin(struct adapter *sc); 153 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 154 uint32_t *); 155 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 156 uint32_t position_memwin(struct adapter *, int, uint32_t); 157 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data, 158 uint_t count); 159 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data, 160 uint_t count); 161 static int init_driver_props(struct adapter *sc, struct driver_properties *p); 162 static int remove_extra_props(struct adapter *sc, int n10g, int n1g); 163 static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 164 struct intrs_and_queues *iaq); 165 static int add_child_node(struct adapter *sc, int idx); 166 static int remove_child_node(struct adapter *sc, int idx); 167 static kstat_t *setup_kstats(struct adapter *sc); 168 static kstat_t *setup_wc_kstats(struct adapter *); 169 static int update_wc_kstats(kstat_t *, int); 170 #ifdef TCP_OFFLOAD_ENABLE 171 static int toe_capability(struct port_info *pi, int enable); 172 static int activate_uld(struct adapter *sc, int id, struct uld_softc *usc); 173 static int deactivate_uld(struct uld_softc *usc); 174 #endif 175 static kmutex_t t4_adapter_list_lock; 176 static SLIST_HEAD(, adapter) t4_adapter_list; 177 #ifdef TCP_OFFLOAD_ENABLE 178 static kmutex_t t4_uld_list_lock; 179 static SLIST_HEAD(, uld_info) t4_uld_list; 180 #endif 181 182 int 183 _init(void) 184 { 185 int rc; 186 187 rc = ddi_soft_state_init(&t4_list, sizeof (struct adapter), 0); 188 if (rc != 0) 189 return (rc); 190 191 rc = mod_install(&modlinkage); 192 if (rc != 0) 193 ddi_soft_state_fini(&t4_list); 194 195 mutex_init(&t4_adapter_list_lock, NULL, MUTEX_DRIVER, NULL); 196 SLIST_INIT(&t4_adapter_list); 197 198 #ifdef TCP_OFFLOAD_ENABLE 199 mutex_init(&t4_uld_list_lock, NULL, MUTEX_DRIVER, NULL); 200 SLIST_INIT(&t4_uld_list); 201 #endif 202 203 return (rc); 204 } 205 206 int 207 _fini(void) 208 { 209 int rc; 210 211 rc = mod_remove(&modlinkage); 212 if (rc != 0) 213 return (rc); 214 215 ddi_soft_state_fini(&t4_list); 216 return (0); 217 } 218 219 int 220 _info(struct modinfo *mi) 221 { 222 return (mod_info(&modlinkage, mi)); 223 } 224 225 /* ARGSUSED */ 226 static int 227 t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp) 228 { 229 struct adapter *sc; 230 minor_t minor; 231 232 minor = getminor((dev_t)arg); /* same as instance# in our case */ 233 234 if (cmd == DDI_INFO_DEVT2DEVINFO) { 235 sc = ddi_get_soft_state(t4_list, minor); 236 if (sc == NULL) 237 return (DDI_FAILURE); 238 239 ASSERT(sc->dev == (dev_t)arg); 240 *rp = (void *)sc->dip; 241 } else if (cmd == DDI_INFO_DEVT2INSTANCE) 242 *rp = (void *) (unsigned long) minor; 243 else 244 ASSERT(0); 245 246 return (DDI_SUCCESS); 247 } 248 249 static int 250 t4_devo_probe(dev_info_t *dip) 251 { 252 int rc, id, *reg; 253 uint_t n, pf; 254 255 id = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 256 "device-id", 0xffff); 257 if (id == 0xffff) 258 return (DDI_PROBE_DONTCARE); 259 260 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 261 "reg", ®, &n); 262 if (rc != DDI_SUCCESS) 263 return (DDI_PROBE_DONTCARE); 264 265 pf = PCI_REG_FUNC_G(reg[0]); 266 ddi_prop_free(reg); 267 268 /* Prevent driver attachment on any PF except 0 on the FPGA */ 269 if (id == 0xa000 && pf != 0) 270 return (DDI_PROBE_FAILURE); 271 272 return (DDI_PROBE_DONTCARE); 273 } 274 275 static int 276 t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 277 { 278 struct adapter *sc = NULL; 279 struct sge *s; 280 int i, instance, rc = DDI_SUCCESS, rqidx, tqidx, q; 281 int irq = 0, nxg, n100g, n40g, n25g, n10g, n1g; 282 #ifdef TCP_OFFLOAD_ENABLE 283 int ofld_rqidx, ofld_tqidx; 284 #endif 285 char name[16]; 286 struct driver_properties *prp; 287 struct intrs_and_queues iaq; 288 ddi_device_acc_attr_t da = { 289 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 290 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 291 .devacc_attr_dataorder = DDI_UNORDERED_OK_ACC 292 }; 293 ddi_device_acc_attr_t da1 = { 294 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 295 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 296 .devacc_attr_dataorder = DDI_MERGING_OK_ACC 297 }; 298 299 if (cmd != DDI_ATTACH) 300 return (DDI_FAILURE); 301 302 /* 303 * Allocate space for soft state. 304 */ 305 instance = ddi_get_instance(dip); 306 rc = ddi_soft_state_zalloc(t4_list, instance); 307 if (rc != DDI_SUCCESS) { 308 cxgb_printf(dip, CE_WARN, 309 "failed to allocate soft state: %d", rc); 310 return (DDI_FAILURE); 311 } 312 313 sc = ddi_get_soft_state(t4_list, instance); 314 sc->dip = dip; 315 sc->dev = makedevice(ddi_driver_major(dip), instance); 316 mutex_init(&sc->lock, NULL, MUTEX_DRIVER, NULL); 317 cv_init(&sc->cv, NULL, CV_DRIVER, NULL); 318 mutex_init(&sc->sfl_lock, NULL, MUTEX_DRIVER, NULL); 319 320 mutex_enter(&t4_adapter_list_lock); 321 SLIST_INSERT_HEAD(&t4_adapter_list, sc, link); 322 mutex_exit(&t4_adapter_list_lock); 323 324 sc->pf = getpf(sc); 325 if (sc->pf > 8) { 326 rc = EINVAL; 327 cxgb_printf(dip, CE_WARN, 328 "failed to determine PCI PF# of device"); 329 goto done; 330 } 331 sc->mbox = sc->pf; 332 333 /* Initialize the driver properties */ 334 prp = &sc->props; 335 (void)init_driver_props(sc, prp); 336 337 /* 338 * Enable access to the PCI config space. 339 */ 340 rc = pci_config_setup(dip, &sc->pci_regh); 341 if (rc != DDI_SUCCESS) { 342 cxgb_printf(dip, CE_WARN, 343 "failed to enable PCI config space access: %d", rc); 344 goto done; 345 } 346 347 /* TODO: Set max read request to 4K */ 348 349 /* 350 * Enable MMIO access. 351 */ 352 rc = ddi_regs_map_setup(dip, 1, &sc->regp, 0, 0, &da, &sc->regh); 353 if (rc != DDI_SUCCESS) { 354 cxgb_printf(dip, CE_WARN, 355 "failed to map device registers: %d", rc); 356 goto done; 357 } 358 359 (void) memset(sc->chan_map, 0xff, sizeof (sc->chan_map)); 360 361 /* 362 * Initialize cpl handler. 363 */ 364 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) { 365 sc->cpl_handler[i] = cpl_not_handled; 366 } 367 368 for (i = 0; i < ARRAY_SIZE(sc->fw_msg_handler); i++) { 369 sc->fw_msg_handler[i] = fw_msg_not_handled; 370 } 371 372 for (i = 0; i < NCHAN; i++) { 373 (void) snprintf(name, sizeof (name), "%s-%d", 374 "reclaim", i); 375 sc->tq[i] = ddi_taskq_create(sc->dip, 376 name, 1, TASKQ_DEFAULTPRI, 0); 377 378 if (sc->tq[i] == NULL) { 379 cxgb_printf(dip, CE_WARN, 380 "failed to create task queues"); 381 rc = DDI_FAILURE; 382 goto done; 383 } 384 } 385 386 /* 387 * Prepare the adapter for operation. 388 */ 389 rc = -t4_prep_adapter(sc, false); 390 if (rc != 0) { 391 cxgb_printf(dip, CE_WARN, "failed to prepare adapter: %d", rc); 392 goto done; 393 } 394 395 /* 396 * Enable BAR1 access. 397 */ 398 sc->doorbells |= DOORBELL_KDB; 399 rc = ddi_regs_map_setup(dip, 2, &sc->reg1p, 0, 0, &da1, &sc->reg1h); 400 if (rc != DDI_SUCCESS) { 401 cxgb_printf(dip, CE_WARN, 402 "failed to map BAR1 device registers: %d", rc); 403 goto done; 404 } else { 405 if (is_t5(sc->params.chip)) { 406 sc->doorbells |= DOORBELL_UDB; 407 if (prp->wc) { 408 /* 409 * Enable write combining on BAR2. This is the 410 * userspace doorbell BAR and is split into 128B 411 * (UDBS_SEG_SIZE) doorbell regions, each associated 412 * with an egress queue. The first 64B has the doorbell 413 * and the second 64B can be used to submit a tx work 414 * request with an implicit doorbell. 415 */ 416 sc->doorbells &= ~DOORBELL_UDB; 417 sc->doorbells |= (DOORBELL_WCWR | 418 DOORBELL_UDBWC); 419 t4_write_reg(sc, A_SGE_STAT_CFG, 420 V_STATSOURCE_T5(7) | V_STATMODE(0)); 421 } 422 } 423 } 424 425 /* 426 * Do this really early. Note that minor number = instance. 427 */ 428 (void) snprintf(name, sizeof (name), "%s,%d", T4_NEXUS_NAME, instance); 429 rc = ddi_create_minor_node(dip, name, S_IFCHR, instance, 430 DDI_NT_NEXUS, 0); 431 if (rc != DDI_SUCCESS) { 432 cxgb_printf(dip, CE_WARN, 433 "failed to create device node: %d", rc); 434 rc = DDI_SUCCESS; /* carry on */ 435 } 436 437 /* Do this early. Memory window is required for loading config file. */ 438 setup_memwin(sc); 439 440 /* Prepare the firmware for operation */ 441 rc = prep_firmware(sc); 442 if (rc != 0) 443 goto done; /* error message displayed already */ 444 445 rc = adap__pre_init_tweaks(sc); 446 if (rc != 0) 447 goto done; 448 449 rc = get_params__pre_init(sc); 450 if (rc != 0) 451 goto done; /* error message displayed already */ 452 453 t4_sge_init(sc); 454 455 if (sc->flags & MASTER_PF) { 456 /* get basic stuff going */ 457 rc = -t4_fw_initialize(sc, sc->mbox); 458 if (rc != 0) { 459 cxgb_printf(sc->dip, CE_WARN, 460 "early init failed: %d.\n", rc); 461 goto done; 462 } 463 } 464 465 rc = get_params__post_init(sc); 466 if (rc != 0) 467 goto done; /* error message displayed already */ 468 469 rc = set_params__post_init(sc); 470 if (rc != 0) 471 goto done; /* error message displayed already */ 472 473 /* 474 * TODO: This is the place to call t4_set_filter_mode() 475 */ 476 477 /* tweak some settings */ 478 t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) | 479 V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) | 480 V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9)); 481 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 482 483 /* 484 * Work-around for bug 2619 485 * Set DisableVlan field in TP_RSS_CONFIG_VRT register so that the 486 * VLAN tag extraction is disabled. 487 */ 488 t4_set_reg_field(sc, A_TP_RSS_CONFIG_VRT, F_DISABLEVLAN, F_DISABLEVLAN); 489 490 /* Store filter mode */ 491 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 492 A_TP_VLAN_PRI_MAP); 493 494 /* 495 * First pass over all the ports - allocate VIs and initialize some 496 * basic parameters like mac address, port type, etc. We also figure 497 * out whether a port is 10G or 1G and use that information when 498 * calculating how many interrupts to attempt to allocate. 499 */ 500 n100g = n40g = n25g = n10g = n1g = 0; 501 for_each_port(sc, i) { 502 struct port_info *pi; 503 504 pi = kmem_zalloc(sizeof (*pi), KM_SLEEP); 505 sc->port[i] = pi; 506 507 /* These must be set before t4_port_init */ 508 pi->adapter = sc; 509 /* LINTED: E_ASSIGN_NARROW_CONV */ 510 pi->port_id = i; 511 } 512 513 /* Allocate the vi and initialize parameters like mac addr */ 514 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0); 515 if (rc) { 516 cxgb_printf(dip, CE_WARN, 517 "unable to initialize port: %d", rc); 518 goto done; 519 } 520 521 for_each_port(sc, i) { 522 struct port_info *pi = sc->port[i]; 523 524 mutex_init(&pi->lock, NULL, MUTEX_DRIVER, NULL); 525 pi->mtu = ETHERMTU; 526 527 if (is_100G_port(pi)) { 528 n100g++; 529 pi->tmr_idx = prp->tmr_idx_10g; 530 pi->pktc_idx = prp->pktc_idx_10g; 531 } else if (is_40G_port(pi)) { 532 n40g++; 533 pi->tmr_idx = prp->tmr_idx_10g; 534 pi->pktc_idx = prp->pktc_idx_10g; 535 } else if (is_25G_port(pi)) { 536 n25g++; 537 pi->tmr_idx = prp->tmr_idx_10g; 538 pi->pktc_idx = prp->pktc_idx_10g; 539 } else if (is_10G_port(pi)) { 540 n10g++; 541 pi->tmr_idx = prp->tmr_idx_10g; 542 pi->pktc_idx = prp->pktc_idx_10g; 543 } else { 544 n1g++; 545 pi->tmr_idx = prp->tmr_idx_1g; 546 pi->pktc_idx = prp->pktc_idx_1g; 547 } 548 549 pi->xact_addr_filt = -1; 550 t4_mc_init(pi); 551 552 setbit(&sc->registered_device_map, i); 553 } 554 555 nxg = n10g + n25g + n40g + n100g; 556 (void) remove_extra_props(sc, nxg, n1g); 557 558 if (sc->registered_device_map == 0) { 559 cxgb_printf(dip, CE_WARN, "no usable ports"); 560 rc = DDI_FAILURE; 561 goto done; 562 } 563 564 rc = cfg_itype_and_nqueues(sc, nxg, n1g, &iaq); 565 if (rc != 0) 566 goto done; /* error message displayed already */ 567 568 sc->intr_type = iaq.intr_type; 569 sc->intr_count = iaq.nirq; 570 571 if (sc->props.multi_rings && (sc->intr_type != DDI_INTR_TYPE_MSIX)) { 572 sc->props.multi_rings = 0; 573 cxgb_printf(dip, CE_WARN, 574 "Multiple rings disabled as interrupt type is not MSI-X"); 575 } 576 577 if (sc->props.multi_rings && iaq.intr_fwd) { 578 sc->props.multi_rings = 0; 579 cxgb_printf(dip, CE_WARN, 580 "Multiple rings disabled as interrupts are forwarded"); 581 } 582 583 if (!sc->props.multi_rings) { 584 iaq.ntxq10g = 1; 585 iaq.ntxq1g = 1; 586 } 587 s = &sc->sge; 588 s->nrxq = nxg * iaq.nrxq10g + n1g * iaq.nrxq1g; 589 s->ntxq = nxg * iaq.ntxq10g + n1g * iaq.ntxq1g; 590 s->neq = s->ntxq + s->nrxq; /* the fl in an rxq is an eq */ 591 #ifdef TCP_OFFLOAD_ENABLE 592 /* control queues, 1 per port + 1 mgmtq */ 593 s->neq += sc->params.nports + 1; 594 #endif 595 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 596 if (iaq.intr_fwd != 0) 597 sc->flags |= INTR_FWD; 598 #ifdef TCP_OFFLOAD_ENABLE 599 if (is_offload(sc) != 0) { 600 601 s->nofldrxq = nxg * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 602 s->nofldtxq = nxg * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 603 s->neq += s->nofldtxq + s->nofldrxq; 604 s->niq += s->nofldrxq; 605 606 s->ofld_rxq = kmem_zalloc(s->nofldrxq * 607 sizeof (struct sge_ofld_rxq), KM_SLEEP); 608 s->ofld_txq = kmem_zalloc(s->nofldtxq * 609 sizeof (struct sge_wrq), KM_SLEEP); 610 s->ctrlq = kmem_zalloc(sc->params.nports * 611 sizeof (struct sge_wrq), KM_SLEEP); 612 613 } 614 #endif 615 s->rxq = kmem_zalloc(s->nrxq * sizeof (struct sge_rxq), KM_SLEEP); 616 s->txq = kmem_zalloc(s->ntxq * sizeof (struct sge_txq), KM_SLEEP); 617 s->iqmap = kmem_zalloc(s->niq * sizeof (struct sge_iq *), KM_SLEEP); 618 s->eqmap = kmem_zalloc(s->neq * sizeof (struct sge_eq *), KM_SLEEP); 619 620 sc->intr_handle = kmem_zalloc(sc->intr_count * 621 sizeof (ddi_intr_handle_t), KM_SLEEP); 622 623 /* 624 * Second pass over the ports. This time we know the number of rx and 625 * tx queues that each port should get. 626 */ 627 rqidx = tqidx = 0; 628 #ifdef TCP_OFFLOAD_ENABLE 629 ofld_rqidx = ofld_tqidx = 0; 630 #endif 631 for_each_port(sc, i) { 632 struct port_info *pi = sc->port[i]; 633 634 if (pi == NULL) 635 continue; 636 637 t4_mc_cb_init(pi); 638 /* LINTED: E_ASSIGN_NARROW_CONV */ 639 pi->first_rxq = rqidx; 640 /* LINTED: E_ASSIGN_NARROW_CONV */ 641 pi->nrxq = (is_10XG_port(pi)) ? iaq.nrxq10g 642 : iaq.nrxq1g; 643 /* LINTED: E_ASSIGN_NARROW_CONV */ 644 pi->first_txq = tqidx; 645 /* LINTED: E_ASSIGN_NARROW_CONV */ 646 pi->ntxq = (is_10XG_port(pi)) ? iaq.ntxq10g 647 : iaq.ntxq1g; 648 649 rqidx += pi->nrxq; 650 tqidx += pi->ntxq; 651 652 #ifdef TCP_OFFLOAD_ENABLE 653 if (is_offload(sc) != 0) { 654 /* LINTED: E_ASSIGN_NARROW_CONV */ 655 pi->first_ofld_rxq = ofld_rqidx; 656 pi->nofldrxq = max(1, pi->nrxq / 4); 657 658 /* LINTED: E_ASSIGN_NARROW_CONV */ 659 pi->first_ofld_txq = ofld_tqidx; 660 pi->nofldtxq = max(1, pi->ntxq / 2); 661 662 ofld_rqidx += pi->nofldrxq; 663 ofld_tqidx += pi->nofldtxq; 664 } 665 #endif 666 667 /* 668 * Enable hw checksumming and LSO for all ports by default. 669 * They can be disabled using ndd (hw_csum and hw_lso). 670 */ 671 pi->features |= (CXGBE_HW_CSUM | CXGBE_HW_LSO); 672 } 673 674 #ifdef TCP_OFFLOAD_ENABLE 675 sc->l2t = t4_init_l2t(sc); 676 #endif 677 678 /* 679 * Setup Interrupts. 680 */ 681 682 i = 0; 683 rc = ddi_intr_alloc(dip, sc->intr_handle, sc->intr_type, 0, 684 sc->intr_count, &i, DDI_INTR_ALLOC_STRICT); 685 if (rc != DDI_SUCCESS) { 686 cxgb_printf(dip, CE_WARN, 687 "failed to allocate %d interrupt(s) of type %d: %d, %d", 688 sc->intr_count, sc->intr_type, rc, i); 689 goto done; 690 } 691 ASSERT(sc->intr_count == i); /* allocation was STRICT */ 692 (void) ddi_intr_get_cap(sc->intr_handle[0], &sc->intr_cap); 693 (void) ddi_intr_get_pri(sc->intr_handle[0], &sc->intr_pri); 694 if (sc->intr_count == 1) { 695 ASSERT(sc->flags & INTR_FWD); 696 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_all, sc, 697 &s->fwq); 698 } else { 699 /* Multiple interrupts. The first one is always error intr */ 700 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_err, sc, 701 NULL); 702 irq++; 703 704 /* The second one is always the firmware event queue */ 705 (void) ddi_intr_add_handler(sc->intr_handle[1], t4_intr, sc, 706 &s->fwq); 707 irq++; 708 /* 709 * Note that if INTR_FWD is set then either the NIC rx 710 * queues or (exclusive or) the TOE rx queueus will be taking 711 * direct interrupts. 712 * 713 * There is no need to check for is_offload(sc) as nofldrxq 714 * will be 0 if offload is disabled. 715 */ 716 for_each_port(sc, i) { 717 struct port_info *pi = sc->port[i]; 718 struct sge_rxq *rxq; 719 #ifdef TCP_OFFLOAD_ENABLE 720 struct sge_ofld_rxq *ofld_rxq; 721 722 /* 723 * Skip over the NIC queues if they aren't taking direct 724 * interrupts. 725 */ 726 if ((sc->flags & INTR_FWD) && 727 pi->nofldrxq > pi->nrxq) 728 goto ofld_queues; 729 #endif 730 rxq = &s->rxq[pi->first_rxq]; 731 for (q = 0; q < pi->nrxq; q++, rxq++) { 732 (void) ddi_intr_add_handler( 733 sc->intr_handle[irq], t4_intr, sc, 734 &rxq->iq); 735 irq++; 736 } 737 738 #ifdef TCP_OFFLOAD_ENABLE 739 /* 740 * Skip over the offload queues if they aren't taking 741 * direct interrupts. 742 */ 743 if ((sc->flags & INTR_FWD)) 744 continue; 745 ofld_queues: 746 ofld_rxq = &s->ofld_rxq[pi->first_ofld_rxq]; 747 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 748 (void) ddi_intr_add_handler( 749 sc->intr_handle[irq], t4_intr, sc, 750 &ofld_rxq->iq); 751 irq++; 752 } 753 #endif 754 } 755 756 } 757 sc->flags |= INTR_ALLOCATED; 758 759 ASSERT(rc == DDI_SUCCESS); 760 ddi_report_dev(dip); 761 762 /* 763 * Hardware/Firmware/etc. Version/Revision IDs. 764 */ 765 t4_dump_version_info(sc); 766 767 if (n100g) { 768 cxgb_printf(dip, CE_NOTE, 769 "%dx100G (%d rxq, %d txq total) %d %s.", 770 n100g, rqidx, tqidx, sc->intr_count, 771 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 772 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 773 "fixed interrupt"); 774 } else if (n40g) { 775 cxgb_printf(dip, CE_NOTE, 776 "%dx40G (%d rxq, %d txq total) %d %s.", 777 n40g, rqidx, tqidx, sc->intr_count, 778 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 779 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 780 "fixed interrupt"); 781 } else if (n25g) { 782 cxgb_printf(dip, CE_NOTE, 783 "%dx25G (%d rxq, %d txq total) %d %s.", 784 n25g, rqidx, tqidx, sc->intr_count, 785 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 786 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 787 "fixed interrupt"); 788 } else if (n10g && n1g) { 789 cxgb_printf(dip, CE_NOTE, 790 "%dx10G %dx1G (%d rxq, %d txq total) %d %s.", 791 n10g, n1g, rqidx, tqidx, sc->intr_count, 792 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 793 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 794 "fixed interrupt"); 795 } else { 796 cxgb_printf(dip, CE_NOTE, 797 "%dx%sG (%d rxq, %d txq per port) %d %s.", 798 n10g ? n10g : n1g, 799 n10g ? "10" : "1", 800 n10g ? iaq.nrxq10g : iaq.nrxq1g, 801 n10g ? iaq.ntxq10g : iaq.ntxq1g, 802 sc->intr_count, 803 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 804 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 805 "fixed interrupt"); 806 } 807 808 sc->ksp = setup_kstats(sc); 809 sc->ksp_stat = setup_wc_kstats(sc); 810 sc->params.drv_memwin = MEMWIN_NIC; 811 812 done: 813 if (rc != DDI_SUCCESS) { 814 (void) t4_devo_detach(dip, DDI_DETACH); 815 816 /* rc may have errno style errors or DDI errors */ 817 rc = DDI_FAILURE; 818 } 819 820 return (rc); 821 } 822 823 static int 824 t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 825 { 826 int instance, i; 827 struct adapter *sc; 828 struct port_info *pi; 829 struct sge *s; 830 831 if (cmd != DDI_DETACH) 832 return (DDI_FAILURE); 833 834 instance = ddi_get_instance(dip); 835 sc = ddi_get_soft_state(t4_list, instance); 836 if (sc == NULL) 837 return (DDI_SUCCESS); 838 839 if (sc->flags & FULL_INIT_DONE) { 840 t4_intr_disable(sc); 841 for_each_port(sc, i) { 842 pi = sc->port[i]; 843 if (pi && pi->flags & PORT_INIT_DONE) 844 (void) port_full_uninit(pi); 845 } 846 (void) adapter_full_uninit(sc); 847 } 848 849 /* Safe to call no matter what */ 850 ddi_prop_remove_all(dip); 851 ddi_remove_minor_node(dip, NULL); 852 853 for (i = 0; i < NCHAN; i++) { 854 if (sc->tq[i]) { 855 ddi_taskq_wait(sc->tq[i]); 856 ddi_taskq_destroy(sc->tq[i]); 857 } 858 } 859 860 if (sc->ksp != NULL) 861 kstat_delete(sc->ksp); 862 if (sc->ksp_stat != NULL) 863 kstat_delete(sc->ksp_stat); 864 865 s = &sc->sge; 866 if (s->rxq != NULL) 867 kmem_free(s->rxq, s->nrxq * sizeof (struct sge_rxq)); 868 #ifdef TCP_OFFLOAD_ENABLE 869 if (s->ofld_txq != NULL) 870 kmem_free(s->ofld_txq, s->nofldtxq * sizeof (struct sge_wrq)); 871 if (s->ofld_rxq != NULL) 872 kmem_free(s->ofld_rxq, 873 s->nofldrxq * sizeof (struct sge_ofld_rxq)); 874 if (s->ctrlq != NULL) 875 kmem_free(s->ctrlq, 876 sc->params.nports * sizeof (struct sge_wrq)); 877 #endif 878 if (s->txq != NULL) 879 kmem_free(s->txq, s->ntxq * sizeof (struct sge_txq)); 880 if (s->iqmap != NULL) 881 kmem_free(s->iqmap, s->niq * sizeof (struct sge_iq *)); 882 if (s->eqmap != NULL) 883 kmem_free(s->eqmap, s->neq * sizeof (struct sge_eq *)); 884 885 if (s->rxbuf_cache != NULL) 886 rxbuf_cache_destroy(s->rxbuf_cache); 887 888 if (sc->flags & INTR_ALLOCATED) { 889 for (i = 0; i < sc->intr_count; i++) { 890 (void) ddi_intr_remove_handler(sc->intr_handle[i]); 891 (void) ddi_intr_free(sc->intr_handle[i]); 892 } 893 sc->flags &= ~INTR_ALLOCATED; 894 } 895 896 if (sc->intr_handle != NULL) { 897 kmem_free(sc->intr_handle, 898 sc->intr_count * sizeof (*sc->intr_handle)); 899 } 900 901 for_each_port(sc, i) { 902 pi = sc->port[i]; 903 if (pi != NULL) { 904 mutex_destroy(&pi->lock); 905 kmem_free(pi, sizeof (*pi)); 906 clrbit(&sc->registered_device_map, i); 907 } 908 } 909 910 if (sc->flags & FW_OK) 911 (void) t4_fw_bye(sc, sc->mbox); 912 913 if (sc->reg1h != NULL) 914 ddi_regs_map_free(&sc->reg1h); 915 916 if (sc->regh != NULL) 917 ddi_regs_map_free(&sc->regh); 918 919 if (sc->pci_regh != NULL) 920 pci_config_teardown(&sc->pci_regh); 921 922 mutex_enter(&t4_adapter_list_lock); 923 SLIST_REMOVE_HEAD(&t4_adapter_list, link); 924 mutex_exit(&t4_adapter_list_lock); 925 926 mutex_destroy(&sc->lock); 927 cv_destroy(&sc->cv); 928 mutex_destroy(&sc->sfl_lock); 929 930 #ifdef DEBUG 931 bzero(sc, sizeof (*sc)); 932 #endif 933 ddi_soft_state_free(t4_list, instance); 934 935 return (DDI_SUCCESS); 936 } 937 938 static int 939 t4_devo_quiesce(dev_info_t *dip) 940 { 941 int instance; 942 struct adapter *sc; 943 944 instance = ddi_get_instance(dip); 945 sc = ddi_get_soft_state(t4_list, instance); 946 if (sc == NULL) 947 return (DDI_SUCCESS); 948 949 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 950 t4_intr_disable(sc); 951 t4_write_reg(sc, A_PL_RST, F_PIORSTMODE | F_PIORST); 952 953 return (DDI_SUCCESS); 954 } 955 956 static int 957 t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg, 958 void *result) 959 { 960 char s[4]; 961 struct port_info *pi; 962 dev_info_t *child = (dev_info_t *)arg; 963 964 switch (op) { 965 case DDI_CTLOPS_REPORTDEV: 966 pi = ddi_get_parent_data(rdip); 967 pi->instance = ddi_get_instance(dip); 968 pi->child_inst = ddi_get_instance(rdip); 969 cmn_err(CE_CONT, "?%s%d is port %s on %s%d\n", 970 ddi_node_name(rdip), ddi_get_instance(rdip), 971 ddi_get_name_addr(rdip), ddi_driver_name(dip), 972 ddi_get_instance(dip)); 973 return (DDI_SUCCESS); 974 975 case DDI_CTLOPS_INITCHILD: 976 pi = ddi_get_parent_data(child); 977 if (pi == NULL) 978 return (DDI_NOT_WELL_FORMED); 979 (void) snprintf(s, sizeof (s), "%d", pi->port_id); 980 ddi_set_name_addr(child, s); 981 return (DDI_SUCCESS); 982 983 case DDI_CTLOPS_UNINITCHILD: 984 ddi_set_name_addr(child, NULL); 985 return (DDI_SUCCESS); 986 987 case DDI_CTLOPS_ATTACH: 988 case DDI_CTLOPS_DETACH: 989 return (DDI_SUCCESS); 990 991 default: 992 return (ddi_ctlops(dip, rdip, op, arg, result)); 993 } 994 } 995 996 static int 997 t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, void *arg, 998 dev_info_t **cdipp) 999 { 1000 int instance, i; 1001 struct adapter *sc; 1002 1003 instance = ddi_get_instance(dip); 1004 sc = ddi_get_soft_state(t4_list, instance); 1005 1006 if (op == BUS_CONFIG_ONE) { 1007 char *c; 1008 1009 /* 1010 * arg is something like "cxgb@0" where 0 is the port_id hanging 1011 * off this nexus. 1012 */ 1013 1014 c = arg; 1015 while (*(c + 1)) 1016 c++; 1017 1018 /* There should be exactly 1 digit after '@' */ 1019 if (*(c - 1) != '@') 1020 return (NDI_FAILURE); 1021 1022 i = *c - '0'; 1023 1024 if (add_child_node(sc, i) != 0) 1025 return (NDI_FAILURE); 1026 1027 flags |= NDI_ONLINE_ATTACH; 1028 1029 } else if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) { 1030 /* Allocate and bind all child device nodes */ 1031 for_each_port(sc, i) 1032 (void) add_child_node(sc, i); 1033 flags |= NDI_ONLINE_ATTACH; 1034 } 1035 1036 return (ndi_busop_bus_config(dip, flags, op, arg, cdipp, 0)); 1037 } 1038 1039 static int 1040 t4_bus_unconfig(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, 1041 void *arg) 1042 { 1043 int instance, i, rc; 1044 struct adapter *sc; 1045 1046 instance = ddi_get_instance(dip); 1047 sc = ddi_get_soft_state(t4_list, instance); 1048 1049 if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ALL || 1050 op == BUS_UNCONFIG_DRIVER) 1051 flags |= NDI_UNCONFIG; 1052 1053 rc = ndi_busop_bus_unconfig(dip, flags, op, arg); 1054 if (rc != 0) 1055 return (rc); 1056 1057 if (op == BUS_UNCONFIG_ONE) { 1058 char *c; 1059 1060 c = arg; 1061 while (*(c + 1)) 1062 c++; 1063 1064 if (*(c - 1) != '@') 1065 return (NDI_SUCCESS); 1066 1067 i = *c - '0'; 1068 1069 rc = remove_child_node(sc, i); 1070 1071 } else if (op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) { 1072 1073 for_each_port(sc, i) 1074 (void) remove_child_node(sc, i); 1075 } 1076 1077 return (rc); 1078 } 1079 1080 /* ARGSUSED */ 1081 static int 1082 t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp) 1083 { 1084 struct adapter *sc; 1085 1086 if (otyp != OTYP_CHR) 1087 return (EINVAL); 1088 1089 sc = ddi_get_soft_state(t4_list, getminor(*devp)); 1090 if (sc == NULL) 1091 return (ENXIO); 1092 1093 return (atomic_cas_uint(&sc->open, 0, EBUSY)); 1094 } 1095 1096 /* ARGSUSED */ 1097 static int 1098 t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp) 1099 { 1100 struct adapter *sc; 1101 1102 sc = ddi_get_soft_state(t4_list, getminor(dev)); 1103 if (sc == NULL) 1104 return (EINVAL); 1105 1106 (void) atomic_swap_uint(&sc->open, 0); 1107 return (0); 1108 } 1109 1110 /* ARGSUSED */ 1111 static int 1112 t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, int *rp) 1113 { 1114 int instance; 1115 struct adapter *sc; 1116 void *data = (void *)d; 1117 1118 if (crgetuid(credp) != 0) 1119 return (EPERM); 1120 1121 instance = getminor(dev); 1122 sc = ddi_get_soft_state(t4_list, instance); 1123 if (sc == NULL) 1124 return (EINVAL); 1125 1126 return (t4_ioctl(sc, cmd, data, mode)); 1127 } 1128 1129 static unsigned int 1130 getpf(struct adapter *sc) 1131 { 1132 int rc, *data; 1133 uint_t n, pf; 1134 1135 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip, 1136 DDI_PROP_DONTPASS, "reg", &data, &n); 1137 if (rc != DDI_SUCCESS) { 1138 cxgb_printf(sc->dip, CE_WARN, 1139 "failed to lookup \"reg\" property: %d", rc); 1140 return (0xff); 1141 } 1142 1143 pf = PCI_REG_FUNC_G(data[0]); 1144 ddi_prop_free(data); 1145 1146 return (pf); 1147 } 1148 1149 1150 static struct fw_info * 1151 find_fw_info(int chip) 1152 { 1153 u32 i; 1154 1155 fi[0].chip = CHELSIO_T4; 1156 fi[0].fw_hdr.chip = FW_HDR_CHIP_T4; 1157 fi[0].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T4)); 1158 fi[0].fw_hdr.intfver_nic = FW_INTFVER(T4, NIC); 1159 fi[0].fw_hdr.intfver_vnic = FW_INTFVER(T4, VNIC); 1160 fi[0].fw_hdr.intfver_ofld = FW_INTFVER(T4, OFLD); 1161 fi[0].fw_hdr.intfver_ri = FW_INTFVER(T4, RI); 1162 fi[0].fw_hdr.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU); 1163 fi[0].fw_hdr.intfver_iscsi = FW_INTFVER(T4, ISCSI); 1164 fi[0].fw_hdr.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU); 1165 fi[0].fw_hdr.intfver_fcoe = FW_INTFVER(T4, FCOE); 1166 1167 fi[1].chip = CHELSIO_T5; 1168 fi[1].fw_hdr.chip = FW_HDR_CHIP_T5; 1169 fi[1].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T5)); 1170 fi[1].fw_hdr.intfver_nic = FW_INTFVER(T5, NIC); 1171 fi[1].fw_hdr.intfver_vnic = FW_INTFVER(T5, VNIC); 1172 fi[1].fw_hdr.intfver_ofld = FW_INTFVER(T5, OFLD); 1173 fi[1].fw_hdr.intfver_ri = FW_INTFVER(T5, RI); 1174 fi[1].fw_hdr.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU); 1175 fi[1].fw_hdr.intfver_iscsi = FW_INTFVER(T5, ISCSI); 1176 fi[1].fw_hdr.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU); 1177 fi[1].fw_hdr.intfver_fcoe = FW_INTFVER(T5, FCOE); 1178 1179 fi[2].chip = CHELSIO_T6; 1180 fi[2].fw_hdr.chip = FW_HDR_CHIP_T6; 1181 fi[2].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T6)); 1182 fi[2].fw_hdr.intfver_nic = FW_INTFVER(T6, NIC); 1183 fi[2].fw_hdr.intfver_vnic = FW_INTFVER(T6, VNIC); 1184 fi[2].fw_hdr.intfver_ofld = FW_INTFVER(T6, OFLD); 1185 fi[2].fw_hdr.intfver_ri = FW_INTFVER(T6, RI); 1186 fi[2].fw_hdr.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU); 1187 fi[2].fw_hdr.intfver_iscsi = FW_INTFVER(T6, ISCSI); 1188 fi[2].fw_hdr.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU); 1189 fi[2].fw_hdr.intfver_fcoe = FW_INTFVER(T6, FCOE); 1190 1191 for (i = 0; i < ARRAY_SIZE(fi); i++) { 1192 if (fi[i].chip == chip) 1193 return &fi[i]; 1194 } 1195 1196 return NULL; 1197 } 1198 1199 /* 1200 * Install a compatible firmware (if required), establish contact with it, 1201 * become the master, and reset the device. 1202 */ 1203 static int 1204 prep_firmware(struct adapter *sc) 1205 { 1206 int rc; 1207 int fw_size; 1208 int reset = 1; 1209 enum dev_state state; 1210 unsigned char *fw_data; 1211 struct fw_info *fw_info; 1212 struct fw_hdr *card_fw; 1213 1214 struct driver_properties *p = &sc->props; 1215 1216 /* Contact firmware, request master */ 1217 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state); 1218 if (rc < 0) { 1219 rc = -rc; 1220 cxgb_printf(sc->dip, CE_WARN, 1221 "failed to connect to the firmware: %d.", rc); 1222 return (rc); 1223 } 1224 1225 if (rc == sc->mbox) 1226 sc->flags |= MASTER_PF; 1227 1228 /* We may need FW version info for later reporting */ 1229 t4_get_version_info(sc); 1230 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(sc->params.chip)); 1231 /* allocate memory to read the header of the firmware on the 1232 * card 1233 */ 1234 if (!fw_info) { 1235 cxgb_printf(sc->dip, CE_WARN, 1236 "unable to look up firmware information for chip %d.\n", 1237 CHELSIO_CHIP_VERSION(sc->params.chip)); 1238 return EINVAL; 1239 } 1240 card_fw = kmem_zalloc(sizeof(*card_fw), KM_SLEEP); 1241 if(!card_fw) { 1242 cxgb_printf(sc->dip, CE_WARN, 1243 "Memory allocation for card FW header failed\n"); 1244 return ENOMEM; 1245 } 1246 switch(CHELSIO_CHIP_VERSION(sc->params.chip)) { 1247 case CHELSIO_T4: 1248 fw_data = t4fw_data; 1249 fw_size = t4fw_size; 1250 break; 1251 case CHELSIO_T5: 1252 fw_data = t5fw_data; 1253 fw_size = t5fw_size; 1254 break; 1255 case CHELSIO_T6: 1256 fw_data = t6fw_data; 1257 fw_size = t6fw_size; 1258 break; 1259 default: 1260 cxgb_printf(sc->dip, CE_WARN, "Adapter type not supported\n"); 1261 kmem_free(card_fw, sizeof(*card_fw)); 1262 return EINVAL; 1263 } 1264 1265 rc = -t4_prep_fw(sc, fw_info, fw_data, fw_size, card_fw, 1266 p->t4_fw_install, state, &reset); 1267 1268 kmem_free(card_fw, sizeof(*card_fw)); 1269 1270 if (rc != 0) { 1271 cxgb_printf(sc->dip, CE_WARN, 1272 "failed to install firmware: %d", rc); 1273 return (rc); 1274 } else { 1275 /* refresh */ 1276 (void) t4_check_fw_version(sc); 1277 } 1278 1279 /* Reset device */ 1280 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1281 if (rc != 0) { 1282 cxgb_printf(sc->dip, CE_WARN, 1283 "firmware reset failed: %d.", rc); 1284 if (rc != ETIMEDOUT && rc != EIO) 1285 (void) t4_fw_bye(sc, sc->mbox); 1286 return (rc); 1287 } 1288 1289 /* Partition adapter resources as specified in the config file. */ 1290 if (sc->flags & MASTER_PF) { 1291 /* Handle default vs special T4 config file */ 1292 1293 rc = partition_resources(sc); 1294 if (rc != 0) 1295 goto err; /* error message displayed already */ 1296 } 1297 1298 sc->flags |= FW_OK; 1299 return (0); 1300 err: 1301 return (rc); 1302 1303 } 1304 1305 static const struct memwin t4_memwin[] = { 1306 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1307 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1308 { MEMWIN2_BASE, MEMWIN2_APERTURE } 1309 }; 1310 1311 static const struct memwin t5_memwin[] = { 1312 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1313 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1314 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1315 }; 1316 1317 #define FW_PARAM_DEV(param) \ 1318 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1319 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1320 #define FW_PARAM_PFVF(param) \ 1321 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1322 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1323 1324 /* 1325 * Verify that the memory range specified by the memtype/offset/len pair is 1326 * valid and lies entirely within the memtype specified. The global address of 1327 * the start of the range is returned in addr. 1328 */ 1329 int 1330 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1331 uint32_t *addr) 1332 { 1333 uint32_t em, addr_len, maddr, mlen; 1334 1335 /* Memory can only be accessed in naturally aligned 4 byte units */ 1336 if (off & 3 || len & 3 || len == 0) 1337 return (EINVAL); 1338 1339 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1340 switch (mtype) { 1341 case MEM_EDC0: 1342 if (!(em & F_EDRAM0_ENABLE)) 1343 return (EINVAL); 1344 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1345 maddr = G_EDRAM0_BASE(addr_len) << 20; 1346 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1347 break; 1348 case MEM_EDC1: 1349 if (!(em & F_EDRAM1_ENABLE)) 1350 return (EINVAL); 1351 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1352 maddr = G_EDRAM1_BASE(addr_len) << 20; 1353 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1354 break; 1355 case MEM_MC: 1356 if (!(em & F_EXT_MEM_ENABLE)) 1357 return (EINVAL); 1358 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1359 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1360 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1361 break; 1362 case MEM_MC1: 1363 if (is_t4(sc->params.chip) || !(em & F_EXT_MEM1_ENABLE)) 1364 return (EINVAL); 1365 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1366 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1367 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1368 break; 1369 default: 1370 return (EINVAL); 1371 } 1372 1373 if (mlen > 0 && off < mlen && off + len <= mlen) { 1374 *addr = maddr + off; /* global address */ 1375 return (0); 1376 } 1377 1378 return (EFAULT); 1379 } 1380 1381 void 1382 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1383 { 1384 const struct memwin *mw; 1385 1386 if (is_t4(sc->params.chip)) { 1387 mw = &t4_memwin[win]; 1388 } else { 1389 mw = &t5_memwin[win]; 1390 } 1391 1392 if (base != NULL) 1393 *base = mw->base; 1394 if (aperture != NULL) 1395 *aperture = mw->aperture; 1396 } 1397 1398 /* 1399 * Upload configuration file to card's memory. 1400 */ 1401 static int 1402 upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma) 1403 { 1404 int rc = 0, cflen; 1405 u_int i, n; 1406 uint32_t param, val, addr, mtype, maddr; 1407 uint32_t off, mw_base, mw_aperture; 1408 const uint32_t *cfdata; 1409 1410 /* Figure out where the firmware wants us to upload it. */ 1411 param = FW_PARAM_DEV(CF); 1412 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1413 if (rc != 0) { 1414 /* Firmwares without config file support will fail this way */ 1415 cxgb_printf(sc->dip, CE_WARN, 1416 "failed to query config file location: %d.\n", rc); 1417 return (rc); 1418 } 1419 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1420 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1421 1422 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 1423 case CHELSIO_T4: 1424 cflen = t4cfg_size & ~3; 1425 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1426 cfdata = (const uint32_t *)t4cfg_data; 1427 break; 1428 case CHELSIO_T5: 1429 cflen = t5cfg_size & ~3; 1430 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1431 cfdata = (const uint32_t *)t5cfg_data; 1432 break; 1433 case CHELSIO_T6: 1434 cflen = t6cfg_size & ~3; 1435 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1436 cfdata = (const uint32_t *)t6cfg_data; 1437 break; 1438 default: 1439 cxgb_printf(sc->dip, CE_WARN, 1440 "Invalid Adapter detected\n"); 1441 return EINVAL; 1442 } 1443 1444 if (cflen > FLASH_CFG_MAX_SIZE) { 1445 cxgb_printf(sc->dip, CE_WARN, 1446 "config file too long (%d, max allowed is %d). ", 1447 cflen, FLASH_CFG_MAX_SIZE); 1448 return (EFBIG); 1449 } 1450 1451 rc = validate_mt_off_len(sc, mtype, maddr, cflen, &addr); 1452 if (rc != 0) { 1453 1454 cxgb_printf(sc->dip, CE_WARN, 1455 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 1456 "Will try to use the config on the card, if any.\n", 1457 __func__, mtype, maddr, cflen, rc); 1458 return (EFAULT); 1459 } 1460 1461 memwin_info(sc, 2, &mw_base, &mw_aperture); 1462 while (cflen) { 1463 off = position_memwin(sc, 2, addr); 1464 n = min(cflen, mw_aperture - off); 1465 for (i = 0; i < n; i += 4) 1466 t4_write_reg(sc, mw_base + off + i, *cfdata++); 1467 cflen -= n; 1468 addr += n; 1469 } 1470 1471 return (rc); 1472 } 1473 1474 /* 1475 * Partition chip resources for use between various PFs, VFs, etc. This is done 1476 * by uploading the firmware configuration file to the adapter and instructing 1477 * the firmware to process it. 1478 */ 1479 static int 1480 partition_resources(struct adapter *sc) 1481 { 1482 int rc; 1483 struct fw_caps_config_cmd caps; 1484 uint32_t mtype, maddr, finicsum, cfcsum; 1485 1486 rc = upload_config_file(sc, &mtype, &maddr); 1487 if (rc != 0) { 1488 mtype = FW_MEMTYPE_CF_FLASH; 1489 maddr = t4_flash_cfg_addr(sc); 1490 } 1491 1492 bzero(&caps, sizeof (caps)); 1493 caps.op_to_write = BE_32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1494 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1495 caps.cfvalid_to_len16 = BE_32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1496 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1497 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1498 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps); 1499 if (rc != 0) { 1500 cxgb_printf(sc->dip, CE_WARN, 1501 "failed to pre-process config file: %d.\n", rc); 1502 return (rc); 1503 } 1504 1505 finicsum = ntohl(caps.finicsum); 1506 cfcsum = ntohl(caps.cfcsum); 1507 if (finicsum != cfcsum) { 1508 cxgb_printf(sc->dip, CE_WARN, 1509 "WARNING: config file checksum mismatch: %08x %08x\n", 1510 finicsum, cfcsum); 1511 } 1512 sc->cfcsum = cfcsum; 1513 1514 /* TODO: Need to configure this correctly */ 1515 caps.toecaps = htons(FW_CAPS_CONFIG_TOE); 1516 caps.iscsicaps = 0; 1517 caps.rdmacaps = 0; 1518 caps.fcoecaps = 0; 1519 /* TODO: Disable VNIC cap for now */ 1520 caps.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM); 1521 1522 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1523 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1524 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps)); 1525 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), NULL); 1526 if (rc != 0) { 1527 cxgb_printf(sc->dip, CE_WARN, 1528 "failed to process config file: %d.\n", rc); 1529 return (rc); 1530 } 1531 1532 return (0); 1533 } 1534 1535 /* 1536 * Tweak configuration based on module parameters, etc. Most of these have 1537 * defaults assigned to them by Firmware Configuration Files (if we're using 1538 * them) but need to be explicitly set if we're using hard-coded 1539 * initialization. But even in the case of using Firmware Configuration 1540 * Files, we'd like to expose the ability to change these via module 1541 * parameters so these are essentially common tweaks/settings for 1542 * Configuration Files and hard-coded initialization ... 1543 */ 1544 static int 1545 adap__pre_init_tweaks(struct adapter *sc) 1546 { 1547 int rx_dma_offset = 2; /* Offset of RX packets into DMA buffers */ 1548 1549 /* 1550 * Fix up various Host-Dependent Parameters like Page Size, Cache 1551 * Line Size, etc. The firmware default is for a 4KB Page Size and 1552 * 64B Cache Line Size ... 1553 */ 1554 (void) t4_fixup_host_params_compat(sc, PAGE_SIZE, CACHE_LINE, T5_LAST_REV); 1555 1556 t4_set_reg_field(sc, A_SGE_CONTROL, 1557 V_PKTSHIFT(M_PKTSHIFT), V_PKTSHIFT(rx_dma_offset)); 1558 1559 return 0; 1560 } 1561 /* 1562 * Retrieve parameters that are needed (or nice to have) prior to calling 1563 * t4_sge_init and t4_fw_initialize. 1564 */ 1565 static int 1566 get_params__pre_init(struct adapter *sc) 1567 { 1568 int rc; 1569 uint32_t param[2], val[2]; 1570 struct fw_devlog_cmd cmd; 1571 struct devlog_params *dlog = &sc->params.devlog; 1572 1573 /* 1574 * Grab the raw VPD parameters. 1575 */ 1576 rc = -t4_get_raw_vpd_params(sc, &sc->params.vpd); 1577 if (rc != 0) { 1578 cxgb_printf(sc->dip, CE_WARN, 1579 "failed to query VPD parameters (pre_init): %d.\n", rc); 1580 return (rc); 1581 } 1582 1583 param[0] = FW_PARAM_DEV(PORTVEC); 1584 param[1] = FW_PARAM_DEV(CCLK); 1585 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1586 if (rc != 0) { 1587 cxgb_printf(sc->dip, CE_WARN, 1588 "failed to query parameters (pre_init): %d.\n", rc); 1589 return (rc); 1590 } 1591 1592 sc->params.portvec = val[0]; 1593 sc->params.nports = 0; 1594 while (val[0]) { 1595 sc->params.nports++; 1596 val[0] &= val[0] - 1; 1597 } 1598 1599 sc->params.vpd.cclk = val[1]; 1600 1601 /* Read device log parameters. */ 1602 bzero(&cmd, sizeof (cmd)); 1603 cmd.op_to_write = htonl(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1604 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1605 cmd.retval_len16 = htonl(FW_LEN16(cmd)); 1606 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof (cmd), &cmd); 1607 if (rc != 0) { 1608 cxgb_printf(sc->dip, CE_WARN, 1609 "failed to get devlog parameters: %d.\n", rc); 1610 bzero(dlog, sizeof (*dlog)); 1611 rc = 0; /* devlog isn't critical for device operation */ 1612 } else { 1613 val[0] = ntohl(cmd.memtype_devlog_memaddr16_devlog); 1614 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1615 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1616 dlog->size = ntohl(cmd.memsize_devlog); 1617 } 1618 1619 return (rc); 1620 } 1621 1622 /* 1623 * Retrieve various parameters that are of interest to the driver. The device 1624 * has been initialized by the firmware at this point. 1625 */ 1626 static int 1627 get_params__post_init(struct adapter *sc) 1628 { 1629 int rc; 1630 uint32_t param[7], val[7]; 1631 struct fw_caps_config_cmd caps; 1632 1633 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1634 param[1] = FW_PARAM_PFVF(EQ_START); 1635 param[2] = FW_PARAM_PFVF(FILTER_START); 1636 param[3] = FW_PARAM_PFVF(FILTER_END); 1637 param[4] = FW_PARAM_PFVF(L2T_START); 1638 param[5] = FW_PARAM_PFVF(L2T_END); 1639 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1640 if (rc != 0) { 1641 cxgb_printf(sc->dip, CE_WARN, 1642 "failed to query parameters (post_init): %d.\n", rc); 1643 return (rc); 1644 } 1645 1646 /* LINTED: E_ASSIGN_NARROW_CONV */ 1647 sc->sge.iq_start = val[0]; 1648 sc->sge.eq_start = val[1]; 1649 sc->tids.ftid_base = val[2]; 1650 sc->tids.nftids = val[3] - val[2] + 1; 1651 sc->vres.l2t.start = val[4]; 1652 sc->vres.l2t.size = val[5] - val[4] + 1; 1653 1654 /* get capabilites */ 1655 bzero(&caps, sizeof (caps)); 1656 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1657 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1658 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps)); 1659 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps); 1660 if (rc != 0) { 1661 cxgb_printf(sc->dip, CE_WARN, 1662 "failed to get card capabilities: %d.\n", rc); 1663 return (rc); 1664 } 1665 1666 if (caps.toecaps != 0) { 1667 /* query offload-related parameters */ 1668 param[0] = FW_PARAM_DEV(NTID); 1669 param[1] = FW_PARAM_PFVF(SERVER_START); 1670 param[2] = FW_PARAM_PFVF(SERVER_END); 1671 param[3] = FW_PARAM_PFVF(TDDP_START); 1672 param[4] = FW_PARAM_PFVF(TDDP_END); 1673 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1674 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1675 if (rc != 0) { 1676 cxgb_printf(sc->dip, CE_WARN, 1677 "failed to query TOE parameters: %d.\n", rc); 1678 return (rc); 1679 } 1680 sc->tids.ntids = val[0]; 1681 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1682 sc->tids.stid_base = val[1]; 1683 sc->tids.nstids = val[2] - val[1] + 1; 1684 sc->vres.ddp.start = val[3]; 1685 sc->vres.ddp.size = val[4] - val[3] + 1; 1686 sc->params.ofldq_wr_cred = val[5]; 1687 sc->params.offload = 1; 1688 } 1689 1690 /* These are finalized by FW initialization, load their values now */ 1691 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1692 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1693 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1694 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1695 1696 return (rc); 1697 } 1698 1699 static int 1700 set_params__post_init(struct adapter *sc) 1701 { 1702 uint32_t param, val; 1703 1704 /* ask for encapsulated CPLs */ 1705 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 1706 val = 1; 1707 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1708 1709 return (0); 1710 } 1711 1712 /* TODO: verify */ 1713 static void 1714 setup_memwin(struct adapter *sc) 1715 { 1716 pci_regspec_t *data; 1717 int rc; 1718 uint_t n; 1719 uintptr_t bar0; 1720 uintptr_t mem_win0_base, mem_win1_base, mem_win2_base; 1721 uintptr_t mem_win2_aperture; 1722 1723 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip, 1724 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&data, &n); 1725 if (rc != DDI_SUCCESS) { 1726 cxgb_printf(sc->dip, CE_WARN, 1727 "failed to lookup \"assigned-addresses\" property: %d", rc); 1728 return; 1729 } 1730 n /= sizeof (*data); 1731 1732 bar0 = ((uint64_t)data[0].pci_phys_mid << 32) | data[0].pci_phys_low; 1733 ddi_prop_free(data); 1734 1735 if (is_t4(sc->params.chip)) { 1736 mem_win0_base = bar0 + MEMWIN0_BASE; 1737 mem_win1_base = bar0 + MEMWIN1_BASE; 1738 mem_win2_base = bar0 + MEMWIN2_BASE; 1739 mem_win2_aperture = MEMWIN2_APERTURE; 1740 } else { 1741 /* For T5, only relative offset inside the PCIe BAR is passed */ 1742 mem_win0_base = MEMWIN0_BASE; 1743 mem_win1_base = MEMWIN1_BASE; 1744 mem_win2_base = MEMWIN2_BASE_T5; 1745 mem_win2_aperture = MEMWIN2_APERTURE_T5; 1746 } 1747 1748 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1749 mem_win0_base | V_BIR(0) | 1750 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1751 1752 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1753 mem_win1_base | V_BIR(0) | 1754 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1755 1756 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1757 mem_win2_base | V_BIR(0) | 1758 V_WINDOW(ilog2(mem_win2_aperture) - 10)); 1759 1760 /* flush */ 1761 (void)t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1762 } 1763 1764 /* 1765 * Positions the memory window such that it can be used to access the specified 1766 * address in the chip's address space. The return value is the offset of addr 1767 * from the start of the window. 1768 */ 1769 uint32_t 1770 position_memwin(struct adapter *sc, int n, uint32_t addr) 1771 { 1772 uint32_t start, pf; 1773 uint32_t reg; 1774 1775 if (addr & 3) { 1776 cxgb_printf(sc->dip, CE_WARN, 1777 "addr (0x%x) is not at a 4B boundary.\n", addr); 1778 return (EFAULT); 1779 } 1780 1781 if (is_t4(sc->params.chip)) { 1782 pf = 0; 1783 start = addr & ~0xf; /* start must be 16B aligned */ 1784 } else { 1785 pf = V_PFNUM(sc->pf); 1786 start = addr & ~0x7f; /* start must be 128B aligned */ 1787 } 1788 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1789 1790 t4_write_reg(sc, reg, start | pf); 1791 (void) t4_read_reg(sc, reg); 1792 1793 return (addr - start); 1794 } 1795 1796 1797 /* 1798 * Reads the named property and fills up the "data" array (which has at least 1799 * "count" elements). We first try and lookup the property for our dev_t and 1800 * then retry with DDI_DEV_T_ANY if it's not found. 1801 * 1802 * Returns non-zero if the property was found and "data" has been updated. 1803 */ 1804 static int 1805 prop_lookup_int_array(struct adapter *sc, char *name, int *data, uint_t count) 1806 { 1807 dev_info_t *dip = sc->dip; 1808 dev_t dev = sc->dev; 1809 int rc, *d; 1810 uint_t i, n; 1811 1812 rc = ddi_prop_lookup_int_array(dev, dip, DDI_PROP_DONTPASS, 1813 name, &d, &n); 1814 if (rc == DDI_PROP_SUCCESS) 1815 goto found; 1816 1817 if (rc != DDI_PROP_NOT_FOUND) { 1818 cxgb_printf(dip, CE_WARN, 1819 "failed to lookup property %s for minor %d: %d.", 1820 name, getminor(dev), rc); 1821 return (0); 1822 } 1823 1824 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1825 name, &d, &n); 1826 if (rc == DDI_PROP_SUCCESS) 1827 goto found; 1828 1829 if (rc != DDI_PROP_NOT_FOUND) { 1830 cxgb_printf(dip, CE_WARN, 1831 "failed to lookup property %s: %d.", name, rc); 1832 return (0); 1833 } 1834 1835 return (0); 1836 1837 found: 1838 if (n > count) { 1839 cxgb_printf(dip, CE_NOTE, 1840 "property %s has too many elements (%d), ignoring extras", 1841 name, n); 1842 } 1843 1844 for (i = 0; i < n && i < count; i++) 1845 data[i] = d[i]; 1846 ddi_prop_free(d); 1847 1848 return (1); 1849 } 1850 1851 static int 1852 prop_lookup_int(struct adapter *sc, char *name, int defval) 1853 { 1854 int rc; 1855 1856 rc = ddi_prop_get_int(sc->dev, sc->dip, DDI_PROP_DONTPASS, name, -1); 1857 if (rc != -1) 1858 return (rc); 1859 1860 return (ddi_prop_get_int(DDI_DEV_T_ANY, sc->dip, DDI_PROP_DONTPASS, 1861 name, defval)); 1862 } 1863 1864 static int 1865 init_driver_props(struct adapter *sc, struct driver_properties *p) 1866 { 1867 dev_t dev = sc->dev; 1868 dev_info_t *dip = sc->dip; 1869 int i, *data; 1870 uint_t tmr[SGE_NTIMERS] = {5, 10, 20, 50, 100, 200}; 1871 uint_t cnt[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 1872 1873 /* 1874 * Holdoff timer 1875 */ 1876 data = &p->timer_val[0]; 1877 for (i = 0; i < SGE_NTIMERS; i++) 1878 data[i] = tmr[i]; 1879 (void) prop_lookup_int_array(sc, "holdoff-timer-values", data, 1880 SGE_NTIMERS); 1881 for (i = 0; i < SGE_NTIMERS; i++) { 1882 int limit = 200U; 1883 if (data[i] > limit) { 1884 cxgb_printf(dip, CE_WARN, 1885 "holdoff timer %d is too high (%d), lowered to %d.", 1886 i, data[i], limit); 1887 data[i] = limit; 1888 } 1889 } 1890 (void) ddi_prop_update_int_array(dev, dip, "holdoff-timer-values", 1891 data, SGE_NTIMERS); 1892 1893 /* 1894 * Holdoff packet counter 1895 */ 1896 data = &p->counter_val[0]; 1897 for (i = 0; i < SGE_NCOUNTERS; i++) 1898 data[i] = cnt[i]; 1899 (void) prop_lookup_int_array(sc, "holdoff-pkt-counter-values", data, 1900 SGE_NCOUNTERS); 1901 for (i = 0; i < SGE_NCOUNTERS; i++) { 1902 int limit = M_THRESHOLD_0; 1903 if (data[i] > limit) { 1904 cxgb_printf(dip, CE_WARN, 1905 "holdoff pkt-counter %d is too high (%d), " 1906 "lowered to %d.", i, data[i], limit); 1907 data[i] = limit; 1908 } 1909 } 1910 (void) ddi_prop_update_int_array(dev, dip, "holdoff-pkt-counter-values", 1911 data, SGE_NCOUNTERS); 1912 1913 /* 1914 * Maximum # of tx and rx queues to use for each 1915 * 100G, 40G, 25G, 10G and 1G port. 1916 */ 1917 p->max_ntxq_10g = prop_lookup_int(sc, "max-ntxq-10G-port", 8); 1918 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port", 1919 p->max_ntxq_10g); 1920 1921 p->max_nrxq_10g = prop_lookup_int(sc, "max-nrxq-10G-port", 8); 1922 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port", 1923 p->max_nrxq_10g); 1924 1925 p->max_ntxq_1g = prop_lookup_int(sc, "max-ntxq-1G-port", 2); 1926 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port", 1927 p->max_ntxq_1g); 1928 1929 p->max_nrxq_1g = prop_lookup_int(sc, "max-nrxq-1G-port", 2); 1930 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port", 1931 p->max_nrxq_1g); 1932 1933 #ifdef TCP_OFFLOAD_ENABLE 1934 p->max_nofldtxq_10g = prop_lookup_int(sc, "max-nofldtxq-10G-port", 8); 1935 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port", 1936 p->max_nofldtxq_10g); 1937 1938 p->max_nofldrxq_10g = prop_lookup_int(sc, "max-nofldrxq-10G-port", 2); 1939 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port", 1940 p->max_nofldrxq_10g); 1941 1942 p->max_nofldtxq_1g = prop_lookup_int(sc, "max-nofldtxq-1G-port", 2); 1943 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port", 1944 p->max_nofldtxq_1g); 1945 1946 p->max_nofldrxq_1g = prop_lookup_int(sc, "max-nofldrxq-1G-port", 1); 1947 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port", 1948 p->max_nofldrxq_1g); 1949 #endif 1950 1951 /* 1952 * Holdoff parameters for 10G and 1G ports. 1953 */ 1954 p->tmr_idx_10g = prop_lookup_int(sc, "holdoff-timer-idx-10G", 0); 1955 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-10G", 1956 p->tmr_idx_10g); 1957 1958 p->pktc_idx_10g = prop_lookup_int(sc, "holdoff-pktc-idx-10G", 2); 1959 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-10G", 1960 p->pktc_idx_10g); 1961 1962 p->tmr_idx_1g = prop_lookup_int(sc, "holdoff-timer-idx-1G", 0); 1963 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-1G", 1964 p->tmr_idx_1g); 1965 1966 p->pktc_idx_1g = prop_lookup_int(sc, "holdoff-pktc-idx-1G", 2); 1967 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-1G", 1968 p->pktc_idx_1g); 1969 1970 /* 1971 * Size (number of entries) of each tx and rx queue. 1972 */ 1973 i = prop_lookup_int(sc, "qsize-txq", TX_EQ_QSIZE); 1974 p->qsize_txq = max(i, 128); 1975 if (p->qsize_txq != i) { 1976 cxgb_printf(dip, CE_WARN, 1977 "using %d instead of %d as the tx queue size", 1978 p->qsize_txq, i); 1979 } 1980 (void) ddi_prop_update_int(dev, dip, "qsize-txq", p->qsize_txq); 1981 1982 i = prop_lookup_int(sc, "qsize-rxq", RX_IQ_QSIZE); 1983 p->qsize_rxq = max(i, 128); 1984 while (p->qsize_rxq & 7) 1985 p->qsize_rxq--; 1986 if (p->qsize_rxq != i) { 1987 cxgb_printf(dip, CE_WARN, 1988 "using %d instead of %d as the rx queue size", 1989 p->qsize_rxq, i); 1990 } 1991 (void) ddi_prop_update_int(dev, dip, "qsize-rxq", p->qsize_rxq); 1992 1993 /* 1994 * Interrupt types allowed. 1995 * Bits 0, 1, 2 = INTx, MSI, MSI-X respectively. See sys/ddi_intr.h 1996 */ 1997 p->intr_types = prop_lookup_int(sc, "interrupt-types", 1998 DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_FIXED); 1999 (void) ddi_prop_update_int(dev, dip, "interrupt-types", p->intr_types); 2000 2001 /* 2002 * Forwarded interrupt queues. Create this property to force the driver 2003 * to use forwarded interrupt queues. 2004 */ 2005 if (ddi_prop_exists(dev, dip, DDI_PROP_DONTPASS, 2006 "interrupt-forwarding") != 0 || 2007 ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2008 "interrupt-forwarding") != 0) { 2009 UNIMPLEMENTED(); 2010 (void) ddi_prop_create(dev, dip, DDI_PROP_CANSLEEP, 2011 "interrupt-forwarding", NULL, 0); 2012 } 2013 2014 /* 2015 * Write combining 2016 * 0 to disable, 1 to enable 2017 */ 2018 p->wc = prop_lookup_int(sc, "write-combine", 1); 2019 cxgb_printf(dip, CE_WARN, "write-combine: using of %d", p->wc); 2020 if (p->wc != 0 && p->wc != 1) { 2021 cxgb_printf(dip, CE_WARN, 2022 "write-combine: using 1 instead of %d", p->wc); 2023 p->wc = 1; 2024 } 2025 (void) ddi_prop_update_int(dev, dip, "write-combine", p->wc); 2026 2027 p->t4_fw_install = prop_lookup_int(sc, "t4_fw_install", 1); 2028 if (p->t4_fw_install != 0 && p->t4_fw_install != 2) 2029 p->t4_fw_install = 1; 2030 (void) ddi_prop_update_int(dev, dip, "t4_fw_install", p->t4_fw_install); 2031 2032 /* Multiple Rings */ 2033 p->multi_rings = prop_lookup_int(sc, "multi-rings", 1); 2034 if (p->multi_rings != 0 && p->multi_rings != 1) { 2035 cxgb_printf(dip, CE_NOTE, 2036 "multi-rings: using value 1 instead of %d", p->multi_rings); 2037 p->multi_rings = 1; 2038 } 2039 2040 (void) ddi_prop_update_int(dev, dip, "multi-rings", p->multi_rings); 2041 2042 return (0); 2043 } 2044 2045 static int 2046 remove_extra_props(struct adapter *sc, int n10g, int n1g) 2047 { 2048 if (n10g == 0) { 2049 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-10G-port"); 2050 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-10G-port"); 2051 (void) ddi_prop_remove(sc->dev, sc->dip, 2052 "holdoff-timer-idx-10G"); 2053 (void) ddi_prop_remove(sc->dev, sc->dip, 2054 "holdoff-pktc-idx-10G"); 2055 } 2056 2057 if (n1g == 0) { 2058 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-1G-port"); 2059 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-1G-port"); 2060 (void) ddi_prop_remove(sc->dev, sc->dip, 2061 "holdoff-timer-idx-1G"); 2062 (void) ddi_prop_remove(sc->dev, sc->dip, "holdoff-pktc-idx-1G"); 2063 } 2064 2065 return (0); 2066 } 2067 2068 static int 2069 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 2070 struct intrs_and_queues *iaq) 2071 { 2072 struct driver_properties *p = &sc->props; 2073 int rc, itype, itypes, navail, nc, nrxq10g, nrxq1g, n; 2074 int nofldrxq10g = 0, nofldrxq1g = 0; 2075 2076 bzero(iaq, sizeof (*iaq)); 2077 nc = ncpus; /* our snapshot of the number of CPUs */ 2078 iaq->ntxq10g = min(nc, p->max_ntxq_10g); 2079 iaq->ntxq1g = min(nc, p->max_ntxq_1g); 2080 iaq->nrxq10g = nrxq10g = min(nc, p->max_nrxq_10g); 2081 iaq->nrxq1g = nrxq1g = min(nc, p->max_nrxq_1g); 2082 #ifdef TCP_OFFLOAD_ENABLE 2083 iaq->nofldtxq10g = min(nc, p->max_nofldtxq_10g); 2084 iaq->nofldtxq1g = min(nc, p->max_nofldtxq_1g); 2085 iaq->nofldrxq10g = nofldrxq10g = min(nc, p->max_nofldrxq_10g); 2086 iaq->nofldrxq1g = nofldrxq1g = min(nc, p->max_nofldrxq_1g); 2087 #endif 2088 2089 rc = ddi_intr_get_supported_types(sc->dip, &itypes); 2090 if (rc != DDI_SUCCESS) { 2091 cxgb_printf(sc->dip, CE_WARN, 2092 "failed to determine supported interrupt types: %d", rc); 2093 return (rc); 2094 } 2095 2096 for (itype = DDI_INTR_TYPE_MSIX; itype; itype >>= 1) { 2097 ASSERT(itype == DDI_INTR_TYPE_MSIX || 2098 itype == DDI_INTR_TYPE_MSI || 2099 itype == DDI_INTR_TYPE_FIXED); 2100 2101 if ((itype & itypes & p->intr_types) == 0) 2102 continue; /* not supported or not allowed */ 2103 2104 navail = 0; 2105 rc = ddi_intr_get_navail(sc->dip, itype, &navail); 2106 if (rc != DDI_SUCCESS || navail == 0) { 2107 cxgb_printf(sc->dip, CE_WARN, 2108 "failed to get # of interrupts for type %d: %d", 2109 itype, rc); 2110 continue; /* carry on */ 2111 } 2112 2113 iaq->intr_type = itype; 2114 if (navail == 0) 2115 continue; 2116 2117 /* 2118 * Best option: an interrupt vector for errors, one for the 2119 * firmware event queue, and one each for each rxq (NIC as well 2120 * as offload). 2121 */ 2122 iaq->nirq = T4_EXTRA_INTR; 2123 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 2124 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 2125 2126 if (iaq->nirq <= navail && 2127 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) { 2128 iaq->intr_fwd = 0; 2129 goto allocate; 2130 } 2131 2132 /* 2133 * Second best option: an interrupt vector for errors, one for 2134 * the firmware event queue, and one each for either NIC or 2135 * offload rxq's. 2136 */ 2137 iaq->nirq = T4_EXTRA_INTR; 2138 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 2139 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 2140 if (iaq->nirq <= navail && 2141 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) { 2142 iaq->intr_fwd = 1; 2143 goto allocate; 2144 } 2145 2146 /* 2147 * Next best option: an interrupt vector for errors, one for the 2148 * firmware event queue, and at least one per port. At this 2149 * point we know we'll have to downsize nrxq or nofldrxq to fit 2150 * what's available to us. 2151 */ 2152 iaq->nirq = T4_EXTRA_INTR; 2153 iaq->nirq += n10g + n1g; 2154 if (iaq->nirq <= navail) { 2155 int leftover = navail - iaq->nirq; 2156 2157 if (n10g > 0) { 2158 int target = max(nrxq10g, nofldrxq10g); 2159 2160 n = 1; 2161 while (n < target && leftover >= n10g) { 2162 leftover -= n10g; 2163 iaq->nirq += n10g; 2164 n++; 2165 } 2166 iaq->nrxq10g = min(n, nrxq10g); 2167 #ifdef TCP_OFFLOAD_ENABLE 2168 iaq->nofldrxq10g = min(n, nofldrxq10g); 2169 #endif 2170 } 2171 2172 if (n1g > 0) { 2173 int target = max(nrxq1g, nofldrxq1g); 2174 2175 n = 1; 2176 while (n < target && leftover >= n1g) { 2177 leftover -= n1g; 2178 iaq->nirq += n1g; 2179 n++; 2180 } 2181 iaq->nrxq1g = min(n, nrxq1g); 2182 #ifdef TCP_OFFLOAD_ENABLE 2183 iaq->nofldrxq1g = min(n, nofldrxq1g); 2184 #endif 2185 } 2186 2187 /* We have arrived at a minimum value required to enable 2188 * per queue irq(either NIC or offload). Thus for non- 2189 * offload case, we will get a vector per queue, while 2190 * offload case, we will get a vector per offload/NIC q. 2191 * Hence enable Interrupt forwarding only for offload 2192 * case. 2193 */ 2194 #ifdef TCP_OFFLOAD_ENABLE 2195 if (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq)) { 2196 iaq->intr_fwd = 1; 2197 #else 2198 if (itype != DDI_INTR_TYPE_MSI) { 2199 #endif 2200 goto allocate; 2201 } 2202 } 2203 2204 /* 2205 * Least desirable option: one interrupt vector for everything. 2206 */ 2207 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2208 #ifdef TCP_OFFLOAD_ENABLE 2209 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2210 #endif 2211 iaq->intr_fwd = 1; 2212 2213 allocate: 2214 return (0); 2215 } 2216 2217 cxgb_printf(sc->dip, CE_WARN, 2218 "failed to find a usable interrupt type. supported=%d, allowed=%d", 2219 itypes, p->intr_types); 2220 return (DDI_FAILURE); 2221 } 2222 2223 static int 2224 add_child_node(struct adapter *sc, int idx) 2225 { 2226 int rc; 2227 struct port_info *pi; 2228 2229 if (idx < 0 || idx >= sc->params.nports) 2230 return (EINVAL); 2231 2232 pi = sc->port[idx]; 2233 if (pi == NULL) 2234 return (ENODEV); /* t4_port_init failed earlier */ 2235 2236 PORT_LOCK(pi); 2237 if (pi->dip != NULL) { 2238 rc = 0; /* EEXIST really, but then bus_config fails */ 2239 goto done; 2240 } 2241 2242 rc = ndi_devi_alloc(sc->dip, T4_PORT_NAME, DEVI_SID_NODEID, &pi->dip); 2243 if (rc != DDI_SUCCESS || pi->dip == NULL) { 2244 rc = ENOMEM; 2245 goto done; 2246 } 2247 2248 (void) ddi_set_parent_data(pi->dip, pi); 2249 (void) ndi_devi_bind_driver(pi->dip, 0); 2250 rc = 0; 2251 done: 2252 PORT_UNLOCK(pi); 2253 return (rc); 2254 } 2255 2256 static int 2257 remove_child_node(struct adapter *sc, int idx) 2258 { 2259 int rc; 2260 struct port_info *pi; 2261 2262 if (idx < 0 || idx >= sc->params.nports) 2263 return (EINVAL); 2264 2265 pi = sc->port[idx]; 2266 if (pi == NULL) 2267 return (ENODEV); 2268 2269 PORT_LOCK(pi); 2270 if (pi->dip == NULL) { 2271 rc = ENODEV; 2272 goto done; 2273 } 2274 2275 rc = ndi_devi_free(pi->dip); 2276 if (rc == 0) 2277 pi->dip = NULL; 2278 done: 2279 PORT_UNLOCK(pi); 2280 return (rc); 2281 } 2282 2283 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 2284 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 2285 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 2286 #define KS_C_SET(x, ...) \ 2287 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 2288 2289 /* 2290 * t4nex:X:config 2291 */ 2292 struct t4_kstats { 2293 kstat_named_t chip_ver; 2294 kstat_named_t fw_vers; 2295 kstat_named_t tp_vers; 2296 kstat_named_t driver_version; 2297 kstat_named_t serial_number; 2298 kstat_named_t ec_level; 2299 kstat_named_t id; 2300 kstat_named_t bus_type; 2301 kstat_named_t bus_width; 2302 kstat_named_t bus_speed; 2303 kstat_named_t core_clock; 2304 kstat_named_t port_cnt; 2305 kstat_named_t port_type; 2306 kstat_named_t pci_vendor_id; 2307 kstat_named_t pci_device_id; 2308 }; 2309 static kstat_t * 2310 setup_kstats(struct adapter *sc) 2311 { 2312 kstat_t *ksp; 2313 struct t4_kstats *kstatp; 2314 int ndata; 2315 struct pci_params *p = &sc->params.pci; 2316 struct vpd_params *v = &sc->params.vpd; 2317 uint16_t pci_vendor, pci_device; 2318 2319 ndata = sizeof (struct t4_kstats) / sizeof (kstat_named_t); 2320 2321 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "config", 2322 "nexus", KSTAT_TYPE_NAMED, ndata, 0); 2323 if (ksp == NULL) { 2324 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats."); 2325 return (NULL); 2326 } 2327 2328 kstatp = (struct t4_kstats *)ksp->ks_data; 2329 2330 KS_UINIT(chip_ver); 2331 KS_CINIT(fw_vers); 2332 KS_CINIT(tp_vers); 2333 KS_CINIT(driver_version); 2334 KS_CINIT(serial_number); 2335 KS_CINIT(ec_level); 2336 KS_CINIT(id); 2337 KS_CINIT(bus_type); 2338 KS_CINIT(bus_width); 2339 KS_CINIT(bus_speed); 2340 KS_UINIT(core_clock); 2341 KS_UINIT(port_cnt); 2342 KS_CINIT(port_type); 2343 KS_CINIT(pci_vendor_id); 2344 KS_CINIT(pci_device_id); 2345 2346 KS_U_SET(chip_ver, sc->params.chip); 2347 KS_C_SET(fw_vers, "%d.%d.%d.%d", 2348 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2349 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2350 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2351 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2352 KS_C_SET(tp_vers, "%d.%d.%d.%d", 2353 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 2354 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 2355 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 2356 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 2357 KS_C_SET(driver_version, DRV_VERSION); 2358 KS_C_SET(serial_number, "%s", v->sn); 2359 KS_C_SET(ec_level, "%s", v->ec); 2360 KS_C_SET(id, "%s", v->id); 2361 KS_C_SET(bus_type, "pci-express"); 2362 KS_C_SET(bus_width, "x%d lanes", p->width); 2363 KS_C_SET(bus_speed, "%d", p->speed); 2364 KS_U_SET(core_clock, v->cclk); 2365 KS_U_SET(port_cnt, sc->params.nports); 2366 2367 t4_os_pci_read_cfg2(sc, PCI_CONF_VENID, &pci_vendor); 2368 KS_C_SET(pci_vendor_id, "0x%x", pci_vendor); 2369 2370 t4_os_pci_read_cfg2(sc, PCI_CONF_DEVID, &pci_device); 2371 KS_C_SET(pci_device_id, "0x%x", pci_device); 2372 2373 KS_C_SET(port_type, "%s/%s/%s/%s", 2374 print_port_speed(sc->port[0]), 2375 print_port_speed(sc->port[1]), 2376 print_port_speed(sc->port[2]), 2377 print_port_speed(sc->port[3])); 2378 2379 /* Do NOT set ksp->ks_update. These kstats do not change. */ 2380 2381 /* Install the kstat */ 2382 ksp->ks_private = (void *)sc; 2383 kstat_install(ksp); 2384 2385 return (ksp); 2386 } 2387 2388 /* 2389 * t4nex:X:stat 2390 */ 2391 struct t4_wc_kstats { 2392 kstat_named_t write_coal_success; 2393 kstat_named_t write_coal_failure; 2394 }; 2395 static kstat_t * 2396 setup_wc_kstats(struct adapter *sc) 2397 { 2398 kstat_t *ksp; 2399 struct t4_wc_kstats *kstatp; 2400 int ndata; 2401 2402 ndata = sizeof(struct t4_wc_kstats) / sizeof(kstat_named_t); 2403 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "stats", 2404 "nexus", KSTAT_TYPE_NAMED, ndata, 0); 2405 if (ksp == NULL) { 2406 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats."); 2407 return (NULL); 2408 } 2409 2410 kstatp = (struct t4_wc_kstats *)ksp->ks_data; 2411 2412 KS_UINIT(write_coal_success); 2413 KS_UINIT(write_coal_failure); 2414 2415 ksp->ks_update = update_wc_kstats; 2416 /* Install the kstat */ 2417 ksp->ks_private = (void *)sc; 2418 kstat_install(ksp); 2419 2420 return (ksp); 2421 } 2422 2423 static int 2424 update_wc_kstats(kstat_t *ksp, int rw) 2425 { 2426 struct t4_wc_kstats *kstatp = (struct t4_wc_kstats *)ksp->ks_data; 2427 struct adapter *sc = ksp->ks_private; 2428 uint32_t wc_total, wc_success, wc_failure; 2429 2430 if (rw == KSTAT_WRITE) 2431 return (0); 2432 2433 if (is_t5(sc->params.chip)) { 2434 wc_total = t4_read_reg(sc, A_SGE_STAT_TOTAL); 2435 wc_failure = t4_read_reg(sc, A_SGE_STAT_MATCH); 2436 wc_success = wc_total - wc_failure; 2437 } else { 2438 wc_success = 0; 2439 wc_failure = 0; 2440 } 2441 2442 KS_U_SET(write_coal_success, wc_success); 2443 KS_U_SET(write_coal_failure, wc_failure); 2444 2445 return (0); 2446 } 2447 2448 int 2449 adapter_full_init(struct adapter *sc) 2450 { 2451 int i, rc = 0; 2452 2453 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2454 2455 rc = t4_setup_adapter_queues(sc); 2456 if (rc != 0) 2457 goto done; 2458 2459 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK) 2460 (void) ddi_intr_block_enable(sc->intr_handle, sc->intr_count); 2461 else { 2462 for (i = 0; i < sc->intr_count; i++) 2463 (void) ddi_intr_enable(sc->intr_handle[i]); 2464 } 2465 t4_intr_enable(sc); 2466 sc->flags |= FULL_INIT_DONE; 2467 2468 #ifdef TCP_OFFLOAD_ENABLE 2469 /* TODO: wrong place to enable TOE capability */ 2470 if (is_offload(sc) != 0) { 2471 for_each_port(sc, i) { 2472 struct port_info *pi = sc->port[i]; 2473 rc = toe_capability(pi, 1); 2474 if (rc != 0) { 2475 cxgb_printf(pi->dip, CE_WARN, 2476 "Failed to activate toe capability: %d", 2477 rc); 2478 rc = 0; /* not a fatal error */ 2479 } 2480 } 2481 } 2482 #endif 2483 2484 done: 2485 if (rc != 0) 2486 (void) adapter_full_uninit(sc); 2487 2488 return (rc); 2489 } 2490 2491 int 2492 adapter_full_uninit(struct adapter *sc) 2493 { 2494 int i, rc = 0; 2495 2496 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2497 2498 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK) 2499 (void) ddi_intr_block_disable(sc->intr_handle, sc->intr_count); 2500 else { 2501 for (i = 0; i < sc->intr_count; i++) 2502 (void) ddi_intr_disable(sc->intr_handle[i]); 2503 } 2504 2505 rc = t4_teardown_adapter_queues(sc); 2506 if (rc != 0) 2507 return (rc); 2508 2509 sc->flags &= ~FULL_INIT_DONE; 2510 2511 return (0); 2512 } 2513 2514 int 2515 port_full_init(struct port_info *pi) 2516 { 2517 struct adapter *sc = pi->adapter; 2518 uint16_t *rss; 2519 struct sge_rxq *rxq; 2520 int rc, i; 2521 2522 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2523 ASSERT((pi->flags & PORT_INIT_DONE) == 0); 2524 2525 /* 2526 * Allocate tx/rx/fl queues for this port. 2527 */ 2528 rc = t4_setup_port_queues(pi); 2529 if (rc != 0) 2530 goto done; /* error message displayed already */ 2531 2532 /* 2533 * Setup RSS for this port. 2534 */ 2535 rss = kmem_zalloc(pi->nrxq * sizeof (*rss), KM_SLEEP); 2536 for_each_rxq(pi, i, rxq) { 2537 rss[i] = rxq->iq.abs_id; 2538 } 2539 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2540 pi->rss_size, rss, pi->nrxq); 2541 kmem_free(rss, pi->nrxq * sizeof (*rss)); 2542 if (rc != 0) { 2543 cxgb_printf(pi->dip, CE_WARN, "rss_config failed: %d", rc); 2544 goto done; 2545 } 2546 2547 pi->flags |= PORT_INIT_DONE; 2548 done: 2549 if (rc != 0) 2550 (void) port_full_uninit(pi); 2551 2552 return (rc); 2553 } 2554 2555 /* 2556 * Idempotent. 2557 */ 2558 int 2559 port_full_uninit(struct port_info *pi) 2560 { 2561 2562 ASSERT(pi->flags & PORT_INIT_DONE); 2563 2564 (void) t4_teardown_port_queues(pi); 2565 pi->flags &= ~PORT_INIT_DONE; 2566 2567 return (0); 2568 } 2569 2570 void 2571 enable_port_queues(struct port_info *pi) 2572 { 2573 struct adapter *sc = pi->adapter; 2574 int i; 2575 struct sge_iq *iq; 2576 struct sge_rxq *rxq; 2577 #ifdef TCP_OFFLOAD_ENABLE 2578 struct sge_ofld_rxq *ofld_rxq; 2579 #endif 2580 2581 ASSERT(pi->flags & PORT_INIT_DONE); 2582 2583 /* 2584 * TODO: whatever was queued up after we set iq->state to IQS_DISABLED 2585 * back in disable_port_queues will be processed now, after an unbounded 2586 * delay. This can't be good. 2587 */ 2588 2589 #ifdef TCP_OFFLOAD_ENABLE 2590 for_each_ofld_rxq(pi, i, ofld_rxq) { 2591 iq = &ofld_rxq->iq; 2592 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) != 2593 IQS_DISABLED) 2594 panic("%s: iq %p wasn't disabled", __func__, 2595 (void *)iq); 2596 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 2597 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); 2598 } 2599 #endif 2600 2601 for_each_rxq(pi, i, rxq) { 2602 iq = &rxq->iq; 2603 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) != 2604 IQS_DISABLED) 2605 panic("%s: iq %p wasn't disabled", __func__, 2606 (void *) iq); 2607 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 2608 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); 2609 } 2610 } 2611 2612 void 2613 disable_port_queues(struct port_info *pi) 2614 { 2615 int i; 2616 struct adapter *sc = pi->adapter; 2617 struct sge_rxq *rxq; 2618 #ifdef TCP_OFFLOAD_ENABLE 2619 struct sge_ofld_rxq *ofld_rxq; 2620 #endif 2621 2622 ASSERT(pi->flags & PORT_INIT_DONE); 2623 2624 /* 2625 * TODO: need proper implementation for all tx queues (ctrl, eth, ofld). 2626 */ 2627 2628 #ifdef TCP_OFFLOAD_ENABLE 2629 for_each_ofld_rxq(pi, i, ofld_rxq) { 2630 while (atomic_cas_uint(&ofld_rxq->iq.state, IQS_IDLE, 2631 IQS_DISABLED) != IQS_IDLE) 2632 msleep(1); 2633 } 2634 #endif 2635 2636 for_each_rxq(pi, i, rxq) { 2637 while (atomic_cas_uint(&rxq->iq.state, IQS_IDLE, 2638 IQS_DISABLED) != IQS_IDLE) 2639 msleep(1); 2640 } 2641 2642 mutex_enter(&sc->sfl_lock); 2643 #ifdef TCP_OFFLOAD_ENABLE 2644 for_each_ofld_rxq(pi, i, ofld_rxq) 2645 ofld_rxq->fl.flags |= FL_DOOMED; 2646 #endif 2647 for_each_rxq(pi, i, rxq) 2648 rxq->fl.flags |= FL_DOOMED; 2649 mutex_exit(&sc->sfl_lock); 2650 /* TODO: need to wait for all fl's to be removed from sc->sfl */ 2651 } 2652 2653 void 2654 t4_fatal_err(struct adapter *sc) 2655 { 2656 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 2657 t4_intr_disable(sc); 2658 cxgb_printf(sc->dip, CE_WARN, 2659 "encountered fatal error, adapter stopped."); 2660 } 2661 2662 int 2663 t4_os_find_pci_capability(struct adapter *sc, int cap) 2664 { 2665 uint16_t stat; 2666 uint8_t cap_ptr, cap_id; 2667 2668 t4_os_pci_read_cfg2(sc, PCI_CONF_STAT, &stat); 2669 if ((stat & PCI_STAT_CAP) == 0) 2670 return (0); /* does not implement capabilities */ 2671 2672 t4_os_pci_read_cfg1(sc, PCI_CONF_CAP_PTR, &cap_ptr); 2673 while (cap_ptr) { 2674 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_ID, &cap_id); 2675 if (cap_id == cap) 2676 return (cap_ptr); /* found */ 2677 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_NEXT_PTR, &cap_ptr); 2678 } 2679 2680 return (0); /* not found */ 2681 } 2682 2683 void 2684 t4_os_portmod_changed(const struct adapter *sc, int idx) 2685 { 2686 static const char *mod_str[] = { 2687 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 2688 }; 2689 const struct port_info *pi = sc->port[idx]; 2690 2691 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 2692 cxgb_printf(pi->dip, CE_NOTE, "transceiver unplugged."); 2693 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 2694 cxgb_printf(pi->dip, CE_NOTE, 2695 "unknown transceiver inserted.\n"); 2696 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 2697 cxgb_printf(pi->dip, CE_NOTE, 2698 "unsupported transceiver inserted.\n"); 2699 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 2700 cxgb_printf(pi->dip, CE_NOTE, "%s transceiver inserted.\n", 2701 mod_str[pi->mod_type]); 2702 else 2703 cxgb_printf(pi->dip, CE_NOTE, "transceiver (type %d) inserted.", 2704 pi->mod_type); 2705 } 2706 2707 /* ARGSUSED */ 2708 static int 2709 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 2710 { 2711 if (m != NULL) 2712 freemsg(m); 2713 return (0); 2714 } 2715 2716 int 2717 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 2718 { 2719 uint_t *loc, new; 2720 2721 if (opcode >= ARRAY_SIZE(sc->cpl_handler)) 2722 return (EINVAL); 2723 2724 new = (uint_t)(unsigned long) (h ? h : cpl_not_handled); 2725 loc = (uint_t *)&sc->cpl_handler[opcode]; 2726 (void) atomic_swap_uint(loc, new); 2727 2728 return (0); 2729 } 2730 2731 static int 2732 fw_msg_not_handled(struct adapter *sc, const __be64 *data) 2733 { 2734 struct cpl_fw6_msg *cpl = container_of(data, struct cpl_fw6_msg, data); 2735 2736 cxgb_printf(sc->dip, CE_WARN, "%s fw_msg type %d", __func__, cpl->type); 2737 return (0); 2738 } 2739 2740 int 2741 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 2742 { 2743 fw_msg_handler_t *loc, new; 2744 2745 if (type >= ARRAY_SIZE(sc->fw_msg_handler)) 2746 return (EINVAL); 2747 2748 /* 2749 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 2750 * handler dispatch table. Reject any attempt to install a handler for 2751 * this subtype. 2752 */ 2753 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 2754 return (EINVAL); 2755 2756 new = h ? h : fw_msg_not_handled; 2757 loc = &sc->fw_msg_handler[type]; 2758 (void)atomic_swap_ptr(loc, (void *)new); 2759 2760 return (0); 2761 } 2762 2763 #ifdef TCP_OFFLOAD_ENABLE 2764 static int 2765 toe_capability(struct port_info *pi, int enable) 2766 { 2767 int rc; 2768 struct adapter *sc = pi->adapter; 2769 2770 if (!is_offload(sc)) 2771 return (ENODEV); 2772 2773 if (enable != 0) { 2774 if (isset(&sc->offload_map, pi->port_id) != 0) 2775 return (0); 2776 2777 if (sc->offload_map == 0) { 2778 rc = activate_uld(sc, ULD_TOM, &sc->tom); 2779 if (rc != 0) 2780 return (rc); 2781 } 2782 2783 setbit(&sc->offload_map, pi->port_id); 2784 } else { 2785 if (!isset(&sc->offload_map, pi->port_id)) 2786 return (0); 2787 2788 clrbit(&sc->offload_map, pi->port_id); 2789 2790 if (sc->offload_map == 0) { 2791 rc = deactivate_uld(&sc->tom); 2792 if (rc != 0) { 2793 setbit(&sc->offload_map, pi->port_id); 2794 return (rc); 2795 } 2796 } 2797 } 2798 2799 return (0); 2800 } 2801 2802 /* 2803 * Add an upper layer driver to the global list. 2804 */ 2805 int 2806 t4_register_uld(struct uld_info *ui) 2807 { 2808 int rc = 0; 2809 struct uld_info *u; 2810 2811 mutex_enter(&t4_uld_list_lock); 2812 SLIST_FOREACH(u, &t4_uld_list, link) { 2813 if (u->uld_id == ui->uld_id) { 2814 rc = EEXIST; 2815 goto done; 2816 } 2817 } 2818 2819 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 2820 ui->refcount = 0; 2821 done: 2822 mutex_exit(&t4_uld_list_lock); 2823 return (rc); 2824 } 2825 2826 int 2827 t4_unregister_uld(struct uld_info *ui) 2828 { 2829 int rc = EINVAL; 2830 struct uld_info *u; 2831 2832 mutex_enter(&t4_uld_list_lock); 2833 2834 SLIST_FOREACH(u, &t4_uld_list, link) { 2835 if (u == ui) { 2836 if (ui->refcount > 0) { 2837 rc = EBUSY; 2838 goto done; 2839 } 2840 2841 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 2842 rc = 0; 2843 goto done; 2844 } 2845 } 2846 done: 2847 mutex_exit(&t4_uld_list_lock); 2848 return (rc); 2849 } 2850 2851 static int 2852 activate_uld(struct adapter *sc, int id, struct uld_softc *usc) 2853 { 2854 int rc = EAGAIN; 2855 struct uld_info *ui; 2856 2857 mutex_enter(&t4_uld_list_lock); 2858 2859 SLIST_FOREACH(ui, &t4_uld_list, link) { 2860 if (ui->uld_id == id) { 2861 rc = ui->attach(sc, &usc->softc); 2862 if (rc == 0) { 2863 ASSERT(usc->softc != NULL); 2864 ui->refcount++; 2865 usc->uld = ui; 2866 } 2867 goto done; 2868 } 2869 } 2870 done: 2871 mutex_exit(&t4_uld_list_lock); 2872 2873 return (rc); 2874 } 2875 2876 static int 2877 deactivate_uld(struct uld_softc *usc) 2878 { 2879 int rc; 2880 2881 mutex_enter(&t4_uld_list_lock); 2882 2883 if (usc->uld == NULL || usc->softc == NULL) { 2884 rc = EINVAL; 2885 goto done; 2886 } 2887 2888 rc = usc->uld->detach(usc->softc); 2889 if (rc == 0) { 2890 ASSERT(usc->uld->refcount > 0); 2891 usc->uld->refcount--; 2892 usc->uld = NULL; 2893 usc->softc = NULL; 2894 } 2895 done: 2896 mutex_exit(&t4_uld_list_lock); 2897 2898 return (rc); 2899 } 2900 2901 void 2902 t4_iterate(void (*func)(int, void *), void *arg) 2903 { 2904 struct adapter *sc; 2905 2906 mutex_enter(&t4_adapter_list_lock); 2907 SLIST_FOREACH(sc, &t4_adapter_list, link) { 2908 /* 2909 * func should not make any assumptions about what state sc is 2910 * in - the only guarantee is that sc->sc_lock is a valid lock. 2911 */ 2912 func(ddi_get_instance(sc->dip), arg); 2913 } 2914 mutex_exit(&t4_adapter_list_lock); 2915 } 2916 2917 #endif 2918