1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source. A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * This file is part of the Chelsio T4 support code.
14  *
15  * Copyright (C) 2010-2013 Chelsio Communications.  All rights reserved.
16  *
17  * This program is distributed in the hope that it will be useful, but WITHOUT
18  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
20  * release for licensing terms and conditions.
21  */
22 
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/modctl.h>
27 #include <sys/conf.h>
28 #include <sys/devops.h>
29 #include <sys/pci.h>
30 #include <sys/atomic.h>
31 #include <sys/types.h>
32 #include <sys/file.h>
33 #include <sys/errno.h>
34 #include <sys/open.h>
35 #include <sys/cred.h>
36 #include <sys/stat.h>
37 #include <sys/mkdev.h>
38 #include <sys/queue.h>
39 #include <sys/containerof.h>
40 #include <sys/sensors.h>
41 
42 #include "version.h"
43 #include "common/common.h"
44 #include "common/t4_msg.h"
45 #include "common/t4_regs.h"
46 #include "firmware/t4_fw.h"
47 #include "firmware/t4_cfg.h"
48 #include "firmware/t5_fw.h"
49 #include "firmware/t5_cfg.h"
50 #include "firmware/t6_fw.h"
51 #include "firmware/t6_cfg.h"
52 #include "t4_l2t.h"
53 
54 static int t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp);
55 static int t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp);
56 static int t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp,
57     int *rp);
58 struct cb_ops t4_cb_ops = {
59 	.cb_open =		t4_cb_open,
60 	.cb_close =		t4_cb_close,
61 	.cb_strategy =		nodev,
62 	.cb_print =		nodev,
63 	.cb_dump =		nodev,
64 	.cb_read =		nodev,
65 	.cb_write =		nodev,
66 	.cb_ioctl =		t4_cb_ioctl,
67 	.cb_devmap =		nodev,
68 	.cb_mmap =		nodev,
69 	.cb_segmap =		nodev,
70 	.cb_chpoll =		nochpoll,
71 	.cb_prop_op =		ddi_prop_op,
72 	.cb_flag =		D_MP,
73 	.cb_rev =		CB_REV,
74 	.cb_aread =		nodev,
75 	.cb_awrite =		nodev
76 };
77 
78 static int t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op,
79     void *arg, void *result);
80 static int t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
81     void *arg, dev_info_t **cdipp);
82 static int t4_bus_unconfig(dev_info_t *dip, uint_t flags,
83     ddi_bus_config_op_t op, void *arg);
84 struct bus_ops t4_bus_ops = {
85 	.busops_rev =		BUSO_REV,
86 	.bus_ctl =		t4_bus_ctl,
87 	.bus_prop_op =		ddi_bus_prop_op,
88 	.bus_config =		t4_bus_config,
89 	.bus_unconfig =		t4_bus_unconfig,
90 };
91 
92 static int t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
93     void **rp);
94 static int t4_devo_probe(dev_info_t *dip);
95 static int t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
96 static int t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
97 static int t4_devo_quiesce(dev_info_t *dip);
98 struct dev_ops t4_dev_ops = {
99 	.devo_rev =		DEVO_REV,
100 	.devo_getinfo =		t4_devo_getinfo,
101 	.devo_identify =	nulldev,
102 	.devo_probe =		t4_devo_probe,
103 	.devo_attach =		t4_devo_attach,
104 	.devo_detach =		t4_devo_detach,
105 	.devo_reset =		nodev,
106 	.devo_cb_ops =		&t4_cb_ops,
107 	.devo_bus_ops =		&t4_bus_ops,
108 	.devo_quiesce =		&t4_devo_quiesce,
109 };
110 
111 static struct modldrv modldrv = {
112 	.drv_modops =		&mod_driverops,
113 	.drv_linkinfo =		"Chelsio T4 nexus " DRV_VERSION,
114 	.drv_dev_ops =		&t4_dev_ops
115 };
116 
117 static struct modlinkage modlinkage = {
118 	.ml_rev =		MODREV_1,
119 	.ml_linkage =		{&modldrv, NULL},
120 };
121 
122 void *t4_list;
123 
124 struct intrs_and_queues {
125 	int intr_type;		/* DDI_INTR_TYPE_* */
126 	int nirq;		/* Number of vectors */
127 	int intr_fwd;		/* Interrupts forwarded */
128 	int ntxq10g;		/* # of NIC txq's for each 10G port */
129 	int nrxq10g;		/* # of NIC rxq's for each 10G port */
130 	int ntxq1g;		/* # of NIC txq's for each 1G port */
131 	int nrxq1g;		/* # of NIC rxq's for each 1G port */
132 #ifdef TCP_OFFLOAD_ENABLE
133 	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
134 	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
135 	int nofldtxq1g;		/* # of TOE txq's for each 1G port */
136 	int nofldrxq1g;		/* # of TOE rxq's for each 1G port */
137 #endif
138 };
139 
140 struct fw_info fi[3];
141 
142 static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss,
143     mblk_t *m);
144 static int fw_msg_not_handled(struct adapter *, const __be64 *);
145 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
146 static unsigned int getpf(struct adapter *sc);
147 static int prep_firmware(struct adapter *sc);
148 static int upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma);
149 static int partition_resources(struct adapter *sc);
150 static int adap__pre_init_tweaks(struct adapter *sc);
151 static int get_params__pre_init(struct adapter *sc);
152 static int get_params__post_init(struct adapter *sc);
153 static int set_params__post_init(struct adapter *);
154 static void setup_memwin(struct adapter *sc);
155 static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
156     uint32_t *);
157 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
158 uint32_t position_memwin(struct adapter *, int, uint32_t);
159 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
160     uint_t count);
161 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
162     uint_t count);
163 static int init_driver_props(struct adapter *sc, struct driver_properties *p);
164 static int remove_extra_props(struct adapter *sc, int n10g, int n1g);
165 static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
166     struct intrs_and_queues *iaq);
167 static int add_child_node(struct adapter *sc, int idx);
168 static int remove_child_node(struct adapter *sc, int idx);
169 static kstat_t *setup_kstats(struct adapter *sc);
170 static kstat_t *setup_wc_kstats(struct adapter *);
171 static int update_wc_kstats(kstat_t *, int);
172 #ifdef TCP_OFFLOAD_ENABLE
173 static int toe_capability(struct port_info *pi, int enable);
174 static int activate_uld(struct adapter *sc, int id, struct uld_softc *usc);
175 static int deactivate_uld(struct uld_softc *usc);
176 #endif
177 static kmutex_t t4_adapter_list_lock;
178 static SLIST_HEAD(, adapter) t4_adapter_list;
179 #ifdef TCP_OFFLOAD_ENABLE
180 static kmutex_t t4_uld_list_lock;
181 static SLIST_HEAD(, uld_info) t4_uld_list;
182 #endif
183 
184 static int t4_temperature_read(void *, sensor_ioctl_scalar_t *);
185 static int t4_voltage_read(void *, sensor_ioctl_scalar_t *);
186 static const ksensor_ops_t t4_temp_ops = {
187 	.kso_kind = ksensor_kind_temperature,
188 	.kso_scalar = t4_temperature_read
189 };
190 
191 static const ksensor_ops_t t4_volt_ops = {
192 	.kso_kind = ksensor_kind_voltage,
193 	.kso_scalar = t4_voltage_read
194 };
195 
196 static int t4_ufm_getcaps(ddi_ufm_handle_t *, void *, ddi_ufm_cap_t *);
197 static int t4_ufm_fill_image(ddi_ufm_handle_t *, void *, uint_t,
198     ddi_ufm_image_t *);
199 static int t4_ufm_fill_slot(ddi_ufm_handle_t *, void *, uint_t, uint_t,
200     ddi_ufm_slot_t *);
201 static ddi_ufm_ops_t t4_ufm_ops = {
202 	.ddi_ufm_op_fill_image = t4_ufm_fill_image,
203 	.ddi_ufm_op_fill_slot = t4_ufm_fill_slot,
204 	.ddi_ufm_op_getcaps = t4_ufm_getcaps
205 };
206 
207 int
_init(void)208 _init(void)
209 {
210 	int rc;
211 
212 	rc = ddi_soft_state_init(&t4_list, sizeof (struct adapter), 0);
213 	if (rc != 0)
214 		return (rc);
215 
216 	rc = mod_install(&modlinkage);
217 	if (rc != 0)
218 		ddi_soft_state_fini(&t4_list);
219 
220 	mutex_init(&t4_adapter_list_lock, NULL, MUTEX_DRIVER, NULL);
221 	SLIST_INIT(&t4_adapter_list);
222 
223 #ifdef TCP_OFFLOAD_ENABLE
224 	mutex_init(&t4_uld_list_lock, NULL, MUTEX_DRIVER, NULL);
225 	SLIST_INIT(&t4_uld_list);
226 #endif
227 
228 	return (rc);
229 }
230 
231 int
_fini(void)232 _fini(void)
233 {
234 	int rc;
235 
236 	rc = mod_remove(&modlinkage);
237 	if (rc != 0)
238 		return (rc);
239 
240 	ddi_soft_state_fini(&t4_list);
241 	return (0);
242 }
243 
244 int
_info(struct modinfo * mi)245 _info(struct modinfo *mi)
246 {
247 	return (mod_info(&modlinkage, mi));
248 }
249 
250 /* ARGSUSED */
251 static int
t4_devo_getinfo(dev_info_t * dip,ddi_info_cmd_t cmd,void * arg,void ** rp)252 t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp)
253 {
254 	struct adapter *sc;
255 	minor_t minor;
256 
257 	minor = getminor((dev_t)arg);	/* same as instance# in our case */
258 
259 	if (cmd == DDI_INFO_DEVT2DEVINFO) {
260 		sc = ddi_get_soft_state(t4_list, minor);
261 		if (sc == NULL)
262 			return (DDI_FAILURE);
263 
264 		ASSERT(sc->dev == (dev_t)arg);
265 		*rp = (void *)sc->dip;
266 	} else if (cmd == DDI_INFO_DEVT2INSTANCE)
267 		*rp = (void *) (unsigned long) minor;
268 	else
269 		ASSERT(0);
270 
271 	return (DDI_SUCCESS);
272 }
273 
274 static int
t4_devo_probe(dev_info_t * dip)275 t4_devo_probe(dev_info_t *dip)
276 {
277 	int rc, id, *reg;
278 	uint_t n, pf;
279 
280 	id = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
281 	    "device-id", 0xffff);
282 	if (id == 0xffff)
283 		return (DDI_PROBE_DONTCARE);
284 
285 	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
286 	    "reg", &reg, &n);
287 	if (rc != DDI_SUCCESS)
288 		return (DDI_PROBE_DONTCARE);
289 
290 	pf = PCI_REG_FUNC_G(reg[0]);
291 	ddi_prop_free(reg);
292 
293 	/* Prevent driver attachment on any PF except 0 on the FPGA */
294 	if (id == 0xa000 && pf != 0)
295 		return (DDI_PROBE_FAILURE);
296 
297 	return (DDI_PROBE_DONTCARE);
298 }
299 
300 static int
t4_devo_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)301 t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
302 {
303 	struct adapter *sc = NULL;
304 	struct sge *s;
305 	int i, instance, rc = DDI_SUCCESS, rqidx, tqidx, q;
306 	int irq = 0, nxg = 0, n1g = 0;
307 #ifdef TCP_OFFLOAD_ENABLE
308 	int ofld_rqidx, ofld_tqidx;
309 #endif
310 	char name[16];
311 	struct driver_properties *prp;
312 	struct intrs_and_queues iaq;
313 	ddi_device_acc_attr_t da = {
314 		.devacc_attr_version = DDI_DEVICE_ATTR_V0,
315 		.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
316 		.devacc_attr_dataorder = DDI_STRICTORDER_ACC
317 	};
318 	ddi_device_acc_attr_t da1 = {
319 		.devacc_attr_version = DDI_DEVICE_ATTR_V0,
320 		.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
321 		.devacc_attr_dataorder = DDI_STRICTORDER_ACC
322 	};
323 
324 	if (cmd != DDI_ATTACH)
325 		return (DDI_FAILURE);
326 
327 	/*
328 	 * Allocate space for soft state.
329 	 */
330 	instance = ddi_get_instance(dip);
331 	rc = ddi_soft_state_zalloc(t4_list, instance);
332 	if (rc != DDI_SUCCESS) {
333 		cxgb_printf(dip, CE_WARN,
334 		    "failed to allocate soft state: %d", rc);
335 		return (DDI_FAILURE);
336 	}
337 
338 	sc = ddi_get_soft_state(t4_list, instance);
339 	sc->dip = dip;
340 	sc->dev = makedevice(ddi_driver_major(dip), instance);
341 	mutex_init(&sc->lock, NULL, MUTEX_DRIVER, NULL);
342 	cv_init(&sc->cv, NULL, CV_DRIVER, NULL);
343 	mutex_init(&sc->sfl_lock, NULL, MUTEX_DRIVER, NULL);
344 	TAILQ_INIT(&sc->sfl);
345 	mutex_init(&sc->mbox_lock, NULL, MUTEX_DRIVER, NULL);
346 	STAILQ_INIT(&sc->mbox_list);
347 
348 	mutex_enter(&t4_adapter_list_lock);
349 	SLIST_INSERT_HEAD(&t4_adapter_list, sc, link);
350 	mutex_exit(&t4_adapter_list_lock);
351 
352 	sc->pf = getpf(sc);
353 	if (sc->pf > 8) {
354 		rc = EINVAL;
355 		cxgb_printf(dip, CE_WARN,
356 		    "failed to determine PCI PF# of device");
357 		goto done;
358 	}
359 	sc->mbox = sc->pf;
360 
361 	/* Initialize the driver properties */
362 	prp = &sc->props;
363 	(void)init_driver_props(sc, prp);
364 
365 	/*
366 	 * Enable access to the PCI config space.
367 	 */
368 	rc = pci_config_setup(dip, &sc->pci_regh);
369 	if (rc != DDI_SUCCESS) {
370 		cxgb_printf(dip, CE_WARN,
371 		    "failed to enable PCI config space access: %d", rc);
372 		goto done;
373 	}
374 
375 	/* TODO: Set max read request to 4K */
376 
377 	/*
378 	 * Enable MMIO access.
379 	 */
380 	rc = ddi_regs_map_setup(dip, 1, &sc->regp, 0, 0, &da, &sc->regh);
381 	if (rc != DDI_SUCCESS) {
382 		cxgb_printf(dip, CE_WARN,
383 		    "failed to map device registers: %d", rc);
384 		goto done;
385 	}
386 
387 	(void) memset(sc->chan_map, 0xff, sizeof (sc->chan_map));
388 
389 	/*
390 	 * Initialize cpl handler.
391 	 */
392 	for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) {
393 		sc->cpl_handler[i] = cpl_not_handled;
394 	}
395 
396 	for (i = 0; i < ARRAY_SIZE(sc->fw_msg_handler); i++) {
397 		sc->fw_msg_handler[i] = fw_msg_not_handled;
398 	}
399 
400 	for (i = 0; i < NCHAN; i++) {
401 		(void) snprintf(name, sizeof (name), "%s-%d",
402 				"reclaim", i);
403 		sc->tq[i] = ddi_taskq_create(sc->dip,
404 		   name, 1, TASKQ_DEFAULTPRI, 0);
405 
406 		if (sc->tq[i] == NULL) {
407 			cxgb_printf(dip, CE_WARN,
408 				   "failed to create task queues");
409 			rc = DDI_FAILURE;
410 			goto done;
411 		}
412 	}
413 
414 	/*
415 	 * Prepare the adapter for operation.
416 	 */
417 	rc = -t4_prep_adapter(sc, false);
418 	if (rc != 0) {
419 		cxgb_printf(dip, CE_WARN, "failed to prepare adapter: %d", rc);
420 		goto done;
421 	}
422 
423 	/*
424 	 * Enable BAR1 access.
425 	 */
426 	sc->doorbells |= DOORBELL_KDB;
427 	rc = ddi_regs_map_setup(dip, 2, &sc->reg1p, 0, 0, &da1, &sc->reg1h);
428 	if (rc != DDI_SUCCESS) {
429 		cxgb_printf(dip, CE_WARN,
430 		    "failed to map BAR1 device registers: %d", rc);
431 		goto done;
432 	} else {
433 		if (is_t5(sc->params.chip)) {
434 			sc->doorbells |= DOORBELL_UDB;
435 			if (prp->wc) {
436 				/*
437 				 * Enable write combining on BAR2.  This is the
438 				 * userspace doorbell BAR and is split into 128B
439 				 * (UDBS_SEG_SIZE) doorbell regions, each associated
440 				 * with an egress queue.  The first 64B has the doorbell
441 				 * and the second 64B can be used to submit a tx work
442 				 * request with an implicit doorbell.
443 				 */
444 				sc->doorbells &= ~DOORBELL_UDB;
445 				sc->doorbells |= (DOORBELL_WCWR |
446 				    DOORBELL_UDBWC);
447 				t4_write_reg(sc, A_SGE_STAT_CFG,
448 				    V_STATSOURCE_T5(7) | V_STATMODE(0));
449 			}
450 		}
451 	}
452 
453 	/*
454 	 * Do this really early.  Note that minor number = instance.
455 	 */
456 	(void) snprintf(name, sizeof (name), "%s,%d", T4_NEXUS_NAME, instance);
457 	rc = ddi_create_minor_node(dip, name, S_IFCHR, instance,
458 	    DDI_NT_NEXUS, 0);
459 	if (rc != DDI_SUCCESS) {
460 		cxgb_printf(dip, CE_WARN,
461 		    "failed to create device node: %d", rc);
462 		rc = DDI_SUCCESS; /* carry on */
463 	}
464 
465 	/* Do this early. Memory window is required for loading config file. */
466 	setup_memwin(sc);
467 
468 	/* Prepare the firmware for operation */
469 	rc = prep_firmware(sc);
470 	if (rc != 0)
471 		goto done; /* error message displayed already */
472 
473 	rc = adap__pre_init_tweaks(sc);
474 	if (rc != 0)
475 		goto done;
476 
477 	rc = get_params__pre_init(sc);
478 	if (rc != 0)
479 		goto done; /* error message displayed already */
480 
481 	t4_sge_init(sc);
482 
483 	if (sc->flags & MASTER_PF) {
484 		/* get basic stuff going */
485 		rc = -t4_fw_initialize(sc, sc->mbox);
486 		if (rc != 0) {
487 			cxgb_printf(sc->dip, CE_WARN,
488 			    "early init failed: %d.\n", rc);
489 			goto done;
490 		}
491 	}
492 
493 	rc = get_params__post_init(sc);
494 	if (rc != 0)
495 		goto done; /* error message displayed already */
496 
497 	rc = set_params__post_init(sc);
498 	if (rc != 0)
499 		goto done; /* error message displayed already */
500 
501 	/*
502 	 * TODO: This is the place to call t4_set_filter_mode()
503 	 */
504 
505 	/* tweak some settings */
506 	t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) |
507 	    V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) |
508 	    V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9));
509 	t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
510 
511 	/*
512 	 * Work-around for bug 2619
513 	 * Set DisableVlan field in TP_RSS_CONFIG_VRT register so that the
514 	 * VLAN tag extraction is disabled.
515 	 */
516 	t4_set_reg_field(sc, A_TP_RSS_CONFIG_VRT, F_DISABLEVLAN, F_DISABLEVLAN);
517 
518 	/* Store filter mode */
519 	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
520 	    A_TP_VLAN_PRI_MAP);
521 
522 	/*
523 	 * First pass over all the ports - allocate VIs and initialize some
524 	 * basic parameters like mac address, port type, etc.  We also figure
525 	 * out whether a port is 10G or 1G and use that information when
526 	 * calculating how many interrupts to attempt to allocate.
527 	 */
528 	for_each_port(sc, i) {
529 		struct port_info *pi;
530 
531 		pi = kmem_zalloc(sizeof (*pi), KM_SLEEP);
532 		sc->port[i] = pi;
533 
534 		/* These must be set before t4_port_init */
535 		pi->adapter = sc;
536 		/* LINTED: E_ASSIGN_NARROW_CONV */
537 		pi->port_id = i;
538 	}
539 
540 	/* Allocate the vi and initialize parameters like mac addr */
541 	rc = -t4_port_init(sc, sc->mbox, sc->pf, 0);
542 	if (rc) {
543 		cxgb_printf(dip, CE_WARN,
544 			    "unable to initialize port: %d", rc);
545 		goto done;
546 	}
547 
548 	for_each_port(sc, i) {
549 		struct port_info *pi = sc->port[i];
550 
551 		mutex_init(&pi->lock, NULL, MUTEX_DRIVER, NULL);
552 		pi->mtu = ETHERMTU;
553 
554 		if (is_10XG_port(pi)) {
555 			nxg++;
556 			pi->tmr_idx = prp->tmr_idx_10g;
557 			pi->pktc_idx = prp->pktc_idx_10g;
558 		} else {
559 			n1g++;
560 			pi->tmr_idx = prp->tmr_idx_1g;
561 			pi->pktc_idx = prp->pktc_idx_1g;
562 		}
563 
564 		pi->xact_addr_filt = -1;
565 		t4_mc_init(pi);
566 
567 		setbit(&sc->registered_device_map, i);
568 	}
569 
570 	(void) remove_extra_props(sc, nxg, n1g);
571 
572 	if (sc->registered_device_map == 0) {
573 		cxgb_printf(dip, CE_WARN, "no usable ports");
574 		rc = DDI_FAILURE;
575 		goto done;
576 	}
577 
578 	rc = cfg_itype_and_nqueues(sc, nxg, n1g, &iaq);
579 	if (rc != 0)
580 		goto done; /* error message displayed already */
581 
582 	sc->intr_type = iaq.intr_type;
583 	sc->intr_count = iaq.nirq;
584 
585 	if (sc->props.multi_rings && (sc->intr_type != DDI_INTR_TYPE_MSIX)) {
586 		sc->props.multi_rings = 0;
587 		cxgb_printf(dip, CE_WARN,
588 		    "Multiple rings disabled as interrupt type is not MSI-X");
589 	}
590 
591 	if (sc->props.multi_rings && iaq.intr_fwd) {
592 		sc->props.multi_rings = 0;
593 		cxgb_printf(dip, CE_WARN,
594 		    "Multiple rings disabled as interrupts are forwarded");
595 	}
596 
597 	if (!sc->props.multi_rings) {
598 		iaq.ntxq10g = 1;
599 		iaq.ntxq1g = 1;
600 	}
601 	s = &sc->sge;
602 	s->nrxq = nxg * iaq.nrxq10g + n1g * iaq.nrxq1g;
603 	s->ntxq = nxg * iaq.ntxq10g + n1g * iaq.ntxq1g;
604 	s->neq = s->ntxq + s->nrxq;	/* the fl in an rxq is an eq */
605 #ifdef TCP_OFFLOAD_ENABLE
606 	/* control queues, 1 per port + 1 mgmtq */
607 	s->neq += sc->params.nports + 1;
608 #endif
609 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
610 	if (iaq.intr_fwd != 0)
611 		sc->flags |= INTR_FWD;
612 #ifdef TCP_OFFLOAD_ENABLE
613 	if (is_offload(sc) != 0) {
614 
615 		s->nofldrxq = nxg * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
616 		s->nofldtxq = nxg * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
617 		s->neq += s->nofldtxq + s->nofldrxq;
618 		s->niq += s->nofldrxq;
619 
620 		s->ofld_rxq = kmem_zalloc(s->nofldrxq *
621 		    sizeof (struct sge_ofld_rxq), KM_SLEEP);
622 		s->ofld_txq = kmem_zalloc(s->nofldtxq *
623 		    sizeof (struct sge_wrq), KM_SLEEP);
624 		s->ctrlq = kmem_zalloc(sc->params.nports *
625 		    sizeof (struct sge_wrq), KM_SLEEP);
626 
627 	}
628 #endif
629 	s->rxq = kmem_zalloc(s->nrxq * sizeof (struct sge_rxq), KM_SLEEP);
630 	s->txq = kmem_zalloc(s->ntxq * sizeof (struct sge_txq), KM_SLEEP);
631 	s->iqmap = kmem_zalloc(s->iqmap_sz * sizeof (struct sge_iq *), KM_SLEEP);
632 	s->eqmap = kmem_zalloc(s->eqmap_sz * sizeof (struct sge_eq *), KM_SLEEP);
633 
634 	sc->intr_handle = kmem_zalloc(sc->intr_count *
635 	    sizeof (ddi_intr_handle_t), KM_SLEEP);
636 
637 	/*
638 	 * Second pass over the ports.  This time we know the number of rx and
639 	 * tx queues that each port should get.
640 	 */
641 	rqidx = tqidx = 0;
642 #ifdef TCP_OFFLOAD_ENABLE
643 	ofld_rqidx = ofld_tqidx = 0;
644 #endif
645 	for_each_port(sc, i) {
646 		struct port_info *pi = sc->port[i];
647 
648 		if (pi == NULL)
649 			continue;
650 
651 		t4_mc_cb_init(pi);
652 		/* LINTED: E_ASSIGN_NARROW_CONV */
653 		pi->first_rxq = rqidx;
654 		/* LINTED: E_ASSIGN_NARROW_CONV */
655 		pi->nrxq = (is_10XG_port(pi)) ? iaq.nrxq10g
656 		    : iaq.nrxq1g;
657 		/* LINTED: E_ASSIGN_NARROW_CONV */
658 		pi->first_txq = tqidx;
659 		/* LINTED: E_ASSIGN_NARROW_CONV */
660 		pi->ntxq = (is_10XG_port(pi)) ? iaq.ntxq10g
661 		    : iaq.ntxq1g;
662 
663 		rqidx += pi->nrxq;
664 		tqidx += pi->ntxq;
665 
666 #ifdef TCP_OFFLOAD_ENABLE
667 		if (is_offload(sc) != 0) {
668 			/* LINTED: E_ASSIGN_NARROW_CONV */
669 			pi->first_ofld_rxq = ofld_rqidx;
670 			pi->nofldrxq = max(1, pi->nrxq / 4);
671 
672 			/* LINTED: E_ASSIGN_NARROW_CONV */
673 			pi->first_ofld_txq = ofld_tqidx;
674 			pi->nofldtxq = max(1, pi->ntxq / 2);
675 
676 			ofld_rqidx += pi->nofldrxq;
677 			ofld_tqidx += pi->nofldtxq;
678 		}
679 #endif
680 
681 		/*
682 		 * Enable hw checksumming and LSO for all ports by default.
683 		 * They can be disabled using ndd (hw_csum and hw_lso).
684 		 */
685 		pi->features |= (CXGBE_HW_CSUM | CXGBE_HW_LSO);
686 	}
687 
688 #ifdef TCP_OFFLOAD_ENABLE
689 		sc->l2t = t4_init_l2t(sc);
690 #endif
691 
692 	/*
693 	 * Setup Interrupts.
694 	 */
695 
696 	i = 0;
697 	rc = ddi_intr_alloc(dip, sc->intr_handle, sc->intr_type, 0,
698 	    sc->intr_count, &i, DDI_INTR_ALLOC_STRICT);
699 	if (rc != DDI_SUCCESS) {
700 		cxgb_printf(dip, CE_WARN,
701 		    "failed to allocate %d interrupt(s) of type %d: %d, %d",
702 		    sc->intr_count, sc->intr_type, rc, i);
703 		goto done;
704 	}
705 	ASSERT(sc->intr_count == i); /* allocation was STRICT */
706 	(void) ddi_intr_get_cap(sc->intr_handle[0], &sc->intr_cap);
707 	(void) ddi_intr_get_pri(sc->intr_handle[0], &sc->intr_pri);
708 	if (sc->intr_count == 1) {
709 		ASSERT(sc->flags & INTR_FWD);
710 		(void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_all, sc,
711 		    &s->fwq);
712 	} else {
713 		/* Multiple interrupts.  The first one is always error intr */
714 		(void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_err, sc,
715 		    NULL);
716 		irq++;
717 
718 		/* The second one is always the firmware event queue */
719 		(void) ddi_intr_add_handler(sc->intr_handle[1], t4_intr, sc,
720 		    &s->fwq);
721 		irq++;
722 		/*
723 		 * Note that if INTR_FWD is set then either the NIC rx
724 		 * queues or (exclusive or) the TOE rx queueus will be taking
725 		 * direct interrupts.
726 		 *
727 		 * There is no need to check for is_offload(sc) as nofldrxq
728 		 * will be 0 if offload is disabled.
729 		 */
730 		for_each_port(sc, i) {
731 			struct port_info *pi = sc->port[i];
732 			struct sge_rxq *rxq;
733 #ifdef TCP_OFFLOAD_ENABLE
734 			struct sge_ofld_rxq *ofld_rxq;
735 
736 			/*
737 			 * Skip over the NIC queues if they aren't taking direct
738 			 * interrupts.
739 			 */
740 			if ((sc->flags & INTR_FWD) &&
741 			    pi->nofldrxq > pi->nrxq)
742 				goto ofld_queues;
743 #endif
744 			rxq = &s->rxq[pi->first_rxq];
745 			for (q = 0; q < pi->nrxq; q++, rxq++) {
746 				(void) ddi_intr_add_handler(
747 				    sc->intr_handle[irq], t4_intr, sc,
748 				    &rxq->iq);
749 				irq++;
750 			}
751 
752 #ifdef TCP_OFFLOAD_ENABLE
753 			/*
754 			 * Skip over the offload queues if they aren't taking
755 			 * direct interrupts.
756 			 */
757 			if ((sc->flags & INTR_FWD))
758 				continue;
759 ofld_queues:
760 			ofld_rxq = &s->ofld_rxq[pi->first_ofld_rxq];
761 			for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
762 				(void) ddi_intr_add_handler(
763 				    sc->intr_handle[irq], t4_intr, sc,
764 				    &ofld_rxq->iq);
765 				irq++;
766 			}
767 #endif
768 		}
769 
770 	}
771 	sc->flags |= INTR_ALLOCATED;
772 
773 	if ((rc = ksensor_create_scalar_pcidev(dip, SENSOR_KIND_TEMPERATURE,
774 	    &t4_temp_ops, sc, "temp", &sc->temp_sensor)) != 0) {
775 		cxgb_printf(dip, CE_WARN, "failed to create temperature "
776 		    "sensor: %d", rc);
777 		rc = DDI_FAILURE;
778 		goto done;
779 	}
780 
781 	if ((rc = ksensor_create_scalar_pcidev(dip, SENSOR_KIND_VOLTAGE,
782 	    &t4_volt_ops, sc, "vdd", &sc->volt_sensor)) != 0) {
783 		cxgb_printf(dip, CE_WARN, "failed to create voltage "
784 		    "sensor: %d", rc);
785 		rc = DDI_FAILURE;
786 		goto done;
787 	}
788 
789 
790 	if ((rc = ddi_ufm_init(dip, DDI_UFM_CURRENT_VERSION, &t4_ufm_ops,
791 	    &sc->ufm_hdl, sc)) != 0) {
792 		cxgb_printf(dip, CE_WARN, "failed to enable UFM ops: %d", rc);
793 		rc = DDI_FAILURE;
794 		goto done;
795 	}
796 	ddi_ufm_update(sc->ufm_hdl);
797 	ddi_report_dev(dip);
798 
799 	/*
800 	 * Hardware/Firmware/etc. Version/Revision IDs.
801 	 */
802 	t4_dump_version_info(sc);
803 
804 	cxgb_printf(dip, CE_NOTE,
805 		    "(%d rxq, %d txq total) %d %s.",
806 		    rqidx, tqidx, sc->intr_count,
807 		    sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
808 		    sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
809 		    "fixed interrupt");
810 
811 	sc->ksp = setup_kstats(sc);
812 	sc->ksp_stat = setup_wc_kstats(sc);
813 	sc->params.drv_memwin = MEMWIN_NIC;
814 
815 done:
816 	if (rc != DDI_SUCCESS) {
817 		(void) t4_devo_detach(dip, DDI_DETACH);
818 
819 		/* rc may have errno style errors or DDI errors */
820 		rc = DDI_FAILURE;
821 	}
822 
823 	return (rc);
824 }
825 
826 static int
t4_devo_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)827 t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
828 {
829 	int instance, i;
830 	struct adapter *sc;
831 	struct port_info *pi;
832 	struct sge *s;
833 
834 	if (cmd != DDI_DETACH)
835 		return (DDI_FAILURE);
836 
837 	instance = ddi_get_instance(dip);
838 	sc = ddi_get_soft_state(t4_list, instance);
839 	if (sc == NULL)
840 		return (DDI_SUCCESS);
841 
842 	if (sc->flags & FULL_INIT_DONE) {
843 		t4_intr_disable(sc);
844 		for_each_port(sc, i) {
845 			pi = sc->port[i];
846 			if (pi && pi->flags & PORT_INIT_DONE)
847 				(void) port_full_uninit(pi);
848 		}
849 		(void) adapter_full_uninit(sc);
850 	}
851 
852 	/* Safe to call no matter what */
853 	if (sc->ufm_hdl != NULL) {
854 		ddi_ufm_fini(sc->ufm_hdl);
855 		sc->ufm_hdl = NULL;
856 	}
857 	(void) ksensor_remove(dip, KSENSOR_ALL_IDS);
858 	ddi_prop_remove_all(dip);
859 	ddi_remove_minor_node(dip, NULL);
860 
861 	for (i = 0; i < NCHAN; i++) {
862 		if (sc->tq[i]) {
863 			ddi_taskq_wait(sc->tq[i]);
864 			ddi_taskq_destroy(sc->tq[i]);
865 		}
866 	}
867 
868 	if (sc->ksp != NULL)
869 		kstat_delete(sc->ksp);
870 	if (sc->ksp_stat != NULL)
871 		kstat_delete(sc->ksp_stat);
872 
873 	s = &sc->sge;
874 	if (s->rxq != NULL)
875 		kmem_free(s->rxq, s->nrxq * sizeof (struct sge_rxq));
876 #ifdef TCP_OFFLOAD_ENABLE
877 	if (s->ofld_txq != NULL)
878 		kmem_free(s->ofld_txq, s->nofldtxq * sizeof (struct sge_wrq));
879 	if (s->ofld_rxq != NULL)
880 		kmem_free(s->ofld_rxq,
881 		    s->nofldrxq * sizeof (struct sge_ofld_rxq));
882 	if (s->ctrlq != NULL)
883 		kmem_free(s->ctrlq,
884 		    sc->params.nports * sizeof (struct sge_wrq));
885 #endif
886 	if (s->txq != NULL)
887 		kmem_free(s->txq, s->ntxq * sizeof (struct sge_txq));
888 	if (s->iqmap != NULL)
889 		kmem_free(s->iqmap, s->iqmap_sz * sizeof (struct sge_iq *));
890 	if (s->eqmap != NULL)
891 		kmem_free(s->eqmap, s->eqmap_sz * sizeof (struct sge_eq *));
892 
893 	if (s->rxbuf_cache != NULL)
894 		rxbuf_cache_destroy(s->rxbuf_cache);
895 
896 	if (sc->flags & INTR_ALLOCATED) {
897 		for (i = 0; i < sc->intr_count; i++) {
898 			(void) ddi_intr_remove_handler(sc->intr_handle[i]);
899 			(void) ddi_intr_free(sc->intr_handle[i]);
900 		}
901 		sc->flags &= ~INTR_ALLOCATED;
902 	}
903 
904 	if (sc->intr_handle != NULL) {
905 		kmem_free(sc->intr_handle,
906 		    sc->intr_count * sizeof (*sc->intr_handle));
907 	}
908 
909 	for_each_port(sc, i) {
910 		pi = sc->port[i];
911 		if (pi != NULL) {
912 			mutex_destroy(&pi->lock);
913 			kmem_free(pi, sizeof (*pi));
914 			clrbit(&sc->registered_device_map, i);
915 		}
916 	}
917 
918 	if (sc->flags & FW_OK)
919 		(void) t4_fw_bye(sc, sc->mbox);
920 
921 	if (sc->reg1h != NULL)
922 		ddi_regs_map_free(&sc->reg1h);
923 
924 	if (sc->regh != NULL)
925 		ddi_regs_map_free(&sc->regh);
926 
927 	if (sc->pci_regh != NULL)
928 		pci_config_teardown(&sc->pci_regh);
929 
930 	mutex_enter(&t4_adapter_list_lock);
931 	SLIST_REMOVE(&t4_adapter_list, sc, adapter, link);
932 	mutex_exit(&t4_adapter_list_lock);
933 
934 	mutex_destroy(&sc->mbox_lock);
935 	mutex_destroy(&sc->lock);
936 	cv_destroy(&sc->cv);
937 	mutex_destroy(&sc->sfl_lock);
938 
939 #ifdef DEBUG
940 	bzero(sc, sizeof (*sc));
941 #endif
942 	ddi_soft_state_free(t4_list, instance);
943 
944 	return (DDI_SUCCESS);
945 }
946 
947 static int
t4_devo_quiesce(dev_info_t * dip)948 t4_devo_quiesce(dev_info_t *dip)
949 {
950 	int instance;
951 	struct adapter *sc;
952 
953 	instance = ddi_get_instance(dip);
954 	sc = ddi_get_soft_state(t4_list, instance);
955 	if (sc == NULL)
956 		return (DDI_SUCCESS);
957 
958 	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
959 	t4_intr_disable(sc);
960 	t4_write_reg(sc, A_PL_RST, F_PIORSTMODE | F_PIORST);
961 
962 	return (DDI_SUCCESS);
963 }
964 
965 static int
t4_bus_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_ctl_enum_t op,void * arg,void * result)966 t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg,
967     void *result)
968 {
969 	char s[4];
970 	struct port_info *pi;
971 	dev_info_t *child = (dev_info_t *)arg;
972 
973 	switch (op) {
974 	case DDI_CTLOPS_REPORTDEV:
975 		pi = ddi_get_parent_data(rdip);
976 		pi->instance = ddi_get_instance(dip);
977 		pi->child_inst = ddi_get_instance(rdip);
978 		cmn_err(CE_CONT, "?%s%d is port %s on %s%d\n",
979 		    ddi_node_name(rdip), ddi_get_instance(rdip),
980 		    ddi_get_name_addr(rdip), ddi_driver_name(dip),
981 		    ddi_get_instance(dip));
982 		return (DDI_SUCCESS);
983 
984 	case DDI_CTLOPS_INITCHILD:
985 		pi = ddi_get_parent_data(child);
986 		if (pi == NULL)
987 			return (DDI_NOT_WELL_FORMED);
988 		(void) snprintf(s, sizeof (s), "%d", pi->port_id);
989 		ddi_set_name_addr(child, s);
990 		return (DDI_SUCCESS);
991 
992 	case DDI_CTLOPS_UNINITCHILD:
993 		ddi_set_name_addr(child, NULL);
994 		return (DDI_SUCCESS);
995 
996 	case DDI_CTLOPS_ATTACH:
997 	case DDI_CTLOPS_DETACH:
998 		return (DDI_SUCCESS);
999 
1000 	default:
1001 		return (ddi_ctlops(dip, rdip, op, arg, result));
1002 	}
1003 }
1004 
1005 static int
t4_bus_config(dev_info_t * dip,uint_t flags,ddi_bus_config_op_t op,void * arg,dev_info_t ** cdipp)1006 t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, void *arg,
1007     dev_info_t **cdipp)
1008 {
1009 	int instance, i;
1010 	struct adapter *sc;
1011 
1012 	instance = ddi_get_instance(dip);
1013 	sc = ddi_get_soft_state(t4_list, instance);
1014 
1015 	if (op == BUS_CONFIG_ONE) {
1016 		char *c;
1017 
1018 		/*
1019 		 * arg is something like "cxgb@0" where 0 is the port_id hanging
1020 		 * off this nexus.
1021 		 */
1022 
1023 		c = arg;
1024 		while (*(c + 1))
1025 			c++;
1026 
1027 		/* There should be exactly 1 digit after '@' */
1028 		if (*(c - 1) != '@')
1029 			return (NDI_FAILURE);
1030 
1031 		i = *c - '0';
1032 
1033 		if (add_child_node(sc, i) != 0)
1034 			return (NDI_FAILURE);
1035 
1036 		flags |= NDI_ONLINE_ATTACH;
1037 
1038 	} else if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) {
1039 		/* Allocate and bind all child device nodes */
1040 		for_each_port(sc, i)
1041 		    (void) add_child_node(sc, i);
1042 		flags |= NDI_ONLINE_ATTACH;
1043 	}
1044 
1045 	return (ndi_busop_bus_config(dip, flags, op, arg, cdipp, 0));
1046 }
1047 
1048 static int
t4_bus_unconfig(dev_info_t * dip,uint_t flags,ddi_bus_config_op_t op,void * arg)1049 t4_bus_unconfig(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
1050     void *arg)
1051 {
1052 	int instance, i, rc;
1053 	struct adapter *sc;
1054 
1055 	instance = ddi_get_instance(dip);
1056 	sc = ddi_get_soft_state(t4_list, instance);
1057 
1058 	if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ALL ||
1059 	    op == BUS_UNCONFIG_DRIVER)
1060 		flags |= NDI_UNCONFIG;
1061 
1062 	rc = ndi_busop_bus_unconfig(dip, flags, op, arg);
1063 	if (rc != 0)
1064 		return (rc);
1065 
1066 	if (op == BUS_UNCONFIG_ONE) {
1067 		char *c;
1068 
1069 		c = arg;
1070 		while (*(c + 1))
1071 			c++;
1072 
1073 		if (*(c - 1) != '@')
1074 			return (NDI_SUCCESS);
1075 
1076 		i = *c - '0';
1077 
1078 		rc = remove_child_node(sc, i);
1079 
1080 	} else if (op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) {
1081 
1082 		for_each_port(sc, i)
1083 		    (void) remove_child_node(sc, i);
1084 	}
1085 
1086 	return (rc);
1087 }
1088 
1089 /* ARGSUSED */
1090 static int
t4_cb_open(dev_t * devp,int flag,int otyp,cred_t * credp)1091 t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp)
1092 {
1093 	struct adapter *sc;
1094 
1095 	if (otyp != OTYP_CHR)
1096 		return (EINVAL);
1097 
1098 	sc = ddi_get_soft_state(t4_list, getminor(*devp));
1099 	if (sc == NULL)
1100 		return (ENXIO);
1101 
1102 	return (atomic_cas_uint(&sc->open, 0, EBUSY));
1103 }
1104 
1105 /* ARGSUSED */
1106 static int
t4_cb_close(dev_t dev,int flag,int otyp,cred_t * credp)1107 t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp)
1108 {
1109 	struct adapter *sc;
1110 
1111 	sc = ddi_get_soft_state(t4_list, getminor(dev));
1112 	if (sc == NULL)
1113 		return (EINVAL);
1114 
1115 	(void) atomic_swap_uint(&sc->open, 0);
1116 	return (0);
1117 }
1118 
1119 /* ARGSUSED */
1120 static int
t4_cb_ioctl(dev_t dev,int cmd,intptr_t d,int mode,cred_t * credp,int * rp)1121 t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, int *rp)
1122 {
1123 	int instance;
1124 	struct adapter *sc;
1125 	void *data = (void *)d;
1126 
1127 	if (crgetuid(credp) != 0)
1128 		return (EPERM);
1129 
1130 	instance = getminor(dev);
1131 	sc = ddi_get_soft_state(t4_list, instance);
1132 	if (sc == NULL)
1133 		return (EINVAL);
1134 
1135 	return (t4_ioctl(sc, cmd, data, mode));
1136 }
1137 
1138 static unsigned int
getpf(struct adapter * sc)1139 getpf(struct adapter *sc)
1140 {
1141 	int rc, *data;
1142 	uint_t n, pf;
1143 
1144 	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1145 	    DDI_PROP_DONTPASS, "reg", &data, &n);
1146 	if (rc != DDI_SUCCESS) {
1147 		cxgb_printf(sc->dip, CE_WARN,
1148 		    "failed to lookup \"reg\" property: %d", rc);
1149 		return (0xff);
1150 	}
1151 
1152 	pf = PCI_REG_FUNC_G(data[0]);
1153 	ddi_prop_free(data);
1154 
1155 	return (pf);
1156 }
1157 
1158 
1159 static struct fw_info *
find_fw_info(int chip)1160 find_fw_info(int chip)
1161 {
1162 	u32 i;
1163 
1164 	fi[0].chip = CHELSIO_T4;
1165 	fi[0].fw_hdr.chip = FW_HDR_CHIP_T4;
1166 	fi[0].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T4));
1167 	fi[0].fw_hdr.intfver_nic = FW_INTFVER(T4, NIC);
1168 	fi[0].fw_hdr.intfver_vnic = FW_INTFVER(T4, VNIC);
1169 	fi[0].fw_hdr.intfver_ofld = FW_INTFVER(T4, OFLD);
1170 	fi[0].fw_hdr.intfver_ri = FW_INTFVER(T4, RI);
1171 	fi[0].fw_hdr.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU);
1172 	fi[0].fw_hdr.intfver_iscsi = FW_INTFVER(T4, ISCSI);
1173 	fi[0].fw_hdr.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU);
1174 	fi[0].fw_hdr.intfver_fcoe = FW_INTFVER(T4, FCOE);
1175 
1176 	fi[1].chip = CHELSIO_T5;
1177 	fi[1].fw_hdr.chip = FW_HDR_CHIP_T5;
1178 	fi[1].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T5));
1179 	fi[1].fw_hdr.intfver_nic = FW_INTFVER(T5, NIC);
1180 	fi[1].fw_hdr.intfver_vnic = FW_INTFVER(T5, VNIC);
1181 	fi[1].fw_hdr.intfver_ofld = FW_INTFVER(T5, OFLD);
1182 	fi[1].fw_hdr.intfver_ri = FW_INTFVER(T5, RI);
1183 	fi[1].fw_hdr.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU);
1184 	fi[1].fw_hdr.intfver_iscsi = FW_INTFVER(T5, ISCSI);
1185 	fi[1].fw_hdr.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU);
1186 	fi[1].fw_hdr.intfver_fcoe = FW_INTFVER(T5, FCOE);
1187 
1188 	fi[2].chip = CHELSIO_T6;
1189 	fi[2].fw_hdr.chip = FW_HDR_CHIP_T6;
1190 	fi[2].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T6));
1191 	fi[2].fw_hdr.intfver_nic = FW_INTFVER(T6, NIC);
1192 	fi[2].fw_hdr.intfver_vnic = FW_INTFVER(T6, VNIC);
1193 	fi[2].fw_hdr.intfver_ofld = FW_INTFVER(T6, OFLD);
1194 	fi[2].fw_hdr.intfver_ri = FW_INTFVER(T6, RI);
1195 	fi[2].fw_hdr.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU);
1196 	fi[2].fw_hdr.intfver_iscsi = FW_INTFVER(T6, ISCSI);
1197 	fi[2].fw_hdr.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU);
1198 	fi[2].fw_hdr.intfver_fcoe = FW_INTFVER(T6, FCOE);
1199 
1200 	for (i = 0; i < ARRAY_SIZE(fi); i++) {
1201 		if (fi[i].chip == chip)
1202 			return &fi[i];
1203 	}
1204 
1205 	return NULL;
1206 }
1207 
1208 /*
1209  * Install a compatible firmware (if required), establish contact with it,
1210  * become the master, and reset the device.
1211  */
1212 static int
prep_firmware(struct adapter * sc)1213 prep_firmware(struct adapter *sc)
1214 {
1215 	int rc;
1216 	int fw_size;
1217 	int reset = 1;
1218 	enum dev_state state;
1219 	unsigned char *fw_data;
1220 	struct fw_info *fw_info;
1221 	struct fw_hdr *card_fw;
1222 
1223 	struct driver_properties *p = &sc->props;
1224 
1225 	/* Contact firmware, request master */
1226 	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state);
1227 	if (rc < 0) {
1228 		rc = -rc;
1229 		cxgb_printf(sc->dip, CE_WARN,
1230 		    "failed to connect to the firmware: %d.", rc);
1231 		return (rc);
1232 	}
1233 
1234 	if (rc == sc->mbox)
1235 		sc->flags |= MASTER_PF;
1236 
1237 	/* We may need FW version info for later reporting */
1238 	t4_get_version_info(sc);
1239 	fw_info = find_fw_info(CHELSIO_CHIP_VERSION(sc->params.chip));
1240 	/* allocate memory to read the header of the firmware on the
1241 	 * card
1242 	 */
1243 	if (!fw_info) {
1244 		cxgb_printf(sc->dip, CE_WARN,
1245 			    "unable to look up firmware information for chip %d.\n",
1246 			    CHELSIO_CHIP_VERSION(sc->params.chip));
1247 		return EINVAL;
1248 	}
1249 	card_fw = kmem_zalloc(sizeof(*card_fw), KM_SLEEP);
1250 	if(!card_fw) {
1251 		cxgb_printf(sc->dip, CE_WARN,
1252 			    "Memory allocation for card FW header failed\n");
1253 		return ENOMEM;
1254 	}
1255 	switch(CHELSIO_CHIP_VERSION(sc->params.chip)) {
1256 	case CHELSIO_T4:
1257 		fw_data = t4fw_data;
1258 		fw_size = t4fw_size;
1259 		break;
1260 	case CHELSIO_T5:
1261 		fw_data = t5fw_data;
1262 		fw_size = t5fw_size;
1263 		break;
1264 	case CHELSIO_T6:
1265 		fw_data = t6fw_data;
1266 		fw_size = t6fw_size;
1267 		break;
1268 	default:
1269 		cxgb_printf(sc->dip, CE_WARN, "Adapter type not supported\n");
1270 		kmem_free(card_fw, sizeof(*card_fw));
1271 		return EINVAL;
1272 	}
1273 
1274 	rc = -t4_prep_fw(sc, fw_info, fw_data, fw_size, card_fw,
1275 			 p->t4_fw_install, state, &reset);
1276 
1277 	kmem_free(card_fw, sizeof(*card_fw));
1278 
1279 	if (rc != 0) {
1280 		cxgb_printf(sc->dip, CE_WARN,
1281 		    "failed to install firmware: %d", rc);
1282 		return (rc);
1283 	} else {
1284 		/* refresh */
1285 		(void) t4_check_fw_version(sc);
1286 	}
1287 
1288 	/* Reset device */
1289 	rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1290 	if (rc != 0) {
1291 		cxgb_printf(sc->dip, CE_WARN,
1292 		    "firmware reset failed: %d.", rc);
1293 		if (rc != ETIMEDOUT && rc != EIO)
1294 			(void) t4_fw_bye(sc, sc->mbox);
1295 		return (rc);
1296 	}
1297 
1298 	/* Partition adapter resources as specified in the config file. */
1299 	if (sc->flags & MASTER_PF) {
1300 		/* Handle default vs special T4 config file */
1301 
1302 		rc = partition_resources(sc);
1303 		if (rc != 0)
1304 			goto err;	/* error message displayed already */
1305 	}
1306 
1307 	sc->flags |= FW_OK;
1308 	return (0);
1309 err:
1310 	return (rc);
1311 
1312 }
1313 
1314 static const struct memwin t4_memwin[] = {
1315 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1316 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1317 	{ MEMWIN2_BASE, MEMWIN2_APERTURE }
1318 };
1319 
1320 static const struct memwin t5_memwin[] = {
1321 	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1322 	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1323 	{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1324 };
1325 
1326 #define	FW_PARAM_DEV(param) \
1327 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1328 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1329 #define	FW_PARAM_PFVF(param) \
1330 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1331 	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1332 
1333 /*
1334  * Verify that the memory range specified by the memtype/offset/len pair is
1335  * valid and lies entirely within the memtype specified.  The global address of
1336  * the start of the range is returned in addr.
1337  */
1338 int
validate_mt_off_len(struct adapter * sc,int mtype,uint32_t off,int len,uint32_t * addr)1339 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1340 	uint32_t *addr)
1341 {
1342 	uint32_t em, addr_len, maddr, mlen;
1343 
1344 	/* Memory can only be accessed in naturally aligned 4 byte units */
1345 	if (off & 3 || len & 3 || len == 0)
1346 		return (EINVAL);
1347 
1348 	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1349 	switch (mtype) {
1350 		case MEM_EDC0:
1351 			if (!(em & F_EDRAM0_ENABLE))
1352 				return (EINVAL);
1353 			addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1354 			maddr = G_EDRAM0_BASE(addr_len) << 20;
1355 			mlen = G_EDRAM0_SIZE(addr_len) << 20;
1356 			break;
1357 		case MEM_EDC1:
1358 			if (!(em & F_EDRAM1_ENABLE))
1359 				return (EINVAL);
1360 			addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1361 			maddr = G_EDRAM1_BASE(addr_len) << 20;
1362 			mlen = G_EDRAM1_SIZE(addr_len) << 20;
1363 			break;
1364 		case MEM_MC:
1365 			if (!(em & F_EXT_MEM_ENABLE))
1366 				return (EINVAL);
1367 			addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1368 			maddr = G_EXT_MEM_BASE(addr_len) << 20;
1369 			mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1370 			break;
1371 		case MEM_MC1:
1372 			if (is_t4(sc->params.chip) || !(em & F_EXT_MEM1_ENABLE))
1373 				return (EINVAL);
1374 			addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1375 			maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1376 			mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1377 			break;
1378 		default:
1379 			return (EINVAL);
1380 	}
1381 
1382 	if (mlen > 0 && off < mlen && off + len <= mlen) {
1383 		*addr = maddr + off;    /* global address */
1384 		return (0);
1385 	}
1386 
1387 	return (EFAULT);
1388 }
1389 
1390 void
memwin_info(struct adapter * sc,int win,uint32_t * base,uint32_t * aperture)1391 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1392 {
1393 	const struct memwin *mw;
1394 
1395 	if (is_t4(sc->params.chip)) {
1396 		mw = &t4_memwin[win];
1397 	} else {
1398 		mw = &t5_memwin[win];
1399 	}
1400 
1401 	if (base != NULL)
1402 		*base = mw->base;
1403 	if (aperture != NULL)
1404 		*aperture = mw->aperture;
1405 }
1406 
1407 /*
1408  * Upload configuration file to card's memory.
1409  */
1410 static int
upload_config_file(struct adapter * sc,uint32_t * mt,uint32_t * ma)1411 upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma)
1412 {
1413 	int rc = 0, cflen;
1414 	u_int i, n;
1415 	uint32_t param, val, addr, mtype, maddr;
1416 	uint32_t off, mw_base, mw_aperture;
1417 	const uint32_t *cfdata;
1418 
1419 	/* Figure out where the firmware wants us to upload it. */
1420 	param = FW_PARAM_DEV(CF);
1421 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1422 	if (rc != 0) {
1423 		/* Firmwares without config file support will fail this way */
1424 		cxgb_printf(sc->dip, CE_WARN,
1425 		    "failed to query config file location: %d.\n", rc);
1426 		return (rc);
1427 	}
1428 	*mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1429 	*ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1430 
1431 	switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
1432 	case CHELSIO_T4:
1433 		cflen = t4cfg_size & ~3;
1434 		/* LINTED: E_BAD_PTR_CAST_ALIGN */
1435 		cfdata = (const uint32_t *)t4cfg_data;
1436 		break;
1437 	case CHELSIO_T5:
1438 		cflen = t5cfg_size & ~3;
1439 		/* LINTED: E_BAD_PTR_CAST_ALIGN */
1440 		cfdata = (const uint32_t *)t5cfg_data;
1441 		break;
1442 	case CHELSIO_T6:
1443 		cflen = t6cfg_size & ~3;
1444 		/* LINTED: E_BAD_PTR_CAST_ALIGN */
1445 		cfdata = (const uint32_t *)t6cfg_data;
1446 		break;
1447 	default:
1448 		cxgb_printf(sc->dip, CE_WARN,
1449 			    "Invalid Adapter detected\n");
1450 		return EINVAL;
1451 	}
1452 
1453 	if (cflen > FLASH_CFG_MAX_SIZE) {
1454 		cxgb_printf(sc->dip, CE_WARN,
1455 		    "config file too long (%d, max allowed is %d).  ",
1456 		    cflen, FLASH_CFG_MAX_SIZE);
1457 		return (EFBIG);
1458 	}
1459 
1460 	rc = validate_mt_off_len(sc, mtype, maddr, cflen, &addr);
1461 	if (rc != 0) {
1462 
1463 		cxgb_printf(sc->dip, CE_WARN,
1464 		    "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
1465 		    "Will try to use the config on the card, if any.\n",
1466 		    __func__, mtype, maddr, cflen, rc);
1467 		return (EFAULT);
1468 	}
1469 
1470 	memwin_info(sc, 2, &mw_base, &mw_aperture);
1471 	while (cflen) {
1472 		off = position_memwin(sc, 2, addr);
1473 		n = min(cflen, mw_aperture - off);
1474 		for (i = 0; i < n; i += 4)
1475 			t4_write_reg(sc, mw_base + off + i, *cfdata++);
1476 		cflen -= n;
1477 		addr += n;
1478 	}
1479 
1480 	return (rc);
1481 }
1482 
1483 /*
1484  * Partition chip resources for use between various PFs, VFs, etc.  This is done
1485  * by uploading the firmware configuration file to the adapter and instructing
1486  * the firmware to process it.
1487  */
1488 static int
partition_resources(struct adapter * sc)1489 partition_resources(struct adapter *sc)
1490 {
1491 	int rc;
1492 	struct fw_caps_config_cmd caps;
1493 	uint32_t mtype, maddr, finicsum, cfcsum;
1494 
1495 	rc = upload_config_file(sc, &mtype, &maddr);
1496 	if (rc != 0) {
1497 		mtype = FW_MEMTYPE_CF_FLASH;
1498 		maddr = t4_flash_cfg_addr(sc);
1499 	}
1500 
1501 	bzero(&caps, sizeof (caps));
1502 	caps.op_to_write = BE_32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1503 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1504 	caps.cfvalid_to_len16 = BE_32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1505 	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1506 	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1507 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1508 	if (rc != 0) {
1509 		cxgb_printf(sc->dip, CE_WARN,
1510 		    "failed to pre-process config file: %d.\n", rc);
1511 		return (rc);
1512 	}
1513 
1514 	finicsum = ntohl(caps.finicsum);
1515 	cfcsum = ntohl(caps.cfcsum);
1516 	if (finicsum != cfcsum) {
1517 		cxgb_printf(sc->dip, CE_WARN,
1518 		    "WARNING: config file checksum mismatch: %08x %08x\n",
1519 		    finicsum, cfcsum);
1520 	}
1521 	sc->cfcsum = cfcsum;
1522 
1523 	/* TODO: Need to configure this correctly */
1524 	caps.toecaps = htons(FW_CAPS_CONFIG_TOE);
1525 	caps.iscsicaps = 0;
1526 	caps.rdmacaps = 0;
1527 	caps.fcoecaps = 0;
1528 	/* TODO: Disable VNIC cap for now */
1529 	caps.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
1530 
1531 	caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1532 	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1533 	caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1534 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), NULL);
1535 	if (rc != 0) {
1536 		cxgb_printf(sc->dip, CE_WARN,
1537 		    "failed to process config file: %d.\n", rc);
1538 		return (rc);
1539 	}
1540 
1541 	return (0);
1542 }
1543 
1544 /*
1545  * Tweak configuration based on module parameters, etc.  Most of these have
1546  * defaults assigned to them by Firmware Configuration Files (if we're using
1547  * them) but need to be explicitly set if we're using hard-coded
1548  * initialization.  But even in the case of using Firmware Configuration
1549  * Files, we'd like to expose the ability to change these via module
1550  * parameters so these are essentially common tweaks/settings for
1551  * Configuration Files and hard-coded initialization ...
1552  */
1553 static int
adap__pre_init_tweaks(struct adapter * sc)1554 adap__pre_init_tweaks(struct adapter *sc)
1555 {
1556 	int rx_dma_offset = 2; /* Offset of RX packets into DMA buffers */
1557 
1558 	/*
1559 	 * Fix up various Host-Dependent Parameters like Page Size, Cache
1560 	 * Line Size, etc.  The firmware default is for a 4KB Page Size and
1561 	 * 64B Cache Line Size ...
1562 	 */
1563 	(void) t4_fixup_host_params_compat(sc, PAGE_SIZE, CACHE_LINE, T5_LAST_REV);
1564 
1565 	t4_set_reg_field(sc, A_SGE_CONTROL,
1566 			 V_PKTSHIFT(M_PKTSHIFT), V_PKTSHIFT(rx_dma_offset));
1567 
1568 	return 0;
1569 }
1570 /*
1571  * Retrieve parameters that are needed (or nice to have) prior to calling
1572  * t4_sge_init and t4_fw_initialize.
1573  */
1574 static int
get_params__pre_init(struct adapter * sc)1575 get_params__pre_init(struct adapter *sc)
1576 {
1577 	int rc;
1578 	uint32_t param[2], val[2];
1579 	struct fw_devlog_cmd cmd;
1580 	struct devlog_params *dlog = &sc->params.devlog;
1581 
1582 	/*
1583 	 * Grab the raw VPD parameters.
1584 	 */
1585 	rc = -t4_get_raw_vpd_params(sc, &sc->params.vpd);
1586 	if (rc != 0) {
1587 		cxgb_printf(sc->dip, CE_WARN,
1588 		    "failed to query VPD parameters (pre_init): %d.\n", rc);
1589 		return (rc);
1590 	}
1591 
1592 	param[0] = FW_PARAM_DEV(PORTVEC);
1593 	param[1] = FW_PARAM_DEV(CCLK);
1594 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1595 	if (rc != 0) {
1596 		cxgb_printf(sc->dip, CE_WARN,
1597 		    "failed to query parameters (pre_init): %d.\n", rc);
1598 		return (rc);
1599 	}
1600 
1601 	sc->params.portvec = val[0];
1602 	sc->params.nports = 0;
1603 	while (val[0]) {
1604 		sc->params.nports++;
1605 		val[0] &= val[0] - 1;
1606 	}
1607 
1608 	sc->params.vpd.cclk = val[1];
1609 
1610 	/* Read device log parameters. */
1611 	bzero(&cmd, sizeof (cmd));
1612 	cmd.op_to_write = htonl(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1613 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1614 	cmd.retval_len16 = htonl(FW_LEN16(cmd));
1615 	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof (cmd), &cmd);
1616 	if (rc != 0) {
1617 		cxgb_printf(sc->dip, CE_WARN,
1618 		    "failed to get devlog parameters: %d.\n", rc);
1619 		bzero(dlog, sizeof (*dlog));
1620 		rc = 0;	/* devlog isn't critical for device operation */
1621 	} else {
1622 		val[0] = ntohl(cmd.memtype_devlog_memaddr16_devlog);
1623 		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1624 		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1625 		dlog->size = ntohl(cmd.memsize_devlog);
1626 	}
1627 
1628 	return (rc);
1629 }
1630 
1631 /*
1632  * Retrieve various parameters that are of interest to the driver.  The device
1633  * has been initialized by the firmware at this point.
1634  */
1635 static int
get_params__post_init(struct adapter * sc)1636 get_params__post_init(struct adapter *sc)
1637 {
1638 	int rc;
1639 	uint32_t param[7], val[7];
1640 	struct fw_caps_config_cmd caps;
1641 
1642 	param[0] = FW_PARAM_PFVF(IQFLINT_START);
1643 	param[1] = FW_PARAM_PFVF(EQ_START);
1644 	param[2] = FW_PARAM_PFVF(FILTER_START);
1645 	param[3] = FW_PARAM_PFVF(FILTER_END);
1646 	param[4] = FW_PARAM_PFVF(L2T_START);
1647 	param[5] = FW_PARAM_PFVF(L2T_END);
1648 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1649 	if (rc != 0) {
1650 		cxgb_printf(sc->dip, CE_WARN,
1651 		    "failed to query parameters (post_init): %d.\n", rc);
1652 		return (rc);
1653 	}
1654 
1655 	/* LINTED: E_ASSIGN_NARROW_CONV */
1656 	sc->sge.iq_start = val[0];
1657 	sc->sge.eq_start = val[1];
1658 	sc->tids.ftid_base = val[2];
1659 	sc->tids.nftids = val[3] - val[2] + 1;
1660 	sc->vres.l2t.start = val[4];
1661 	sc->vres.l2t.size = val[5] - val[4] + 1;
1662 
1663 	param[0] = FW_PARAM_PFVF(IQFLINT_END);
1664 	param[1] = FW_PARAM_PFVF(EQ_END);
1665 	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1666 	if (rc != 0) {
1667 		cxgb_printf(sc->dip, CE_WARN,
1668 			    "failed to query eq/iq map size parameters (post_init): %d.\n",
1669 			    rc);
1670 		return (rc);
1671 	}
1672 
1673 	sc->sge.iqmap_sz = val[0] - sc->sge.iq_start + 1;
1674 	sc->sge.eqmap_sz = val[1] - sc->sge.eq_start + 1;
1675 
1676 	/* get capabilites */
1677 	bzero(&caps, sizeof (caps));
1678 	caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1679 	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1680 	caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1681 	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1682 	if (rc != 0) {
1683 		cxgb_printf(sc->dip, CE_WARN,
1684 		    "failed to get card capabilities: %d.\n", rc);
1685 		return (rc);
1686 	}
1687 
1688 	if (caps.toecaps != 0) {
1689 		/* query offload-related parameters */
1690 		param[0] = FW_PARAM_DEV(NTID);
1691 		param[1] = FW_PARAM_PFVF(SERVER_START);
1692 		param[2] = FW_PARAM_PFVF(SERVER_END);
1693 		param[3] = FW_PARAM_PFVF(TDDP_START);
1694 		param[4] = FW_PARAM_PFVF(TDDP_END);
1695 		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1696 		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1697 		if (rc != 0) {
1698 			cxgb_printf(sc->dip, CE_WARN,
1699 			    "failed to query TOE parameters: %d.\n", rc);
1700 			return (rc);
1701 		}
1702 		sc->tids.ntids = val[0];
1703 		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1704 		sc->tids.stid_base = val[1];
1705 		sc->tids.nstids = val[2] - val[1] + 1;
1706 		sc->vres.ddp.start = val[3];
1707 		sc->vres.ddp.size = val[4] - val[3] + 1;
1708 		sc->params.ofldq_wr_cred = val[5];
1709 		sc->params.offload = 1;
1710 	}
1711 
1712 	rc = -t4_get_pfres(sc);
1713 	if (rc != 0) {
1714 		cxgb_printf(sc->dip, CE_WARN,
1715 			    "failed to query PF resource params: %d.\n", rc);
1716 		return (rc);
1717 	}
1718 
1719 	/* These are finalized by FW initialization, load their values now */
1720 	val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1721 	sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1722 	sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1723 	t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1724 
1725 	return (rc);
1726 }
1727 
1728 static int
set_params__post_init(struct adapter * sc)1729 set_params__post_init(struct adapter *sc)
1730 {
1731 	uint32_t param, val;
1732 
1733 	/* ask for encapsulated CPLs */
1734 	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
1735 	val = 1;
1736 	(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1737 
1738 	return (0);
1739 }
1740 
1741 /* TODO: verify */
1742 static void
setup_memwin(struct adapter * sc)1743 setup_memwin(struct adapter *sc)
1744 {
1745 	pci_regspec_t *data;
1746 	int rc;
1747 	uint_t n;
1748 	uintptr_t bar0;
1749 	uintptr_t mem_win0_base, mem_win1_base, mem_win2_base;
1750 	uintptr_t mem_win2_aperture;
1751 
1752 	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1753 	    DDI_PROP_DONTPASS, "assigned-addresses", (int **)&data, &n);
1754 	if (rc != DDI_SUCCESS) {
1755 		cxgb_printf(sc->dip, CE_WARN,
1756 		    "failed to lookup \"assigned-addresses\" property: %d", rc);
1757 		return;
1758 	}
1759 	n /= sizeof (*data);
1760 
1761 	bar0 = ((uint64_t)data[0].pci_phys_mid << 32) | data[0].pci_phys_low;
1762 	ddi_prop_free(data);
1763 
1764 	if (is_t4(sc->params.chip)) {
1765 		mem_win0_base = bar0 + MEMWIN0_BASE;
1766 		mem_win1_base = bar0 + MEMWIN1_BASE;
1767 		mem_win2_base = bar0 + MEMWIN2_BASE;
1768 		mem_win2_aperture = MEMWIN2_APERTURE;
1769 	} else {
1770 		/* For T5, only relative offset inside the PCIe BAR is passed */
1771 		mem_win0_base = MEMWIN0_BASE;
1772 		mem_win1_base = MEMWIN1_BASE;
1773 		mem_win2_base = MEMWIN2_BASE_T5;
1774 		mem_win2_aperture = MEMWIN2_APERTURE_T5;
1775 	}
1776 
1777 	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1778 	    mem_win0_base | V_BIR(0) |
1779 	    V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1780 
1781 	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1782 	    mem_win1_base | V_BIR(0) |
1783 	    V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1784 
1785 	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1786 	    mem_win2_base | V_BIR(0) |
1787 	    V_WINDOW(ilog2(mem_win2_aperture) - 10));
1788 
1789 	/* flush */
1790 	(void)t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1791 }
1792 
1793 /*
1794  * Positions the memory window such that it can be used to access the specified
1795  * address in the chip's address space.  The return value is the offset of addr
1796  * from the start of the window.
1797  */
1798 uint32_t
position_memwin(struct adapter * sc,int n,uint32_t addr)1799 position_memwin(struct adapter *sc, int n, uint32_t addr)
1800 {
1801 	uint32_t start, pf;
1802 	uint32_t reg;
1803 
1804 	if (addr & 3) {
1805 		cxgb_printf(sc->dip, CE_WARN,
1806 		    "addr (0x%x) is not at a 4B boundary.\n", addr);
1807 		return (EFAULT);
1808 	}
1809 
1810 	if (is_t4(sc->params.chip)) {
1811 		pf = 0;
1812 		start = addr & ~0xf;    /* start must be 16B aligned */
1813 	} else {
1814 		pf = V_PFNUM(sc->pf);
1815 		start = addr & ~0x7f;   /* start must be 128B aligned */
1816 	}
1817 	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1818 
1819 	t4_write_reg(sc, reg, start | pf);
1820 	(void) t4_read_reg(sc, reg);
1821 
1822 	return (addr - start);
1823 }
1824 
1825 
1826 /*
1827  * Reads the named property and fills up the "data" array (which has at least
1828  * "count" elements).  We first try and lookup the property for our dev_t and
1829  * then retry with DDI_DEV_T_ANY if it's not found.
1830  *
1831  * Returns non-zero if the property was found and "data" has been updated.
1832  */
1833 static int
prop_lookup_int_array(struct adapter * sc,char * name,int * data,uint_t count)1834 prop_lookup_int_array(struct adapter *sc, char *name, int *data, uint_t count)
1835 {
1836 	dev_info_t *dip = sc->dip;
1837 	dev_t dev = sc->dev;
1838 	int rc, *d;
1839 	uint_t i, n;
1840 
1841 	rc = ddi_prop_lookup_int_array(dev, dip, DDI_PROP_DONTPASS,
1842 	    name, &d, &n);
1843 	if (rc == DDI_PROP_SUCCESS)
1844 		goto found;
1845 
1846 	if (rc != DDI_PROP_NOT_FOUND) {
1847 		cxgb_printf(dip, CE_WARN,
1848 		    "failed to lookup property %s for minor %d: %d.",
1849 		    name, getminor(dev), rc);
1850 		return (0);
1851 	}
1852 
1853 	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1854 	    name, &d, &n);
1855 	if (rc == DDI_PROP_SUCCESS)
1856 		goto found;
1857 
1858 	if (rc != DDI_PROP_NOT_FOUND) {
1859 		cxgb_printf(dip, CE_WARN,
1860 		    "failed to lookup property %s: %d.", name, rc);
1861 		return (0);
1862 	}
1863 
1864 	return (0);
1865 
1866 found:
1867 	if (n > count) {
1868 		cxgb_printf(dip, CE_NOTE,
1869 		    "property %s has too many elements (%d), ignoring extras",
1870 		    name, n);
1871 	}
1872 
1873 	for (i = 0; i < n && i < count; i++)
1874 		data[i] = d[i];
1875 	ddi_prop_free(d);
1876 
1877 	return (1);
1878 }
1879 
1880 static int
prop_lookup_int(struct adapter * sc,char * name,int defval)1881 prop_lookup_int(struct adapter *sc, char *name, int defval)
1882 {
1883 	int rc;
1884 
1885 	rc = ddi_prop_get_int(sc->dev, sc->dip, DDI_PROP_DONTPASS, name, -1);
1886 	if (rc != -1)
1887 		return (rc);
1888 
1889 	return (ddi_prop_get_int(DDI_DEV_T_ANY, sc->dip, DDI_PROP_DONTPASS,
1890 	    name, defval));
1891 }
1892 
1893 static int
init_driver_props(struct adapter * sc,struct driver_properties * p)1894 init_driver_props(struct adapter *sc, struct driver_properties *p)
1895 {
1896 	dev_t dev = sc->dev;
1897 	dev_info_t *dip = sc->dip;
1898 	int i, *data;
1899 	uint_t tmr[SGE_NTIMERS] = {5, 10, 20, 50, 100, 200};
1900 	uint_t cnt[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
1901 
1902 	/*
1903 	 * Holdoff timer
1904 	 */
1905 	data = &p->timer_val[0];
1906 	for (i = 0; i < SGE_NTIMERS; i++)
1907 		data[i] = tmr[i];
1908 	(void) prop_lookup_int_array(sc, "holdoff-timer-values", data,
1909 	    SGE_NTIMERS);
1910 	for (i = 0; i < SGE_NTIMERS; i++) {
1911 		int limit = 200U;
1912 		if (data[i] > limit) {
1913 			cxgb_printf(dip, CE_WARN,
1914 			    "holdoff timer %d is too high (%d), lowered to %d.",
1915 			    i, data[i], limit);
1916 			data[i] = limit;
1917 		}
1918 	}
1919 	(void) ddi_prop_update_int_array(dev, dip, "holdoff-timer-values",
1920 	    data, SGE_NTIMERS);
1921 
1922 	/*
1923 	 * Holdoff packet counter
1924 	 */
1925 	data = &p->counter_val[0];
1926 	for (i = 0; i < SGE_NCOUNTERS; i++)
1927 		data[i] = cnt[i];
1928 	(void) prop_lookup_int_array(sc, "holdoff-pkt-counter-values", data,
1929 	    SGE_NCOUNTERS);
1930 	for (i = 0; i < SGE_NCOUNTERS; i++) {
1931 		int limit = M_THRESHOLD_0;
1932 		if (data[i] > limit) {
1933 			cxgb_printf(dip, CE_WARN,
1934 			    "holdoff pkt-counter %d is too high (%d), "
1935 			    "lowered to %d.", i, data[i], limit);
1936 			data[i] = limit;
1937 		}
1938 	}
1939 	(void) ddi_prop_update_int_array(dev, dip, "holdoff-pkt-counter-values",
1940 	    data, SGE_NCOUNTERS);
1941 
1942 	/*
1943 	 * Maximum # of tx and rx queues to use for each
1944 	 * 100G, 40G, 25G, 10G and 1G port.
1945 	 */
1946 	p->max_ntxq_10g = prop_lookup_int(sc, "max-ntxq-10G-port", 8);
1947 	(void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1948 	    p->max_ntxq_10g);
1949 
1950 	p->max_nrxq_10g = prop_lookup_int(sc, "max-nrxq-10G-port", 8);
1951 	(void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1952 	    p->max_nrxq_10g);
1953 
1954 	p->max_ntxq_1g = prop_lookup_int(sc, "max-ntxq-1G-port", 2);
1955 	(void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1956 	    p->max_ntxq_1g);
1957 
1958 	p->max_nrxq_1g = prop_lookup_int(sc, "max-nrxq-1G-port", 2);
1959 	(void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1960 	    p->max_nrxq_1g);
1961 
1962 #ifdef TCP_OFFLOAD_ENABLE
1963 	p->max_nofldtxq_10g = prop_lookup_int(sc, "max-nofldtxq-10G-port", 8);
1964 	(void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1965 	    p->max_nofldtxq_10g);
1966 
1967 	p->max_nofldrxq_10g = prop_lookup_int(sc, "max-nofldrxq-10G-port", 2);
1968 	(void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1969 	    p->max_nofldrxq_10g);
1970 
1971 	p->max_nofldtxq_1g = prop_lookup_int(sc, "max-nofldtxq-1G-port", 2);
1972 	(void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1973 	    p->max_nofldtxq_1g);
1974 
1975 	p->max_nofldrxq_1g = prop_lookup_int(sc, "max-nofldrxq-1G-port", 1);
1976 	(void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1977 	    p->max_nofldrxq_1g);
1978 #endif
1979 
1980 	/*
1981 	 * Holdoff parameters for 10G and 1G ports.
1982 	 */
1983 	p->tmr_idx_10g = prop_lookup_int(sc, "holdoff-timer-idx-10G", 0);
1984 	(void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-10G",
1985 	    p->tmr_idx_10g);
1986 
1987 	p->pktc_idx_10g = prop_lookup_int(sc, "holdoff-pktc-idx-10G", 2);
1988 	(void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-10G",
1989 	    p->pktc_idx_10g);
1990 
1991 	p->tmr_idx_1g = prop_lookup_int(sc, "holdoff-timer-idx-1G", 0);
1992 	(void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-1G",
1993 	    p->tmr_idx_1g);
1994 
1995 	p->pktc_idx_1g = prop_lookup_int(sc, "holdoff-pktc-idx-1G", 2);
1996 	(void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-1G",
1997 	    p->pktc_idx_1g);
1998 
1999 	/*
2000 	 * Size (number of entries) of each tx and rx queue.
2001 	 */
2002 	i = prop_lookup_int(sc, "qsize-txq", TX_EQ_QSIZE);
2003 	p->qsize_txq = max(i, 128);
2004 	if (p->qsize_txq != i) {
2005 		cxgb_printf(dip, CE_WARN,
2006 		    "using %d instead of %d as the tx queue size",
2007 		    p->qsize_txq, i);
2008 	}
2009 	(void) ddi_prop_update_int(dev, dip, "qsize-txq", p->qsize_txq);
2010 
2011 	i = prop_lookup_int(sc, "qsize-rxq", RX_IQ_QSIZE);
2012 	p->qsize_rxq = max(i, 128);
2013 	while (p->qsize_rxq & 7)
2014 		p->qsize_rxq--;
2015 	if (p->qsize_rxq != i) {
2016 		cxgb_printf(dip, CE_WARN,
2017 		    "using %d instead of %d as the rx queue size",
2018 		    p->qsize_rxq, i);
2019 	}
2020 	(void) ddi_prop_update_int(dev, dip, "qsize-rxq", p->qsize_rxq);
2021 
2022 	/*
2023 	 * Interrupt types allowed.
2024 	 * Bits 0, 1, 2 = INTx, MSI, MSI-X respectively.  See sys/ddi_intr.h
2025 	 */
2026 	p->intr_types = prop_lookup_int(sc, "interrupt-types",
2027 	    DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_FIXED);
2028 	(void) ddi_prop_update_int(dev, dip, "interrupt-types", p->intr_types);
2029 
2030 	/*
2031 	 * Forwarded interrupt queues.  Create this property to force the driver
2032 	 * to use forwarded interrupt queues.
2033 	 */
2034 	if (ddi_prop_exists(dev, dip, DDI_PROP_DONTPASS,
2035 	    "interrupt-forwarding") != 0 ||
2036 	    ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2037 	    "interrupt-forwarding") != 0) {
2038 		UNIMPLEMENTED();
2039 		(void) ddi_prop_create(dev, dip, DDI_PROP_CANSLEEP,
2040 		    "interrupt-forwarding", NULL, 0);
2041 	}
2042 
2043 	/*
2044 	 * Write combining
2045 	 * 0 to disable, 1 to enable
2046 	 */
2047 	p->wc = prop_lookup_int(sc, "write-combine", 1);
2048 	cxgb_printf(dip, CE_WARN, "write-combine: using of %d", p->wc);
2049 	if (p->wc != 0 && p->wc != 1) {
2050 		cxgb_printf(dip, CE_WARN,
2051 		    "write-combine: using 1 instead of %d", p->wc);
2052 		p->wc = 1;
2053 	}
2054 	(void) ddi_prop_update_int(dev, dip, "write-combine", p->wc);
2055 
2056 	p->t4_fw_install = prop_lookup_int(sc, "t4_fw_install", 1);
2057 	if (p->t4_fw_install != 0 && p->t4_fw_install != 2)
2058 		p->t4_fw_install = 1;
2059 	(void) ddi_prop_update_int(dev, dip, "t4_fw_install", p->t4_fw_install);
2060 
2061 	/* Multiple Rings */
2062 	p->multi_rings = prop_lookup_int(sc, "multi-rings", 1);
2063 	if (p->multi_rings != 0 && p->multi_rings != 1) {
2064 		cxgb_printf(dip, CE_NOTE,
2065 			   "multi-rings: using value 1 instead of %d", p->multi_rings);
2066 		p->multi_rings = 1;
2067 	}
2068 
2069 	(void) ddi_prop_update_int(dev, dip, "multi-rings", p->multi_rings);
2070 
2071 	return (0);
2072 }
2073 
2074 static int
remove_extra_props(struct adapter * sc,int n10g,int n1g)2075 remove_extra_props(struct adapter *sc, int n10g, int n1g)
2076 {
2077 	if (n10g == 0) {
2078 		(void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-10G-port");
2079 		(void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-10G-port");
2080 		(void) ddi_prop_remove(sc->dev, sc->dip,
2081 		    "holdoff-timer-idx-10G");
2082 		(void) ddi_prop_remove(sc->dev, sc->dip,
2083 		    "holdoff-pktc-idx-10G");
2084 	}
2085 
2086 	if (n1g == 0) {
2087 		(void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-1G-port");
2088 		(void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-1G-port");
2089 		(void) ddi_prop_remove(sc->dev, sc->dip,
2090 		    "holdoff-timer-idx-1G");
2091 		(void) ddi_prop_remove(sc->dev, sc->dip, "holdoff-pktc-idx-1G");
2092 	}
2093 
2094 	return (0);
2095 }
2096 
2097 static int
cfg_itype_and_nqueues(struct adapter * sc,int n10g,int n1g,struct intrs_and_queues * iaq)2098 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
2099     struct intrs_and_queues *iaq)
2100 {
2101 	struct driver_properties *p = &sc->props;
2102 	int rc, itype, itypes, navail, nc, n;
2103 	int pfres_rxq, pfres_txq, pfresq;
2104 
2105 	bzero(iaq, sizeof (*iaq));
2106 	nc = ncpus;	/* our snapshot of the number of CPUs */
2107 	iaq->ntxq10g = min(nc, p->max_ntxq_10g);
2108 	iaq->ntxq1g = min(nc, p->max_ntxq_1g);
2109 	iaq->nrxq10g = min(nc, p->max_nrxq_10g);
2110 	iaq->nrxq1g = min(nc, p->max_nrxq_1g);
2111 #ifdef TCP_OFFLOAD_ENABLE
2112 	iaq->nofldtxq10g = min(nc, p->max_nofldtxq_10g);
2113 	iaq->nofldtxq1g = min(nc, p->max_nofldtxq_1g);
2114 	iaq->nofldrxq10g = min(nc, p->max_nofldrxq_10g);
2115 	iaq->nofldrxq1g = min(nc, p->max_nofldrxq_1g);
2116 #endif
2117 
2118 	pfres_rxq = iaq->nrxq10g * n10g + iaq->nrxq1g * n1g;
2119 	pfres_txq = iaq->ntxq10g * n10g + iaq->ntxq1g * n1g;
2120 #ifdef TCP_OFFLOAD_ENABLE
2121 	pfres_rxq += iaq->nofldrxq10g * n10g + iaq->nofldrxq1g * n1g;
2122 	pfres_txq += iaq->nofldtxq10g * n10g + iaq->nofldtxq1g * n1g;
2123 #endif
2124 
2125 	/* If current configuration of max number of Rxqs and Txqs exceed
2126 	 * the max available for all the ports under this PF, then shrink
2127 	 * the queues to max available. Reduce them in a way that each
2128 	 * port under this PF has equally distributed number of queues.
2129 	 * Must guarantee at least 1 queue for each port for both NIC
2130 	 * and Offload queues.
2131 	 *
2132 	 * neq - fixed max number of Egress queues on Tx path and Free List
2133 	 * queues that hold Rx payload data on Rx path. Half are reserved
2134 	 * for Egress queues and the other half for Free List queues.
2135 	 * Hence, the division by 2.
2136 	 *
2137 	 * niqflint - max number of Ingress queues with interrupts on Rx
2138 	 * path to receive completions that indicate Rx payload has been
2139 	 * posted in its associated Free List queue. Also handles Tx
2140 	 * completions for packets successfully transmitted on Tx path.
2141 	 *
2142 	 * nethctrl - max number of Egress queues only for Tx path. This
2143 	 * number is usually half of neq. However, if it became less than
2144 	 * neq due to lack of resources based on firmware configuration,
2145 	 * then take the lower value.
2146 	 */
2147 	while (pfres_rxq >
2148 	       min(sc->params.pfres.neq / 2, sc->params.pfres.niqflint)) {
2149 		pfresq = pfres_rxq;
2150 
2151 		if (iaq->nrxq10g > 1) {
2152 			iaq->nrxq10g--;
2153 			pfres_rxq -= n10g;
2154 		}
2155 
2156 		if (iaq->nrxq1g > 1) {
2157 			iaq->nrxq1g--;
2158 			pfres_rxq -= n1g;
2159 		}
2160 
2161 #ifdef TCP_OFFLOAD_ENABLE
2162 		if (iaq->nofldrxq10g > 1) {
2163 			iaq->nofldrxq10g--;
2164 			pfres_rxq -= n10g;
2165 		}
2166 
2167 		if (iaq->nofldrxq1g > 1) {
2168 			iaq->nofldrxq1g--;
2169 			pfres_rxq -= n1g;
2170 		}
2171 #endif
2172 
2173 		/* Break if nothing changed */
2174 		if (pfresq == pfres_rxq)
2175 			break;
2176 	}
2177 
2178 	while (pfres_txq >
2179 	       min(sc->params.pfres.neq / 2, sc->params.pfres.nethctrl)) {
2180 		pfresq = pfres_txq;
2181 
2182 		if (iaq->ntxq10g > 1) {
2183 			iaq->ntxq10g--;
2184 			pfres_txq -= n10g;
2185 		}
2186 
2187 		if (iaq->ntxq1g > 1) {
2188 			iaq->ntxq1g--;
2189 			pfres_txq -= n1g;
2190 		}
2191 
2192 #ifdef TCP_OFFLOAD_ENABLE
2193 		if (iaq->nofldtxq10g > 1) {
2194 			iaq->nofldtxq10g--;
2195 			pfres_txq -= n10g;
2196 		}
2197 
2198 		if (iaq->nofldtxq1g > 1) {
2199 			iaq->nofldtxq1g--;
2200 			pfres_txq -= n1g;
2201 		}
2202 #endif
2203 
2204 		/* Break if nothing changed */
2205 		if (pfresq == pfres_txq)
2206 			break;
2207 	}
2208 
2209 	rc = ddi_intr_get_supported_types(sc->dip, &itypes);
2210 	if (rc != DDI_SUCCESS) {
2211 		cxgb_printf(sc->dip, CE_WARN,
2212 		    "failed to determine supported interrupt types: %d", rc);
2213 		return (rc);
2214 	}
2215 
2216 	for (itype = DDI_INTR_TYPE_MSIX; itype; itype >>= 1) {
2217 		ASSERT(itype == DDI_INTR_TYPE_MSIX ||
2218 		    itype == DDI_INTR_TYPE_MSI ||
2219 		    itype == DDI_INTR_TYPE_FIXED);
2220 
2221 		if ((itype & itypes & p->intr_types) == 0)
2222 			continue;	/* not supported or not allowed */
2223 
2224 		navail = 0;
2225 		rc = ddi_intr_get_navail(sc->dip, itype, &navail);
2226 		if (rc != DDI_SUCCESS || navail == 0) {
2227 			cxgb_printf(sc->dip, CE_WARN,
2228 			    "failed to get # of interrupts for type %d: %d",
2229 			    itype, rc);
2230 			continue;	/* carry on */
2231 		}
2232 
2233 		iaq->intr_type = itype;
2234 		if (navail == 0)
2235 			continue;
2236 
2237 		/*
2238 		 * Best option: an interrupt vector for errors, one for the
2239 		 * firmware event queue, and one each for each rxq (NIC as well
2240 		 * as offload).
2241 		 */
2242 		iaq->nirq = T4_EXTRA_INTR;
2243 		iaq->nirq += n10g * iaq->nrxq10g;
2244 		iaq->nirq += n1g * iaq->nrxq1g;
2245 #ifdef TCP_OFFLOAD_ENABLE
2246 		iaq->nirq += n10g * iaq->nofldrxq10g;
2247 		iaq->nirq += n1g * iaq->nofldrxq1g;
2248 #endif
2249 
2250 		if (iaq->nirq <= navail &&
2251 		    (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2252 			iaq->intr_fwd = 0;
2253 			goto allocate;
2254 		}
2255 
2256 		/*
2257 		 * Second best option: an interrupt vector for errors, one for
2258 		 * the firmware event queue, and one each for either NIC or
2259 		 * offload rxq's.
2260 		 */
2261 		iaq->nirq = T4_EXTRA_INTR;
2262 #ifdef TCP_OFFLOAD_ENABLE
2263 		iaq->nirq += n10g * max(iaq->nrxq10g, iaq->nofldrxq10g);
2264 		iaq->nirq += n1g * max(iaq->nrxq1g, iaq->nofldrxq1g);
2265 #else
2266 		iaq->nirq += n10g * iaq->nrxq10g;
2267 		iaq->nirq += n1g * iaq->nrxq1g;
2268 #endif
2269 		if (iaq->nirq <= navail &&
2270 		    (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2271 			iaq->intr_fwd = 1;
2272 			goto allocate;
2273 		}
2274 
2275 		/*
2276 		 * Next best option: an interrupt vector for errors, one for the
2277 		 * firmware event queue, and at least one per port.  At this
2278 		 * point we know we'll have to downsize nrxq or nofldrxq to fit
2279 		 * what's available to us.
2280 		 */
2281 		iaq->nirq = T4_EXTRA_INTR;
2282 		iaq->nirq += n10g + n1g;
2283 		if (iaq->nirq <= navail) {
2284 			int leftover = navail - iaq->nirq;
2285 
2286 			if (n10g > 0) {
2287 				int target = iaq->nrxq10g;
2288 
2289 #ifdef TCP_OFFLOAD_ENABLE
2290 				target = max(target, iaq->nofldrxq10g);
2291 #endif
2292 				n = 1;
2293 				while (n < target && leftover >= n10g) {
2294 					leftover -= n10g;
2295 					iaq->nirq += n10g;
2296 					n++;
2297 				}
2298 				iaq->nrxq10g = min(n, iaq->nrxq10g);
2299 #ifdef TCP_OFFLOAD_ENABLE
2300 				iaq->nofldrxq10g = min(n, iaq->nofldrxq10g);
2301 #endif
2302 			}
2303 
2304 			if (n1g > 0) {
2305 				int target = iaq->nrxq1g;
2306 
2307 #ifdef TCP_OFFLOAD_ENABLE
2308 				target = max(target, iaq->nofldrxq1g);
2309 #endif
2310 				n = 1;
2311 				while (n < target && leftover >= n1g) {
2312 					leftover -= n1g;
2313 					iaq->nirq += n1g;
2314 					n++;
2315 				}
2316 				iaq->nrxq1g = min(n, iaq->nrxq1g);
2317 #ifdef TCP_OFFLOAD_ENABLE
2318 				iaq->nofldrxq1g = min(n, iaq->nofldrxq1g);
2319 #endif
2320 			}
2321 
2322 			/* We have arrived at a minimum value required to enable
2323 			 * per queue irq(either NIC or offload). Thus for non-
2324 			 * offload case, we will get a vector per queue, while
2325 			 * offload case, we will get a vector per offload/NIC q.
2326 			 * Hence enable Interrupt forwarding only for offload
2327 			 * case.
2328 			 */
2329 #ifdef TCP_OFFLOAD_ENABLE
2330 			if (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq)) {
2331 				iaq->intr_fwd = 1;
2332 #else
2333 			if (itype != DDI_INTR_TYPE_MSI) {
2334 #endif
2335 				goto allocate;
2336 			}
2337 		}
2338 
2339 		/*
2340 		 * Least desirable option: one interrupt vector for everything.
2341 		 */
2342 		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2343 #ifdef TCP_OFFLOAD_ENABLE
2344 		iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2345 #endif
2346 		iaq->intr_fwd = 1;
2347 
2348 allocate:
2349 		return (0);
2350 	}
2351 
2352 	cxgb_printf(sc->dip, CE_WARN,
2353 	    "failed to find a usable interrupt type.  supported=%d, allowed=%d",
2354 	    itypes, p->intr_types);
2355 	return (DDI_FAILURE);
2356 }
2357 
2358 static int
2359 add_child_node(struct adapter *sc, int idx)
2360 {
2361 	int rc;
2362 	struct port_info *pi;
2363 
2364 	if (idx < 0 || idx >= sc->params.nports)
2365 		return (EINVAL);
2366 
2367 	pi = sc->port[idx];
2368 	if (pi == NULL)
2369 		return (ENODEV);	/* t4_port_init failed earlier */
2370 
2371 	PORT_LOCK(pi);
2372 	if (pi->dip != NULL) {
2373 		rc = 0;		/* EEXIST really, but then bus_config fails */
2374 		goto done;
2375 	}
2376 
2377 	rc = ndi_devi_alloc(sc->dip, T4_PORT_NAME, DEVI_SID_NODEID, &pi->dip);
2378 	if (rc != DDI_SUCCESS || pi->dip == NULL) {
2379 		rc = ENOMEM;
2380 		goto done;
2381 	}
2382 
2383 	(void) ddi_set_parent_data(pi->dip, pi);
2384 	(void) ndi_devi_bind_driver(pi->dip, 0);
2385 	rc = 0;
2386 done:
2387 	PORT_UNLOCK(pi);
2388 	return (rc);
2389 }
2390 
2391 static int
2392 remove_child_node(struct adapter *sc, int idx)
2393 {
2394 	int rc;
2395 	struct port_info *pi;
2396 
2397 	if (idx < 0 || idx >= sc->params.nports)
2398 		return (EINVAL);
2399 
2400 	pi = sc->port[idx];
2401 	if (pi == NULL)
2402 		return (ENODEV);
2403 
2404 	PORT_LOCK(pi);
2405 	if (pi->dip == NULL) {
2406 		rc = ENODEV;
2407 		goto done;
2408 	}
2409 
2410 	rc = ndi_devi_free(pi->dip);
2411 	if (rc == 0)
2412 		pi->dip = NULL;
2413 done:
2414 	PORT_UNLOCK(pi);
2415 	return (rc);
2416 }
2417 
2418 static char *
2419 print_port_speed(const struct port_info *pi)
2420 {
2421 	if (!pi)
2422 		return "-";
2423 
2424 	if (is_100G_port(pi))
2425 		return "100G";
2426 	else if (is_50G_port(pi))
2427 		return "50G";
2428 	else if (is_40G_port(pi))
2429 		return "40G";
2430 	else if (is_25G_port(pi))
2431 		return "25G";
2432 	else if (is_10G_port(pi))
2433 		return "10G";
2434 	else
2435 		return "1G";
2436 }
2437 
2438 #define	KS_UINIT(x)	kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
2439 #define	KS_CINIT(x)	kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
2440 #define	KS_U_SET(x, y)	kstatp->x.value.ul = (y)
2441 #define	KS_C_SET(x, ...)	\
2442 			(void) snprintf(kstatp->x.value.c, 16,  __VA_ARGS__)
2443 
2444 /*
2445  * t4nex:X:config
2446  */
2447 struct t4_kstats {
2448 	kstat_named_t chip_ver;
2449 	kstat_named_t fw_vers;
2450 	kstat_named_t tp_vers;
2451 	kstat_named_t driver_version;
2452 	kstat_named_t serial_number;
2453 	kstat_named_t ec_level;
2454 	kstat_named_t id;
2455 	kstat_named_t bus_type;
2456 	kstat_named_t bus_width;
2457 	kstat_named_t bus_speed;
2458 	kstat_named_t core_clock;
2459 	kstat_named_t port_cnt;
2460 	kstat_named_t port_type;
2461 	kstat_named_t pci_vendor_id;
2462 	kstat_named_t pci_device_id;
2463 };
2464 static kstat_t *
2465 setup_kstats(struct adapter *sc)
2466 {
2467 	kstat_t *ksp;
2468 	struct t4_kstats *kstatp;
2469 	int ndata;
2470 	struct pci_params *p = &sc->params.pci;
2471 	struct vpd_params *v = &sc->params.vpd;
2472 	uint16_t pci_vendor, pci_device;
2473 
2474 	ndata = sizeof (struct t4_kstats) / sizeof (kstat_named_t);
2475 
2476 	ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "config",
2477 	    "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2478 	if (ksp == NULL) {
2479 		cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2480 		return (NULL);
2481 	}
2482 
2483 	kstatp = (struct t4_kstats *)ksp->ks_data;
2484 
2485 	KS_UINIT(chip_ver);
2486 	KS_CINIT(fw_vers);
2487 	KS_CINIT(tp_vers);
2488 	KS_CINIT(driver_version);
2489 	KS_CINIT(serial_number);
2490 	KS_CINIT(ec_level);
2491 	KS_CINIT(id);
2492 	KS_CINIT(bus_type);
2493 	KS_CINIT(bus_width);
2494 	KS_CINIT(bus_speed);
2495 	KS_UINIT(core_clock);
2496 	KS_UINIT(port_cnt);
2497 	KS_CINIT(port_type);
2498 	KS_CINIT(pci_vendor_id);
2499 	KS_CINIT(pci_device_id);
2500 
2501 	KS_U_SET(chip_ver, sc->params.chip);
2502 	KS_C_SET(fw_vers, "%d.%d.%d.%d",
2503 	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2504 	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2505 	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2506 	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2507 	KS_C_SET(tp_vers, "%d.%d.%d.%d",
2508 	    G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
2509 	    G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
2510 	    G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
2511 	    G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
2512 	KS_C_SET(driver_version, DRV_VERSION);
2513 	KS_C_SET(serial_number, "%s", v->sn);
2514 	KS_C_SET(ec_level, "%s", v->ec);
2515 	KS_C_SET(id, "%s", v->id);
2516 	KS_C_SET(bus_type, "pci-express");
2517 	KS_C_SET(bus_width, "x%d lanes", p->width);
2518 	KS_C_SET(bus_speed, "%d", p->speed);
2519 	KS_U_SET(core_clock, v->cclk);
2520 	KS_U_SET(port_cnt, sc->params.nports);
2521 
2522 	t4_os_pci_read_cfg2(sc, PCI_CONF_VENID, &pci_vendor);
2523 	KS_C_SET(pci_vendor_id, "0x%x", pci_vendor);
2524 
2525 	t4_os_pci_read_cfg2(sc, PCI_CONF_DEVID, &pci_device);
2526 	KS_C_SET(pci_device_id, "0x%x", pci_device);
2527 
2528 	KS_C_SET(port_type, "%s/%s/%s/%s",
2529 		 print_port_speed(sc->port[0]),
2530 		 print_port_speed(sc->port[1]),
2531 		 print_port_speed(sc->port[2]),
2532 		 print_port_speed(sc->port[3]));
2533 
2534 	/* Do NOT set ksp->ks_update.  These kstats do not change. */
2535 
2536 	/* Install the kstat */
2537 	ksp->ks_private = (void *)sc;
2538 	kstat_install(ksp);
2539 
2540 	return (ksp);
2541 }
2542 
2543 /*
2544  * t4nex:X:stat
2545  */
2546 struct t4_wc_kstats {
2547 	kstat_named_t write_coal_success;
2548 	kstat_named_t write_coal_failure;
2549 };
2550 static kstat_t *
2551 setup_wc_kstats(struct adapter *sc)
2552 {
2553 	kstat_t *ksp;
2554 	struct t4_wc_kstats *kstatp;
2555 	int ndata;
2556 
2557 	ndata = sizeof(struct t4_wc_kstats) / sizeof(kstat_named_t);
2558 	ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "stats",
2559 	    "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2560 	if (ksp == NULL) {
2561 		cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2562 		return (NULL);
2563 	}
2564 
2565 	kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2566 
2567 	KS_UINIT(write_coal_success);
2568 	KS_UINIT(write_coal_failure);
2569 
2570 	ksp->ks_update = update_wc_kstats;
2571 	/* Install the kstat */
2572 	ksp->ks_private = (void *)sc;
2573 	kstat_install(ksp);
2574 
2575 	return (ksp);
2576 }
2577 
2578 static int
2579 update_wc_kstats(kstat_t *ksp, int rw)
2580 {
2581 	struct t4_wc_kstats *kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2582 	struct adapter *sc = ksp->ks_private;
2583 	uint32_t wc_total, wc_success, wc_failure;
2584 
2585 	if (rw == KSTAT_WRITE)
2586 		return (0);
2587 
2588 	if (is_t5(sc->params.chip)) {
2589 		wc_total = t4_read_reg(sc, A_SGE_STAT_TOTAL);
2590 		wc_failure = t4_read_reg(sc, A_SGE_STAT_MATCH);
2591 		wc_success = wc_total - wc_failure;
2592 	} else {
2593 		wc_success = 0;
2594 		wc_failure = 0;
2595 	}
2596 
2597 	KS_U_SET(write_coal_success, wc_success);
2598 	KS_U_SET(write_coal_failure, wc_failure);
2599 
2600 	return (0);
2601 }
2602 
2603 int
2604 adapter_full_init(struct adapter *sc)
2605 {
2606 	int i, rc = 0;
2607 
2608 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2609 
2610 	rc = t4_setup_adapter_queues(sc);
2611 	if (rc != 0)
2612 		goto done;
2613 
2614 	if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2615 		(void) ddi_intr_block_enable(sc->intr_handle, sc->intr_count);
2616 	else {
2617 		for (i = 0; i < sc->intr_count; i++)
2618 			(void) ddi_intr_enable(sc->intr_handle[i]);
2619 	}
2620 	t4_intr_enable(sc);
2621 	sc->flags |= FULL_INIT_DONE;
2622 
2623 #ifdef TCP_OFFLOAD_ENABLE
2624 	/* TODO: wrong place to enable TOE capability */
2625 	if (is_offload(sc) != 0) {
2626 		for_each_port(sc, i) {
2627 			struct port_info *pi = sc->port[i];
2628 			rc = toe_capability(pi, 1);
2629 			if (rc != 0) {
2630 				cxgb_printf(pi->dip, CE_WARN,
2631 				    "Failed to activate toe capability: %d",
2632 				    rc);
2633 				rc = 0;		/* not a fatal error */
2634 			}
2635 		}
2636 	}
2637 #endif
2638 
2639 done:
2640 	if (rc != 0)
2641 		(void) adapter_full_uninit(sc);
2642 
2643 	return (rc);
2644 }
2645 
2646 int
2647 adapter_full_uninit(struct adapter *sc)
2648 {
2649 	int i, rc = 0;
2650 
2651 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2652 
2653 	if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2654 		(void) ddi_intr_block_disable(sc->intr_handle, sc->intr_count);
2655 	else {
2656 		for (i = 0; i < sc->intr_count; i++)
2657 			(void) ddi_intr_disable(sc->intr_handle[i]);
2658 	}
2659 
2660 	rc = t4_teardown_adapter_queues(sc);
2661 	if (rc != 0)
2662 		return (rc);
2663 
2664 	sc->flags &= ~FULL_INIT_DONE;
2665 
2666 	return (0);
2667 }
2668 
2669 int
2670 port_full_init(struct port_info *pi)
2671 {
2672 	struct adapter *sc = pi->adapter;
2673 	uint16_t *rss;
2674 	struct sge_rxq *rxq;
2675 	int rc, i;
2676 
2677 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2678 	ASSERT((pi->flags & PORT_INIT_DONE) == 0);
2679 
2680 	/*
2681 	 * Allocate tx/rx/fl queues for this port.
2682 	 */
2683 	rc = t4_setup_port_queues(pi);
2684 	if (rc != 0)
2685 		goto done;	/* error message displayed already */
2686 
2687 	/*
2688 	 * Setup RSS for this port.
2689 	 */
2690 	rss = kmem_zalloc(pi->nrxq * sizeof (*rss), KM_SLEEP);
2691 	for_each_rxq(pi, i, rxq) {
2692 		rss[i] = rxq->iq.abs_id;
2693 	}
2694 	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2695 	    pi->rss_size, rss, pi->nrxq);
2696 	kmem_free(rss, pi->nrxq * sizeof (*rss));
2697 	if (rc != 0) {
2698 		cxgb_printf(pi->dip, CE_WARN, "rss_config failed: %d", rc);
2699 		goto done;
2700 	}
2701 
2702 	pi->flags |= PORT_INIT_DONE;
2703 done:
2704 	if (rc != 0)
2705 		(void) port_full_uninit(pi);
2706 
2707 	return (rc);
2708 }
2709 
2710 /*
2711  * Idempotent.
2712  */
2713 int
2714 port_full_uninit(struct port_info *pi)
2715 {
2716 
2717 	ASSERT(pi->flags & PORT_INIT_DONE);
2718 
2719 	(void) t4_teardown_port_queues(pi);
2720 	pi->flags &= ~PORT_INIT_DONE;
2721 
2722 	return (0);
2723 }
2724 
2725 void
2726 enable_port_queues(struct port_info *pi)
2727 {
2728 	struct adapter *sc = pi->adapter;
2729 	int i;
2730 	struct sge_iq *iq;
2731 	struct sge_rxq *rxq;
2732 #ifdef TCP_OFFLOAD_ENABLE
2733 	struct sge_ofld_rxq *ofld_rxq;
2734 #endif
2735 
2736 	ASSERT(pi->flags & PORT_INIT_DONE);
2737 
2738 	/*
2739 	 * TODO: whatever was queued up after we set iq->state to IQS_DISABLED
2740 	 * back in disable_port_queues will be processed now, after an unbounded
2741 	 * delay.  This can't be good.
2742 	 */
2743 
2744 #ifdef TCP_OFFLOAD_ENABLE
2745 	for_each_ofld_rxq(pi, i, ofld_rxq) {
2746 		iq = &ofld_rxq->iq;
2747 		if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2748 		    IQS_DISABLED)
2749 			panic("%s: iq %p wasn't disabled", __func__,
2750 			    (void *)iq);
2751 		t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2752 		    V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2753 	}
2754 #endif
2755 
2756 	for_each_rxq(pi, i, rxq) {
2757 		iq = &rxq->iq;
2758 		if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2759 		    IQS_DISABLED)
2760 			panic("%s: iq %p wasn't disabled", __func__,
2761 			    (void *) iq);
2762 		t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2763 		    V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2764 	}
2765 }
2766 
2767 void
2768 disable_port_queues(struct port_info *pi)
2769 {
2770 	int i;
2771 	struct adapter *sc = pi->adapter;
2772 	struct sge_rxq *rxq;
2773 #ifdef TCP_OFFLOAD_ENABLE
2774 	struct sge_ofld_rxq *ofld_rxq;
2775 #endif
2776 
2777 	ASSERT(pi->flags & PORT_INIT_DONE);
2778 
2779 	/*
2780 	 * TODO: need proper implementation for all tx queues (ctrl, eth, ofld).
2781 	 */
2782 
2783 #ifdef TCP_OFFLOAD_ENABLE
2784 	for_each_ofld_rxq(pi, i, ofld_rxq) {
2785 		while (atomic_cas_uint(&ofld_rxq->iq.state, IQS_IDLE,
2786 		    IQS_DISABLED) != IQS_IDLE)
2787 			msleep(1);
2788 	}
2789 #endif
2790 
2791 	for_each_rxq(pi, i, rxq) {
2792 		while (atomic_cas_uint(&rxq->iq.state, IQS_IDLE,
2793 		    IQS_DISABLED) != IQS_IDLE)
2794 			msleep(1);
2795 	}
2796 
2797 	mutex_enter(&sc->sfl_lock);
2798 #ifdef TCP_OFFLOAD_ENABLE
2799 	for_each_ofld_rxq(pi, i, ofld_rxq)
2800 	    ofld_rxq->fl.flags |= FL_DOOMED;
2801 #endif
2802 	for_each_rxq(pi, i, rxq)
2803 	    rxq->fl.flags |= FL_DOOMED;
2804 	mutex_exit(&sc->sfl_lock);
2805 	/* TODO: need to wait for all fl's to be removed from sc->sfl */
2806 }
2807 
2808 void
2809 t4_fatal_err(struct adapter *sc)
2810 {
2811 	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
2812 	t4_intr_disable(sc);
2813 	cxgb_printf(sc->dip, CE_WARN,
2814 	    "encountered fatal error, adapter stopped.");
2815 }
2816 
2817 int
2818 t4_os_find_pci_capability(struct adapter *sc, int cap)
2819 {
2820 	uint16_t stat;
2821 	uint8_t cap_ptr, cap_id;
2822 
2823 	t4_os_pci_read_cfg2(sc, PCI_CONF_STAT, &stat);
2824 	if ((stat & PCI_STAT_CAP) == 0)
2825 		return (0); /* does not implement capabilities */
2826 
2827 	t4_os_pci_read_cfg1(sc, PCI_CONF_CAP_PTR, &cap_ptr);
2828 	while (cap_ptr) {
2829 		t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_ID, &cap_id);
2830 		if (cap_id == cap)
2831 			return (cap_ptr); /* found */
2832 		t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_NEXT_PTR, &cap_ptr);
2833 	}
2834 
2835 	return (0); /* not found */
2836 }
2837 
2838 void
2839 t4_os_portmod_changed(struct adapter *sc, int idx)
2840 {
2841 	static const char *mod_str[] = {
2842 		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
2843 	};
2844 	struct port_info *pi = sc->port[idx];
2845 
2846 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
2847 		cxgb_printf(pi->dip, CE_NOTE, "transceiver unplugged.");
2848 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
2849 		cxgb_printf(pi->dip, CE_NOTE,
2850 		    "unknown transceiver inserted.\n");
2851 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
2852 		cxgb_printf(pi->dip, CE_NOTE,
2853 		    "unsupported transceiver inserted.\n");
2854 	else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
2855 		cxgb_printf(pi->dip, CE_NOTE, "%s transceiver inserted.\n",
2856 		    mod_str[pi->mod_type]);
2857 	else
2858 		cxgb_printf(pi->dip, CE_NOTE, "transceiver (type %d) inserted.",
2859 		    pi->mod_type);
2860 
2861 	if ((isset(&sc->open_device_map, pi->port_id) != 0) &&
2862 	    pi->link_cfg.new_module)
2863 		pi->link_cfg.redo_l1cfg = true;
2864 }
2865 
2866 /* ARGSUSED */
2867 static int
2868 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
2869 {
2870 	if (m != NULL)
2871 		freemsg(m);
2872 	return (0);
2873 }
2874 
2875 int
2876 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
2877 {
2878 	uint_t *loc, new;
2879 
2880 	if (opcode >= ARRAY_SIZE(sc->cpl_handler))
2881 		return (EINVAL);
2882 
2883 	new = (uint_t)(unsigned long) (h ? h : cpl_not_handled);
2884 	loc = (uint_t *)&sc->cpl_handler[opcode];
2885 	(void) atomic_swap_uint(loc, new);
2886 
2887 	return (0);
2888 }
2889 
2890 static int
2891 fw_msg_not_handled(struct adapter *sc, const __be64 *data)
2892 {
2893 	struct cpl_fw6_msg *cpl;
2894 
2895 	cpl = __containerof((void *)data, struct cpl_fw6_msg, data);
2896 
2897 	cxgb_printf(sc->dip, CE_WARN, "%s fw_msg type %d", __func__, cpl->type);
2898 	return (0);
2899 }
2900 
2901 int
2902 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
2903 {
2904 	fw_msg_handler_t *loc, new;
2905 
2906 	if (type >= ARRAY_SIZE(sc->fw_msg_handler))
2907 		return (EINVAL);
2908 
2909 	/*
2910 	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
2911 	 * handler dispatch table.  Reject any attempt to install a handler for
2912 	 * this subtype.
2913 	 */
2914 	if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
2915 		return (EINVAL);
2916 
2917 	new = h ? h : fw_msg_not_handled;
2918 	loc = &sc->fw_msg_handler[type];
2919 	(void)atomic_swap_ptr(loc, (void *)new);
2920 
2921 	return (0);
2922 }
2923 
2924 #ifdef TCP_OFFLOAD_ENABLE
2925 static int
2926 toe_capability(struct port_info *pi, int enable)
2927 {
2928 	int rc;
2929 	struct adapter *sc = pi->adapter;
2930 
2931 	if (!is_offload(sc))
2932 		return (ENODEV);
2933 
2934 	if (enable != 0) {
2935 		if (isset(&sc->offload_map, pi->port_id) != 0)
2936 			return (0);
2937 
2938 		if (sc->offload_map == 0) {
2939 			rc = activate_uld(sc, ULD_TOM, &sc->tom);
2940 			if (rc != 0)
2941 				return (rc);
2942 		}
2943 
2944 		setbit(&sc->offload_map, pi->port_id);
2945 	} else {
2946 		if (!isset(&sc->offload_map, pi->port_id))
2947 			return (0);
2948 
2949 		clrbit(&sc->offload_map, pi->port_id);
2950 
2951 		if (sc->offload_map == 0) {
2952 			rc = deactivate_uld(&sc->tom);
2953 			if (rc != 0) {
2954 				setbit(&sc->offload_map, pi->port_id);
2955 				return (rc);
2956 			}
2957 		}
2958 	}
2959 
2960 	return (0);
2961 }
2962 
2963 /*
2964  * Add an upper layer driver to the global list.
2965  */
2966 int
2967 t4_register_uld(struct uld_info *ui)
2968 {
2969 	int rc = 0;
2970 	struct uld_info *u;
2971 
2972 	mutex_enter(&t4_uld_list_lock);
2973 	SLIST_FOREACH(u, &t4_uld_list, link) {
2974 		if (u->uld_id == ui->uld_id) {
2975 			rc = EEXIST;
2976 			goto done;
2977 		}
2978 	}
2979 
2980 	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
2981 	ui->