1/*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12/*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications.  All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23#include <sys/ddi.h>
24#include <sys/sunddi.h>
25#include <sys/sunndi.h>
26#include <sys/modctl.h>
27#include <sys/conf.h>
28#include <sys/devops.h>
29#include <sys/pci.h>
30#include <sys/atomic.h>
31#include <sys/types.h>
32#include <sys/file.h>
33#include <sys/errno.h>
34#include <sys/open.h>
35#include <sys/cred.h>
36#include <sys/stat.h>
37#include <sys/mkdev.h>
38#include <sys/queue.h>
39#include <sys/containerof.h>
40#include <sys/sensors.h>
41
42#include "version.h"
43#include "common/common.h"
44#include "common/t4_msg.h"
45#include "common/t4_regs.h"
46#include "firmware/t4_fw.h"
47#include "firmware/t4_cfg.h"
48#include "firmware/t5_fw.h"
49#include "firmware/t5_cfg.h"
50#include "firmware/t6_fw.h"
51#include "firmware/t6_cfg.h"
52#include "t4_l2t.h"
53
54static int t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp);
55static int t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp);
56static int t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp,
57    int *rp);
58struct cb_ops t4_cb_ops = {
59	.cb_open =		t4_cb_open,
60	.cb_close =		t4_cb_close,
61	.cb_strategy =		nodev,
62	.cb_print =		nodev,
63	.cb_dump =		nodev,
64	.cb_read =		nodev,
65	.cb_write =		nodev,
66	.cb_ioctl =		t4_cb_ioctl,
67	.cb_devmap =		nodev,
68	.cb_mmap =		nodev,
69	.cb_segmap =		nodev,
70	.cb_chpoll =		nochpoll,
71	.cb_prop_op =		ddi_prop_op,
72	.cb_flag =		D_MP,
73	.cb_rev =		CB_REV,
74	.cb_aread =		nodev,
75	.cb_awrite =		nodev
76};
77
78static int t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op,
79    void *arg, void *result);
80static int t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
81    void *arg, dev_info_t **cdipp);
82static int t4_bus_unconfig(dev_info_t *dip, uint_t flags,
83    ddi_bus_config_op_t op, void *arg);
84struct bus_ops t4_bus_ops = {
85	.busops_rev =		BUSO_REV,
86	.bus_ctl =		t4_bus_ctl,
87	.bus_prop_op =		ddi_bus_prop_op,
88	.bus_config =		t4_bus_config,
89	.bus_unconfig =		t4_bus_unconfig,
90};
91
92static int t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
93    void **rp);
94static int t4_devo_probe(dev_info_t *dip);
95static int t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
96static int t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
97static int t4_devo_quiesce(dev_info_t *dip);
98struct dev_ops t4_dev_ops = {
99	.devo_rev =		DEVO_REV,
100	.devo_getinfo =		t4_devo_getinfo,
101	.devo_identify =	nulldev,
102	.devo_probe =		t4_devo_probe,
103	.devo_attach =		t4_devo_attach,
104	.devo_detach =		t4_devo_detach,
105	.devo_reset =		nodev,
106	.devo_cb_ops =		&t4_cb_ops,
107	.devo_bus_ops =		&t4_bus_ops,
108	.devo_quiesce =		&t4_devo_quiesce,
109};
110
111static struct modldrv modldrv = {
112	.drv_modops =		&mod_driverops,
113	.drv_linkinfo =		"Chelsio T4 nexus " DRV_VERSION,
114	.drv_dev_ops =		&t4_dev_ops
115};
116
117static struct modlinkage modlinkage = {
118	.ml_rev =		MODREV_1,
119	.ml_linkage =		{&modldrv, NULL},
120};
121
122void *t4_list;
123
124struct intrs_and_queues {
125	int intr_type;		/* DDI_INTR_TYPE_* */
126	int nirq;		/* Number of vectors */
127	int intr_fwd;		/* Interrupts forwarded */
128	int ntxq10g;		/* # of NIC txq's for each 10G port */
129	int nrxq10g;		/* # of NIC rxq's for each 10G port */
130	int ntxq1g;		/* # of NIC txq's for each 1G port */
131	int nrxq1g;		/* # of NIC rxq's for each 1G port */
132#ifdef TCP_OFFLOAD_ENABLE
133	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
134	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
135	int nofldtxq1g;		/* # of TOE txq's for each 1G port */
136	int nofldrxq1g;		/* # of TOE rxq's for each 1G port */
137#endif
138};
139
140struct fw_info fi[3];
141
142static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss,
143    mblk_t *m);
144static int fw_msg_not_handled(struct adapter *, const __be64 *);
145int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
146static unsigned int getpf(struct adapter *sc);
147static int prep_firmware(struct adapter *sc);
148static int upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma);
149static int partition_resources(struct adapter *sc);
150static int adap__pre_init_tweaks(struct adapter *sc);
151static int get_params__pre_init(struct adapter *sc);
152static int get_params__post_init(struct adapter *sc);
153static int set_params__post_init(struct adapter *);
154static void setup_memwin(struct adapter *sc);
155static int validate_mt_off_len(struct adapter *, int, uint32_t, int,
156    uint32_t *);
157void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
158uint32_t position_memwin(struct adapter *, int, uint32_t);
159static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
160    uint_t count);
161static int prop_lookup_int_array(struct adapter *sc, char *name, int *data,
162    uint_t count);
163static int init_driver_props(struct adapter *sc, struct driver_properties *p);
164static int remove_extra_props(struct adapter *sc, int n10g, int n1g);
165static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
166    struct intrs_and_queues *iaq);
167static int add_child_node(struct adapter *sc, int idx);
168static int remove_child_node(struct adapter *sc, int idx);
169static kstat_t *setup_kstats(struct adapter *sc);
170static kstat_t *setup_wc_kstats(struct adapter *);
171static int update_wc_kstats(kstat_t *, int);
172#ifdef TCP_OFFLOAD_ENABLE
173static int toe_capability(struct port_info *pi, int enable);
174static int activate_uld(struct adapter *sc, int id, struct uld_softc *usc);
175static int deactivate_uld(struct uld_softc *usc);
176#endif
177static kmutex_t t4_adapter_list_lock;
178static SLIST_HEAD(, adapter) t4_adapter_list;
179#ifdef TCP_OFFLOAD_ENABLE
180static kmutex_t t4_uld_list_lock;
181static SLIST_HEAD(, uld_info) t4_uld_list;
182#endif
183
184static int t4_temperature_read(void *, sensor_ioctl_scalar_t *);
185static int t4_voltage_read(void *, sensor_ioctl_scalar_t *);
186static const ksensor_ops_t t4_temp_ops = {
187	.kso_kind = ksensor_kind_temperature,
188	.kso_scalar = t4_temperature_read
189};
190
191static const ksensor_ops_t t4_volt_ops = {
192	.kso_kind = ksensor_kind_voltage,
193	.kso_scalar = t4_voltage_read
194};
195
196int
197_init(void)
198{
199	int rc;
200
201	rc = ddi_soft_state_init(&t4_list, sizeof (struct adapter), 0);
202	if (rc != 0)
203		return (rc);
204
205	rc = mod_install(&modlinkage);
206	if (rc != 0)
207		ddi_soft_state_fini(&t4_list);
208
209	mutex_init(&t4_adapter_list_lock, NULL, MUTEX_DRIVER, NULL);
210	SLIST_INIT(&t4_adapter_list);
211
212#ifdef TCP_OFFLOAD_ENABLE
213	mutex_init(&t4_uld_list_lock, NULL, MUTEX_DRIVER, NULL);
214	SLIST_INIT(&t4_uld_list);
215#endif
216
217	return (rc);
218}
219
220int
221_fini(void)
222{
223	int rc;
224
225	rc = mod_remove(&modlinkage);
226	if (rc != 0)
227		return (rc);
228
229	ddi_soft_state_fini(&t4_list);
230	return (0);
231}
232
233int
234_info(struct modinfo *mi)
235{
236	return (mod_info(&modlinkage, mi));
237}
238
239/* ARGSUSED */
240static int
241t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp)
242{
243	struct adapter *sc;
244	minor_t minor;
245
246	minor = getminor((dev_t)arg);	/* same as instance# in our case */
247
248	if (cmd == DDI_INFO_DEVT2DEVINFO) {
249		sc = ddi_get_soft_state(t4_list, minor);
250		if (sc == NULL)
251			return (DDI_FAILURE);
252
253		ASSERT(sc->dev == (dev_t)arg);
254		*rp = (void *)sc->dip;
255	} else if (cmd == DDI_INFO_DEVT2INSTANCE)
256		*rp = (void *) (unsigned long) minor;
257	else
258		ASSERT(0);
259
260	return (DDI_SUCCESS);
261}
262
263static int
264t4_devo_probe(dev_info_t *dip)
265{
266	int rc, id, *reg;
267	uint_t n, pf;
268
269	id = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
270	    "device-id", 0xffff);
271	if (id == 0xffff)
272		return (DDI_PROBE_DONTCARE);
273
274	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
275	    "reg", &reg, &n);
276	if (rc != DDI_SUCCESS)
277		return (DDI_PROBE_DONTCARE);
278
279	pf = PCI_REG_FUNC_G(reg[0]);
280	ddi_prop_free(reg);
281
282	/* Prevent driver attachment on any PF except 0 on the FPGA */
283	if (id == 0xa000 && pf != 0)
284		return (DDI_PROBE_FAILURE);
285
286	return (DDI_PROBE_DONTCARE);
287}
288
289static int
290t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
291{
292	struct adapter *sc = NULL;
293	struct sge *s;
294	int i, instance, rc = DDI_SUCCESS, rqidx, tqidx, q;
295	int irq = 0, nxg, n100g, n40g, n25g, n10g, n1g;
296#ifdef TCP_OFFLOAD_ENABLE
297	int ofld_rqidx, ofld_tqidx;
298#endif
299	char name[16];
300	struct driver_properties *prp;
301	struct intrs_and_queues iaq;
302	ddi_device_acc_attr_t da = {
303		.devacc_attr_version = DDI_DEVICE_ATTR_V0,
304		.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
305		.devacc_attr_dataorder = DDI_UNORDERED_OK_ACC
306	};
307	ddi_device_acc_attr_t da1 = {
308		.devacc_attr_version = DDI_DEVICE_ATTR_V0,
309		.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC,
310		.devacc_attr_dataorder = DDI_MERGING_OK_ACC
311	};
312
313	if (cmd != DDI_ATTACH)
314		return (DDI_FAILURE);
315
316	/*
317	 * Allocate space for soft state.
318	 */
319	instance = ddi_get_instance(dip);
320	rc = ddi_soft_state_zalloc(t4_list, instance);
321	if (rc != DDI_SUCCESS) {
322		cxgb_printf(dip, CE_WARN,
323		    "failed to allocate soft state: %d", rc);
324		return (DDI_FAILURE);
325	}
326
327	sc = ddi_get_soft_state(t4_list, instance);
328	sc->dip = dip;
329	sc->dev = makedevice(ddi_driver_major(dip), instance);
330	mutex_init(&sc->lock, NULL, MUTEX_DRIVER, NULL);
331	cv_init(&sc->cv, NULL, CV_DRIVER, NULL);
332	mutex_init(&sc->sfl_lock, NULL, MUTEX_DRIVER, NULL);
333	TAILQ_INIT(&sc->sfl);
334
335	mutex_enter(&t4_adapter_list_lock);
336	SLIST_INSERT_HEAD(&t4_adapter_list, sc, link);
337	mutex_exit(&t4_adapter_list_lock);
338
339	sc->pf = getpf(sc);
340	if (sc->pf > 8) {
341		rc = EINVAL;
342		cxgb_printf(dip, CE_WARN,
343		    "failed to determine PCI PF# of device");
344		goto done;
345	}
346	sc->mbox = sc->pf;
347
348	/* Initialize the driver properties */
349	prp = &sc->props;
350	(void)init_driver_props(sc, prp);
351
352	/*
353	 * Enable access to the PCI config space.
354	 */
355	rc = pci_config_setup(dip, &sc->pci_regh);
356	if (rc != DDI_SUCCESS) {
357		cxgb_printf(dip, CE_WARN,
358		    "failed to enable PCI config space access: %d", rc);
359		goto done;
360	}
361
362	/* TODO: Set max read request to 4K */
363
364	/*
365	 * Enable MMIO access.
366	 */
367	rc = ddi_regs_map_setup(dip, 1, &sc->regp, 0, 0, &da, &sc->regh);
368	if (rc != DDI_SUCCESS) {
369		cxgb_printf(dip, CE_WARN,
370		    "failed to map device registers: %d", rc);
371		goto done;
372	}
373
374	(void) memset(sc->chan_map, 0xff, sizeof (sc->chan_map));
375
376	/*
377	 * Initialize cpl handler.
378	 */
379	for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) {
380		sc->cpl_handler[i] = cpl_not_handled;
381	}
382
383	for (i = 0; i < ARRAY_SIZE(sc->fw_msg_handler); i++) {
384		sc->fw_msg_handler[i] = fw_msg_not_handled;
385	}
386
387	for (i = 0; i < NCHAN; i++) {
388		(void) snprintf(name, sizeof (name), "%s-%d",
389				"reclaim", i);
390		sc->tq[i] = ddi_taskq_create(sc->dip,
391		   name, 1, TASKQ_DEFAULTPRI, 0);
392
393		if (sc->tq[i] == NULL) {
394			cxgb_printf(dip, CE_WARN,
395				   "failed to create task queues");
396			rc = DDI_FAILURE;
397			goto done;
398		}
399	}
400
401	/*
402	 * Prepare the adapter for operation.
403	 */
404	rc = -t4_prep_adapter(sc, false);
405	if (rc != 0) {
406		cxgb_printf(dip, CE_WARN, "failed to prepare adapter: %d", rc);
407		goto done;
408	}
409
410	/*
411	 * Enable BAR1 access.
412	 */
413	sc->doorbells |= DOORBELL_KDB;
414	rc = ddi_regs_map_setup(dip, 2, &sc->reg1p, 0, 0, &da1, &sc->reg1h);
415	if (rc != DDI_SUCCESS) {
416		cxgb_printf(dip, CE_WARN,
417		    "failed to map BAR1 device registers: %d", rc);
418		goto done;
419	} else {
420		if (is_t5(sc->params.chip)) {
421			sc->doorbells |= DOORBELL_UDB;
422			if (prp->wc) {
423				/*
424				 * Enable write combining on BAR2.  This is the
425				 * userspace doorbell BAR and is split into 128B
426				 * (UDBS_SEG_SIZE) doorbell regions, each associated
427				 * with an egress queue.  The first 64B has the doorbell
428				 * and the second 64B can be used to submit a tx work
429				 * request with an implicit doorbell.
430				 */
431				sc->doorbells &= ~DOORBELL_UDB;
432				sc->doorbells |= (DOORBELL_WCWR |
433				    DOORBELL_UDBWC);
434				t4_write_reg(sc, A_SGE_STAT_CFG,
435				    V_STATSOURCE_T5(7) | V_STATMODE(0));
436			}
437		}
438	}
439
440	/*
441	 * Do this really early.  Note that minor number = instance.
442	 */
443	(void) snprintf(name, sizeof (name), "%s,%d", T4_NEXUS_NAME, instance);
444	rc = ddi_create_minor_node(dip, name, S_IFCHR, instance,
445	    DDI_NT_NEXUS, 0);
446	if (rc != DDI_SUCCESS) {
447		cxgb_printf(dip, CE_WARN,
448		    "failed to create device node: %d", rc);
449		rc = DDI_SUCCESS; /* carry on */
450	}
451
452	/* Do this early. Memory window is required for loading config file. */
453	setup_memwin(sc);
454
455	/* Prepare the firmware for operation */
456	rc = prep_firmware(sc);
457	if (rc != 0)
458		goto done; /* error message displayed already */
459
460	rc = adap__pre_init_tweaks(sc);
461	if (rc != 0)
462		goto done;
463
464	rc = get_params__pre_init(sc);
465	if (rc != 0)
466		goto done; /* error message displayed already */
467
468	t4_sge_init(sc);
469
470	if (sc->flags & MASTER_PF) {
471		/* get basic stuff going */
472		rc = -t4_fw_initialize(sc, sc->mbox);
473		if (rc != 0) {
474			cxgb_printf(sc->dip, CE_WARN,
475			    "early init failed: %d.\n", rc);
476			goto done;
477		}
478	}
479
480	rc = get_params__post_init(sc);
481	if (rc != 0)
482		goto done; /* error message displayed already */
483
484	rc = set_params__post_init(sc);
485	if (rc != 0)
486		goto done; /* error message displayed already */
487
488	/*
489	 * TODO: This is the place to call t4_set_filter_mode()
490	 */
491
492	/* tweak some settings */
493	t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) |
494	    V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) |
495	    V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9));
496	t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
497
498	/*
499	 * Work-around for bug 2619
500	 * Set DisableVlan field in TP_RSS_CONFIG_VRT register so that the
501	 * VLAN tag extraction is disabled.
502	 */
503	t4_set_reg_field(sc, A_TP_RSS_CONFIG_VRT, F_DISABLEVLAN, F_DISABLEVLAN);
504
505	/* Store filter mode */
506	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
507	    A_TP_VLAN_PRI_MAP);
508
509	/*
510	 * First pass over all the ports - allocate VIs and initialize some
511	 * basic parameters like mac address, port type, etc.  We also figure
512	 * out whether a port is 10G or 1G and use that information when
513	 * calculating how many interrupts to attempt to allocate.
514	 */
515	n100g = n40g = n25g = n10g = n1g = 0;
516	for_each_port(sc, i) {
517		struct port_info *pi;
518
519		pi = kmem_zalloc(sizeof (*pi), KM_SLEEP);
520		sc->port[i] = pi;
521
522		/* These must be set before t4_port_init */
523		pi->adapter = sc;
524		/* LINTED: E_ASSIGN_NARROW_CONV */
525		pi->port_id = i;
526	}
527
528	/* Allocate the vi and initialize parameters like mac addr */
529	rc = -t4_port_init(sc, sc->mbox, sc->pf, 0);
530	if (rc) {
531		cxgb_printf(dip, CE_WARN,
532			    "unable to initialize port: %d", rc);
533		goto done;
534	}
535
536	for_each_port(sc, i) {
537		struct port_info *pi = sc->port[i];
538
539		mutex_init(&pi->lock, NULL, MUTEX_DRIVER, NULL);
540		pi->mtu = ETHERMTU;
541
542		if (is_100G_port(pi)) {
543			n100g++;
544			pi->tmr_idx = prp->tmr_idx_10g;
545			pi->pktc_idx = prp->pktc_idx_10g;
546		} else if (is_40G_port(pi)) {
547			n40g++;
548			pi->tmr_idx = prp->tmr_idx_10g;
549			pi->pktc_idx = prp->pktc_idx_10g;
550		} else if (is_25G_port(pi)) {
551			n25g++;
552			pi->tmr_idx = prp->tmr_idx_10g;
553			pi->pktc_idx = prp->pktc_idx_10g;
554		} else if (is_10G_port(pi)) {
555			n10g++;
556			pi->tmr_idx = prp->tmr_idx_10g;
557			pi->pktc_idx = prp->pktc_idx_10g;
558		} else {
559			n1g++;
560			pi->tmr_idx = prp->tmr_idx_1g;
561			pi->pktc_idx = prp->pktc_idx_1g;
562		}
563
564		pi->xact_addr_filt = -1;
565		t4_mc_init(pi);
566
567		setbit(&sc->registered_device_map, i);
568	}
569
570	nxg = n10g + n25g + n40g + n100g;
571	(void) remove_extra_props(sc, nxg, n1g);
572
573	if (sc->registered_device_map == 0) {
574		cxgb_printf(dip, CE_WARN, "no usable ports");
575		rc = DDI_FAILURE;
576		goto done;
577	}
578
579	rc = cfg_itype_and_nqueues(sc, nxg, n1g, &iaq);
580	if (rc != 0)
581		goto done; /* error message displayed already */
582
583	sc->intr_type = iaq.intr_type;
584	sc->intr_count = iaq.nirq;
585
586	if (sc->props.multi_rings && (sc->intr_type != DDI_INTR_TYPE_MSIX)) {
587		sc->props.multi_rings = 0;
588		cxgb_printf(dip, CE_WARN,
589		    "Multiple rings disabled as interrupt type is not MSI-X");
590	}
591
592	if (sc->props.multi_rings && iaq.intr_fwd) {
593		sc->props.multi_rings = 0;
594		cxgb_printf(dip, CE_WARN,
595		    "Multiple rings disabled as interrupts are forwarded");
596	}
597
598	if (!sc->props.multi_rings) {
599		iaq.ntxq10g = 1;
600		iaq.ntxq1g = 1;
601	}
602	s = &sc->sge;
603	s->nrxq = nxg * iaq.nrxq10g + n1g * iaq.nrxq1g;
604	s->ntxq = nxg * iaq.ntxq10g + n1g * iaq.ntxq1g;
605	s->neq = s->ntxq + s->nrxq;	/* the fl in an rxq is an eq */
606#ifdef TCP_OFFLOAD_ENABLE
607	/* control queues, 1 per port + 1 mgmtq */
608	s->neq += sc->params.nports + 1;
609#endif
610	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
611	if (iaq.intr_fwd != 0)
612		sc->flags |= INTR_FWD;
613#ifdef TCP_OFFLOAD_ENABLE
614	if (is_offload(sc) != 0) {
615
616		s->nofldrxq = nxg * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
617		s->nofldtxq = nxg * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
618		s->neq += s->nofldtxq + s->nofldrxq;
619		s->niq += s->nofldrxq;
620
621		s->ofld_rxq = kmem_zalloc(s->nofldrxq *
622		    sizeof (struct sge_ofld_rxq), KM_SLEEP);
623		s->ofld_txq = kmem_zalloc(s->nofldtxq *
624		    sizeof (struct sge_wrq), KM_SLEEP);
625		s->ctrlq = kmem_zalloc(sc->params.nports *
626		    sizeof (struct sge_wrq), KM_SLEEP);
627
628	}
629#endif
630	s->rxq = kmem_zalloc(s->nrxq * sizeof (struct sge_rxq), KM_SLEEP);
631	s->txq = kmem_zalloc(s->ntxq * sizeof (struct sge_txq), KM_SLEEP);
632	s->iqmap = kmem_zalloc(s->niq * sizeof (struct sge_iq *), KM_SLEEP);
633	s->eqmap = kmem_zalloc(s->neq * sizeof (struct sge_eq *), KM_SLEEP);
634
635	sc->intr_handle = kmem_zalloc(sc->intr_count *
636	    sizeof (ddi_intr_handle_t), KM_SLEEP);
637
638	/*
639	 * Second pass over the ports.  This time we know the number of rx and
640	 * tx queues that each port should get.
641	 */
642	rqidx = tqidx = 0;
643#ifdef TCP_OFFLOAD_ENABLE
644	ofld_rqidx = ofld_tqidx = 0;
645#endif
646	for_each_port(sc, i) {
647		struct port_info *pi = sc->port[i];
648
649		if (pi == NULL)
650			continue;
651
652		t4_mc_cb_init(pi);
653		/* LINTED: E_ASSIGN_NARROW_CONV */
654		pi->first_rxq = rqidx;
655		/* LINTED: E_ASSIGN_NARROW_CONV */
656		pi->nrxq = (is_10XG_port(pi)) ? iaq.nrxq10g
657		    : iaq.nrxq1g;
658		/* LINTED: E_ASSIGN_NARROW_CONV */
659		pi->first_txq = tqidx;
660		/* LINTED: E_ASSIGN_NARROW_CONV */
661		pi->ntxq = (is_10XG_port(pi)) ? iaq.ntxq10g
662		    : iaq.ntxq1g;
663
664		rqidx += pi->nrxq;
665		tqidx += pi->ntxq;
666
667#ifdef TCP_OFFLOAD_ENABLE
668		if (is_offload(sc) != 0) {
669			/* LINTED: E_ASSIGN_NARROW_CONV */
670			pi->first_ofld_rxq = ofld_rqidx;
671			pi->nofldrxq = max(1, pi->nrxq / 4);
672
673			/* LINTED: E_ASSIGN_NARROW_CONV */
674			pi->first_ofld_txq = ofld_tqidx;
675			pi->nofldtxq = max(1, pi->ntxq / 2);
676
677			ofld_rqidx += pi->nofldrxq;
678			ofld_tqidx += pi->nofldtxq;
679		}
680#endif
681
682		/*
683		 * Enable hw checksumming and LSO for all ports by default.
684		 * They can be disabled using ndd (hw_csum and hw_lso).
685		 */
686		pi->features |= (CXGBE_HW_CSUM | CXGBE_HW_LSO);
687	}
688
689#ifdef TCP_OFFLOAD_ENABLE
690		sc->l2t = t4_init_l2t(sc);
691#endif
692
693	/*
694	 * Setup Interrupts.
695	 */
696
697	i = 0;
698	rc = ddi_intr_alloc(dip, sc->intr_handle, sc->intr_type, 0,
699	    sc->intr_count, &i, DDI_INTR_ALLOC_STRICT);
700	if (rc != DDI_SUCCESS) {
701		cxgb_printf(dip, CE_WARN,
702		    "failed to allocate %d interrupt(s) of type %d: %d, %d",
703		    sc->intr_count, sc->intr_type, rc, i);
704		goto done;
705	}
706	ASSERT(sc->intr_count == i); /* allocation was STRICT */
707	(void) ddi_intr_get_cap(sc->intr_handle[0], &sc->intr_cap);
708	(void) ddi_intr_get_pri(sc->intr_handle[0], &sc->intr_pri);
709	if (sc->intr_count == 1) {
710		ASSERT(sc->flags & INTR_FWD);
711		(void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_all, sc,
712		    &s->fwq);
713	} else {
714		/* Multiple interrupts.  The first one is always error intr */
715		(void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_err, sc,
716		    NULL);
717		irq++;
718
719		/* The second one is always the firmware event queue */
720		(void) ddi_intr_add_handler(sc->intr_handle[1], t4_intr, sc,
721		    &s->fwq);
722		irq++;
723		/*
724		 * Note that if INTR_FWD is set then either the NIC rx
725		 * queues or (exclusive or) the TOE rx queueus will be taking
726		 * direct interrupts.
727		 *
728		 * There is no need to check for is_offload(sc) as nofldrxq
729		 * will be 0 if offload is disabled.
730		 */
731		for_each_port(sc, i) {
732			struct port_info *pi = sc->port[i];
733			struct sge_rxq *rxq;
734#ifdef TCP_OFFLOAD_ENABLE
735			struct sge_ofld_rxq *ofld_rxq;
736
737			/*
738			 * Skip over the NIC queues if they aren't taking direct
739			 * interrupts.
740			 */
741			if ((sc->flags & INTR_FWD) &&
742			    pi->nofldrxq > pi->nrxq)
743				goto ofld_queues;
744#endif
745			rxq = &s->rxq[pi->first_rxq];
746			for (q = 0; q < pi->nrxq; q++, rxq++) {
747				(void) ddi_intr_add_handler(
748				    sc->intr_handle[irq], t4_intr, sc,
749				    &rxq->iq);
750				irq++;
751			}
752
753#ifdef TCP_OFFLOAD_ENABLE
754			/*
755			 * Skip over the offload queues if they aren't taking
756			 * direct interrupts.
757			 */
758			if ((sc->flags & INTR_FWD))
759				continue;
760ofld_queues:
761			ofld_rxq = &s->ofld_rxq[pi->first_ofld_rxq];
762			for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
763				(void) ddi_intr_add_handler(
764				    sc->intr_handle[irq], t4_intr, sc,
765				    &ofld_rxq->iq);
766				irq++;
767			}
768#endif
769		}
770
771	}
772	sc->flags |= INTR_ALLOCATED;
773
774	if ((rc = ksensor_create_scalar_pcidev(dip, SENSOR_KIND_TEMPERATURE,
775	    &t4_temp_ops, sc, "temp", &sc->temp_sensor)) != 0) {
776		cxgb_printf(dip, CE_WARN, "failed to create temperature "
777		    "sensor: %d", rc);
778		rc = DDI_FAILURE;
779		goto done;
780	}
781
782	if ((rc = ksensor_create_scalar_pcidev(dip, SENSOR_KIND_VOLTAGE,
783	    &t4_volt_ops, sc, "vdd", &sc->volt_sensor)) != 0) {
784		cxgb_printf(dip, CE_WARN, "failed to create voltage "
785		    "sensor: %d", rc);
786		rc = DDI_FAILURE;
787		goto done;
788	}
789
790
791	ddi_report_dev(dip);
792
793	/*
794	 * Hardware/Firmware/etc. Version/Revision IDs.
795	 */
796	t4_dump_version_info(sc);
797
798	if (n100g) {
799		cxgb_printf(dip, CE_NOTE,
800		    "%dx100G (%d rxq, %d txq total) %d %s.",
801		    n100g, rqidx, tqidx, sc->intr_count,
802		    sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
803		    sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
804		    "fixed interrupt");
805	} else if (n40g) {
806		cxgb_printf(dip, CE_NOTE,
807		    "%dx40G (%d rxq, %d txq total) %d %s.",
808		    n40g, rqidx, tqidx, sc->intr_count,
809		    sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
810		    sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
811		    "fixed interrupt");
812	} else if (n25g) {
813		cxgb_printf(dip, CE_NOTE,
814		    "%dx25G (%d rxq, %d txq total) %d %s.",
815		    n25g, rqidx, tqidx, sc->intr_count,
816		    sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
817		    sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
818		    "fixed interrupt");
819	} else if (n10g && n1g) {
820		cxgb_printf(dip, CE_NOTE,
821		    "%dx10G %dx1G (%d rxq, %d txq total) %d %s.",
822		    n10g, n1g, rqidx, tqidx, sc->intr_count,
823		    sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
824		    sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
825		    "fixed interrupt");
826	} else {
827		cxgb_printf(dip, CE_NOTE,
828		    "%dx%sG (%d rxq, %d txq per port) %d %s.",
829		    n10g ? n10g : n1g,
830		    n10g ? "10" : "1",
831		    n10g ? iaq.nrxq10g : iaq.nrxq1g,
832		    n10g ? iaq.ntxq10g : iaq.ntxq1g,
833		    sc->intr_count,
834		    sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" :
835		    sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" :
836		    "fixed interrupt");
837	}
838
839	sc->ksp = setup_kstats(sc);
840	sc->ksp_stat = setup_wc_kstats(sc);
841	sc->params.drv_memwin = MEMWIN_NIC;
842
843done:
844	if (rc != DDI_SUCCESS) {
845		(void) t4_devo_detach(dip, DDI_DETACH);
846
847		/* rc may have errno style errors or DDI errors */
848		rc = DDI_FAILURE;
849	}
850
851	return (rc);
852}
853
854static int
855t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
856{
857	int instance, i;
858	struct adapter *sc;
859	struct port_info *pi;
860	struct sge *s;
861
862	if (cmd != DDI_DETACH)
863		return (DDI_FAILURE);
864
865	instance = ddi_get_instance(dip);
866	sc = ddi_get_soft_state(t4_list, instance);
867	if (sc == NULL)
868		return (DDI_SUCCESS);
869
870	if (sc->flags & FULL_INIT_DONE) {
871		t4_intr_disable(sc);
872		for_each_port(sc, i) {
873			pi = sc->port[i];
874			if (pi && pi->flags & PORT_INIT_DONE)
875				(void) port_full_uninit(pi);
876		}
877		(void) adapter_full_uninit(sc);
878	}
879
880	/* Safe to call no matter what */
881	(void) ksensor_remove(dip, KSENSOR_ALL_IDS);
882	ddi_prop_remove_all(dip);
883	ddi_remove_minor_node(dip, NULL);
884
885	for (i = 0; i < NCHAN; i++) {
886		if (sc->tq[i]) {
887			ddi_taskq_wait(sc->tq[i]);
888			ddi_taskq_destroy(sc->tq[i]);
889		}
890	}
891
892	if (sc->ksp != NULL)
893		kstat_delete(sc->ksp);
894	if (sc->ksp_stat != NULL)
895		kstat_delete(sc->ksp_stat);
896
897	s = &sc->sge;
898	if (s->rxq != NULL)
899		kmem_free(s->rxq, s->nrxq * sizeof (struct sge_rxq));
900#ifdef TCP_OFFLOAD_ENABLE
901	if (s->ofld_txq != NULL)
902		kmem_free(s->ofld_txq, s->nofldtxq * sizeof (struct sge_wrq));
903	if (s->ofld_rxq != NULL)
904		kmem_free(s->ofld_rxq,
905		    s->nofldrxq * sizeof (struct sge_ofld_rxq));
906	if (s->ctrlq != NULL)
907		kmem_free(s->ctrlq,
908		    sc->params.nports * sizeof (struct sge_wrq));
909#endif
910	if (s->txq != NULL)
911		kmem_free(s->txq, s->ntxq * sizeof (struct sge_txq));
912	if (s->iqmap != NULL)
913		kmem_free(s->iqmap, s->niq * sizeof (struct sge_iq *));
914	if (s->eqmap != NULL)
915		kmem_free(s->eqmap, s->neq * sizeof (struct sge_eq *));
916
917	if (s->rxbuf_cache != NULL)
918		rxbuf_cache_destroy(s->rxbuf_cache);
919
920	if (sc->flags & INTR_ALLOCATED) {
921		for (i = 0; i < sc->intr_count; i++) {
922			(void) ddi_intr_remove_handler(sc->intr_handle[i]);
923			(void) ddi_intr_free(sc->intr_handle[i]);
924		}
925		sc->flags &= ~INTR_ALLOCATED;
926	}
927
928	if (sc->intr_handle != NULL) {
929		kmem_free(sc->intr_handle,
930		    sc->intr_count * sizeof (*sc->intr_handle));
931	}
932
933	for_each_port(sc, i) {
934		pi = sc->port[i];
935		if (pi != NULL) {
936			mutex_destroy(&pi->lock);
937			kmem_free(pi, sizeof (*pi));
938			clrbit(&sc->registered_device_map, i);
939		}
940	}
941
942	if (sc->flags & FW_OK)
943		(void) t4_fw_bye(sc, sc->mbox);
944
945	if (sc->reg1h != NULL)
946		ddi_regs_map_free(&sc->reg1h);
947
948	if (sc->regh != NULL)
949		ddi_regs_map_free(&sc->regh);
950
951	if (sc->pci_regh != NULL)
952		pci_config_teardown(&sc->pci_regh);
953
954	mutex_enter(&t4_adapter_list_lock);
955	SLIST_REMOVE_HEAD(&t4_adapter_list, link);
956	mutex_exit(&t4_adapter_list_lock);
957
958	mutex_destroy(&sc->lock);
959	cv_destroy(&sc->cv);
960	mutex_destroy(&sc->sfl_lock);
961
962#ifdef DEBUG
963	bzero(sc, sizeof (*sc));
964#endif
965	ddi_soft_state_free(t4_list, instance);
966
967	return (DDI_SUCCESS);
968}
969
970static int
971t4_devo_quiesce(dev_info_t *dip)
972{
973	int instance;
974	struct adapter *sc;
975
976	instance = ddi_get_instance(dip);
977	sc = ddi_get_soft_state(t4_list, instance);
978	if (sc == NULL)
979		return (DDI_SUCCESS);
980
981	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
982	t4_intr_disable(sc);
983	t4_write_reg(sc, A_PL_RST, F_PIORSTMODE | F_PIORST);
984
985	return (DDI_SUCCESS);
986}
987
988static int
989t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg,
990    void *result)
991{
992	char s[4];
993	struct port_info *pi;
994	dev_info_t *child = (dev_info_t *)arg;
995
996	switch (op) {
997	case DDI_CTLOPS_REPORTDEV:
998		pi = ddi_get_parent_data(rdip);
999		pi->instance = ddi_get_instance(dip);
1000		pi->child_inst = ddi_get_instance(rdip);
1001		cmn_err(CE_CONT, "?%s%d is port %s on %s%d\n",
1002		    ddi_node_name(rdip), ddi_get_instance(rdip),
1003		    ddi_get_name_addr(rdip), ddi_driver_name(dip),
1004		    ddi_get_instance(dip));
1005		return (DDI_SUCCESS);
1006
1007	case DDI_CTLOPS_INITCHILD:
1008		pi = ddi_get_parent_data(child);
1009		if (pi == NULL)
1010			return (DDI_NOT_WELL_FORMED);
1011		(void) snprintf(s, sizeof (s), "%d", pi->port_id);
1012		ddi_set_name_addr(child, s);
1013		return (DDI_SUCCESS);
1014
1015	case DDI_CTLOPS_UNINITCHILD:
1016		ddi_set_name_addr(child, NULL);
1017		return (DDI_SUCCESS);
1018
1019	case DDI_CTLOPS_ATTACH:
1020	case DDI_CTLOPS_DETACH:
1021		return (DDI_SUCCESS);
1022
1023	default:
1024		return (ddi_ctlops(dip, rdip, op, arg, result));
1025	}
1026}
1027
1028static int
1029t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, void *arg,
1030    dev_info_t **cdipp)
1031{
1032	int instance, i;
1033	struct adapter *sc;
1034
1035	instance = ddi_get_instance(dip);
1036	sc = ddi_get_soft_state(t4_list, instance);
1037
1038	if (op == BUS_CONFIG_ONE) {
1039		char *c;
1040
1041		/*
1042		 * arg is something like "cxgb@0" where 0 is the port_id hanging
1043		 * off this nexus.
1044		 */
1045
1046		c = arg;
1047		while (*(c + 1))
1048			c++;
1049
1050		/* There should be exactly 1 digit after '@' */
1051		if (*(c - 1) != '@')
1052			return (NDI_FAILURE);
1053
1054		i = *c - '0';
1055
1056		if (add_child_node(sc, i) != 0)
1057			return (NDI_FAILURE);
1058
1059		flags |= NDI_ONLINE_ATTACH;
1060
1061	} else if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) {
1062		/* Allocate and bind all child device nodes */
1063		for_each_port(sc, i)
1064		    (void) add_child_node(sc, i);
1065		flags |= NDI_ONLINE_ATTACH;
1066	}
1067
1068	return (ndi_busop_bus_config(dip, flags, op, arg, cdipp, 0));
1069}
1070
1071static int
1072t4_bus_unconfig(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op,
1073    void *arg)
1074{
1075	int instance, i, rc;
1076	struct adapter *sc;
1077
1078	instance = ddi_get_instance(dip);
1079	sc = ddi_get_soft_state(t4_list, instance);
1080
1081	if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ALL ||
1082	    op == BUS_UNCONFIG_DRIVER)
1083		flags |= NDI_UNCONFIG;
1084
1085	rc = ndi_busop_bus_unconfig(dip, flags, op, arg);
1086	if (rc != 0)
1087		return (rc);
1088
1089	if (op == BUS_UNCONFIG_ONE) {
1090		char *c;
1091
1092		c = arg;
1093		while (*(c + 1))
1094			c++;
1095
1096		if (*(c - 1) != '@')
1097			return (NDI_SUCCESS);
1098
1099		i = *c - '0';
1100
1101		rc = remove_child_node(sc, i);
1102
1103	} else if (op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) {
1104
1105		for_each_port(sc, i)
1106		    (void) remove_child_node(sc, i);
1107	}
1108
1109	return (rc);
1110}
1111
1112/* ARGSUSED */
1113static int
1114t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp)
1115{
1116	struct adapter *sc;
1117
1118	if (otyp != OTYP_CHR)
1119		return (EINVAL);
1120
1121	sc = ddi_get_soft_state(t4_list, getminor(*devp));
1122	if (sc == NULL)
1123		return (ENXIO);
1124
1125	return (atomic_cas_uint(&sc->open, 0, EBUSY));
1126}
1127
1128/* ARGSUSED */
1129static int
1130t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp)
1131{
1132	struct adapter *sc;
1133
1134	sc = ddi_get_soft_state(t4_list, getminor(dev));
1135	if (sc == NULL)
1136		return (EINVAL);
1137
1138	(void) atomic_swap_uint(&sc->open, 0);
1139	return (0);
1140}
1141
1142/* ARGSUSED */
1143static int
1144t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, int *rp)
1145{
1146	int instance;
1147	struct adapter *sc;
1148	void *data = (void *)d;
1149
1150	if (crgetuid(credp) != 0)
1151		return (EPERM);
1152
1153	instance = getminor(dev);
1154	sc = ddi_get_soft_state(t4_list, instance);
1155	if (sc == NULL)
1156		return (EINVAL);
1157
1158	return (t4_ioctl(sc, cmd, data, mode));
1159}
1160
1161static unsigned int
1162getpf(struct adapter *sc)
1163{
1164	int rc, *data;
1165	uint_t n, pf;
1166
1167	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1168	    DDI_PROP_DONTPASS, "reg", &data, &n);
1169	if (rc != DDI_SUCCESS) {
1170		cxgb_printf(sc->dip, CE_WARN,
1171		    "failed to lookup \"reg\" property: %d", rc);
1172		return (0xff);
1173	}
1174
1175	pf = PCI_REG_FUNC_G(data[0]);
1176	ddi_prop_free(data);
1177
1178	return (pf);
1179}
1180
1181
1182static struct fw_info *
1183find_fw_info(int chip)
1184{
1185	u32 i;
1186
1187	fi[0].chip = CHELSIO_T4;
1188	fi[0].fw_hdr.chip = FW_HDR_CHIP_T4;
1189	fi[0].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T4));
1190	fi[0].fw_hdr.intfver_nic = FW_INTFVER(T4, NIC);
1191	fi[0].fw_hdr.intfver_vnic = FW_INTFVER(T4, VNIC);
1192	fi[0].fw_hdr.intfver_ofld = FW_INTFVER(T4, OFLD);
1193	fi[0].fw_hdr.intfver_ri = FW_INTFVER(T4, RI);
1194	fi[0].fw_hdr.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU);
1195	fi[0].fw_hdr.intfver_iscsi = FW_INTFVER(T4, ISCSI);
1196	fi[0].fw_hdr.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU);
1197	fi[0].fw_hdr.intfver_fcoe = FW_INTFVER(T4, FCOE);
1198
1199	fi[1].chip = CHELSIO_T5;
1200	fi[1].fw_hdr.chip = FW_HDR_CHIP_T5;
1201	fi[1].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T5));
1202	fi[1].fw_hdr.intfver_nic = FW_INTFVER(T5, NIC);
1203	fi[1].fw_hdr.intfver_vnic = FW_INTFVER(T5, VNIC);
1204	fi[1].fw_hdr.intfver_ofld = FW_INTFVER(T5, OFLD);
1205	fi[1].fw_hdr.intfver_ri = FW_INTFVER(T5, RI);
1206	fi[1].fw_hdr.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU);
1207	fi[1].fw_hdr.intfver_iscsi = FW_INTFVER(T5, ISCSI);
1208	fi[1].fw_hdr.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU);
1209	fi[1].fw_hdr.intfver_fcoe = FW_INTFVER(T5, FCOE);
1210
1211	fi[2].chip = CHELSIO_T6;
1212	fi[2].fw_hdr.chip = FW_HDR_CHIP_T6;
1213	fi[2].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T6));
1214	fi[2].fw_hdr.intfver_nic = FW_INTFVER(T6, NIC);
1215	fi[2].fw_hdr.intfver_vnic = FW_INTFVER(T6, VNIC);
1216	fi[2].fw_hdr.intfver_ofld = FW_INTFVER(T6, OFLD);
1217	fi[2].fw_hdr.intfver_ri = FW_INTFVER(T6, RI);
1218	fi[2].fw_hdr.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU);
1219	fi[2].fw_hdr.intfver_iscsi = FW_INTFVER(T6, ISCSI);
1220	fi[2].fw_hdr.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU);
1221	fi[2].fw_hdr.intfver_fcoe = FW_INTFVER(T6, FCOE);
1222
1223	for (i = 0; i < ARRAY_SIZE(fi); i++) {
1224		if (fi[i].chip == chip)
1225			return &fi[i];
1226	}
1227
1228	return NULL;
1229}
1230
1231/*
1232 * Install a compatible firmware (if required), establish contact with it,
1233 * become the master, and reset the device.
1234 */
1235static int
1236prep_firmware(struct adapter *sc)
1237{
1238	int rc;
1239	int fw_size;
1240	int reset = 1;
1241	enum dev_state state;
1242	unsigned char *fw_data;
1243	struct fw_info *fw_info;
1244	struct fw_hdr *card_fw;
1245
1246	struct driver_properties *p = &sc->props;
1247
1248	/* Contact firmware, request master */
1249	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state);
1250	if (rc < 0) {
1251		rc = -rc;
1252		cxgb_printf(sc->dip, CE_WARN,
1253		    "failed to connect to the firmware: %d.", rc);
1254		return (rc);
1255	}
1256
1257	if (rc == sc->mbox)
1258		sc->flags |= MASTER_PF;
1259
1260	/* We may need FW version info for later reporting */
1261	t4_get_version_info(sc);
1262	fw_info = find_fw_info(CHELSIO_CHIP_VERSION(sc->params.chip));
1263	/* allocate memory to read the header of the firmware on the
1264	 * card
1265	 */
1266	if (!fw_info) {
1267		cxgb_printf(sc->dip, CE_WARN,
1268			    "unable to look up firmware information for chip %d.\n",
1269			    CHELSIO_CHIP_VERSION(sc->params.chip));
1270		return EINVAL;
1271	}
1272	card_fw = kmem_zalloc(sizeof(*card_fw), KM_SLEEP);
1273	if(!card_fw) {
1274		cxgb_printf(sc->dip, CE_WARN,
1275			    "Memory allocation for card FW header failed\n");
1276		return ENOMEM;
1277	}
1278	switch(CHELSIO_CHIP_VERSION(sc->params.chip)) {
1279	case CHELSIO_T4:
1280		fw_data = t4fw_data;
1281		fw_size = t4fw_size;
1282		break;
1283	case CHELSIO_T5:
1284		fw_data = t5fw_data;
1285		fw_size = t5fw_size;
1286		break;
1287	case CHELSIO_T6:
1288		fw_data = t6fw_data;
1289		fw_size = t6fw_size;
1290		break;
1291	default:
1292		cxgb_printf(sc->dip, CE_WARN, "Adapter type not supported\n");
1293		kmem_free(card_fw, sizeof(*card_fw));
1294		return EINVAL;
1295	}
1296
1297	rc = -t4_prep_fw(sc, fw_info, fw_data, fw_size, card_fw,
1298			 p->t4_fw_install, state, &reset);
1299
1300	kmem_free(card_fw, sizeof(*card_fw));
1301
1302	if (rc != 0) {
1303		cxgb_printf(sc->dip, CE_WARN,
1304		    "failed to install firmware: %d", rc);
1305		return (rc);
1306	} else {
1307		/* refresh */
1308		(void) t4_check_fw_version(sc);
1309	}
1310
1311	/* Reset device */
1312	rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1313	if (rc != 0) {
1314		cxgb_printf(sc->dip, CE_WARN,
1315		    "firmware reset failed: %d.", rc);
1316		if (rc != ETIMEDOUT && rc != EIO)
1317			(void) t4_fw_bye(sc, sc->mbox);
1318		return (rc);
1319	}
1320
1321	/* Partition adapter resources as specified in the config file. */
1322	if (sc->flags & MASTER_PF) {
1323		/* Handle default vs special T4 config file */
1324
1325		rc = partition_resources(sc);
1326		if (rc != 0)
1327			goto err;	/* error message displayed already */
1328	}
1329
1330	sc->flags |= FW_OK;
1331	return (0);
1332err:
1333	return (rc);
1334
1335}
1336
1337static const struct memwin t4_memwin[] = {
1338	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1339	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1340	{ MEMWIN2_BASE, MEMWIN2_APERTURE }
1341};
1342
1343static const struct memwin t5_memwin[] = {
1344	{ MEMWIN0_BASE, MEMWIN0_APERTURE },
1345	{ MEMWIN1_BASE, MEMWIN1_APERTURE },
1346	{ MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
1347};
1348
1349#define	FW_PARAM_DEV(param) \
1350	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1351	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1352#define	FW_PARAM_PFVF(param) \
1353	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1354	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1355
1356/*
1357 * Verify that the memory range specified by the memtype/offset/len pair is
1358 * valid and lies entirely within the memtype specified.  The global address of
1359 * the start of the range is returned in addr.
1360 */
1361int
1362validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len,
1363	uint32_t *addr)
1364{
1365	uint32_t em, addr_len, maddr, mlen;
1366
1367	/* Memory can only be accessed in naturally aligned 4 byte units */
1368	if (off & 3 || len & 3 || len == 0)
1369		return (EINVAL);
1370
1371	em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1372	switch (mtype) {
1373		case MEM_EDC0:
1374			if (!(em & F_EDRAM0_ENABLE))
1375				return (EINVAL);
1376			addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1377			maddr = G_EDRAM0_BASE(addr_len) << 20;
1378			mlen = G_EDRAM0_SIZE(addr_len) << 20;
1379			break;
1380		case MEM_EDC1:
1381			if (!(em & F_EDRAM1_ENABLE))
1382				return (EINVAL);
1383			addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1384			maddr = G_EDRAM1_BASE(addr_len) << 20;
1385			mlen = G_EDRAM1_SIZE(addr_len) << 20;
1386			break;
1387		case MEM_MC:
1388			if (!(em & F_EXT_MEM_ENABLE))
1389				return (EINVAL);
1390			addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1391			maddr = G_EXT_MEM_BASE(addr_len) << 20;
1392			mlen = G_EXT_MEM_SIZE(addr_len) << 20;
1393			break;
1394		case MEM_MC1:
1395			if (is_t4(sc->params.chip) || !(em & F_EXT_MEM1_ENABLE))
1396				return (EINVAL);
1397			addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
1398			maddr = G_EXT_MEM1_BASE(addr_len) << 20;
1399			mlen = G_EXT_MEM1_SIZE(addr_len) << 20;
1400			break;
1401		default:
1402			return (EINVAL);
1403	}
1404
1405	if (mlen > 0 && off < mlen && off + len <= mlen) {
1406		*addr = maddr + off;    /* global address */
1407		return (0);
1408	}
1409
1410	return (EFAULT);
1411}
1412
1413void
1414memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture)
1415{
1416	const struct memwin *mw;
1417
1418	if (is_t4(sc->params.chip)) {
1419		mw = &t4_memwin[win];
1420	} else {
1421		mw = &t5_memwin[win];
1422	}
1423
1424	if (base != NULL)
1425		*base = mw->base;
1426	if (aperture != NULL)
1427		*aperture = mw->aperture;
1428}
1429
1430/*
1431 * Upload configuration file to card's memory.
1432 */
1433static int
1434upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma)
1435{
1436	int rc = 0, cflen;
1437	u_int i, n;
1438	uint32_t param, val, addr, mtype, maddr;
1439	uint32_t off, mw_base, mw_aperture;
1440	const uint32_t *cfdata;
1441
1442	/* Figure out where the firmware wants us to upload it. */
1443	param = FW_PARAM_DEV(CF);
1444	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1445	if (rc != 0) {
1446		/* Firmwares without config file support will fail this way */
1447		cxgb_printf(sc->dip, CE_WARN,
1448		    "failed to query config file location: %d.\n", rc);
1449		return (rc);
1450	}
1451	*mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1452	*ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1453
1454	switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
1455	case CHELSIO_T4:
1456		cflen = t4cfg_size & ~3;
1457		/* LINTED: E_BAD_PTR_CAST_ALIGN */
1458		cfdata = (const uint32_t *)t4cfg_data;
1459		break;
1460	case CHELSIO_T5:
1461		cflen = t5cfg_size & ~3;
1462		/* LINTED: E_BAD_PTR_CAST_ALIGN */
1463		cfdata = (const uint32_t *)t5cfg_data;
1464		break;
1465	case CHELSIO_T6:
1466		cflen = t6cfg_size & ~3;
1467		/* LINTED: E_BAD_PTR_CAST_ALIGN */
1468		cfdata = (const uint32_t *)t6cfg_data;
1469		break;
1470	default:
1471		cxgb_printf(sc->dip, CE_WARN,
1472			    "Invalid Adapter detected\n");
1473		return EINVAL;
1474	}
1475
1476	if (cflen > FLASH_CFG_MAX_SIZE) {
1477		cxgb_printf(sc->dip, CE_WARN,
1478		    "config file too long (%d, max allowed is %d).  ",
1479		    cflen, FLASH_CFG_MAX_SIZE);
1480		return (EFBIG);
1481	}
1482
1483	rc = validate_mt_off_len(sc, mtype, maddr, cflen, &addr);
1484	if (rc != 0) {
1485
1486		cxgb_printf(sc->dip, CE_WARN,
1487		    "%s: addr (%d/0x%x) or len %d is not valid: %d.  "
1488		    "Will try to use the config on the card, if any.\n",
1489		    __func__, mtype, maddr, cflen, rc);
1490		return (EFAULT);
1491	}
1492
1493	memwin_info(sc, 2, &mw_base, &mw_aperture);
1494	while (cflen) {
1495		off = position_memwin(sc, 2, addr);
1496		n = min(cflen, mw_aperture - off);
1497		for (i = 0; i < n; i += 4)
1498			t4_write_reg(sc, mw_base + off + i, *cfdata++);
1499		cflen -= n;
1500		addr += n;
1501	}
1502
1503	return (rc);
1504}
1505
1506/*
1507 * Partition chip resources for use between various PFs, VFs, etc.  This is done
1508 * by uploading the firmware configuration file to the adapter and instructing
1509 * the firmware to process it.
1510 */
1511static int
1512partition_resources(struct adapter *sc)
1513{
1514	int rc;
1515	struct fw_caps_config_cmd caps;
1516	uint32_t mtype, maddr, finicsum, cfcsum;
1517
1518	rc = upload_config_file(sc, &mtype, &maddr);
1519	if (rc != 0) {
1520		mtype = FW_MEMTYPE_CF_FLASH;
1521		maddr = t4_flash_cfg_addr(sc);
1522	}
1523
1524	bzero(&caps, sizeof (caps));
1525	caps.op_to_write = BE_32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1526	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1527	caps.cfvalid_to_len16 = BE_32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1528	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1529	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1530	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1531	if (rc != 0) {
1532		cxgb_printf(sc->dip, CE_WARN,
1533		    "failed to pre-process config file: %d.\n", rc);
1534		return (rc);
1535	}
1536
1537	finicsum = ntohl(caps.finicsum);
1538	cfcsum = ntohl(caps.cfcsum);
1539	if (finicsum != cfcsum) {
1540		cxgb_printf(sc->dip, CE_WARN,
1541		    "WARNING: config file checksum mismatch: %08x %08x\n",
1542		    finicsum, cfcsum);
1543	}
1544	sc->cfcsum = cfcsum;
1545
1546	/* TODO: Need to configure this correctly */
1547	caps.toecaps = htons(FW_CAPS_CONFIG_TOE);
1548	caps.iscsicaps = 0;
1549	caps.rdmacaps = 0;
1550	caps.fcoecaps = 0;
1551	/* TODO: Disable VNIC cap for now */
1552	caps.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
1553
1554	caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1555	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1556	caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1557	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), NULL);
1558	if (rc != 0) {
1559		cxgb_printf(sc->dip, CE_WARN,
1560		    "failed to process config file: %d.\n", rc);
1561		return (rc);
1562	}
1563
1564	return (0);
1565}
1566
1567/*
1568 * Tweak configuration based on module parameters, etc.  Most of these have
1569 * defaults assigned to them by Firmware Configuration Files (if we're using
1570 * them) but need to be explicitly set if we're using hard-coded
1571 * initialization.  But even in the case of using Firmware Configuration
1572 * Files, we'd like to expose the ability to change these via module
1573 * parameters so these are essentially common tweaks/settings for
1574 * Configuration Files and hard-coded initialization ...
1575 */
1576static int
1577adap__pre_init_tweaks(struct adapter *sc)
1578{
1579	int rx_dma_offset = 2; /* Offset of RX packets into DMA buffers */
1580
1581	/*
1582	 * Fix up various Host-Dependent Parameters like Page Size, Cache
1583	 * Line Size, etc.  The firmware default is for a 4KB Page Size and
1584	 * 64B Cache Line Size ...
1585	 */
1586	(void) t4_fixup_host_params_compat(sc, PAGE_SIZE, CACHE_LINE, T5_LAST_REV);
1587
1588	t4_set_reg_field(sc, A_SGE_CONTROL,
1589			 V_PKTSHIFT(M_PKTSHIFT), V_PKTSHIFT(rx_dma_offset));
1590
1591	return 0;
1592}
1593/*
1594 * Retrieve parameters that are needed (or nice to have) prior to calling
1595 * t4_sge_init and t4_fw_initialize.
1596 */
1597static int
1598get_params__pre_init(struct adapter *sc)
1599{
1600	int rc;
1601	uint32_t param[2], val[2];
1602	struct fw_devlog_cmd cmd;
1603	struct devlog_params *dlog = &sc->params.devlog;
1604
1605	/*
1606	 * Grab the raw VPD parameters.
1607	 */
1608	rc = -t4_get_raw_vpd_params(sc, &sc->params.vpd);
1609	if (rc != 0) {
1610		cxgb_printf(sc->dip, CE_WARN,
1611		    "failed to query VPD parameters (pre_init): %d.\n", rc);
1612		return (rc);
1613	}
1614
1615	param[0] = FW_PARAM_DEV(PORTVEC);
1616	param[1] = FW_PARAM_DEV(CCLK);
1617	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1618	if (rc != 0) {
1619		cxgb_printf(sc->dip, CE_WARN,
1620		    "failed to query parameters (pre_init): %d.\n", rc);
1621		return (rc);
1622	}
1623
1624	sc->params.portvec = val[0];
1625	sc->params.nports = 0;
1626	while (val[0]) {
1627		sc->params.nports++;
1628		val[0] &= val[0] - 1;
1629	}
1630
1631	sc->params.vpd.cclk = val[1];
1632
1633	/* Read device log parameters. */
1634	bzero(&cmd, sizeof (cmd));
1635	cmd.op_to_write = htonl(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1636	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1637	cmd.retval_len16 = htonl(FW_LEN16(cmd));
1638	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof (cmd), &cmd);
1639	if (rc != 0) {
1640		cxgb_printf(sc->dip, CE_WARN,
1641		    "failed to get devlog parameters: %d.\n", rc);
1642		bzero(dlog, sizeof (*dlog));
1643		rc = 0;	/* devlog isn't critical for device operation */
1644	} else {
1645		val[0] = ntohl(cmd.memtype_devlog_memaddr16_devlog);
1646		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1647		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1648		dlog->size = ntohl(cmd.memsize_devlog);
1649	}
1650
1651	return (rc);
1652}
1653
1654/*
1655 * Retrieve various parameters that are of interest to the driver.  The device
1656 * has been initialized by the firmware at this point.
1657 */
1658static int
1659get_params__post_init(struct adapter *sc)
1660{
1661	int rc;
1662	uint32_t param[7], val[7];
1663	struct fw_caps_config_cmd caps;
1664
1665	param[0] = FW_PARAM_PFVF(IQFLINT_START);
1666	param[1] = FW_PARAM_PFVF(EQ_START);
1667	param[2] = FW_PARAM_PFVF(FILTER_START);
1668	param[3] = FW_PARAM_PFVF(FILTER_END);
1669	param[4] = FW_PARAM_PFVF(L2T_START);
1670	param[5] = FW_PARAM_PFVF(L2T_END);
1671	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1672	if (rc != 0) {
1673		cxgb_printf(sc->dip, CE_WARN,
1674		    "failed to query parameters (post_init): %d.\n", rc);
1675		return (rc);
1676	}
1677
1678	/* LINTED: E_ASSIGN_NARROW_CONV */
1679	sc->sge.iq_start = val[0];
1680	sc->sge.eq_start = val[1];
1681	sc->tids.ftid_base = val[2];
1682	sc->tids.nftids = val[3] - val[2] + 1;
1683	sc->vres.l2t.start = val[4];
1684	sc->vres.l2t.size = val[5] - val[4] + 1;
1685
1686	/* get capabilites */
1687	bzero(&caps, sizeof (caps));
1688	caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1689	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1690	caps.cfvalid_to_len16 = htonl(FW_LEN16(caps));
1691	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps);
1692	if (rc != 0) {
1693		cxgb_printf(sc->dip, CE_WARN,
1694		    "failed to get card capabilities: %d.\n", rc);
1695		return (rc);
1696	}
1697
1698	if (caps.toecaps != 0) {
1699		/* query offload-related parameters */
1700		param[0] = FW_PARAM_DEV(NTID);
1701		param[1] = FW_PARAM_PFVF(SERVER_START);
1702		param[2] = FW_PARAM_PFVF(SERVER_END);
1703		param[3] = FW_PARAM_PFVF(TDDP_START);
1704		param[4] = FW_PARAM_PFVF(TDDP_END);
1705		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1706		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1707		if (rc != 0) {
1708			cxgb_printf(sc->dip, CE_WARN,
1709			    "failed to query TOE parameters: %d.\n", rc);
1710			return (rc);
1711		}
1712		sc->tids.ntids = val[0];
1713		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1714		sc->tids.stid_base = val[1];
1715		sc->tids.nstids = val[2] - val[1] + 1;
1716		sc->vres.ddp.start = val[3];
1717		sc->vres.ddp.size = val[4] - val[3] + 1;
1718		sc->params.ofldq_wr_cred = val[5];
1719		sc->params.offload = 1;
1720	}
1721
1722	/* These are finalized by FW initialization, load their values now */
1723	val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1724	sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1725	sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1726	t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1727
1728	return (rc);
1729}
1730
1731static int
1732set_params__post_init(struct adapter *sc)
1733{
1734	uint32_t param, val;
1735
1736	/* ask for encapsulated CPLs */
1737	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
1738	val = 1;
1739	(void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1740
1741	return (0);
1742}
1743
1744/* TODO: verify */
1745static void
1746setup_memwin(struct adapter *sc)
1747{
1748	pci_regspec_t *data;
1749	int rc;
1750	uint_t n;
1751	uintptr_t bar0;
1752	uintptr_t mem_win0_base, mem_win1_base, mem_win2_base;
1753	uintptr_t mem_win2_aperture;
1754
1755	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip,
1756	    DDI_PROP_DONTPASS, "assigned-addresses", (int **)&data, &n);
1757	if (rc != DDI_SUCCESS) {
1758		cxgb_printf(sc->dip, CE_WARN,
1759		    "failed to lookup \"assigned-addresses\" property: %d", rc);
1760		return;
1761	}
1762	n /= sizeof (*data);
1763
1764	bar0 = ((uint64_t)data[0].pci_phys_mid << 32) | data[0].pci_phys_low;
1765	ddi_prop_free(data);
1766
1767	if (is_t4(sc->params.chip)) {
1768		mem_win0_base = bar0 + MEMWIN0_BASE;
1769		mem_win1_base = bar0 + MEMWIN1_BASE;
1770		mem_win2_base = bar0 + MEMWIN2_BASE;
1771		mem_win2_aperture = MEMWIN2_APERTURE;
1772	} else {
1773		/* For T5, only relative offset inside the PCIe BAR is passed */
1774		mem_win0_base = MEMWIN0_BASE;
1775		mem_win1_base = MEMWIN1_BASE;
1776		mem_win2_base = MEMWIN2_BASE_T5;
1777		mem_win2_aperture = MEMWIN2_APERTURE_T5;
1778	}
1779
1780	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1781	    mem_win0_base | V_BIR(0) |
1782	    V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1783
1784	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1785	    mem_win1_base | V_BIR(0) |
1786	    V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1787
1788	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1789	    mem_win2_base | V_BIR(0) |
1790	    V_WINDOW(ilog2(mem_win2_aperture) - 10));
1791
1792	/* flush */
1793	(void)t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1794}
1795
1796/*
1797 * Positions the memory window such that it can be used to access the specified
1798 * address in the chip's address space.  The return value is the offset of addr
1799 * from the start of the window.
1800 */
1801uint32_t
1802position_memwin(struct adapter *sc, int n, uint32_t addr)
1803{
1804	uint32_t start, pf;
1805	uint32_t reg;
1806
1807	if (addr & 3) {
1808		cxgb_printf(sc->dip, CE_WARN,
1809		    "addr (0x%x) is not at a 4B boundary.\n", addr);
1810		return (EFAULT);
1811	}
1812
1813	if (is_t4(sc->params.chip)) {
1814		pf = 0;
1815		start = addr & ~0xf;    /* start must be 16B aligned */
1816	} else {
1817		pf = V_PFNUM(sc->pf);
1818		start = addr & ~0x7f;   /* start must be 128B aligned */
1819	}
1820	reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n);
1821
1822	t4_write_reg(sc, reg, start | pf);
1823	(void) t4_read_reg(sc, reg);
1824
1825	return (addr - start);
1826}
1827
1828
1829/*
1830 * Reads the named property and fills up the "data" array (which has at least
1831 * "count" elements).  We first try and lookup the property for our dev_t and
1832 * then retry with DDI_DEV_T_ANY if it's not found.
1833 *
1834 * Returns non-zero if the property was found and "data" has been updated.
1835 */
1836static int
1837prop_lookup_int_array(struct adapter *sc, char *name, int *data, uint_t count)
1838{
1839	dev_info_t *dip = sc->dip;
1840	dev_t dev = sc->dev;
1841	int rc, *d;
1842	uint_t i, n;
1843
1844	rc = ddi_prop_lookup_int_array(dev, dip, DDI_PROP_DONTPASS,
1845	    name, &d, &n);
1846	if (rc == DDI_PROP_SUCCESS)
1847		goto found;
1848
1849	if (rc != DDI_PROP_NOT_FOUND) {
1850		cxgb_printf(dip, CE_WARN,
1851		    "failed to lookup property %s for minor %d: %d.",
1852		    name, getminor(dev), rc);
1853		return (0);
1854	}
1855
1856	rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1857	    name, &d, &n);
1858	if (rc == DDI_PROP_SUCCESS)
1859		goto found;
1860
1861	if (rc != DDI_PROP_NOT_FOUND) {
1862		cxgb_printf(dip, CE_WARN,
1863		    "failed to lookup property %s: %d.", name, rc);
1864		return (0);
1865	}
1866
1867	return (0);
1868
1869found:
1870	if (n > count) {
1871		cxgb_printf(dip, CE_NOTE,
1872		    "property %s has too many elements (%d), ignoring extras",
1873		    name, n);
1874	}
1875
1876	for (i = 0; i < n && i < count; i++)
1877		data[i] = d[i];
1878	ddi_prop_free(d);
1879
1880	return (1);
1881}
1882
1883static int
1884prop_lookup_int(struct adapter *sc, char *name, int defval)
1885{
1886	int rc;
1887
1888	rc = ddi_prop_get_int(sc->dev, sc->dip, DDI_PROP_DONTPASS, name, -1);
1889	if (rc != -1)
1890		return (rc);
1891
1892	return (ddi_prop_get_int(DDI_DEV_T_ANY, sc->dip, DDI_PROP_DONTPASS,
1893	    name, defval));
1894}
1895
1896static int
1897init_driver_props(struct adapter *sc, struct driver_properties *p)
1898{
1899	dev_t dev = sc->dev;
1900	dev_info_t *dip = sc->dip;
1901	int i, *data;
1902	uint_t tmr[SGE_NTIMERS] = {5, 10, 20, 50, 100, 200};
1903	uint_t cnt[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
1904
1905	/*
1906	 * Holdoff timer
1907	 */
1908	data = &p->timer_val[0];
1909	for (i = 0; i < SGE_NTIMERS; i++)
1910		data[i] = tmr[i];
1911	(void) prop_lookup_int_array(sc, "holdoff-timer-values", data,
1912	    SGE_NTIMERS);
1913	for (i = 0; i < SGE_NTIMERS; i++) {
1914		int limit = 200U;
1915		if (data[i] > limit) {
1916			cxgb_printf(dip, CE_WARN,
1917			    "holdoff timer %d is too high (%d), lowered to %d.",
1918			    i, data[i], limit);
1919			data[i] = limit;
1920		}
1921	}
1922	(void) ddi_prop_update_int_array(dev, dip, "holdoff-timer-values",
1923	    data, SGE_NTIMERS);
1924
1925	/*
1926	 * Holdoff packet counter
1927	 */
1928	data = &p->counter_val[0];
1929	for (i = 0; i < SGE_NCOUNTERS; i++)
1930		data[i] = cnt[i];
1931	(void) prop_lookup_int_array(sc, "holdoff-pkt-counter-values", data,
1932	    SGE_NCOUNTERS);
1933	for (i = 0; i < SGE_NCOUNTERS; i++) {
1934		int limit = M_THRESHOLD_0;
1935		if (data[i] > limit) {
1936			cxgb_printf(dip, CE_WARN,
1937			    "holdoff pkt-counter %d is too high (%d), "
1938			    "lowered to %d.", i, data[i], limit);
1939			data[i] = limit;
1940		}
1941	}
1942	(void) ddi_prop_update_int_array(dev, dip, "holdoff-pkt-counter-values",
1943	    data, SGE_NCOUNTERS);
1944
1945	/*
1946	 * Maximum # of tx and rx queues to use for each
1947	 * 100G, 40G, 25G, 10G and 1G port.
1948	 */
1949	p->max_ntxq_10g = prop_lookup_int(sc, "max-ntxq-10G-port", 8);
1950	(void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1951	    p->max_ntxq_10g);
1952
1953	p->max_nrxq_10g = prop_lookup_int(sc, "max-nrxq-10G-port", 8);
1954	(void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1955	    p->max_nrxq_10g);
1956
1957	p->max_ntxq_1g = prop_lookup_int(sc, "max-ntxq-1G-port", 2);
1958	(void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1959	    p->max_ntxq_1g);
1960
1961	p->max_nrxq_1g = prop_lookup_int(sc, "max-nrxq-1G-port", 2);
1962	(void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1963	    p->max_nrxq_1g);
1964
1965#ifdef TCP_OFFLOAD_ENABLE
1966	p->max_nofldtxq_10g = prop_lookup_int(sc, "max-nofldtxq-10G-port", 8);
1967	(void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port",
1968	    p->max_nofldtxq_10g);
1969
1970	p->max_nofldrxq_10g = prop_lookup_int(sc, "max-nofldrxq-10G-port", 2);
1971	(void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port",
1972	    p->max_nofldrxq_10g);
1973
1974	p->max_nofldtxq_1g = prop_lookup_int(sc, "max-nofldtxq-1G-port", 2);
1975	(void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port",
1976	    p->max_nofldtxq_1g);
1977
1978	p->max_nofldrxq_1g = prop_lookup_int(sc, "max-nofldrxq-1G-port", 1);
1979	(void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port",
1980	    p->max_nofldrxq_1g);
1981#endif
1982
1983	/*
1984	 * Holdoff parameters for 10G and 1G ports.
1985	 */
1986	p->tmr_idx_10g = prop_lookup_int(sc, "holdoff-timer-idx-10G", 0);
1987	(void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-10G",
1988	    p->tmr_idx_10g);
1989
1990	p->pktc_idx_10g = prop_lookup_int(sc, "holdoff-pktc-idx-10G", 2);
1991	(void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-10G",
1992	    p->pktc_idx_10g);
1993
1994	p->tmr_idx_1g = prop_lookup_int(sc, "holdoff-timer-idx-1G", 0);
1995	(void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-1G",
1996	    p->tmr_idx_1g);
1997
1998	p->pktc_idx_1g = prop_lookup_int(sc, "holdoff-pktc-idx-1G", 2);
1999	(void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-1G",
2000	    p->pktc_idx_1g);
2001
2002	/*
2003	 * Size (number of entries) of each tx and rx queue.
2004	 */
2005	i = prop_lookup_int(sc, "qsize-txq", TX_EQ_QSIZE);
2006	p->qsize_txq = max(i, 128);
2007	if (p->qsize_txq != i) {
2008		cxgb_printf(dip, CE_WARN,
2009		    "using %d instead of %d as the tx queue size",
2010		    p->qsize_txq, i);
2011	}
2012	(void) ddi_prop_update_int(dev, dip, "qsize-txq", p->qsize_txq);
2013
2014	i = prop_lookup_int(sc, "qsize-rxq", RX_IQ_QSIZE);
2015	p->qsize_rxq = max(i, 128);
2016	while (p->qsize_rxq & 7)
2017		p->qsize_rxq--;
2018	if (p->qsize_rxq != i) {
2019		cxgb_printf(dip, CE_WARN,
2020		    "using %d instead of %d as the rx queue size",
2021		    p->qsize_rxq, i);
2022	}
2023	(void) ddi_prop_update_int(dev, dip, "qsize-rxq", p->qsize_rxq);
2024
2025	/*
2026	 * Interrupt types allowed.
2027	 * Bits 0, 1, 2 = INTx, MSI, MSI-X respectively.  See sys/ddi_intr.h
2028	 */
2029	p->intr_types = prop_lookup_int(sc, "interrupt-types",
2030	    DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_FIXED);
2031	(void) ddi_prop_update_int(dev, dip, "interrupt-types", p->intr_types);
2032
2033	/*
2034	 * Forwarded interrupt queues.  Create this property to force the driver
2035	 * to use forwarded interrupt queues.
2036	 */
2037	if (ddi_prop_exists(dev, dip, DDI_PROP_DONTPASS,
2038	    "interrupt-forwarding") != 0 ||
2039	    ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2040	    "interrupt-forwarding") != 0) {
2041		UNIMPLEMENTED();
2042		(void) ddi_prop_create(dev, dip, DDI_PROP_CANSLEEP,
2043		    "interrupt-forwarding", NULL, 0);
2044	}
2045
2046	/*
2047	 * Write combining
2048	 * 0 to disable, 1 to enable
2049	 */
2050	p->wc = prop_lookup_int(sc, "write-combine", 1);
2051	cxgb_printf(dip, CE_WARN, "write-combine: using of %d", p->wc);
2052	if (p->wc != 0 && p->wc != 1) {
2053		cxgb_printf(dip, CE_WARN,
2054		    "write-combine: using 1 instead of %d", p->wc);
2055		p->wc = 1;
2056	}
2057	(void) ddi_prop_update_int(dev, dip, "write-combine", p->wc);
2058
2059	p->t4_fw_install = prop_lookup_int(sc, "t4_fw_install", 1);
2060	if (p->t4_fw_install != 0 && p->t4_fw_install != 2)
2061		p->t4_fw_install = 1;
2062	(void) ddi_prop_update_int(dev, dip, "t4_fw_install", p->t4_fw_install);
2063
2064	/* Multiple Rings */
2065	p->multi_rings = prop_lookup_int(sc, "multi-rings", 1);
2066	if (p->multi_rings != 0 && p->multi_rings != 1) {
2067		cxgb_printf(dip, CE_NOTE,
2068			   "multi-rings: using value 1 instead of %d", p->multi_rings);
2069		p->multi_rings = 1;
2070	}
2071
2072	(void) ddi_prop_update_int(dev, dip, "multi-rings", p->multi_rings);
2073
2074	return (0);
2075}
2076
2077static int
2078remove_extra_props(struct adapter *sc, int n10g, int n1g)
2079{
2080	if (n10g == 0) {
2081		(void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-10G-port");
2082		(void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-10G-port");
2083		(void) ddi_prop_remove(sc->dev, sc->dip,
2084		    "holdoff-timer-idx-10G");
2085		(void) ddi_prop_remove(sc->dev, sc->dip,
2086		    "holdoff-pktc-idx-10G");
2087	}
2088
2089	if (n1g == 0) {
2090		(void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-1G-port");
2091		(void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-1G-port");
2092		(void) ddi_prop_remove(sc->dev, sc->dip,
2093		    "holdoff-timer-idx-1G");
2094		(void) ddi_prop_remove(sc->dev, sc->dip, "holdoff-pktc-idx-1G");
2095	}
2096
2097	return (0);
2098}
2099
2100static int
2101cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
2102    struct intrs_and_queues *iaq)
2103{
2104	struct driver_properties *p = &sc->props;
2105	int rc, itype, itypes, navail, nc, nrxq10g, nrxq1g, n;
2106	int nofldrxq10g = 0, nofldrxq1g = 0;
2107
2108	bzero(iaq, sizeof (*iaq));
2109	nc = ncpus;	/* our snapshot of the number of CPUs */
2110	iaq->ntxq10g = min(nc, p->max_ntxq_10g);
2111	iaq->ntxq1g = min(nc, p->max_ntxq_1g);
2112	iaq->nrxq10g = nrxq10g = min(nc, p->max_nrxq_10g);
2113	iaq->nrxq1g = nrxq1g = min(nc, p->max_nrxq_1g);
2114#ifdef TCP_OFFLOAD_ENABLE
2115	iaq->nofldtxq10g = min(nc, p->max_nofldtxq_10g);
2116	iaq->nofldtxq1g = min(nc, p->max_nofldtxq_1g);
2117	iaq->nofldrxq10g = nofldrxq10g = min(nc, p->max_nofldrxq_10g);
2118	iaq->nofldrxq1g = nofldrxq1g = min(nc, p->max_nofldrxq_1g);
2119#endif
2120
2121	rc = ddi_intr_get_supported_types(sc->dip, &itypes);
2122	if (rc != DDI_SUCCESS) {
2123		cxgb_printf(sc->dip, CE_WARN,
2124		    "failed to determine supported interrupt types: %d", rc);
2125		return (rc);
2126	}
2127
2128	for (itype = DDI_INTR_TYPE_MSIX; itype; itype >>= 1) {
2129		ASSERT(itype == DDI_INTR_TYPE_MSIX ||
2130		    itype == DDI_INTR_TYPE_MSI ||
2131		    itype == DDI_INTR_TYPE_FIXED);
2132
2133		if ((itype & itypes & p->intr_types) == 0)
2134			continue;	/* not supported or not allowed */
2135
2136		navail = 0;
2137		rc = ddi_intr_get_navail(sc->dip, itype, &navail);
2138		if (rc != DDI_SUCCESS || navail == 0) {
2139			cxgb_printf(sc->dip, CE_WARN,
2140			    "failed to get # of interrupts for type %d: %d",
2141			    itype, rc);
2142			continue;	/* carry on */
2143		}
2144
2145		iaq->intr_type = itype;
2146		if (navail == 0)
2147			continue;
2148
2149		/*
2150		 * Best option: an interrupt vector for errors, one for the
2151		 * firmware event queue, and one each for each rxq (NIC as well
2152		 * as offload).
2153		 */
2154		iaq->nirq = T4_EXTRA_INTR;
2155		iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
2156		iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
2157
2158		if (iaq->nirq <= navail &&
2159		    (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2160			iaq->intr_fwd = 0;
2161			goto allocate;
2162		}
2163
2164		/*
2165		 * Second best option: an interrupt vector for errors, one for
2166		 * the firmware event queue, and one each for either NIC or
2167		 * offload rxq's.
2168		 */
2169		iaq->nirq = T4_EXTRA_INTR;
2170		iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
2171		iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
2172		if (iaq->nirq <= navail &&
2173		    (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) {
2174			iaq->intr_fwd = 1;
2175			goto allocate;
2176		}
2177
2178		/*
2179		 * Next best option: an interrupt vector for errors, one for the
2180		 * firmware event queue, and at least one per port.  At this
2181		 * point we know we'll have to downsize nrxq or nofldrxq to fit
2182		 * what's available to us.
2183		 */
2184		iaq->nirq = T4_EXTRA_INTR;
2185		iaq->nirq += n10g + n1g;
2186		if (iaq->nirq <= navail) {
2187			int leftover = navail - iaq->nirq;
2188
2189			if (n10g > 0) {
2190				int target = max(nrxq10g, nofldrxq10g);
2191
2192				n = 1;
2193				while (n < target && leftover >= n10g) {
2194					leftover -= n10g;
2195					iaq->nirq += n10g;
2196					n++;
2197				}
2198				iaq->nrxq10g = min(n, nrxq10g);
2199#ifdef TCP_OFFLOAD_ENABLE
2200				iaq->nofldrxq10g = min(n, nofldrxq10g);
2201#endif
2202			}
2203
2204			if (n1g > 0) {
2205				int target = max(nrxq1g, nofldrxq1g);
2206
2207				n = 1;
2208				while (n < target && leftover >= n1g) {
2209					leftover -= n1g;
2210					iaq->nirq += n1g;
2211					n++;
2212				}
2213				iaq->nrxq1g = min(n, nrxq1g);
2214#ifdef TCP_OFFLOAD_ENABLE
2215				iaq->nofldrxq1g = min(n, nofldrxq1g);
2216#endif
2217			}
2218
2219			/* We have arrived at a minimum value required to enable
2220			 * per queue irq(either NIC or offload). Thus for non-
2221			 * offload case, we will get a vector per queue, while
2222			 * offload case, we will get a vector per offload/NIC q.
2223			 * Hence enable Interrupt forwarding only for offload
2224			 * case.
2225			 */
2226#ifdef TCP_OFFLOAD_ENABLE
2227			if (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq)) {
2228				iaq->intr_fwd = 1;
2229#else
2230			if (itype != DDI_INTR_TYPE_MSI) {
2231#endif
2232				goto allocate;
2233			}
2234		}
2235
2236		/*
2237		 * Least desirable option: one interrupt vector for everything.
2238		 */
2239		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
2240#ifdef TCP_OFFLOAD_ENABLE
2241		iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
2242#endif
2243		iaq->intr_fwd = 1;
2244
2245allocate:
2246		return (0);
2247	}
2248
2249	cxgb_printf(sc->dip, CE_WARN,
2250	    "failed to find a usable interrupt type.  supported=%d, allowed=%d",
2251	    itypes, p->intr_types);
2252	return (DDI_FAILURE);
2253}
2254
2255static int
2256add_child_node(struct adapter *sc, int idx)
2257{
2258	int rc;
2259	struct port_info *pi;
2260
2261	if (idx < 0 || idx >= sc->params.nports)
2262		return (EINVAL);
2263
2264	pi = sc->port[idx];
2265	if (pi == NULL)
2266		return (ENODEV);	/* t4_port_init failed earlier */
2267
2268	PORT_LOCK(pi);
2269	if (pi->dip != NULL) {
2270		rc = 0;		/* EEXIST really, but then bus_config fails */
2271		goto done;
2272	}
2273
2274	rc = ndi_devi_alloc(sc->dip, T4_PORT_NAME, DEVI_SID_NODEID, &pi->dip);
2275	if (rc != DDI_SUCCESS || pi->dip == NULL) {
2276		rc = ENOMEM;
2277		goto done;
2278	}
2279
2280	(void) ddi_set_parent_data(pi->dip, pi);
2281	(void) ndi_devi_bind_driver(pi->dip, 0);
2282	rc = 0;
2283done:
2284	PORT_UNLOCK(pi);
2285	return (rc);
2286}
2287
2288static int
2289remove_child_node(struct adapter *sc, int idx)
2290{
2291	int rc;
2292	struct port_info *pi;
2293
2294	if (idx < 0 || idx >= sc->params.nports)
2295		return (EINVAL);
2296
2297	pi = sc->port[idx];
2298	if (pi == NULL)
2299		return (ENODEV);
2300
2301	PORT_LOCK(pi);
2302	if (pi->dip == NULL) {
2303		rc = ENODEV;
2304		goto done;
2305	}
2306
2307	rc = ndi_devi_free(pi->dip);
2308	if (rc == 0)
2309		pi->dip = NULL;
2310done:
2311	PORT_UNLOCK(pi);
2312	return (rc);
2313}
2314
2315#define	KS_UINIT(x)	kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
2316#define	KS_CINIT(x)	kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
2317#define	KS_U_SET(x, y)	kstatp->x.value.ul = (y)
2318#define	KS_C_SET(x, ...)	\
2319			(void) snprintf(kstatp->x.value.c, 16,  __VA_ARGS__)
2320
2321/*
2322 * t4nex:X:config
2323 */
2324struct t4_kstats {
2325	kstat_named_t chip_ver;
2326	kstat_named_t fw_vers;
2327	kstat_named_t tp_vers;
2328	kstat_named_t driver_version;
2329	kstat_named_t serial_number;
2330	kstat_named_t ec_level;
2331	kstat_named_t id;
2332	kstat_named_t bus_type;
2333	kstat_named_t bus_width;
2334	kstat_named_t bus_speed;
2335	kstat_named_t core_clock;
2336	kstat_named_t port_cnt;
2337	kstat_named_t port_type;
2338	kstat_named_t pci_vendor_id;
2339	kstat_named_t pci_device_id;
2340};
2341static kstat_t *
2342setup_kstats(struct adapter *sc)
2343{
2344	kstat_t *ksp;
2345	struct t4_kstats *kstatp;
2346	int ndata;
2347	struct pci_params *p = &sc->params.pci;
2348	struct vpd_params *v = &sc->params.vpd;
2349	uint16_t pci_vendor, pci_device;
2350
2351	ndata = sizeof (struct t4_kstats) / sizeof (kstat_named_t);
2352
2353	ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "config",
2354	    "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2355	if (ksp == NULL) {
2356		cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2357		return (NULL);
2358	}
2359
2360	kstatp = (struct t4_kstats *)ksp->ks_data;
2361
2362	KS_UINIT(chip_ver);
2363	KS_CINIT(fw_vers);
2364	KS_CINIT(tp_vers);
2365	KS_CINIT(driver_version);
2366	KS_CINIT(serial_number);
2367	KS_CINIT(ec_level);
2368	KS_CINIT(id);
2369	KS_CINIT(bus_type);
2370	KS_CINIT(bus_width);
2371	KS_CINIT(bus_speed);
2372	KS_UINIT(core_clock);
2373	KS_UINIT(port_cnt);
2374	KS_CINIT(port_type);
2375	KS_CINIT(pci_vendor_id);
2376	KS_CINIT(pci_device_id);
2377
2378	KS_U_SET(chip_ver, sc->params.chip);
2379	KS_C_SET(fw_vers, "%d.%d.%d.%d",
2380	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
2381	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
2382	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
2383	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
2384	KS_C_SET(tp_vers, "%d.%d.%d.%d",
2385	    G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
2386	    G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
2387	    G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
2388	    G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
2389	KS_C_SET(driver_version, DRV_VERSION);
2390	KS_C_SET(serial_number, "%s", v->sn);
2391	KS_C_SET(ec_level, "%s", v->ec);
2392	KS_C_SET(id, "%s", v->id);
2393	KS_C_SET(bus_type, "pci-express");
2394	KS_C_SET(bus_width, "x%d lanes", p->width);
2395	KS_C_SET(bus_speed, "%d", p->speed);
2396	KS_U_SET(core_clock, v->cclk);
2397	KS_U_SET(port_cnt, sc->params.nports);
2398
2399	t4_os_pci_read_cfg2(sc, PCI_CONF_VENID, &pci_vendor);
2400	KS_C_SET(pci_vendor_id, "0x%x", pci_vendor);
2401
2402	t4_os_pci_read_cfg2(sc, PCI_CONF_DEVID, &pci_device);
2403	KS_C_SET(pci_device_id, "0x%x", pci_device);
2404
2405	KS_C_SET(port_type, "%s/%s/%s/%s",
2406		 print_port_speed(sc->port[0]),
2407		 print_port_speed(sc->port[1]),
2408		 print_port_speed(sc->port[2]),
2409		 print_port_speed(sc->port[3]));
2410
2411	/* Do NOT set ksp->ks_update.  These kstats do not change. */
2412
2413	/* Install the kstat */
2414	ksp->ks_private = (void *)sc;
2415	kstat_install(ksp);
2416
2417	return (ksp);
2418}
2419
2420/*
2421 * t4nex:X:stat
2422 */
2423struct t4_wc_kstats {
2424	kstat_named_t write_coal_success;
2425	kstat_named_t write_coal_failure;
2426};
2427static kstat_t *
2428setup_wc_kstats(struct adapter *sc)
2429{
2430	kstat_t *ksp;
2431	struct t4_wc_kstats *kstatp;
2432	int ndata;
2433
2434	ndata = sizeof(struct t4_wc_kstats) / sizeof(kstat_named_t);
2435	ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "stats",
2436	    "nexus", KSTAT_TYPE_NAMED, ndata, 0);
2437	if (ksp == NULL) {
2438		cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats.");
2439		return (NULL);
2440	}
2441
2442	kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2443
2444	KS_UINIT(write_coal_success);
2445	KS_UINIT(write_coal_failure);
2446
2447	ksp->ks_update = update_wc_kstats;
2448	/* Install the kstat */
2449	ksp->ks_private = (void *)sc;
2450	kstat_install(ksp);
2451
2452	return (ksp);
2453}
2454
2455static int
2456update_wc_kstats(kstat_t *ksp, int rw)
2457{
2458	struct t4_wc_kstats *kstatp = (struct t4_wc_kstats *)ksp->ks_data;
2459	struct adapter *sc = ksp->ks_private;
2460	uint32_t wc_total, wc_success, wc_failure;
2461
2462	if (rw == KSTAT_WRITE)
2463		return (0);
2464
2465	if (is_t5(sc->params.chip)) {
2466		wc_total = t4_read_reg(sc, A_SGE_STAT_TOTAL);
2467		wc_failure = t4_read_reg(sc, A_SGE_STAT_MATCH);
2468		wc_success = wc_total - wc_failure;
2469	} else {
2470		wc_success = 0;
2471		wc_failure = 0;
2472	}
2473
2474	KS_U_SET(write_coal_success, wc_success);
2475	KS_U_SET(write_coal_failure, wc_failure);
2476
2477	return (0);
2478}
2479
2480int
2481adapter_full_init(struct adapter *sc)
2482{
2483	int i, rc = 0;
2484
2485	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2486
2487	rc = t4_setup_adapter_queues(sc);
2488	if (rc != 0)
2489		goto done;
2490
2491	if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2492		(void) ddi_intr_block_enable(sc->intr_handle, sc->intr_count);
2493	else {
2494		for (i = 0; i < sc->intr_count; i++)
2495			(void) ddi_intr_enable(sc->intr_handle[i]);
2496	}
2497	t4_intr_enable(sc);
2498	sc->flags |= FULL_INIT_DONE;
2499
2500#ifdef TCP_OFFLOAD_ENABLE
2501	/* TODO: wrong place to enable TOE capability */
2502	if (is_offload(sc) != 0) {
2503		for_each_port(sc, i) {
2504			struct port_info *pi = sc->port[i];
2505			rc = toe_capability(pi, 1);
2506			if (rc != 0) {
2507				cxgb_printf(pi->dip, CE_WARN,
2508				    "Failed to activate toe capability: %d",
2509				    rc);
2510				rc = 0;		/* not a fatal error */
2511			}
2512		}
2513	}
2514#endif
2515
2516done:
2517	if (rc != 0)
2518		(void) adapter_full_uninit(sc);
2519
2520	return (rc);
2521}
2522
2523int
2524adapter_full_uninit(struct adapter *sc)
2525{
2526	int i, rc = 0;
2527
2528	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2529
2530	if (sc->intr_cap & DDI_INTR_FLAG_BLOCK)
2531		(void) ddi_intr_block_disable(sc->intr_handle, sc->intr_count);
2532	else {
2533		for (i = 0; i < sc->intr_count; i++)
2534			(void) ddi_intr_disable(sc->intr_handle[i]);
2535	}
2536
2537	rc = t4_teardown_adapter_queues(sc);
2538	if (rc != 0)
2539		return (rc);
2540
2541	sc->flags &= ~FULL_INIT_DONE;
2542
2543	return (0);
2544}
2545
2546int
2547port_full_init(struct port_info *pi)
2548{
2549	struct adapter *sc = pi->adapter;
2550	uint16_t *rss;
2551	struct sge_rxq *rxq;
2552	int rc, i;
2553
2554	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2555	ASSERT((pi->flags & PORT_INIT_DONE) == 0);
2556
2557	/*
2558	 * Allocate tx/rx/fl queues for this port.
2559	 */
2560	rc = t4_setup_port_queues(pi);
2561	if (rc != 0)
2562		goto done;	/* error message displayed already */
2563
2564	/*
2565	 * Setup RSS for this port.
2566	 */
2567	rss = kmem_zalloc(pi->nrxq * sizeof (*rss), KM_SLEEP);
2568	for_each_rxq(pi, i, rxq) {
2569		rss[i] = rxq->iq.abs_id;
2570	}
2571	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2572	    pi->rss_size, rss, pi->nrxq);
2573	kmem_free(rss, pi->nrxq * sizeof (*rss));
2574	if (rc != 0) {
2575		cxgb_printf(pi->dip, CE_WARN, "rss_config failed: %d", rc);
2576		goto done;
2577	}
2578
2579	pi->flags |= PORT_INIT_DONE;
2580done:
2581	if (rc != 0)
2582		(void) port_full_uninit(pi);
2583
2584	return (rc);
2585}
2586
2587/*
2588 * Idempotent.
2589 */
2590int
2591port_full_uninit(struct port_info *pi)
2592{
2593
2594	ASSERT(pi->flags & PORT_INIT_DONE);
2595
2596	(void) t4_teardown_port_queues(pi);
2597	pi->flags &= ~PORT_INIT_DONE;
2598
2599	return (0);
2600}
2601
2602void
2603enable_port_queues(struct port_info *pi)
2604{
2605	struct adapter *sc = pi->adapter;
2606	int i;
2607	struct sge_iq *iq;
2608	struct sge_rxq *rxq;
2609#ifdef TCP_OFFLOAD_ENABLE
2610	struct sge_ofld_rxq *ofld_rxq;
2611#endif
2612
2613	ASSERT(pi->flags & PORT_INIT_DONE);
2614
2615	/*
2616	 * TODO: whatever was queued up after we set iq->state to IQS_DISABLED
2617	 * back in disable_port_queues will be processed now, after an unbounded
2618	 * delay.  This can't be good.
2619	 */
2620
2621#ifdef TCP_OFFLOAD_ENABLE
2622	for_each_ofld_rxq(pi, i, ofld_rxq) {
2623		iq = &ofld_rxq->iq;
2624		if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2625		    IQS_DISABLED)
2626			panic("%s: iq %p wasn't disabled", __func__,
2627			    (void *)iq);
2628		t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2629		    V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2630	}
2631#endif
2632
2633	for_each_rxq(pi, i, rxq) {
2634		iq = &rxq->iq;
2635		if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) !=
2636		    IQS_DISABLED)
2637			panic("%s: iq %p wasn't disabled", __func__,
2638			    (void *) iq);
2639		t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
2640		    V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id));
2641	}
2642}
2643
2644void
2645disable_port_queues(struct port_info *pi)
2646{
2647	int i;
2648	struct adapter *sc = pi->adapter;
2649	struct sge_rxq *rxq;
2650#ifdef TCP_OFFLOAD_ENABLE
2651	struct sge_ofld_rxq *ofld_rxq;
2652#endif
2653
2654	ASSERT(pi->flags & PORT_INIT_DONE);
2655
2656	/*
2657	 * TODO: need proper implementation for all tx queues (ctrl, eth, ofld).
2658	 */
2659
2660#ifdef TCP_OFFLOAD_ENABLE
2661	for_each_ofld_rxq(pi, i, ofld_rxq) {
2662		while (atomic_cas_uint(&ofld_rxq->iq.state, IQS_IDLE,
2663		    IQS_DISABLED) != IQS_IDLE)
2664			msleep(1);
2665	}
2666#endif
2667
2668	for_each_rxq(pi, i, rxq) {
2669		while (atomic_cas_uint(&rxq->iq.state, IQS_IDLE,
2670		    IQS_DISABLED) != IQS_IDLE)
2671			msleep(1);
2672	}
2673
2674	mutex_enter(&sc->sfl_lock);
2675#ifdef TCP_OFFLOAD_ENABLE
2676	for_each_ofld_rxq(pi, i, ofld_rxq)
2677	    ofld_rxq->fl.flags |= FL_DOOMED;
2678#endif
2679	for_each_rxq(pi, i, rxq)
2680	    rxq->fl.flags |= FL_DOOMED;
2681	mutex_exit(&sc->sfl_lock);
2682	/* TODO: need to wait for all fl's to be removed from sc->sfl */
2683}
2684
2685void
2686t4_fatal_err(struct adapter *sc)
2687{
2688	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
2689	t4_intr_disable(sc);
2690	cxgb_printf(sc->dip, CE_WARN,
2691	    "encountered fatal error, adapter stopped.");
2692}
2693
2694int
2695t4_os_find_pci_capability(struct adapter *sc, int cap)
2696{
2697	uint16_t stat;
2698	uint8_t cap_ptr, cap_id;
2699
2700	t4_os_pci_read_cfg2(sc, PCI_CONF_STAT, &stat);
2701	if ((stat & PCI_STAT_CAP) == 0)
2702		return (0); /* does not implement capabilities */
2703
2704	t4_os_pci_read_cfg1(sc, PCI_CONF_CAP_PTR, &cap_ptr);
2705	while (cap_ptr) {
2706		t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_ID, &cap_id);
2707		if (cap_id == cap)
2708			return (cap_ptr); /* found */
2709		t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_NEXT_PTR, &cap_ptr);
2710	}
2711
2712	return (0); /* not found */
2713}
2714
2715void
2716t4_os_portmod_changed(const struct adapter *sc, int idx)
2717{
2718	static const char *mod_str[] = {
2719		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
2720	};
2721	const struct port_info *pi = sc->port[idx];
2722
2723	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
2724		cxgb_printf(pi->dip, CE_NOTE, "transceiver unplugged.");
2725	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
2726		cxgb_printf(pi->dip, CE_NOTE,
2727		    "unknown transceiver inserted.\n");
2728	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
2729		cxgb_printf(pi->dip, CE_NOTE,
2730		    "unsupported transceiver inserted.\n");
2731	else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
2732		cxgb_printf(pi->dip, CE_NOTE, "%s transceiver inserted.\n",
2733		    mod_str[pi->mod_type]);
2734	else
2735		cxgb_printf(pi->dip, CE_NOTE, "transceiver (type %d) inserted.",
2736		    pi->mod_type);
2737}
2738
2739/* ARGSUSED */
2740static int
2741cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
2742{
2743	if (m != NULL)
2744		freemsg(m);
2745	return (0);
2746}
2747
2748int
2749t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
2750{
2751	uint_t *loc, new;
2752
2753	if (opcode >= ARRAY_SIZE(sc->cpl_handler))
2754		return (EINVAL);
2755
2756	new = (uint_t)(unsigned long) (h ? h : cpl_not_handled);
2757	loc = (uint_t *)&sc->cpl_handler[opcode];
2758	(void) atomic_swap_uint(loc, new);
2759
2760	return (0);
2761}
2762
2763static int
2764fw_msg_not_handled(struct adapter *sc, const __be64 *data)
2765{
2766	struct cpl_fw6_msg *cpl;
2767
2768	cpl = __containerof((void *)data, struct cpl_fw6_msg, data);
2769
2770	cxgb_printf(sc->dip, CE_WARN, "%s fw_msg type %d", __func__, cpl->type);
2771	return (0);
2772}
2773
2774int
2775t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
2776{
2777	fw_msg_handler_t *loc, new;
2778
2779	if (type >= ARRAY_SIZE(sc->fw_msg_handler))
2780		return (EINVAL);
2781
2782	/*
2783	 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL
2784	 * handler dispatch table.  Reject any attempt to install a handler for
2785	 * this subtype.
2786	 */
2787	if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL)
2788		return (EINVAL);
2789
2790	new = h ? h : fw_msg_not_handled;
2791	loc = &sc->fw_msg_handler[type];
2792	(void)atomic_swap_ptr(loc, (void *)new);
2793
2794	return (0);
2795}
2796
2797#ifdef TCP_OFFLOAD_ENABLE
2798static int
2799toe_capability(struct port_info *pi, int enable)
2800{
2801	int rc;
2802	struct adapter *sc = pi->adapter;
2803
2804	if (!is_offload(sc))
2805		return (ENODEV);
2806
2807	if (enable != 0) {
2808		if (isset(&sc->offload_map, pi->port_id) != 0)
2809			return (0);
2810
2811		if (sc->offload_map == 0) {
2812			rc = activate_uld(sc, ULD_TOM, &sc->tom);
2813			if (rc != 0)
2814				return (rc);
2815		}
2816
2817		setbit(&sc->offload_map, pi->port_id);
2818	} else {
2819		if (!isset(&sc->offload_map, pi->port_id))
2820			return (0);
2821
2822		clrbit(&sc->offload_map, pi->port_id);
2823
2824		if (sc->offload_map == 0) {
2825			rc = deactivate_uld(&sc->tom);
2826			if (rc != 0) {
2827				setbit(&sc->offload_map, pi->port_id);
2828				return (rc);
2829			}
2830		}
2831	}
2832
2833	return (0);
2834}
2835
2836/*
2837 * Add an upper layer driver to the global list.
2838 */
2839int
2840t4_register_uld(struct uld_info *ui)
2841{
2842	int rc = 0;
2843	struct uld_info *u;
2844
2845	mutex_enter(&t4_uld_list_lock);
2846	SLIST_FOREACH(u, &t4_uld_list, link) {
2847		if (u->uld_id == ui->uld_id) {
2848			rc = EEXIST;
2849			goto done;
2850		}
2851	}
2852
2853	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
2854	ui->refcount = 0;
2855done:
2856	mutex_exit(&t4_uld_list_lock);
2857	return (rc);
2858}
2859
2860int
2861t4_unregister_uld(struct uld_info *ui)
2862{
2863	int rc = EINVAL;
2864	struct uld_info *u;
2865
2866	mutex_enter(&t4_uld_list_lock);
2867
2868	SLIST_FOREACH(u, &t4_uld_list, link) {
2869		if (u == ui) {
2870			if (ui->refcount > 0) {
2871				rc = EBUSY;
2872				goto done;
2873			}
2874
2875			SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
2876			rc = 0;
2877			goto done;
2878		}
2879	}
2880done:
2881	mutex_exit(&t4_uld_list_lock);
2882	return (rc);
2883}
2884
2885static int
2886activate_uld(struct adapter *sc, int id, struct uld_softc *usc)
2887{
2888	int rc = EAGAIN;
2889	struct uld_info *ui;
2890
2891	mutex_enter(&t4_uld_list_lock);
2892
2893	SLIST_FOREACH(ui, &t4_uld_list, link) {
2894		if (ui->uld_id == id) {
2895			rc = ui->attach(sc, &usc->softc);
2896			if (rc == 0) {
2897				ASSERT(usc->softc != NULL);
2898				ui->refcount++;
2899				usc->uld = ui;
2900			}
2901			goto done;
2902		}
2903	}
2904done:
2905	mutex_exit(&t4_uld_list_lock);
2906
2907	return (rc);
2908}
2909
2910static int
2911deactivate_uld(struct uld_softc *usc)
2912{
2913	int rc;
2914
2915	mutex_enter(&t4_uld_list_lock);
2916
2917	if (usc->uld == NULL || usc->softc == NULL) {
2918		rc = EINVAL;
2919		goto done;
2920	}
2921
2922	rc = usc->uld->detach(usc->softc);
2923	if (rc == 0) {
2924		ASSERT(usc->uld->refcount > 0);
2925		usc->uld->refcount--;
2926		usc->uld = NULL;
2927		usc->softc = NULL;
2928	}
2929done:
2930	mutex_exit(&t4_uld_list_lock);
2931
2932	return (rc);
2933}
2934
2935void
2936t4_iterate(void (*func)(int, void *), void *arg)
2937{
2938	struct adapter *sc;
2939
2940	mutex_enter(&t4_adapter_list_lock);
2941	SLIST_FOREACH(sc, &t4_adapter_list, link) {
2942		/*
2943		 * func should not make any assumptions about what state sc is
2944		 * in - the only guarantee is that sc->sc_lock is a valid lock.
2945		 */
2946		func(ddi_get_instance(sc->dip), arg);
2947	}
2948	mutex_exit(&t4_adapter_list_lock);
2949}
2950
2951#endif
2952
2953static int
2954t4_sensor_read(struct adapter *sc, uint32_t diag, uint32_t *valp)
2955{
2956	int rc;
2957	struct port_info *pi = sc->port[0];
2958	uint32_t param, val;
2959
2960	rc = begin_synchronized_op(pi, 1, 1);
2961	if (rc != 0) {
2962		return (rc);
2963	}
2964	param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
2965	    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
2966	    V_FW_PARAMS_PARAM_Y(diag);
2967	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
2968	end_synchronized_op(pi, 1);
2969
2970	if (rc != 0) {
2971		return (rc);
2972	}
2973
2974	if (val == 0) {
2975		return (EIO);
2976	}
2977
2978	*valp = val;
2979	return (0);
2980}
2981
2982static int
2983t4_temperature_read(void *arg, sensor_ioctl_scalar_t *scalar)
2984{
2985	int ret;
2986	struct adapter *sc = arg;
2987	uint32_t val;
2988
2989	ret = t4_sensor_read(sc, FW_PARAM_DEV_DIAG_TMP, &val);
2990	if (ret != 0) {
2991		return (ret);
2992	}
2993
2994	/*
2995	 * The device measures temperature in units of 1 degree Celsius. We
2996	 * don't know its precision.
2997	 */
2998	scalar->sis_unit = SENSOR_UNIT_CELSIUS;
2999	scalar->sis_gran = 1;
3000	scalar->sis_prec = 0;
3001	scalar->sis_value = val;
3002
3003	return (0);
3004}
3005
3006static int
3007t4_voltage_read(void *arg, sensor_ioctl_scalar_t *scalar)
3008{
3009	int ret;
3010	struct adapter *sc = arg;
3011	uint32_t val;
3012
3013	ret = t4_sensor_read(sc, FW_PARAM_DEV_DIAG_VDD, &val);
3014	if (ret != 0) {
3015		return (ret);
3016	}
3017
3018	scalar->sis_unit = SENSOR_UNIT_VOLTS;
3019	scalar->sis_gran = 1000;
3020	scalar->sis_prec = 0;
3021	scalar->sis_value = val;
3022
3023	return (0);
3024}
3025