1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2012, 2016 Chelsio Communications, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include "opt_inet.h"
33
34#include <sys/param.h>
35#include <sys/eventhandler.h>
36
37#include "common.h"
38#include "t4_regs.h"
39#include "t4_regs_values.h"
40#include "firmware/t4fw_interface.h"
41
42#undef msleep
43#define msleep(x) do { \
44	if (cold) \
45		DELAY((x) * 1000); \
46	else \
47		pause("t4hw", (x) * hz / 1000); \
48} while (0)
49
50/**
51 *	t4_wait_op_done_val - wait until an operation is completed
52 *	@adapter: the adapter performing the operation
53 *	@reg: the register to check for completion
54 *	@mask: a single-bit field within @reg that indicates completion
55 *	@polarity: the value of the field when the operation is completed
56 *	@attempts: number of check iterations
57 *	@delay: delay in usecs between iterations
58 *	@valp: where to store the value of the register at completion time
59 *
60 *	Wait until an operation is completed by checking a bit in a register
61 *	up to @attempts times.  If @valp is not NULL the value of the register
62 *	at the time it indicated completion is stored there.  Returns 0 if the
63 *	operation completes and	-EAGAIN	otherwise.
64 */
65static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask,
66			       int polarity, int attempts, int delay, u32 *valp)
67{
68	while (1) {
69		u32 val = t4_read_reg(adapter, reg);
70
71		if (!!(val & mask) == polarity) {
72			if (valp)
73				*valp = val;
74			return 0;
75		}
76		if (--attempts == 0)
77			return -EAGAIN;
78		if (delay)
79			udelay(delay);
80	}
81}
82
83static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
84				  int polarity, int attempts, int delay)
85{
86	return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts,
87				   delay, NULL);
88}
89
90/**
91 *	t4_set_reg_field - set a register field to a value
92 *	@adapter: the adapter to program
93 *	@addr: the register address
94 *	@mask: specifies the portion of the register to modify
95 *	@val: the new value for the register field
96 *
97 *	Sets a register field specified by the supplied mask to the
98 *	given value.
99 */
100void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask,
101		      u32 val)
102{
103	u32 v = t4_read_reg(adapter, addr) & ~mask;
104
105	t4_write_reg(adapter, addr, v | val);
106	(void) t4_read_reg(adapter, addr);      /* flush */
107}
108
109/**
110 *	t4_read_indirect - read indirectly addressed registers
111 *	@adap: the adapter
112 *	@addr_reg: register holding the indirect address
113 *	@data_reg: register holding the value of the indirect register
114 *	@vals: where the read register values are stored
115 *	@nregs: how many indirect registers to read
116 *	@start_idx: index of first indirect register to read
117 *
118 *	Reads registers that are accessed indirectly through an address/data
119 *	register pair.
120 */
121void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
122			     unsigned int data_reg, u32 *vals,
123			     unsigned int nregs, unsigned int start_idx)
124{
125	while (nregs--) {
126		t4_write_reg(adap, addr_reg, start_idx);
127		*vals++ = t4_read_reg(adap, data_reg);
128		start_idx++;
129	}
130}
131
132/**
133 *	t4_write_indirect - write indirectly addressed registers
134 *	@adap: the adapter
135 *	@addr_reg: register holding the indirect addresses
136 *	@data_reg: register holding the value for the indirect registers
137 *	@vals: values to write
138 *	@nregs: how many indirect registers to write
139 *	@start_idx: address of first indirect register to write
140 *
141 *	Writes a sequential block of registers that are accessed indirectly
142 *	through an address/data register pair.
143 */
144void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
145		       unsigned int data_reg, const u32 *vals,
146		       unsigned int nregs, unsigned int start_idx)
147{
148	while (nregs--) {
149		t4_write_reg(adap, addr_reg, start_idx++);
150		t4_write_reg(adap, data_reg, *vals++);
151	}
152}
153
154/*
155 * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor
156 * mechanism.  This guarantees that we get the real value even if we're
157 * operating within a Virtual Machine and the Hypervisor is trapping our
158 * Configuration Space accesses.
159 *
160 * N.B. This routine should only be used as a last resort: the firmware uses
161 *      the backdoor registers on a regular basis and we can end up
162 *      conflicting with it's uses!
163 */
164u32 t4_hw_pci_read_cfg4(adapter_t *adap, int reg)
165{
166	u32 req = V_FUNCTION(adap->pf) | V_REGISTER(reg);
167	u32 val;
168
169	if (chip_id(adap) <= CHELSIO_T5)
170		req |= F_ENABLE;
171	else
172		req |= F_T6_ENABLE;
173
174	if (is_t4(adap))
175		req |= F_LOCALCFG;
176
177	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, req);
178	val = t4_read_reg(adap, A_PCIE_CFG_SPACE_DATA);
179
180	/*
181	 * Reset F_ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a
182	 * Configuration Space read.  (None of the other fields matter when
183	 * F_ENABLE is 0 so a simple register write is easier than a
184	 * read-modify-write via t4_set_reg_field().)
185	 */
186	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, 0);
187
188	return val;
189}
190
191/*
192 * t4_report_fw_error - report firmware error
193 * @adap: the adapter
194 *
195 * The adapter firmware can indicate error conditions to the host.
196 * If the firmware has indicated an error, print out the reason for
197 * the firmware error.
198 */
199static void t4_report_fw_error(struct adapter *adap)
200{
201	static const char *const reason[] = {
202		"Crash",			/* PCIE_FW_EVAL_CRASH */
203		"During Device Preparation",	/* PCIE_FW_EVAL_PREP */
204		"During Device Configuration",	/* PCIE_FW_EVAL_CONF */
205		"During Device Initialization",	/* PCIE_FW_EVAL_INIT */
206		"Unexpected Event",		/* PCIE_FW_EVAL_UNEXPECTEDEVENT */
207		"Insufficient Airflow",		/* PCIE_FW_EVAL_OVERHEAT */
208		"Device Shutdown",		/* PCIE_FW_EVAL_DEVICESHUTDOWN */
209		"Reserved",			/* reserved */
210	};
211	u32 pcie_fw;
212
213	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
214	if (pcie_fw & F_PCIE_FW_ERR) {
215		adap->flags &= ~FW_OK;
216		CH_ERR(adap, "firmware reports adapter error: %s (0x%08x)\n",
217		    reason[G_PCIE_FW_EVAL(pcie_fw)], pcie_fw);
218		if (pcie_fw != 0xffffffff)
219			t4_os_dump_devlog(adap);
220	}
221}
222
223/*
224 * Get the reply to a mailbox command and store it in @rpl in big-endian order.
225 */
226static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit,
227			 u32 mbox_addr)
228{
229	for ( ; nflit; nflit--, mbox_addr += 8)
230		*rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr));
231}
232
233/*
234 * Handle a FW assertion reported in a mailbox.
235 */
236static void fw_asrt(struct adapter *adap, struct fw_debug_cmd *asrt)
237{
238	CH_ALERT(adap,
239		  "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n",
240		  asrt->u.assert.filename_0_7,
241		  be32_to_cpu(asrt->u.assert.line),
242		  be32_to_cpu(asrt->u.assert.x),
243		  be32_to_cpu(asrt->u.assert.y));
244}
245
246struct port_tx_state {
247	uint64_t rx_pause;
248	uint64_t tx_frames;
249};
250
251static void
252read_tx_state_one(struct adapter *sc, int i, struct port_tx_state *tx_state)
253{
254	uint32_t rx_pause_reg, tx_frames_reg;
255
256	if (is_t4(sc)) {
257		tx_frames_reg = PORT_REG(i, A_MPS_PORT_STAT_TX_PORT_FRAMES_L);
258		rx_pause_reg = PORT_REG(i, A_MPS_PORT_STAT_RX_PORT_PAUSE_L);
259	} else {
260		tx_frames_reg = T5_PORT_REG(i, A_MPS_PORT_STAT_TX_PORT_FRAMES_L);
261		rx_pause_reg = T5_PORT_REG(i, A_MPS_PORT_STAT_RX_PORT_PAUSE_L);
262	}
263
264	tx_state->rx_pause = t4_read_reg64(sc, rx_pause_reg);
265	tx_state->tx_frames = t4_read_reg64(sc, tx_frames_reg);
266}
267
268static void
269read_tx_state(struct adapter *sc, struct port_tx_state *tx_state)
270{
271	int i;
272
273	for_each_port(sc, i)
274		read_tx_state_one(sc, i, &tx_state[i]);
275}
276
277static void
278check_tx_state(struct adapter *sc, struct port_tx_state *tx_state)
279{
280	uint32_t port_ctl_reg;
281	uint64_t tx_frames, rx_pause;
282	int i;
283
284	for_each_port(sc, i) {
285		rx_pause = tx_state[i].rx_pause;
286		tx_frames = tx_state[i].tx_frames;
287		read_tx_state_one(sc, i, &tx_state[i]);	/* update */
288
289		if (is_t4(sc))
290			port_ctl_reg = PORT_REG(i, A_MPS_PORT_CTL);
291		else
292			port_ctl_reg = T5_PORT_REG(i, A_MPS_PORT_CTL);
293		if (t4_read_reg(sc, port_ctl_reg) & F_PORTTXEN &&
294		    rx_pause != tx_state[i].rx_pause &&
295		    tx_frames == tx_state[i].tx_frames) {
296			t4_set_reg_field(sc, port_ctl_reg, F_PORTTXEN, 0);
297			mdelay(1);
298			t4_set_reg_field(sc, port_ctl_reg, F_PORTTXEN, F_PORTTXEN);
299		}
300	}
301}
302
303#define X_CIM_PF_NOACCESS 0xeeeeeeee
304/**
305 *	t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox
306 *	@adap: the adapter
307 *	@mbox: index of the mailbox to use
308 *	@cmd: the command to write
309 *	@size: command length in bytes
310 *	@rpl: where to optionally store the reply
311 *	@sleep_ok: if true we may sleep while awaiting command completion
312 *	@timeout: time to wait for command to finish before timing out
313 *		(negative implies @sleep_ok=false)
314 *
315 *	Sends the given command to FW through the selected mailbox and waits
316 *	for the FW to execute the command.  If @rpl is not %NULL it is used to
317 *	store the FW's reply to the command.  The command and its optional
318 *	reply are of the same length.  Some FW commands like RESET and
319 *	INITIALIZE can take a considerable amount of time to execute.
320 *	@sleep_ok determines whether we may sleep while awaiting the response.
321 *	If sleeping is allowed we use progressive backoff otherwise we spin.
322 *	Note that passing in a negative @timeout is an alternate mechanism
323 *	for specifying @sleep_ok=false.  This is useful when a higher level
324 *	interface allows for specification of @timeout but not @sleep_ok ...
325 *
326 *	The return value is 0 on success or a negative errno on failure.  A
327 *	failure can happen either because we are not able to execute the
328 *	command or FW executes it but signals an error.  In the latter case
329 *	the return value is the error code indicated by FW (negated).
330 */
331int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
332			    int size, void *rpl, bool sleep_ok, int timeout)
333{
334	/*
335	 * We delay in small increments at first in an effort to maintain
336	 * responsiveness for simple, fast executing commands but then back
337	 * off to larger delays to a maximum retry delay.
338	 */
339	static const int delay[] = {
340		1, 1, 3, 5, 10, 10, 20, 50, 100
341	};
342	u32 v;
343	u64 res;
344	int i, ms, delay_idx, ret, next_tx_check;
345	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
346	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
347	u32 ctl;
348	__be64 cmd_rpl[MBOX_LEN/8];
349	u32 pcie_fw;
350	struct port_tx_state tx_state[MAX_NPORTS];
351
352	if (adap->flags & CHK_MBOX_ACCESS)
353		ASSERT_SYNCHRONIZED_OP(adap);
354
355	if (size <= 0 || (size & 15) || size > MBOX_LEN)
356		return -EINVAL;
357
358	if (adap->flags & IS_VF) {
359		if (is_t6(adap))
360			data_reg = FW_T6VF_MBDATA_BASE_ADDR;
361		else
362			data_reg = FW_T4VF_MBDATA_BASE_ADDR;
363		ctl_reg = VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL);
364	}
365
366	/*
367	 * If we have a negative timeout, that implies that we can't sleep.
368	 */
369	if (timeout < 0) {
370		sleep_ok = false;
371		timeout = -timeout;
372	}
373
374	/*
375	 * Attempt to gain access to the mailbox.
376	 */
377	for (i = 0; i < 4; i++) {
378		ctl = t4_read_reg(adap, ctl_reg);
379		v = G_MBOWNER(ctl);
380		if (v != X_MBOWNER_NONE)
381			break;
382	}
383
384	/*
385	 * If we were unable to gain access, report the error to our caller.
386	 */
387	if (v != X_MBOWNER_PL) {
388		t4_report_fw_error(adap);
389		ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT;
390		return ret;
391	}
392
393	/*
394	 * If we gain ownership of the mailbox and there's a "valid" message
395	 * in it, this is likely an asynchronous error message from the
396	 * firmware.  So we'll report that and then proceed on with attempting
397	 * to issue our own command ... which may well fail if the error
398	 * presaged the firmware crashing ...
399	 */
400	if (ctl & F_MBMSGVALID) {
401		CH_DUMP_MBOX(adap, mbox, data_reg, "VLD", NULL, true);
402	}
403
404	/*
405	 * Copy in the new mailbox command and send it on its way ...
406	 */
407	memset(cmd_rpl, 0, sizeof(cmd_rpl));
408	memcpy(cmd_rpl, cmd, size);
409	CH_DUMP_MBOX(adap, mbox, 0, "cmd", cmd_rpl, false);
410	for (i = 0; i < ARRAY_SIZE(cmd_rpl); i++)
411		t4_write_reg64(adap, data_reg + i * 8, be64_to_cpu(cmd_rpl[i]));
412
413	if (adap->flags & IS_VF) {
414		/*
415		 * For the VFs, the Mailbox Data "registers" are
416		 * actually backed by T4's "MA" interface rather than
417		 * PL Registers (as is the case for the PFs).  Because
418		 * these are in different coherency domains, the write
419		 * to the VF's PL-register-backed Mailbox Control can
420		 * race in front of the writes to the MA-backed VF
421		 * Mailbox Data "registers".  So we need to do a
422		 * read-back on at least one byte of the VF Mailbox
423		 * Data registers before doing the write to the VF
424		 * Mailbox Control register.
425		 */
426		t4_read_reg(adap, data_reg);
427	}
428
429	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
430	read_tx_state(adap, &tx_state[0]);	/* also flushes the write_reg */
431	next_tx_check = 1000;
432	delay_idx = 0;
433	ms = delay[0];
434
435	/*
436	 * Loop waiting for the reply; bail out if we time out or the firmware
437	 * reports an error.
438	 */
439	pcie_fw = 0;
440	for (i = 0; i < timeout; i += ms) {
441		if (!(adap->flags & IS_VF)) {
442			pcie_fw = t4_read_reg(adap, A_PCIE_FW);
443			if (pcie_fw & F_PCIE_FW_ERR)
444				break;
445		}
446
447		if (i >= next_tx_check) {
448			check_tx_state(adap, &tx_state[0]);
449			next_tx_check = i + 1000;
450		}
451
452		if (sleep_ok) {
453			ms = delay[delay_idx];  /* last element may repeat */
454			if (delay_idx < ARRAY_SIZE(delay) - 1)
455				delay_idx++;
456			msleep(ms);
457		} else {
458			mdelay(ms);
459		}
460
461		v = t4_read_reg(adap, ctl_reg);
462		if (v == X_CIM_PF_NOACCESS)
463			continue;
464		if (G_MBOWNER(v) == X_MBOWNER_PL) {
465			if (!(v & F_MBMSGVALID)) {
466				t4_write_reg(adap, ctl_reg,
467					     V_MBOWNER(X_MBOWNER_NONE));
468				continue;
469			}
470
471			/*
472			 * Retrieve the command reply and release the mailbox.
473			 */
474			get_mbox_rpl(adap, cmd_rpl, MBOX_LEN/8, data_reg);
475			CH_DUMP_MBOX(adap, mbox, 0, "rpl", cmd_rpl, false);
476			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));
477
478			res = be64_to_cpu(cmd_rpl[0]);
479			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
480				fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl);
481				res = V_FW_CMD_RETVAL(EIO);
482			} else if (rpl)
483				memcpy(rpl, cmd_rpl, size);
484			return -G_FW_CMD_RETVAL((int)res);
485		}
486	}
487
488	/*
489	 * We timed out waiting for a reply to our mailbox command.  Report
490	 * the error and also check to see if the firmware reported any
491	 * errors ...
492	 */
493	CH_ERR(adap, "command %#x in mbox %d timed out (0x%08x).\n",
494	    *(const u8 *)cmd, mbox, pcie_fw);
495	CH_DUMP_MBOX(adap, mbox, 0, "cmdsent", cmd_rpl, true);
496	CH_DUMP_MBOX(adap, mbox, data_reg, "current", NULL, true);
497
498	if (pcie_fw & F_PCIE_FW_ERR) {
499		ret = -ENXIO;
500		t4_report_fw_error(adap);
501	} else {
502		ret = -ETIMEDOUT;
503		t4_os_dump_devlog(adap);
504	}
505
506	t4_fatal_err(adap, true);
507	return ret;
508}
509
510int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
511		    void *rpl, bool sleep_ok)
512{
513		return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl,
514					       sleep_ok, FW_CMD_MAX_TIMEOUT);
515
516}
517
518static int t4_edc_err_read(struct adapter *adap, int idx)
519{
520	u32 edc_ecc_err_addr_reg;
521	u32 edc_bist_status_rdata_reg;
522
523	if (is_t4(adap)) {
524		CH_WARN(adap, "%s: T4 NOT supported.\n", __func__);
525		return 0;
526	}
527	if (idx != MEM_EDC0 && idx != MEM_EDC1) {
528		CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx);
529		return 0;
530	}
531
532	edc_ecc_err_addr_reg = EDC_T5_REG(A_EDC_H_ECC_ERR_ADDR, idx);
533	edc_bist_status_rdata_reg = EDC_T5_REG(A_EDC_H_BIST_STATUS_RDATA, idx);
534
535	CH_WARN(adap,
536		"edc%d err addr 0x%x: 0x%x.\n",
537		idx, edc_ecc_err_addr_reg,
538		t4_read_reg(adap, edc_ecc_err_addr_reg));
539	CH_WARN(adap,
540	 	"bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
541		edc_bist_status_rdata_reg,
542		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg),
543		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 8),
544		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 16),
545		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 24),
546		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 32),
547		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 40),
548		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 48),
549		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 56),
550		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 64));
551
552	return 0;
553}
554
555/**
556 *	t4_mc_read - read from MC through backdoor accesses
557 *	@adap: the adapter
558 *	@idx: which MC to access
559 *	@addr: address of first byte requested
560 *	@data: 64 bytes of data containing the requested address
561 *	@ecc: where to store the corresponding 64-bit ECC word
562 *
563 *	Read 64 bytes of data from MC starting at a 64-byte-aligned address
564 *	that covers the requested address @addr.  If @parity is not %NULL it
565 *	is assigned the 64-bit ECC word for the read data.
566 */
567int t4_mc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc)
568{
569	int i;
570	u32 mc_bist_cmd_reg, mc_bist_cmd_addr_reg, mc_bist_cmd_len_reg;
571	u32 mc_bist_status_rdata_reg, mc_bist_data_pattern_reg;
572
573	if (is_t4(adap)) {
574		mc_bist_cmd_reg = A_MC_BIST_CMD;
575		mc_bist_cmd_addr_reg = A_MC_BIST_CMD_ADDR;
576		mc_bist_cmd_len_reg = A_MC_BIST_CMD_LEN;
577		mc_bist_status_rdata_reg = A_MC_BIST_STATUS_RDATA;
578		mc_bist_data_pattern_reg = A_MC_BIST_DATA_PATTERN;
579	} else {
580		mc_bist_cmd_reg = MC_REG(A_MC_P_BIST_CMD, idx);
581		mc_bist_cmd_addr_reg = MC_REG(A_MC_P_BIST_CMD_ADDR, idx);
582		mc_bist_cmd_len_reg = MC_REG(A_MC_P_BIST_CMD_LEN, idx);
583		mc_bist_status_rdata_reg = MC_REG(A_MC_P_BIST_STATUS_RDATA,
584						  idx);
585		mc_bist_data_pattern_reg = MC_REG(A_MC_P_BIST_DATA_PATTERN,
586						  idx);
587	}
588
589	if (t4_read_reg(adap, mc_bist_cmd_reg) & F_START_BIST)
590		return -EBUSY;
591	t4_write_reg(adap, mc_bist_cmd_addr_reg, addr & ~0x3fU);
592	t4_write_reg(adap, mc_bist_cmd_len_reg, 64);
593	t4_write_reg(adap, mc_bist_data_pattern_reg, 0xc);
594	t4_write_reg(adap, mc_bist_cmd_reg, V_BIST_OPCODE(1) |
595		     F_START_BIST | V_BIST_CMD_GAP(1));
596	i = t4_wait_op_done(adap, mc_bist_cmd_reg, F_START_BIST, 0, 10, 1);
597	if (i)
598		return i;
599
600#define MC_DATA(i) MC_BIST_STATUS_REG(mc_bist_status_rdata_reg, i)
601
602	for (i = 15; i >= 0; i--)
603		*data++ = ntohl(t4_read_reg(adap, MC_DATA(i)));
604	if (ecc)
605		*ecc = t4_read_reg64(adap, MC_DATA(16));
606#undef MC_DATA
607	return 0;
608}
609
610/**
611 *	t4_edc_read - read from EDC through backdoor accesses
612 *	@adap: the adapter
613 *	@idx: which EDC to access
614 *	@addr: address of first byte requested
615 *	@data: 64 bytes of data containing the requested address
616 *	@ecc: where to store the corresponding 64-bit ECC word
617 *
618 *	Read 64 bytes of data from EDC starting at a 64-byte-aligned address
619 *	that covers the requested address @addr.  If @parity is not %NULL it
620 *	is assigned the 64-bit ECC word for the read data.
621 */
622int t4_edc_read(struct adapter *adap, int idx, u32 addr, __be32 *data, u64 *ecc)
623{
624	int i;
625	u32 edc_bist_cmd_reg, edc_bist_cmd_addr_reg, edc_bist_cmd_len_reg;
626	u32 edc_bist_cmd_data_pattern, edc_bist_status_rdata_reg;
627
628	if (is_t4(adap)) {
629		edc_bist_cmd_reg = EDC_REG(A_EDC_BIST_CMD, idx);
630		edc_bist_cmd_addr_reg = EDC_REG(A_EDC_BIST_CMD_ADDR, idx);
631		edc_bist_cmd_len_reg = EDC_REG(A_EDC_BIST_CMD_LEN, idx);
632		edc_bist_cmd_data_pattern = EDC_REG(A_EDC_BIST_DATA_PATTERN,
633						    idx);
634		edc_bist_status_rdata_reg = EDC_REG(A_EDC_BIST_STATUS_RDATA,
635						    idx);
636	} else {
637/*
638 * These macro are missing in t4_regs.h file.
639 * Added temporarily for testing.
640 */
641#define EDC_STRIDE_T5 (EDC_T51_BASE_ADDR - EDC_T50_BASE_ADDR)
642#define EDC_REG_T5(reg, idx) (reg + EDC_STRIDE_T5 * idx)
643		edc_bist_cmd_reg = EDC_REG_T5(A_EDC_H_BIST_CMD, idx);
644		edc_bist_cmd_addr_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_ADDR, idx);
645		edc_bist_cmd_len_reg = EDC_REG_T5(A_EDC_H_BIST_CMD_LEN, idx);
646		edc_bist_cmd_data_pattern = EDC_REG_T5(A_EDC_H_BIST_DATA_PATTERN,
647						    idx);
648		edc_bist_status_rdata_reg = EDC_REG_T5(A_EDC_H_BIST_STATUS_RDATA,
649						    idx);
650#undef EDC_REG_T5
651#undef EDC_STRIDE_T5
652	}
653
654	if (t4_read_reg(adap, edc_bist_cmd_reg) & F_START_BIST)
655		return -EBUSY;
656	t4_write_reg(adap, edc_bist_cmd_addr_reg, addr & ~0x3fU);
657	t4_write_reg(adap, edc_bist_cmd_len_reg, 64);
658	t4_write_reg(adap, edc_bist_cmd_data_pattern, 0xc);
659	t4_write_reg(adap, edc_bist_cmd_reg,
660		     V_BIST_OPCODE(1) | V_BIST_CMD_GAP(1) | F_START_BIST);
661	i = t4_wait_op_done(adap, edc_bist_cmd_reg, F_START_BIST, 0, 10, 1);
662	if (i)
663		return i;
664
665#define EDC_DATA(i) EDC_BIST_STATUS_REG(edc_bist_status_rdata_reg, i)
666
667	for (i = 15; i >= 0; i--)
668		*data++ = ntohl(t4_read_reg(adap, EDC_DATA(i)));
669	if (ecc)
670		*ecc = t4_read_reg64(adap, EDC_DATA(16));
671#undef EDC_DATA
672	return 0;
673}
674
675/**
676 *	t4_mem_read - read EDC 0, EDC 1 or MC into buffer
677 *	@adap: the adapter
678 *	@mtype: memory type: MEM_EDC0, MEM_EDC1 or MEM_MC
679 *	@addr: address within indicated memory type
680 *	@len: amount of memory to read
681 *	@buf: host memory buffer
682 *
683 *	Reads an [almost] arbitrary memory region in the firmware: the
684 *	firmware memory address, length and host buffer must be aligned on
685 *	32-bit boudaries.  The memory is returned as a raw byte sequence from
686 *	the firmware's memory.  If this memory contains data structures which
687 *	contain multi-byte integers, it's the callers responsibility to
688 *	perform appropriate byte order conversions.
689 */
690int t4_mem_read(struct adapter *adap, int mtype, u32 addr, u32 len,
691		__be32 *buf)
692{
693	u32 pos, start, end, offset;
694	int ret;
695
696	/*
697	 * Argument sanity checks ...
698	 */
699	if ((addr & 0x3) || (len & 0x3))
700		return -EINVAL;
701
702	/*
703	 * The underlaying EDC/MC read routines read 64 bytes at a time so we
704	 * need to round down the start and round up the end.  We'll start
705	 * copying out of the first line at (addr - start) a word at a time.
706	 */
707	start = rounddown2(addr, 64);
708	end = roundup2(addr + len, 64);
709	offset = (addr - start)/sizeof(__be32);
710
711	for (pos = start; pos < end; pos += 64, offset = 0) {
712		__be32 data[16];
713
714		/*
715		 * Read the chip's memory block and bail if there's an error.
716		 */
717		if ((mtype == MEM_MC) || (mtype == MEM_MC1))
718			ret = t4_mc_read(adap, mtype - MEM_MC, pos, data, NULL);
719		else
720			ret = t4_edc_read(adap, mtype, pos, data, NULL);
721		if (ret)
722			return ret;
723
724		/*
725		 * Copy the data into the caller's memory buffer.
726		 */
727		while (offset < 16 && len > 0) {
728			*buf++ = data[offset++];
729			len -= sizeof(__be32);
730		}
731	}
732
733	return 0;
734}
735
736/*
737 * Return the specified PCI-E Configuration Space register from our Physical
738 * Function.  We try first via a Firmware LDST Command (if fw_attach != 0)
739 * since we prefer to let the firmware own all of these registers, but if that
740 * fails we go for it directly ourselves.
741 */
742u32 t4_read_pcie_cfg4(struct adapter *adap, int reg, int drv_fw_attach)
743{
744
745	/*
746	 * If fw_attach != 0, construct and send the Firmware LDST Command to
747	 * retrieve the specified PCI-E Configuration Space register.
748	 */
749	if (drv_fw_attach != 0) {
750		struct fw_ldst_cmd ldst_cmd;
751		int ret;
752
753		memset(&ldst_cmd, 0, sizeof(ldst_cmd));
754		ldst_cmd.op_to_addrspace =
755			cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
756				    F_FW_CMD_REQUEST |
757				    F_FW_CMD_READ |
758				    V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_FUNC_PCIE));
759		ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd));
760		ldst_cmd.u.pcie.select_naccess = V_FW_LDST_CMD_NACCESS(1);
761		ldst_cmd.u.pcie.ctrl_to_fn =
762			(F_FW_LDST_CMD_LC | V_FW_LDST_CMD_FN(adap->pf));
763		ldst_cmd.u.pcie.r = reg;
764
765		/*
766		 * If the LDST Command succeeds, return the result, otherwise
767		 * fall through to reading it directly ourselves ...
768		 */
769		ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
770				 &ldst_cmd);
771		if (ret == 0)
772			return be32_to_cpu(ldst_cmd.u.pcie.data[0]);
773
774		CH_WARN(adap, "Firmware failed to return "
775			"Configuration Space register %d, err = %d\n",
776			reg, -ret);
777	}
778
779	/*
780	 * Read the desired Configuration Space register via the PCI-E
781	 * Backdoor mechanism.
782	 */
783	return t4_hw_pci_read_cfg4(adap, reg);
784}
785
786/**
787 *	t4_get_regs_len - return the size of the chips register set
788 *	@adapter: the adapter
789 *
790 *	Returns the size of the chip's BAR0 register space.
791 */
792unsigned int t4_get_regs_len(struct adapter *adapter)
793{
794	unsigned int chip_version = chip_id(adapter);
795
796	switch (chip_version) {
797	case CHELSIO_T4:
798		if (adapter->flags & IS_VF)
799			return FW_T4VF_REGMAP_SIZE;
800		return T4_REGMAP_SIZE;
801
802	case CHELSIO_T5:
803	case CHELSIO_T6:
804		if (adapter->flags & IS_VF)
805			return FW_T4VF_REGMAP_SIZE;
806		return T5_REGMAP_SIZE;
807	}
808
809	CH_ERR(adapter,
810		"Unsupported chip version %d\n", chip_version);
811	return 0;
812}
813
814/**
815 *	t4_get_regs - read chip registers into provided buffer
816 *	@adap: the adapter
817 *	@buf: register buffer
818 *	@buf_size: size (in bytes) of register buffer
819 *
820 *	If the provided register buffer isn't large enough for the chip's
821 *	full register range, the register dump will be truncated to the
822 *	register buffer's size.
823 */
824void t4_get_regs(struct adapter *adap, u8 *buf, size_t buf_size)
825{
826	static const unsigned int t4_reg_ranges[] = {
827		0x1008, 0x1108,
828		0x1180, 0x1184,
829		0x1190, 0x1194,
830		0x11a0, 0x11a4,
831		0x11b0, 0x11b4,
832		0x11fc, 0x123c,
833		0x1300, 0x173c,
834		0x1800, 0x18fc,
835		0x3000, 0x30d8,
836		0x30e0, 0x30e4,
837		0x30ec, 0x5910,
838		0x5920, 0x5924,
839		0x5960, 0x5960,
840		0x5968, 0x5968,
841		0x5970, 0x5970,
842		0x5978, 0x5978,
843		0x5980, 0x5980,
844		0x5988, 0x5988,
845		0x5990, 0x5990,
846		0x5998, 0x5998,
847		0x59a0, 0x59d4,
848		0x5a00, 0x5ae0,
849		0x5ae8, 0x5ae8,
850		0x5af0, 0x5af0,
851		0x5af8, 0x5af8,
852		0x6000, 0x6098,
853		0x6100, 0x6150,
854		0x6200, 0x6208,
855		0x6240, 0x6248,
856		0x6280, 0x62b0,
857		0x62c0, 0x6338,
858		0x6370, 0x638c,
859		0x6400, 0x643c,
860		0x6500, 0x6524,
861		0x6a00, 0x6a04,
862		0x6a14, 0x6a38,
863		0x6a60, 0x6a70,
864		0x6a78, 0x6a78,
865		0x6b00, 0x6b0c,
866		0x6b1c, 0x6b84,
867		0x6bf0, 0x6bf8,
868		0x6c00, 0x6c0c,
869		0x6c1c, 0x6c84,
870		0x6cf0, 0x6cf8,
871		0x6d00, 0x6d0c,
872		0x6d1c, 0x6d84,
873		0x6df0, 0x6df8,
874		0x6e00, 0x6e0c,
875		0x6e1c, 0x6e84,
876		0x6ef0, 0x6ef8,
877		0x6f00, 0x6f0c,
878		0x6f1c, 0x6f84,
879		0x6ff0, 0x6ff8,
880		0x7000, 0x700c,
881		0x701c, 0x7084,
882		0x70f0, 0x70f8,
883		0x7100, 0x710c,
884		0x711c, 0x7184,
885		0x71f0, 0x71f8,
886		0x7200, 0x720c,
887		0x721c, 0x7284,
888		0x72f0, 0x72f8,
889		0x7300, 0x730c,
890		0x731c, 0x7384,
891		0x73f0, 0x73f8,
892		0x7400, 0x7450,
893		0x7500, 0x7530,
894		0x7600, 0x760c,
895		0x7614, 0x761c,
896		0x7680, 0x76cc,
897		0x7700, 0x7798,
898		0x77c0, 0x77fc,
899		0x7900, 0x79fc,
900		0x7b00, 0x7b58,
901		0x7b60, 0x7b84,
902		0x7b8c, 0x7c38,
903		0x7d00, 0x7d38,
904		0x7d40, 0x7d80,
905		0x7d8c, 0x7ddc,
906		0x7de4, 0x7e04,
907		0x7e10, 0x7e1c,
908		0x7e24, 0x7e38,
909		0x7e40, 0x7e44,
910		0x7e4c, 0x7e78,
911		0x7e80, 0x7ea4,
912		0x7eac, 0x7edc,
913		0x7ee8, 0x7efc,
914		0x8dc0, 0x8e04,
915		0x8e10, 0x8e1c,
916		0x8e30, 0x8e78,
917		0x8ea0, 0x8eb8,
918		0x8ec0, 0x8f6c,
919		0x8fc0, 0x9008,
920		0x9010, 0x9058,
921		0x9060, 0x9060,
922		0x9068, 0x9074,
923		0x90fc, 0x90fc,
924		0x9400, 0x9408,
925		0x9410, 0x9458,
926		0x9600, 0x9600,
927		0x9608, 0x9638,
928		0x9640, 0x96bc,
929		0x9800, 0x9808,
930		0x9820, 0x983c,
931		0x9850, 0x9864,
932		0x9c00, 0x9c6c,
933		0x9c80, 0x9cec,
934		0x9d00, 0x9d6c,
935		0x9d80, 0x9dec,
936		0x9e00, 0x9e6c,
937		0x9e80, 0x9eec,
938		0x9f00, 0x9f6c,
939		0x9f80, 0x9fec,
940		0xd004, 0xd004,
941		0xd010, 0xd03c,
942		0xdfc0, 0xdfe0,
943		0xe000, 0xea7c,
944		0xf000, 0x11110,
945		0x11118, 0x11190,
946		0x19040, 0x1906c,
947		0x19078, 0x19080,
948		0x1908c, 0x190e4,
949		0x190f0, 0x190f8,
950		0x19100, 0x19110,
951		0x19120, 0x19124,
952		0x19150, 0x19194,
953		0x1919c, 0x191b0,
954		0x191d0, 0x191e8,
955		0x19238, 0x1924c,
956		0x193f8, 0x1943c,
957		0x1944c, 0x19474,
958		0x19490, 0x194e0,
959		0x194f0, 0x194f8,
960		0x19800, 0x19c08,
961		0x19c10, 0x19c90,
962		0x19ca0, 0x19ce4,
963		0x19cf0, 0x19d40,
964		0x19d50, 0x19d94,
965		0x19da0, 0x19de8,
966		0x19df0, 0x19e40,
967		0x19e50, 0x19e90,
968		0x19ea0, 0x19f4c,
969		0x1a000, 0x1a004,
970		0x1a010, 0x1a06c,
971		0x1a0b0, 0x1a0e4,
972		0x1a0ec, 0x1a0f4,
973		0x1a100, 0x1a108,
974		0x1a114, 0x1a120,
975		0x1a128, 0x1a130,
976		0x1a138, 0x1a138,
977		0x1a190, 0x1a1c4,
978		0x1a1fc, 0x1a1fc,
979		0x1e040, 0x1e04c,
980		0x1e284, 0x1e28c,
981		0x1e2c0, 0x1e2c0,
982		0x1e2e0, 0x1e2e0,
983		0x1e300, 0x1e384,
984		0x1e3c0, 0x1e3c8,
985		0x1e440, 0x1e44c,
986		0x1e684, 0x1e68c,
987		0x1e6c0, 0x1e6c0,
988		0x1e6e0, 0x1e6e0,
989		0x1e700, 0x1e784,
990		0x1e7c0, 0x1e7c8,
991		0x1e840, 0x1e84c,
992		0x1ea84, 0x1ea8c,
993		0x1eac0, 0x1eac0,
994		0x1eae0, 0x1eae0,
995		0x1eb00, 0x1eb84,
996		0x1ebc0, 0x1ebc8,
997		0x1ec40, 0x1ec4c,
998		0x1ee84, 0x1ee8c,
999		0x1eec0, 0x1eec0,
1000		0x1eee0, 0x1eee0,
1001		0x1ef00, 0x1ef84,
1002		0x1efc0, 0x1efc8,
1003		0x1f040, 0x1f04c,
1004		0x1f284, 0x1f28c,
1005		0x1f2c0, 0x1f2c0,
1006		0x1f2e0, 0x1f2e0,
1007		0x1f300, 0x1f384,
1008		0x1f3c0, 0x1f3c8,
1009		0x1f440, 0x1f44c,
1010		0x1f684, 0x1f68c,
1011		0x1f6c0, 0x1f6c0,
1012		0x1f6e0, 0x1f6e0,
1013		0x1f700, 0x1f784,
1014		0x1f7c0, 0x1f7c8,
1015		0x1f840, 0x1f84c,
1016		0x1fa84, 0x1fa8c,
1017		0x1fac0, 0x1fac0,
1018		0x1fae0, 0x1fae0,
1019		0x1fb00, 0x1fb84,
1020		0x1fbc0, 0x1fbc8,
1021		0x1fc40, 0x1fc4c,
1022		0x1fe84, 0x1fe8c,
1023		0x1fec0, 0x1fec0,
1024		0x1fee0, 0x1fee0,
1025		0x1ff00, 0x1ff84,
1026		0x1ffc0, 0x1ffc8,
1027		0x20000, 0x2002c,
1028		0x20100, 0x2013c,
1029		0x20190, 0x201a0,
1030		0x201a8, 0x201b8,
1031		0x201c4, 0x201c8,
1032		0x20200, 0x20318,
1033		0x20400, 0x204b4,
1034		0x204c0, 0x20528,
1035		0x20540, 0x20614,
1036		0x21000, 0x21040,
1037		0x2104c, 0x21060,
1038		0x210c0, 0x210ec,
1039		0x21200, 0x21268,
1040		0x21270, 0x21284,
1041		0x212fc, 0x21388,
1042		0x21400, 0x21404,
1043		0x21500, 0x21500,
1044		0x21510, 0x21518,
1045		0x2152c, 0x21530,
1046		0x2153c, 0x2153c,
1047		0x21550, 0x21554,
1048		0x21600, 0x21600,
1049		0x21608, 0x2161c,
1050		0x21624, 0x21628,
1051		0x21630, 0x21634,
1052		0x2163c, 0x2163c,
1053		0x21700, 0x2171c,
1054		0x21780, 0x2178c,
1055		0x21800, 0x21818,
1056		0x21820, 0x21828,
1057		0x21830, 0x21848,
1058		0x21850, 0x21854,
1059		0x21860, 0x21868,
1060		0x21870, 0x21870,
1061		0x21878, 0x21898,
1062		0x218a0, 0x218a8,
1063		0x218b0, 0x218c8,
1064		0x218d0, 0x218d4,
1065		0x218e0, 0x218e8,
1066		0x218f0, 0x218f0,
1067		0x218f8, 0x21a18,
1068		0x21a20, 0x21a28,
1069		0x21a30, 0x21a48,
1070		0x21a50, 0x21a54,
1071		0x21a60, 0x21a68,
1072		0x21a70, 0x21a70,
1073		0x21a78, 0x21a98,
1074		0x21aa0, 0x21aa8,
1075		0x21ab0, 0x21ac8,
1076		0x21ad0, 0x21ad4,
1077		0x21ae0, 0x21ae8,
1078		0x21af0, 0x21af0,
1079		0x21af8, 0x21c18,
1080		0x21c20, 0x21c20,
1081		0x21c28, 0x21c30,
1082		0x21c38, 0x21c38,
1083		0x21c80, 0x21c98,
1084		0x21ca0, 0x21ca8,
1085		0x21cb0, 0x21cc8,
1086		0x21cd0, 0x21cd4,
1087		0x21ce0, 0x21ce8,
1088		0x21cf0, 0x21cf0,
1089		0x21cf8, 0x21d7c,
1090		0x21e00, 0x21e04,
1091		0x22000, 0x2202c,
1092		0x22100, 0x2213c,
1093		0x22190, 0x221a0,
1094		0x221a8, 0x221b8,
1095		0x221c4, 0x221c8,
1096		0x22200, 0x22318,
1097		0x22400, 0x224b4,
1098		0x224c0, 0x22528,
1099		0x22540, 0x22614,
1100		0x23000, 0x23040,
1101		0x2304c, 0x23060,
1102		0x230c0, 0x230ec,
1103		0x23200, 0x23268,
1104		0x23270, 0x23284,
1105		0x232fc, 0x23388,
1106		0x23400, 0x23404,
1107		0x23500, 0x23500,
1108		0x23510, 0x23518,
1109		0x2352c, 0x23530,
1110		0x2353c, 0x2353c,
1111		0x23550, 0x23554,
1112		0x23600, 0x23600,
1113		0x23608, 0x2361c,
1114		0x23624, 0x23628,
1115		0x23630, 0x23634,
1116		0x2363c, 0x2363c,
1117		0x23700, 0x2371c,
1118		0x23780, 0x2378c,
1119		0x23800, 0x23818,
1120		0x23820, 0x23828,
1121		0x23830, 0x23848,
1122		0x23850, 0x23854,
1123		0x23860, 0x23868,
1124		0x23870, 0x23870,
1125		0x23878, 0x23898,
1126		0x238a0, 0x238a8,
1127		0x238b0, 0x238c8,
1128		0x238d0, 0x238d4,
1129		0x238e0, 0x238e8,
1130		0x238f0, 0x238f0,
1131		0x238f8, 0x23a18,
1132		0x23a20, 0x23a28,
1133		0x23a30, 0x23a48,
1134		0x23a50, 0x23a54,
1135		0x23a60, 0x23a68,
1136		0x23a70, 0x23a70,
1137		0x23a78, 0x23a98,
1138		0x23aa0, 0x23aa8,
1139		0x23ab0, 0x23ac8,
1140		0x23ad0, 0x23ad4,
1141		0x23ae0, 0x23ae8,
1142		0x23af0, 0x23af0,
1143		0x23af8, 0x23c18,
1144		0x23c20, 0x23c20,
1145		0x23c28, 0x23c30,
1146		0x23c38, 0x23c38,
1147		0x23c80, 0x23c98,
1148		0x23ca0, 0x23ca8,
1149		0x23cb0, 0x23cc8,
1150		0x23cd0, 0x23cd4,
1151		0x23ce0, 0x23ce8,
1152		0x23cf0, 0x23cf0,
1153		0x23cf8, 0x23d7c,
1154		0x23e00, 0x23e04,
1155		0x24000, 0x2402c,
1156		0x24100, 0x2413c,
1157		0x24190, 0x241a0,
1158		0x241a8, 0x241b8,
1159		0x241c4, 0x241c8,
1160		0x24200, 0x24318,
1161		0x24400, 0x244b4,
1162		0x244c0, 0x24528,
1163		0x24540, 0x24614,
1164		0x25000, 0x25040,
1165		0x2504c, 0x25060,
1166		0x250c0, 0x250ec,
1167		0x25200, 0x25268,
1168		0x25270, 0x25284,
1169		0x252fc, 0x25388,
1170		0x25400, 0x25404,
1171		0x25500, 0x25500,
1172		0x25510, 0x25518,
1173		0x2552c, 0x25530,
1174		0x2553c, 0x2553c,
1175		0x25550, 0x25554,
1176		0x25600, 0x25600,
1177		0x25608, 0x2561c,
1178		0x25624, 0x25628,
1179		0x25630, 0x25634,
1180		0x2563c, 0x2563c,
1181		0x25700, 0x2571c,
1182		0x25780, 0x2578c,
1183		0x25800, 0x25818,
1184		0x25820, 0x25828,
1185		0x25830, 0x25848,
1186		0x25850, 0x25854,
1187		0x25860, 0x25868,
1188		0x25870, 0x25870,
1189		0x25878, 0x25898,
1190		0x258a0, 0x258a8,
1191		0x258b0, 0x258c8,
1192		0x258d0, 0x258d4,
1193		0x258e0, 0x258e8,
1194		0x258f0, 0x258f0,
1195		0x258f8, 0x25a18,
1196		0x25a20, 0x25a28,
1197		0x25a30, 0x25a48,
1198		0x25a50, 0x25a54,
1199		0x25a60, 0x25a68,
1200		0x25a70, 0x25a70,
1201		0x25a78, 0x25a98,
1202		0x25aa0, 0x25aa8,
1203		0x25ab0, 0x25ac8,
1204		0x25ad0, 0x25ad4,
1205		0x25ae0, 0x25ae8,
1206		0x25af0, 0x25af0,
1207		0x25af8, 0x25c18,
1208		0x25c20, 0x25c20,
1209		0x25c28, 0x25c30,
1210		0x25c38, 0x25c38,
1211		0x25c80, 0x25c98,
1212		0x25ca0, 0x25ca8,
1213		0x25cb0, 0x25cc8,
1214		0x25cd0, 0x25cd4,
1215		0x25ce0, 0x25ce8,
1216		0x25cf0, 0x25cf0,
1217		0x25cf8, 0x25d7c,
1218		0x25e00, 0x25e04,
1219		0x26000, 0x2602c,
1220		0x26100, 0x2613c,
1221		0x26190, 0x261a0,
1222		0x261a8, 0x261b8,
1223		0x261c4, 0x261c8,
1224		0x26200, 0x26318,
1225		0x26400, 0x264b4,
1226		0x264c0, 0x26528,
1227		0x26540, 0x26614,
1228		0x27000, 0x27040,
1229		0x2704c, 0x27060,
1230		0x270c0, 0x270ec,
1231		0x27200, 0x27268,
1232		0x27270, 0x27284,
1233		0x272fc, 0x27388,
1234		0x27400, 0x27404,
1235		0x27500, 0x27500,
1236		0x27510, 0x27518,
1237		0x2752c, 0x27530,
1238		0x2753c, 0x2753c,
1239		0x27550, 0x27554,
1240		0x27600, 0x27600,
1241		0x27608, 0x2761c,
1242		0x27624, 0x27628,
1243		0x27630, 0x27634,
1244		0x2763c, 0x2763c,
1245		0x27700, 0x2771c,
1246		0x27780, 0x2778c,
1247		0x27800, 0x27818,
1248		0x27820, 0x27828,
1249		0x27830, 0x27848,
1250		0x27850, 0x27854,
1251		0x27860, 0x27868,
1252		0x27870, 0x27870,
1253		0x27878, 0x27898,
1254		0x278a0, 0x278a8,
1255		0x278b0, 0x278c8,
1256		0x278d0, 0x278d4,
1257		0x278e0, 0x278e8,
1258		0x278f0, 0x278f0,
1259		0x278f8, 0x27a18,
1260		0x27a20, 0x27a28,
1261		0x27a30, 0x27a48,
1262		0x27a50, 0x27a54,
1263		0x27a60, 0x27a68,
1264		0x27a70, 0x27a70,
1265		0x27a78, 0x27a98,
1266		0x27aa0, 0x27aa8,
1267		0x27ab0, 0x27ac8,
1268		0x27ad0, 0x27ad4,
1269		0x27ae0, 0x27ae8,
1270		0x27af0, 0x27af0,
1271		0x27af8, 0x27c18,
1272		0x27c20, 0x27c20,
1273		0x27c28, 0x27c30,
1274		0x27c38, 0x27c38,
1275		0x27c80, 0x27c98,
1276		0x27ca0, 0x27ca8,
1277		0x27cb0, 0x27cc8,
1278		0x27cd0, 0x27cd4,
1279		0x27ce0, 0x27ce8,
1280		0x27cf0, 0x27cf0,
1281		0x27cf8, 0x27d7c,
1282		0x27e00, 0x27e04,
1283	};
1284
1285	static const unsigned int t4vf_reg_ranges[] = {
1286		VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS),
1287		VF_MPS_REG(A_MPS_VF_CTL),
1288		VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H),
1289		VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_WHOAMI),
1290		VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL),
1291		VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS),
1292		FW_T4VF_MBDATA_BASE_ADDR,
1293		FW_T4VF_MBDATA_BASE_ADDR +
1294		((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4),
1295	};
1296
1297	static const unsigned int t5_reg_ranges[] = {
1298		0x1008, 0x10c0,
1299		0x10cc, 0x10f8,
1300		0x1100, 0x1100,
1301		0x110c, 0x1148,
1302		0x1180, 0x1184,
1303		0x1190, 0x1194,
1304		0x11a0, 0x11a4,
1305		0x11b0, 0x11b4,
1306		0x11fc, 0x123c,
1307		0x1280, 0x173c,
1308		0x1800, 0x18fc,
1309		0x3000, 0x3028,
1310		0x3060, 0x30b0,
1311		0x30b8, 0x30d8,
1312		0x30e0, 0x30fc,
1313		0x3140, 0x357c,
1314		0x35a8, 0x35cc,
1315		0x35ec, 0x35ec,
1316		0x3600, 0x5624,
1317		0x56cc, 0x56ec,
1318		0x56f4, 0x5720,
1319		0x5728, 0x575c,
1320		0x580c, 0x5814,
1321		0x5890, 0x589c,
1322		0x58a4, 0x58ac,
1323		0x58b8, 0x58bc,
1324		0x5940, 0x59c8,
1325		0x59d0, 0x59dc,
1326		0x59fc, 0x5a18,
1327		0x5a60, 0x5a70,
1328		0x5a80, 0x5a9c,
1329		0x5b94, 0x5bfc,
1330		0x6000, 0x6020,
1331		0x6028, 0x6040,
1332		0x6058, 0x609c,
1333		0x60a8, 0x614c,
1334		0x7700, 0x7798,
1335		0x77c0, 0x78fc,
1336		0x7b00, 0x7b58,
1337		0x7b60, 0x7b84,
1338		0x7b8c, 0x7c54,
1339		0x7d00, 0x7d38,
1340		0x7d40, 0x7d80,
1341		0x7d8c, 0x7ddc,
1342		0x7de4, 0x7e04,
1343		0x7e10, 0x7e1c,
1344		0x7e24, 0x7e38,
1345		0x7e40, 0x7e44,
1346		0x7e4c, 0x7e78,
1347		0x7e80, 0x7edc,
1348		0x7ee8, 0x7efc,
1349		0x8dc0, 0x8de0,
1350		0x8df8, 0x8e04,
1351		0x8e10, 0x8e84,
1352		0x8ea0, 0x8f84,
1353		0x8fc0, 0x9058,
1354		0x9060, 0x9060,
1355		0x9068, 0x90f8,
1356		0x9400, 0x9408,
1357		0x9410, 0x9470,
1358		0x9600, 0x9600,
1359		0x9608, 0x9638,
1360		0x9640, 0x96f4,
1361		0x9800, 0x9808,
1362		0x9810, 0x9864,
1363		0x9c00, 0x9c6c,
1364		0x9c80, 0x9cec,
1365		0x9d00, 0x9d6c,
1366		0x9d80, 0x9dec,
1367		0x9e00, 0x9e6c,
1368		0x9e80, 0x9eec,
1369		0x9f00, 0x9f6c,
1370		0x9f80, 0xa020,
1371		0xd000, 0xd004,
1372		0xd010, 0xd03c,
1373		0xdfc0, 0xdfe0,
1374		0xe000, 0x1106c,
1375		0x11074, 0x11088,
1376		0x1109c, 0x1117c,
1377		0x11190, 0x11204,
1378		0x19040, 0x1906c,
1379		0x19078, 0x19080,
1380		0x1908c, 0x190e8,
1381		0x190f0, 0x190f8,
1382		0x19100, 0x19110,
1383		0x19120, 0x19124,
1384		0x19150, 0x19194,
1385		0x1919c, 0x191b0,
1386		0x191d0, 0x191e8,
1387		0x19238, 0x19290,
1388		0x193f8, 0x19428,
1389		0x19430, 0x19444,
1390		0x1944c, 0x1946c,
1391		0x19474, 0x19474,
1392		0x19490, 0x194cc,
1393		0x194f0, 0x194f8,
1394		0x19c00, 0x19c08,
1395		0x19c10, 0x19c60,
1396		0x19c94, 0x19ce4,
1397		0x19cf0, 0x19d40,
1398		0x19d50, 0x19d94,
1399		0x19da0, 0x19de8,
1400		0x19df0, 0x19e10,
1401		0x19e50, 0x19e90,
1402		0x19ea0, 0x19f24,
1403		0x19f34, 0x19f34,
1404		0x19f40, 0x19f50,
1405		0x19f90, 0x19fb4,
1406		0x19fc4, 0x19fe4,
1407		0x1a000, 0x1a004,
1408		0x1a010, 0x1a06c,
1409		0x1a0b0, 0x1a0e4,
1410		0x1a0ec, 0x1a0f8,
1411		0x1a100, 0x1a108,
1412		0x1a114, 0x1a130,
1413		0x1a138, 0x1a1c4,
1414		0x1a1fc, 0x1a1fc,
1415		0x1e008, 0x1e00c,
1416		0x1e040, 0x1e044,
1417		0x1e04c, 0x1e04c,
1418		0x1e284, 0x1e290,
1419		0x1e2c0, 0x1e2c0,
1420		0x1e2e0, 0x1e2e0,
1421		0x1e300, 0x1e384,
1422		0x1e3c0, 0x1e3c8,
1423		0x1e408, 0x1e40c,
1424		0x1e440, 0x1e444,
1425		0x1e44c, 0x1e44c,
1426		0x1e684, 0x1e690,
1427		0x1e6c0, 0x1e6c0,
1428		0x1e6e0, 0x1e6e0,
1429		0x1e700, 0x1e784,
1430		0x1e7c0, 0x1e7c8,
1431		0x1e808, 0x1e80c,
1432		0x1e840, 0x1e844,
1433		0x1e84c, 0x1e84c,
1434		0x1ea84, 0x1ea90,
1435		0x1eac0, 0x1eac0,
1436		0x1eae0, 0x1eae0,
1437		0x1eb00, 0x1eb84,
1438		0x1ebc0, 0x1ebc8,
1439		0x1ec08, 0x1ec0c,
1440		0x1ec40, 0x1ec44,
1441		0x1ec4c, 0x1ec4c,
1442		0x1ee84, 0x1ee90,
1443		0x1eec0, 0x1eec0,
1444		0x1eee0, 0x1eee0,
1445		0x1ef00, 0x1ef84,
1446		0x1efc0, 0x1efc8,
1447		0x1f008, 0x1f00c,
1448		0x1f040, 0x1f044,
1449		0x1f04c, 0x1f04c,
1450		0x1f284, 0x1f290,
1451		0x1f2c0, 0x1f2c0,
1452		0x1f2e0, 0x1f2e0,
1453		0x1f300, 0x1f384,
1454		0x1f3c0, 0x1f3c8,
1455		0x1f408, 0x1f40c,
1456		0x1f440, 0x1f444,
1457		0x1f44c, 0x1f44c,
1458		0x1f684, 0x1f690,
1459		0x1f6c0, 0x1f6c0,
1460		0x1f6e0, 0x1f6e0,
1461		0x1f700, 0x1f784,
1462		0x1f7c0, 0x1f7c8,
1463		0x1f808, 0x1f80c,
1464		0x1f840, 0x1f844,
1465		0x1f84c, 0x1f84c,
1466		0x1fa84, 0x1fa90,
1467		0x1fac0, 0x1fac0,
1468		0x1fae0, 0x1fae0,
1469		0x1fb00, 0x1fb84,
1470		0x1fbc0, 0x1fbc8,
1471		0x1fc08, 0x1fc0c,
1472		0x1fc40, 0x1fc44,
1473		0x1fc4c, 0x1fc4c,
1474		0x1fe84, 0x1fe90,
1475		0x1fec0, 0x1fec0,
1476		0x1fee0, 0x1fee0,
1477		0x1ff00, 0x1ff84,
1478		0x1ffc0, 0x1ffc8,
1479		0x30000, 0x30030,
1480		0x30100, 0x30144,
1481		0x30190, 0x301a0,
1482		0x301a8, 0x301b8,
1483		0x301c4, 0x301c8,
1484		0x301d0, 0x301d0,
1485		0x30200, 0x30318,
1486		0x30400, 0x304b4,
1487		0x304c0, 0x3052c,
1488		0x30540, 0x3061c,
1489		0x30800, 0x30828,
1490		0x30834, 0x30834,
1491		0x308c0, 0x30908,
1492		0x30910, 0x309ac,
1493		0x30a00, 0x30a14,
1494		0x30a1c, 0x30a2c,
1495		0x30a44, 0x30a50,
1496		0x30a74, 0x30a74,
1497		0x30a7c, 0x30afc,
1498		0x30b08, 0x30c24,
1499		0x30d00, 0x30d00,
1500		0x30d08, 0x30d14,
1501		0x30d1c, 0x30d20,
1502		0x30d3c, 0x30d3c,
1503		0x30d48, 0x30d50,
1504		0x31200, 0x3120c,
1505		0x31220, 0x31220,
1506		0x31240, 0x31240,
1507		0x31600, 0x3160c,
1508		0x31a00, 0x31a1c,
1509		0x31e00, 0x31e20,
1510		0x31e38, 0x31e3c,
1511		0x31e80, 0x31e80,
1512		0x31e88, 0x31ea8,
1513		0x31eb0, 0x31eb4,
1514		0x31ec8, 0x31ed4,
1515		0x31fb8, 0x32004,
1516		0x32200, 0x32200,
1517		0x32208, 0x32240,
1518		0x32248, 0x32280,
1519		0x32288, 0x322c0,
1520		0x322c8, 0x322fc,
1521		0x32600, 0x32630,
1522		0x32a00, 0x32abc,
1523		0x32b00, 0x32b10,
1524		0x32b20, 0x32b30,
1525		0x32b40, 0x32b50,
1526		0x32b60, 0x32b70,
1527		0x33000, 0x33028,
1528		0x33030, 0x33048,
1529		0x33060, 0x33068,
1530		0x33070, 0x3309c,
1531		0x330f0, 0x33128,
1532		0x33130, 0x33148,
1533		0x33160, 0x33168,
1534		0x33170, 0x3319c,
1535		0x331f0, 0x33238,
1536		0x33240, 0x33240,
1537		0x33248, 0x33250,
1538		0x3325c, 0x33264,
1539		0x33270, 0x332b8,
1540		0x332c0, 0x332e4,
1541		0x332f8, 0x33338,
1542		0x33340, 0x33340,
1543		0x33348, 0x33350,
1544		0x3335c, 0x33364,
1545		0x33370, 0x333b8,
1546		0x333c0, 0x333e4,
1547		0x333f8, 0x33428,
1548		0x33430, 0x33448,
1549		0x33460, 0x33468,
1550		0x33470, 0x3349c,
1551		0x334f0, 0x33528,
1552		0x33530, 0x33548,
1553		0x33560, 0x33568,
1554		0x33570, 0x3359c,
1555		0x335f0, 0x33638,
1556		0x33640, 0x33640,
1557		0x33648, 0x33650,
1558		0x3365c, 0x33664,
1559		0x33670, 0x336b8,
1560		0x336c0, 0x336e4,
1561		0x336f8, 0x33738,
1562		0x33740, 0x33740,
1563		0x33748, 0x33750,
1564		0x3375c, 0x33764,
1565		0x33770, 0x337b8,
1566		0x337c0, 0x337e4,
1567		0x337f8, 0x337fc,
1568		0x33814, 0x33814,
1569		0x3382c, 0x3382c,
1570		0x33880, 0x3388c,
1571		0x338e8, 0x338ec,
1572		0x33900, 0x33928,
1573		0x33930, 0x33948,
1574		0x33960, 0x33968,
1575		0x33970, 0x3399c,
1576		0x339f0, 0x33a38,
1577		0x33a40, 0x33a40,
1578		0x33a48, 0x33a50,
1579		0x33a5c, 0x33a64,
1580		0x33a70, 0x33ab8,
1581		0x33ac0, 0x33ae4,
1582		0x33af8, 0x33b10,
1583		0x33b28, 0x33b28,
1584		0x33b3c, 0x33b50,
1585		0x33bf0, 0x33c10,
1586		0x33c28, 0x33c28,
1587		0x33c3c, 0x33c50,
1588		0x33cf0, 0x33cfc,
1589		0x34000, 0x34030,
1590		0x34100, 0x34144,
1591		0x34190, 0x341a0,
1592		0x341a8, 0x341b8,
1593		0x341c4, 0x341c8,
1594		0x341d0, 0x341d0,
1595		0x34200, 0x34318,
1596		0x34400, 0x344b4,
1597		0x344c0, 0x3452c,
1598		0x34540, 0x3461c,
1599		0x34800, 0x34828,
1600		0x34834, 0x34834,
1601		0x348c0, 0x34908,
1602		0x34910, 0x349ac,
1603		0x34a00, 0x34a14,
1604		0x34a1c, 0x34a2c,
1605		0x34a44, 0x34a50,
1606		0x34a74, 0x34a74,
1607		0x34a7c, 0x34afc,
1608		0x34b08, 0x34c24,
1609		0x34d00, 0x34d00,
1610		0x34d08, 0x34d14,
1611		0x34d1c, 0x34d20,
1612		0x34d3c, 0x34d3c,
1613		0x34d48, 0x34d50,
1614		0x35200, 0x3520c,
1615		0x35220, 0x35220,
1616		0x35240, 0x35240,
1617		0x35600, 0x3560c,
1618		0x35a00, 0x35a1c,
1619		0x35e00, 0x35e20,
1620		0x35e38, 0x35e3c,
1621		0x35e80, 0x35e80,
1622		0x35e88, 0x35ea8,
1623		0x35eb0, 0x35eb4,
1624		0x35ec8, 0x35ed4,
1625		0x35fb8, 0x36004,
1626		0x36200, 0x36200,
1627		0x36208, 0x36240,
1628		0x36248, 0x36280,
1629		0x36288, 0x362c0,
1630		0x362c8, 0x362fc,
1631		0x36600, 0x36630,
1632		0x36a00, 0x36abc,
1633		0x36b00, 0x36b10,
1634		0x36b20, 0x36b30,
1635		0x36b40, 0x36b50,
1636		0x36b60, 0x36b70,
1637		0x37000, 0x37028,
1638		0x37030, 0x37048,
1639		0x37060, 0x37068,
1640		0x37070, 0x3709c,
1641		0x370f0, 0x37128,
1642		0x37130, 0x37148,
1643		0x37160, 0x37168,
1644		0x37170, 0x3719c,
1645		0x371f0, 0x37238,
1646		0x37240, 0x37240,
1647		0x37248, 0x37250,
1648		0x3725c, 0x37264,
1649		0x37270, 0x372b8,
1650		0x372c0, 0x372e4,
1651		0x372f8, 0x37338,
1652		0x37340, 0x37340,
1653		0x37348, 0x37350,
1654		0x3735c, 0x37364,
1655		0x37370, 0x373b8,
1656		0x373c0, 0x373e4,
1657		0x373f8, 0x37428,
1658		0x37430, 0x37448,
1659		0x37460, 0x37468,
1660		0x37470, 0x3749c,
1661		0x374f0, 0x37528,
1662		0x37530, 0x37548,
1663		0x37560, 0x37568,
1664		0x37570, 0x3759c,
1665		0x375f0, 0x37638,
1666		0x37640, 0x37640,
1667		0x37648, 0x37650,
1668		0x3765c, 0x37664,
1669		0x37670, 0x376b8,
1670		0x376c0, 0x376e4,
1671		0x376f8, 0x37738,
1672		0x37740, 0x37740,
1673		0x37748, 0x37750,
1674		0x3775c, 0x37764,
1675		0x37770, 0x377b8,
1676		0x377c0, 0x377e4,
1677		0x377f8, 0x377fc,
1678		0x37814, 0x37814,
1679		0x3782c, 0x3782c,
1680		0x37880, 0x3788c,
1681		0x378e8, 0x378ec,
1682		0x37900, 0x37928,
1683		0x37930, 0x37948,
1684		0x37960, 0x37968,
1685		0x37970, 0x3799c,
1686		0x379f0, 0x37a38,
1687		0x37a40, 0x37a40,
1688		0x37a48, 0x37a50,
1689		0x37a5c, 0x37a64,
1690		0x37a70, 0x37ab8,
1691		0x37ac0, 0x37ae4,
1692		0x37af8, 0x37b10,
1693		0x37b28, 0x37b28,
1694		0x37b3c, 0x37b50,
1695		0x37bf0, 0x37c10,
1696		0x37c28, 0x37c28,
1697		0x37c3c, 0x37c50,
1698		0x37cf0, 0x37cfc,
1699		0x38000, 0x38030,
1700		0x38100, 0x38144,
1701		0x38190, 0x381a0,
1702		0x381a8, 0x381b8,
1703		0x381c4, 0x381c8,
1704		0x381d0, 0x381d0,
1705		0x38200, 0x38318,
1706		0x38400, 0x384b4,
1707		0x384c0, 0x3852c,
1708		0x38540, 0x3861c,
1709		0x38800, 0x38828,
1710		0x38834, 0x38834,
1711		0x388c0, 0x38908,
1712		0x38910, 0x389ac,
1713		0x38a00, 0x38a14,
1714		0x38a1c, 0x38a2c,
1715		0x38a44, 0x38a50,
1716		0x38a74, 0x38a74,
1717		0x38a7c, 0x38afc,
1718		0x38b08, 0x38c24,
1719		0x38d00, 0x38d00,
1720		0x38d08, 0x38d14,
1721		0x38d1c, 0x38d20,
1722		0x38d3c, 0x38d3c,
1723		0x38d48, 0x38d50,
1724		0x39200, 0x3920c,
1725		0x39220, 0x39220,
1726		0x39240, 0x39240,
1727		0x39600, 0x3960c,
1728		0x39a00, 0x39a1c,
1729		0x39e00, 0x39e20,
1730		0x39e38, 0x39e3c,
1731		0x39e80, 0x39e80,
1732		0x39e88, 0x39ea8,
1733		0x39eb0, 0x39eb4,
1734		0x39ec8, 0x39ed4,
1735		0x39fb8, 0x3a004,
1736		0x3a200, 0x3a200,
1737		0x3a208, 0x3a240,
1738		0x3a248, 0x3a280,
1739		0x3a288, 0x3a2c0,
1740		0x3a2c8, 0x3a2fc,
1741		0x3a600, 0x3a630,
1742		0x3aa00, 0x3aabc,
1743		0x3ab00, 0x3ab10,
1744		0x3ab20, 0x3ab30,
1745		0x3ab40, 0x3ab50,
1746		0x3ab60, 0x3ab70,
1747		0x3b000, 0x3b028,
1748		0x3b030, 0x3b048,
1749		0x3b060, 0x3b068,
1750		0x3b070, 0x3b09c,
1751		0x3b0f0, 0x3b128,
1752		0x3b130, 0x3b148,
1753		0x3b160, 0x3b168,
1754		0x3b170, 0x3b19c,
1755		0x3b1f0, 0x3b238,
1756		0x3b240, 0x3b240,
1757		0x3b248, 0x3b250,
1758		0x3b25c, 0x3b264,
1759		0x3b270, 0x3b2b8,
1760		0x3b2c0, 0x3b2e4,
1761		0x3b2f8, 0x3b338,
1762		0x3b340, 0x3b340,
1763		0x3b348, 0x3b350,
1764		0x3b35c, 0x3b364,
1765		0x3b370, 0x3b3b8,
1766		0x3b3c0, 0x3b3e4,
1767		0x3b3f8, 0x3b428,
1768		0x3b430, 0x3b448,
1769		0x3b460, 0x3b468,
1770		0x3b470, 0x3b49c,
1771		0x3b4f0, 0x3b528,
1772		0x3b530, 0x3b548,
1773		0x3b560, 0x3b568,
1774		0x3b570, 0x3b59c,
1775		0x3b5f0, 0x3b638,
1776		0x3b640, 0x3b640,
1777		0x3b648, 0x3b650,
1778		0x3b65c, 0x3b664,
1779		0x3b670, 0x3b6b8,
1780		0x3b6c0, 0x3b6e4,
1781		0x3b6f8, 0x3b738,
1782		0x3b740, 0x3b740,
1783		0x3b748, 0x3b750,
1784		0x3b75c, 0x3b764,
1785		0x3b770, 0x3b7b8,
1786		0x3b7c0, 0x3b7e4,
1787		0x3b7f8, 0x3b7fc,
1788		0x3b814, 0x3b814,
1789		0x3b82c, 0x3b82c,
1790		0x3b880, 0x3b88c,
1791		0x3b8e8, 0x3b8ec,
1792		0x3b900, 0x3b928,
1793		0x3b930, 0x3b948,
1794		0x3b960, 0x3b968,
1795		0x3b970, 0x3b99c,
1796		0x3b9f0, 0x3ba38,
1797		0x3ba40, 0x3ba40,
1798		0x3ba48, 0x3ba50,
1799		0x3ba5c, 0x3ba64,
1800		0x3ba70, 0x3bab8,
1801		0x3bac0, 0x3bae4,
1802		0x3baf8, 0x3bb10,
1803		0x3bb28, 0x3bb28,
1804		0x3bb3c, 0x3bb50,
1805		0x3bbf0, 0x3bc10,
1806		0x3bc28, 0x3bc28,
1807		0x3bc3c, 0x3bc50,
1808		0x3bcf0, 0x3bcfc,
1809		0x3c000, 0x3c030,
1810		0x3c100, 0x3c144,
1811		0x3c190, 0x3c1a0,
1812		0x3c1a8, 0x3c1b8,
1813		0x3c1c4, 0x3c1c8,
1814		0x3c1d0, 0x3c1d0,
1815		0x3c200, 0x3c318,
1816		0x3c400, 0x3c4b4,
1817		0x3c4c0, 0x3c52c,
1818		0x3c540, 0x3c61c,
1819		0x3c800, 0x3c828,
1820		0x3c834, 0x3c834,
1821		0x3c8c0, 0x3c908,
1822		0x3c910, 0x3c9ac,
1823		0x3ca00, 0x3ca14,
1824		0x3ca1c, 0x3ca2c,
1825		0x3ca44, 0x3ca50,
1826		0x3ca74, 0x3ca74,
1827		0x3ca7c, 0x3cafc,
1828		0x3cb08, 0x3cc24,
1829		0x3cd00, 0x3cd00,
1830		0x3cd08, 0x3cd14,
1831		0x3cd1c, 0x3cd20,
1832		0x3cd3c, 0x3cd3c,
1833		0x3cd48, 0x3cd50,
1834		0x3d200, 0x3d20c,
1835		0x3d220, 0x3d220,
1836		0x3d240, 0x3d240,
1837		0x3d600, 0x3d60c,
1838		0x3da00, 0x3da1c,
1839		0x3de00, 0x3de20,
1840		0x3de38, 0x3de3c,
1841		0x3de80, 0x3de80,
1842		0x3de88, 0x3dea8,
1843		0x3deb0, 0x3deb4,
1844		0x3dec8, 0x3ded4,
1845		0x3dfb8, 0x3e004,
1846		0x3e200, 0x3e200,
1847		0x3e208, 0x3e240,
1848		0x3e248, 0x3e280,
1849		0x3e288, 0x3e2c0,
1850		0x3e2c8, 0x3e2fc,
1851		0x3e600, 0x3e630,
1852		0x3ea00, 0x3eabc,
1853		0x3eb00, 0x3eb10,
1854		0x3eb20, 0x3eb30,
1855		0x3eb40, 0x3eb50,
1856		0x3eb60, 0x3eb70,
1857		0x3f000, 0x3f028,
1858		0x3f030, 0x3f048,
1859		0x3f060, 0x3f068,
1860		0x3f070, 0x3f09c,
1861		0x3f0f0, 0x3f128,
1862		0x3f130, 0x3f148,
1863		0x3f160, 0x3f168,
1864		0x3f170, 0x3f19c,
1865		0x3f1f0, 0x3f238,
1866		0x3f240, 0x3f240,
1867		0x3f248, 0x3f250,
1868		0x3f25c, 0x3f264,
1869		0x3f270, 0x3f2b8,
1870		0x3f2c0, 0x3f2e4,
1871		0x3f2f8, 0x3f338,
1872		0x3f340, 0x3f340,
1873		0x3f348, 0x3f350,
1874		0x3f35c, 0x3f364,
1875		0x3f370, 0x3f3b8,
1876		0x3f3c0, 0x3f3e4,
1877		0x3f3f8, 0x3f428,
1878		0x3f430, 0x3f448,
1879		0x3f460, 0x3f468,
1880		0x3f470, 0x3f49c,
1881		0x3f4f0, 0x3f528,
1882		0x3f530, 0x3f548,
1883		0x3f560, 0x3f568,
1884		0x3f570, 0x3f59c,
1885		0x3f5f0, 0x3f638,
1886		0x3f640, 0x3f640,
1887		0x3f648, 0x3f650,
1888		0x3f65c, 0x3f664,
1889		0x3f670, 0x3f6b8,
1890		0x3f6c0, 0x3f6e4,
1891		0x3f6f8, 0x3f738,
1892		0x3f740, 0x3f740,
1893		0x3f748, 0x3f750,
1894		0x3f75c, 0x3f764,
1895		0x3f770, 0x3f7b8,
1896		0x3f7c0, 0x3f7e4,
1897		0x3f7f8, 0x3f7fc,
1898		0x3f814, 0x3f814,
1899		0x3f82c, 0x3f82c,
1900		0x3f880, 0x3f88c,
1901		0x3f8e8, 0x3f8ec,
1902		0x3f900, 0x3f928,
1903		0x3f930, 0x3f948,
1904		0x3f960, 0x3f968,
1905		0x3f970, 0x3f99c,
1906		0x3f9f0, 0x3fa38,
1907		0x3fa40, 0x3fa40,
1908		0x3fa48, 0x3fa50,
1909		0x3fa5c, 0x3fa64,
1910		0x3fa70, 0x3fab8,
1911		0x3fac0, 0x3fae4,
1912		0x3faf8, 0x3fb10,
1913		0x3fb28, 0x3fb28,
1914		0x3fb3c, 0x3fb50,
1915		0x3fbf0, 0x3fc10,
1916		0x3fc28, 0x3fc28,
1917		0x3fc3c, 0x3fc50,
1918		0x3fcf0, 0x3fcfc,
1919		0x40000, 0x4000c,
1920		0x40040, 0x40050,
1921		0x40060, 0x40068,
1922		0x4007c, 0x4008c,
1923		0x40094, 0x400b0,
1924		0x400c0, 0x40144,
1925		0x40180, 0x4018c,
1926		0x40200, 0x40254,
1927		0x40260, 0x40264,
1928		0x40270, 0x40288,
1929		0x40290, 0x40298,
1930		0x402ac, 0x402c8,
1931		0x402d0, 0x402e0,
1932		0x402f0, 0x402f0,
1933		0x40300, 0x4033c,
1934		0x403f8, 0x403fc,
1935		0x41304, 0x413c4,
1936		0x41400, 0x4140c,
1937		0x41414, 0x4141c,
1938		0x41480, 0x414d0,
1939		0x44000, 0x44054,
1940		0x4405c, 0x44078,
1941		0x440c0, 0x44174,
1942		0x44180, 0x441ac,
1943		0x441b4, 0x441b8,
1944		0x441c0, 0x44254,
1945		0x4425c, 0x44278,
1946		0x442c0, 0x44374,
1947		0x44380, 0x443ac,
1948		0x443b4, 0x443b8,
1949		0x443c0, 0x44454,
1950		0x4445c, 0x44478,
1951		0x444c0, 0x44574,
1952		0x44580, 0x445ac,
1953		0x445b4, 0x445b8,
1954		0x445c0, 0x44654,
1955		0x4465c, 0x44678,
1956		0x446c0, 0x44774,
1957		0x44780, 0x447ac,
1958		0x447b4, 0x447b8,
1959		0x447c0, 0x44854,
1960		0x4485c, 0x44878,
1961		0x448c0, 0x44974,
1962		0x44980, 0x449ac,
1963		0x449b4, 0x449b8,
1964		0x449c0, 0x449fc,
1965		0x45000, 0x45004,
1966		0x45010, 0x45030,
1967		0x45040, 0x45060,
1968		0x45068, 0x45068,
1969		0x45080, 0x45084,
1970		0x450a0, 0x450b0,
1971		0x45200, 0x45204,
1972		0x45210, 0x45230,
1973		0x45240, 0x45260,
1974		0x45268, 0x45268,
1975		0x45280, 0x45284,
1976		0x452a0, 0x452b0,
1977		0x460c0, 0x460e4,
1978		0x47000, 0x4703c,
1979		0x47044, 0x4708c,
1980		0x47200, 0x47250,
1981		0x47400, 0x47408,
1982		0x47414, 0x47420,
1983		0x47600, 0x47618,
1984		0x47800, 0x47814,
1985		0x48000, 0x4800c,
1986		0x48040, 0x48050,
1987		0x48060, 0x48068,
1988		0x4807c, 0x4808c,
1989		0x48094, 0x480b0,
1990		0x480c0, 0x48144,
1991		0x48180, 0x4818c,
1992		0x48200, 0x48254,
1993		0x48260, 0x48264,
1994		0x48270, 0x48288,
1995		0x48290, 0x48298,
1996		0x482ac, 0x482c8,
1997		0x482d0, 0x482e0,
1998		0x482f0, 0x482f0,
1999		0x48300, 0x4833c,
2000		0x483f8, 0x483fc,
2001		0x49304, 0x493c4,
2002		0x49400, 0x4940c,
2003		0x49414, 0x4941c,
2004		0x49480, 0x494d0,
2005		0x4c000, 0x4c054,
2006		0x4c05c, 0x4c078,
2007		0x4c0c0, 0x4c174,
2008		0x4c180, 0x4c1ac,
2009		0x4c1b4, 0x4c1b8,
2010		0x4c1c0, 0x4c254,
2011		0x4c25c, 0x4c278,
2012		0x4c2c0, 0x4c374,
2013		0x4c380, 0x4c3ac,
2014		0x4c3b4, 0x4c3b8,
2015		0x4c3c0, 0x4c454,
2016		0x4c45c, 0x4c478,
2017		0x4c4c0, 0x4c574,
2018		0x4c580, 0x4c5ac,
2019		0x4c5b4, 0x4c5b8,
2020		0x4c5c0, 0x4c654,
2021		0x4c65c, 0x4c678,
2022		0x4c6c0, 0x4c774,
2023		0x4c780, 0x4c7ac,
2024		0x4c7b4, 0x4c7b8,
2025		0x4c7c0, 0x4c854,
2026		0x4c85c, 0x4c878,
2027		0x4c8c0, 0x4c974,
2028		0x4c980, 0x4c9ac,
2029		0x4c9b4, 0x4c9b8,
2030		0x4c9c0, 0x4c9fc,
2031		0x4d000, 0x4d004,
2032		0x4d010, 0x4d030,
2033		0x4d040, 0x4d060,
2034		0x4d068, 0x4d068,
2035		0x4d080, 0x4d084,
2036		0x4d0a0, 0x4d0b0,
2037		0x4d200, 0x4d204,
2038		0x4d210, 0x4d230,
2039		0x4d240, 0x4d260,
2040		0x4d268, 0x4d268,
2041		0x4d280, 0x4d284,
2042		0x4d2a0, 0x4d2b0,
2043		0x4e0c0, 0x4e0e4,
2044		0x4f000, 0x4f03c,
2045		0x4f044, 0x4f08c,
2046		0x4f200, 0x4f250,
2047		0x4f400, 0x4f408,
2048		0x4f414, 0x4f420,
2049		0x4f600, 0x4f618,
2050		0x4f800, 0x4f814,
2051		0x50000, 0x50084,
2052		0x50090, 0x500cc,
2053		0x50400, 0x50400,
2054		0x50800, 0x50884,
2055		0x50890, 0x508cc,
2056		0x50c00, 0x50c00,
2057		0x51000, 0x5101c,
2058		0x51300, 0x51308,
2059	};
2060
2061	static const unsigned int t5vf_reg_ranges[] = {
2062		VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS),
2063		VF_MPS_REG(A_MPS_VF_CTL),
2064		VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H),
2065		VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_REVISION),
2066		VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL),
2067		VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS),
2068		FW_T4VF_MBDATA_BASE_ADDR,
2069		FW_T4VF_MBDATA_BASE_ADDR +
2070		((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4),
2071	};
2072
2073	static const unsigned int t6_reg_ranges[] = {
2074		0x1008, 0x101c,
2075		0x1024, 0x10a8,
2076		0x10b4, 0x10f8,
2077		0x1100, 0x1114,
2078		0x111c, 0x112c,
2079		0x1138, 0x113c,
2080		0x1144, 0x114c,
2081		0x1180, 0x1184,
2082		0x1190, 0x1194,
2083		0x11a0, 0x11a4,
2084		0x11b0, 0x11c4,
2085		0x11fc, 0x1274,
2086		0x1280, 0x133c,
2087		0x1800, 0x18fc,
2088		0x3000, 0x302c,
2089		0x3060, 0x30b0,
2090		0x30b8, 0x30d8,
2091		0x30e0, 0x30fc,
2092		0x3140, 0x357c,
2093		0x35a8, 0x35cc,
2094		0x35ec, 0x35ec,
2095		0x3600, 0x5624,
2096		0x56cc, 0x56ec,
2097		0x56f4, 0x5720,
2098		0x5728, 0x575c,
2099		0x580c, 0x5814,
2100		0x5890, 0x589c,
2101		0x58a4, 0x58ac,
2102		0x58b8, 0x58bc,
2103		0x5940, 0x595c,
2104		0x5980, 0x598c,
2105		0x59b0, 0x59c8,
2106		0x59d0, 0x59dc,
2107		0x59fc, 0x5a18,
2108		0x5a60, 0x5a6c,
2109		0x5a80, 0x5a8c,
2110		0x5a94, 0x5a9c,
2111		0x5b94, 0x5bfc,
2112		0x5c10, 0x5e48,
2113		0x5e50, 0x5e94,
2114		0x5ea0, 0x5eb0,
2115		0x5ec0, 0x5ec0,
2116		0x5ec8, 0x5ed0,
2117		0x5ee0, 0x5ee0,
2118		0x5ef0, 0x5ef0,
2119		0x5f00, 0x5f00,
2120		0x6000, 0x6020,
2121		0x6028, 0x6040,
2122		0x6058, 0x609c,
2123		0x60a8, 0x619c,
2124		0x7700, 0x7798,
2125		0x77c0, 0x7880,
2126		0x78cc, 0x78fc,
2127		0x7b00, 0x7b58,
2128		0x7b60, 0x7b84,
2129		0x7b8c, 0x7c54,
2130		0x7d00, 0x7d38,
2131		0x7d40, 0x7d84,
2132		0x7d8c, 0x7ddc,
2133		0x7de4, 0x7e04,
2134		0x7e10, 0x7e1c,
2135		0x7e24, 0x7e38,
2136		0x7e40, 0x7e44,
2137		0x7e4c, 0x7e78,
2138		0x7e80, 0x7edc,
2139		0x7ee8, 0x7efc,
2140		0x8dc0, 0x8de0,
2141		0x8df8, 0x8e04,
2142		0x8e10, 0x8e84,
2143		0x8ea0, 0x8f88,
2144		0x8fb8, 0x9058,
2145		0x9060, 0x9060,
2146		0x9068, 0x90f8,
2147		0x9100, 0x9124,
2148		0x9400, 0x9470,
2149		0x9600, 0x9600,
2150		0x9608, 0x9638,
2151		0x9640, 0x9704,
2152		0x9710, 0x971c,
2153		0x9800, 0x9808,
2154		0x9810, 0x9864,
2155		0x9c00, 0x9c6c,
2156		0x9c80, 0x9cec,
2157		0x9d00, 0x9d6c,
2158		0x9d80, 0x9dec,
2159		0x9e00, 0x9e6c,
2160		0x9e80, 0x9eec,
2161		0x9f00, 0x9f6c,
2162		0x9f80, 0xa020,
2163		0xd000, 0xd03c,
2164		0xd100, 0xd118,
2165		0xd200, 0xd214,
2166		0xd220, 0xd234,
2167		0xd240, 0xd254,
2168		0xd260, 0xd274,
2169		0xd280, 0xd294,
2170		0xd2a0, 0xd2b4,
2171		0xd2c0, 0xd2d4,
2172		0xd2e0, 0xd2f4,
2173		0xd300, 0xd31c,
2174		0xdfc0, 0xdfe0,
2175		0xe000, 0xf008,
2176		0xf010, 0xf018,
2177		0xf020, 0xf028,
2178		0x11000, 0x11014,
2179		0x11048, 0x1106c,
2180		0x11074, 0x11088,
2181		0x11098, 0x11120,
2182		0x1112c, 0x1117c,
2183		0x11190, 0x112e0,
2184		0x11300, 0x1130c,
2185		0x12000, 0x1206c,
2186		0x19040, 0x1906c,
2187		0x19078, 0x19080,
2188		0x1908c, 0x190e8,
2189		0x190f0, 0x190f8,
2190		0x19100, 0x19110,
2191		0x19120, 0x19124,
2192		0x19150, 0x19194,
2193		0x1919c, 0x191b0,
2194		0x191d0, 0x191e8,
2195		0x19238, 0x19290,
2196		0x192a4, 0x192b0,
2197		0x19348, 0x1934c,
2198		0x193f8, 0x19418,
2199		0x19420, 0x19428,
2200		0x19430, 0x19444,
2201		0x1944c, 0x1946c,
2202		0x19474, 0x19474,
2203		0x19490, 0x194cc,
2204		0x194f0, 0x194f8,
2205		0x19c00, 0x19c48,
2206		0x19c50, 0x19c80,
2207		0x19c94, 0x19c98,
2208		0x19ca0, 0x19cbc,
2209		0x19ce4, 0x19ce4,
2210		0x19cf0, 0x19cf8,
2211		0x19d00, 0x19d28,
2212		0x19d50, 0x19d78,
2213		0x19d94, 0x19d98,
2214		0x19da0, 0x19de0,
2215		0x19df0, 0x19e10,
2216		0x19e50, 0x19e6c,
2217		0x19ea0, 0x19ebc,
2218		0x19ec4, 0x19ef4,
2219		0x19f04, 0x19f2c,
2220		0x19f34, 0x19f34,
2221		0x19f40, 0x19f50,
2222		0x19f90, 0x19fac,
2223		0x19fc4, 0x19fc8,
2224		0x19fd0, 0x19fe4,
2225		0x1a000, 0x1a004,
2226		0x1a010, 0x1a06c,
2227		0x1a0b0, 0x1a0e4,
2228		0x1a0ec, 0x1a0f8,
2229		0x1a100, 0x1a108,
2230		0x1a114, 0x1a130,
2231		0x1a138, 0x1a1c4,
2232		0x1a1fc, 0x1a1fc,
2233		0x1e008, 0x1e00c,
2234		0x1e040, 0x1e044,
2235		0x1e04c, 0x1e04c,
2236		0x1e284, 0x1e290,
2237		0x1e2c0, 0x1e2c0,
2238		0x1e2e0, 0x1e2e0,
2239		0x1e300, 0x1e384,
2240		0x1e3c0, 0x1e3c8,
2241		0x1e408, 0x1e40c,
2242		0x1e440, 0x1e444,
2243		0x1e44c, 0x1e44c,
2244		0x1e684, 0x1e690,
2245		0x1e6c0, 0x1e6c0,
2246		0x1e6e0, 0x1e6e0,
2247		0x1e700, 0x1e784,
2248		0x1e7c0, 0x1e7c8,
2249		0x1e808, 0x1e80c,
2250		0x1e840, 0x1e844,
2251		0x1e84c, 0x1e84c,
2252		0x1ea84, 0x1ea90,
2253		0x1eac0, 0x1eac0,
2254		0x1eae0, 0x1eae0,
2255		0x1eb00, 0x1eb84,
2256		0x1ebc0, 0x1ebc8,
2257		0x1ec08, 0x1ec0c,
2258		0x1ec40, 0x1ec44,
2259		0x1ec4c, 0x1ec4c,
2260		0x1ee84, 0x1ee90,
2261		0x1eec0, 0x1eec0,
2262		0x1eee0, 0x1eee0,
2263		0x1ef00, 0x1ef84,
2264		0x1efc0, 0x1efc8,
2265		0x1f008, 0x1f00c,
2266		0x1f040, 0x1f044,
2267		0x1f04c, 0x1f04c,
2268		0x1f284, 0x1f290,
2269		0x1f2c0, 0x1f2c0,
2270		0x1f2e0, 0x1f2e0,
2271		0x1f300, 0x1f384,
2272		0x1f3c0, 0x1f3c8,
2273		0x1f408, 0x1f40c,
2274		0x1f440, 0x1f444,
2275		0x1f44c, 0x1f44c,
2276		0x1f684, 0x1f690,
2277		0x1f6c0, 0x1f6c0,
2278		0x1f6e0, 0x1f6e0,
2279		0x1f700, 0x1f784,
2280		0x1f7c0, 0x1f7c8,
2281		0x1f808, 0x1f80c,
2282		0x1f840, 0x1f844,
2283		0x1f84c, 0x1f84c,
2284		0x1fa84, 0x1fa90,
2285		0x1fac0, 0x1fac0,
2286		0x1fae0, 0x1fae0,
2287		0x1fb00, 0x1fb84,
2288		0x1fbc0, 0x1fbc8,
2289		0x1fc08, 0x1fc0c,
2290		0x1fc40, 0x1fc44,
2291		0x1fc4c, 0x1fc4c,
2292		0x1fe84, 0x1fe90,
2293		0x1fec0, 0x1fec0,
2294		0x1fee0, 0x1fee0,
2295		0x1ff00, 0x1ff84,
2296		0x1ffc0, 0x1ffc8,
2297		0x30000, 0x30030,
2298		0x30100, 0x30168,
2299		0x30190, 0x301a0,
2300		0x301a8, 0x301b8,
2301		0x301c4, 0x301c8,
2302		0x301d0, 0x301d0,
2303		0x30200, 0x30320,
2304		0x30400, 0x304b4,
2305		0x304c0, 0x3052c,
2306		0x30540, 0x3061c,
2307		0x30800, 0x308a0,
2308		0x308c0, 0x30908,
2309		0x30910, 0x309b8,
2310		0x30a00, 0x30a04,
2311		0x30a0c, 0x30a14,
2312		0x30a1c, 0x30a2c,
2313		0x30a44, 0x30a50,
2314		0x30a74, 0x30a74,
2315		0x30a7c, 0x30afc,
2316		0x30b08, 0x30c24,
2317		0x30d00, 0x30d14,
2318		0x30d1c, 0x30d3c,
2319		0x30d44, 0x30d4c,
2320		0x30d54, 0x30d74,
2321		0x30d7c, 0x30d7c,
2322		0x30de0, 0x30de0,
2323		0x30e00, 0x30ed4,
2324		0x30f00, 0x30fa4,
2325		0x30fc0, 0x30fc4,
2326		0x31000, 0x31004,
2327		0x31080, 0x310fc,
2328		0x31208, 0x31220,
2329		0x3123c, 0x31254,
2330		0x31300, 0x31300,
2331		0x31308, 0x3131c,
2332		0x31338, 0x3133c,
2333		0x31380, 0x31380,
2334		0x31388, 0x313a8,
2335		0x313b4, 0x313b4,
2336		0x31400, 0x31420,
2337		0x31438, 0x3143c,
2338		0x31480, 0x31480,
2339		0x314a8, 0x314a8,
2340		0x314b0, 0x314b4,
2341		0x314c8, 0x314d4,
2342		0x31a40, 0x31a4c,
2343		0x31af0, 0x31b20,
2344		0x31b38, 0x31b3c,
2345		0x31b80, 0x31b80,
2346		0x31ba8, 0x31ba8,
2347		0x31bb0, 0x31bb4,
2348		0x31bc8, 0x31bd4,
2349		0x32140, 0x3218c,
2350		0x321f0, 0x321f4,
2351		0x32200, 0x32200,
2352		0x32218, 0x32218,
2353		0x32400, 0x32400,
2354		0x32408, 0x3241c,
2355		0x32618, 0x32620,
2356		0x32664, 0x32664,
2357		0x326a8, 0x326a8,
2358		0x326ec, 0x326ec,
2359		0x32a00, 0x32abc,
2360		0x32b00, 0x32b18,
2361		0x32b20, 0x32b38,
2362		0x32b40, 0x32b58,
2363		0x32b60, 0x32b78,
2364		0x32c00, 0x32c00,
2365		0x32c08, 0x32c3c,
2366		0x33000, 0x3302c,
2367		0x33034, 0x33050,
2368		0x33058, 0x33058,
2369		0x33060, 0x3308c,
2370		0x3309c, 0x330ac,
2371		0x330c0, 0x330c0,
2372		0x330c8, 0x330d0,
2373		0x330d8, 0x330e0,
2374		0x330ec, 0x3312c,
2375		0x33134, 0x33150,
2376		0x33158, 0x33158,
2377		0x33160, 0x3318c,
2378		0x3319c, 0x331ac,
2379		0x331c0, 0x331c0,
2380		0x331c8, 0x331d0,
2381		0x331d8, 0x331e0,
2382		0x331ec, 0x33290,
2383		0x33298, 0x332c4,
2384		0x332e4, 0x33390,
2385		0x33398, 0x333c4,
2386		0x333e4, 0x3342c,
2387		0x33434, 0x33450,
2388		0x33458, 0x33458,
2389		0x33460, 0x3348c,
2390		0x3349c, 0x334ac,
2391		0x334c0, 0x334c0,
2392		0x334c8, 0x334d0,
2393		0x334d8, 0x334e0,
2394		0x334ec, 0x3352c,
2395		0x33534, 0x33550,
2396		0x33558, 0x33558,
2397		0x33560, 0x3358c,
2398		0x3359c, 0x335ac,
2399		0x335c0, 0x335c0,
2400		0x335c8, 0x335d0,
2401		0x335d8, 0x335e0,
2402		0x335ec, 0x33690,
2403		0x33698, 0x336c4,
2404		0x336e4, 0x33790,
2405		0x33798, 0x337c4,
2406		0x337e4, 0x337fc,
2407		0x33814, 0x33814,
2408		0x33854, 0x33868,
2409		0x33880, 0x3388c,
2410		0x338c0, 0x338d0,
2411		0x338e8, 0x338ec,
2412		0x33900, 0x3392c,
2413		0x33934, 0x33950,
2414		0x33958, 0x33958,
2415		0x33960, 0x3398c,
2416		0x3399c, 0x339ac,
2417		0x339c0, 0x339c0,
2418		0x339c8, 0x339d0,
2419		0x339d8, 0x339e0,
2420		0x339ec, 0x33a90,
2421		0x33a98, 0x33ac4,
2422		0x33ae4, 0x33b10,
2423		0x33b24, 0x33b28,
2424		0x33b38, 0x33b50,
2425		0x33bf0, 0x33c10,
2426		0x33c24, 0x33c28,
2427		0x33c38, 0x33c50,
2428		0x33cf0, 0x33cfc,
2429		0x34000, 0x34030,
2430		0x34100, 0x34168,
2431		0x34190, 0x341a0,
2432		0x341a8, 0x341b8,
2433		0x341c4, 0x341c8,
2434		0x341d0, 0x341d0,
2435		0x34200, 0x34320,
2436		0x34400, 0x344b4,
2437		0x344c0, 0x3452c,
2438		0x34540, 0x3461c,
2439		0x34800, 0x348a0,
2440		0x348c0, 0x34908,
2441		0x34910, 0x349b8,
2442		0x34a00, 0x34a04,
2443		0x34a0c, 0x34a14,
2444		0x34a1c, 0x34a2c,
2445		0x34a44, 0x34a50,
2446		0x34a74, 0x34a74,
2447		0x34a7c, 0x34afc,
2448		0x34b08, 0x34c24,
2449		0x34d00, 0x34d14,
2450		0x34d1c, 0x34d3c,
2451		0x34d44, 0x34d4c,
2452		0x34d54, 0x34d74,
2453		0x34d7c, 0x34d7c,
2454		0x34de0, 0x34de0,
2455		0x34e00, 0x34ed4,
2456		0x34f00, 0x34fa4,
2457		0x34fc0, 0x34fc4,
2458		0x35000, 0x35004,
2459		0x35080, 0x350fc,
2460		0x35208, 0x35220,
2461		0x3523c, 0x35254,
2462		0x35300, 0x35300,
2463		0x35308, 0x3531c,
2464		0x35338, 0x3533c,
2465		0x35380, 0x35380,
2466		0x35388, 0x353a8,
2467		0x353b4, 0x353b4,
2468		0x35400, 0x35420,
2469		0x35438, 0x3543c,
2470		0x35480, 0x35480,
2471		0x354a8, 0x354a8,
2472		0x354b0, 0x354b4,
2473		0x354c8, 0x354d4,
2474		0x35a40, 0x35a4c,
2475		0x35af0, 0x35b20,
2476		0x35b38, 0x35b3c,
2477		0x35b80, 0x35b80,
2478		0x35ba8, 0x35ba8,
2479		0x35bb0, 0x35bb4,
2480		0x35bc8, 0x35bd4,
2481		0x36140, 0x3618c,
2482		0x361f0, 0x361f4,
2483		0x36200, 0x36200,
2484		0x36218, 0x36218,
2485		0x36400, 0x36400,
2486		0x36408, 0x3641c,
2487		0x36618, 0x36620,
2488		0x36664, 0x36664,
2489		0x366a8, 0x366a8,
2490		0x366ec, 0x366ec,
2491		0x36a00, 0x36abc,
2492		0x36b00, 0x36b18,
2493		0x36b20, 0x36b38,
2494		0x36b40, 0x36b58,
2495		0x36b60, 0x36b78,
2496		0x36c00, 0x36c00,
2497		0x36c08, 0x36c3c,
2498		0x37000, 0x3702c,
2499		0x37034, 0x37050,
2500		0x37058, 0x37058,
2501		0x37060, 0x3708c,
2502		0x3709c, 0x370ac,
2503		0x370c0, 0x370c0,
2504		0x370c8, 0x370d0,
2505		0x370d8, 0x370e0,
2506		0x370ec, 0x3712c,
2507		0x37134, 0x37150,
2508		0x37158, 0x37158,
2509		0x37160, 0x3718c,
2510		0x3719c, 0x371ac,
2511		0x371c0, 0x371c0,
2512		0x371c8, 0x371d0,
2513		0x371d8, 0x371e0,
2514		0x371ec, 0x37290,
2515		0x37298, 0x372c4,
2516		0x372e4, 0x37390,
2517		0x37398, 0x373c4,
2518		0x373e4, 0x3742c,
2519		0x37434, 0x37450,
2520		0x37458, 0x37458,
2521		0x37460, 0x3748c,
2522		0x3749c, 0x374ac,
2523		0x374c0, 0x374c0,
2524		0x374c8, 0x374d0,
2525		0x374d8, 0x374e0,
2526		0x374ec, 0x3752c,
2527		0x37534, 0x37550,
2528		0x37558, 0x37558,
2529		0x37560, 0x3758c,
2530		0x3759c, 0x375ac,
2531		0x375c0, 0x375c0,
2532		0x375c8, 0x375d0,
2533		0x375d8, 0x375e0,
2534		0x375ec, 0x37690,
2535		0x37698, 0x376c4,
2536		0x376e4, 0x37790,
2537		0x37798, 0x377c4,
2538		0x377e4, 0x377fc,
2539		0x37814, 0x37814,
2540		0x37854, 0x37868,
2541		0x37880, 0x3788c,
2542		0x378c0, 0x378d0,
2543		0x378e8, 0x378ec,
2544		0x37900, 0x3792c,
2545		0x37934, 0x37950,
2546		0x37958, 0x37958,
2547		0x37960, 0x3798c,
2548		0x3799c, 0x379ac,
2549		0x379c0, 0x379c0,
2550		0x379c8, 0x379d0,
2551		0x379d8, 0x379e0,
2552		0x379ec, 0x37a90,
2553		0x37a98, 0x37ac4,
2554		0x37ae4, 0x37b10,
2555		0x37b24, 0x37b28,
2556		0x37b38, 0x37b50,
2557		0x37bf0, 0x37c10,
2558		0x37c24, 0x37c28,
2559		0x37c38, 0x37c50,
2560		0x37cf0, 0x37cfc,
2561		0x40040, 0x40040,
2562		0x40080, 0x40084,
2563		0x40100, 0x40100,
2564		0x40140, 0x401bc,
2565		0x40200, 0x40214,
2566		0x40228, 0x40228,
2567		0x40240, 0x40258,
2568		0x40280, 0x40280,
2569		0x40304, 0x40304,
2570		0x40330, 0x4033c,
2571		0x41304, 0x413c8,
2572		0x413d0, 0x413dc,
2573		0x413f0, 0x413f0,
2574		0x41400, 0x4140c,
2575		0x41414, 0x4141c,
2576		0x41480, 0x414d0,
2577		0x44000, 0x4407c,
2578		0x440c0, 0x441ac,
2579		0x441b4, 0x4427c,
2580		0x442c0, 0x443ac,
2581		0x443b4, 0x4447c,
2582		0x444c0, 0x445ac,
2583		0x445b4, 0x4467c,
2584		0x446c0, 0x447ac,
2585		0x447b4, 0x4487c,
2586		0x448c0, 0x449ac,
2587		0x449b4, 0x44a7c,
2588		0x44ac0, 0x44bac,
2589		0x44bb4, 0x44c7c,
2590		0x44cc0, 0x44dac,
2591		0x44db4, 0x44e7c,
2592		0x44ec0, 0x44fac,
2593		0x44fb4, 0x4507c,
2594		0x450c0, 0x451ac,
2595		0x451b4, 0x451fc,
2596		0x45800, 0x45804,
2597		0x45810, 0x45830,
2598		0x45840, 0x45860,
2599		0x45868, 0x45868,
2600		0x45880, 0x45884,
2601		0x458a0, 0x458b0,
2602		0x45a00, 0x45a04,
2603		0x45a10, 0x45a30,
2604		0x45a40, 0x45a60,
2605		0x45a68, 0x45a68,
2606		0x45a80, 0x45a84,
2607		0x45aa0, 0x45ab0,
2608		0x460c0, 0x460e4,
2609		0x47000, 0x4703c,
2610		0x47044, 0x4708c,
2611		0x47200, 0x47250,
2612		0x47400, 0x47408,
2613		0x47414, 0x47420,
2614		0x47600, 0x47618,
2615		0x47800, 0x47814,
2616		0x47820, 0x4782c,
2617		0x50000, 0x50084,
2618		0x50090, 0x500cc,
2619		0x50300, 0x50384,
2620		0x50400, 0x50400,
2621		0x50800, 0x50884,
2622		0x50890, 0x508cc,
2623		0x50b00, 0x50b84,
2624		0x50c00, 0x50c00,
2625		0x51000, 0x51020,
2626		0x51028, 0x510b0,
2627		0x51300, 0x51324,
2628	};
2629
2630	static const unsigned int t6vf_reg_ranges[] = {
2631		VF_SGE_REG(A_SGE_VF_KDOORBELL), VF_SGE_REG(A_SGE_VF_GTS),
2632		VF_MPS_REG(A_MPS_VF_CTL),
2633		VF_MPS_REG(A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H),
2634		VF_PL_REG(A_PL_VF_WHOAMI), VF_PL_REG(A_PL_VF_REVISION),
2635		VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_CTRL),
2636		VF_CIM_REG(A_CIM_VF_EXT_MAILBOX_STATUS),
2637		FW_T6VF_MBDATA_BASE_ADDR,
2638		FW_T6VF_MBDATA_BASE_ADDR +
2639		((NUM_CIM_PF_MAILBOX_DATA_INSTANCES - 1) * 4),
2640	};
2641
2642	u32 *buf_end = (u32 *)(buf + buf_size);
2643	const unsigned int *reg_ranges;
2644	int reg_ranges_size, range;
2645	unsigned int chip_version = chip_id(adap);
2646
2647	/*
2648	 * Select the right set of register ranges to dump depending on the
2649	 * adapter chip type.
2650	 */
2651	switch (chip_version) {
2652	case CHELSIO_T4:
2653		if (adap->flags & IS_VF) {
2654			reg_ranges = t4vf_reg_ranges;
2655			reg_ranges_size = ARRAY_SIZE(t4vf_reg_ranges);
2656		} else {
2657			reg_ranges = t4_reg_ranges;
2658			reg_ranges_size = ARRAY_SIZE(t4_reg_ranges);
2659		}
2660		break;
2661
2662	case CHELSIO_T5:
2663		if (adap->flags & IS_VF) {
2664			reg_ranges = t5vf_reg_ranges;
2665			reg_ranges_size = ARRAY_SIZE(t5vf_reg_ranges);
2666		} else {
2667			reg_ranges = t5_reg_ranges;
2668			reg_ranges_size = ARRAY_SIZE(t5_reg_ranges);
2669		}
2670		break;
2671
2672	case CHELSIO_T6:
2673		if (adap->flags & IS_VF) {
2674			reg_ranges = t6vf_reg_ranges;
2675			reg_ranges_size = ARRAY_SIZE(t6vf_reg_ranges);
2676		} else {
2677			reg_ranges = t6_reg_ranges;
2678			reg_ranges_size = ARRAY_SIZE(t6_reg_ranges);
2679		}
2680		break;
2681
2682	default:
2683		CH_ERR(adap,
2684			"Unsupported chip version %d\n", chip_version);
2685		return;
2686	}
2687
2688	/*
2689	 * Clear the register buffer and insert the appropriate register
2690	 * values selected by the above register ranges.
2691	 */
2692	memset(buf, 0, buf_size);
2693	for (range = 0; range < reg_ranges_size; range += 2) {
2694		unsigned int reg = reg_ranges[range];
2695		unsigned int last_reg = reg_ranges[range + 1];
2696		u32 *bufp = (u32 *)(buf + reg);
2697
2698		/*
2699		 * Iterate across the register range filling in the register
2700		 * buffer but don't write past the end of the register buffer.
2701		 */
2702		while (reg <= last_reg && bufp < buf_end) {
2703			*bufp++ = t4_read_reg(adap, reg);
2704			reg += sizeof(u32);
2705		}
2706	}
2707}
2708
2709/*
2710 * Partial EEPROM Vital Product Data structure.  The VPD starts with one ID
2711 * header followed by one or more VPD-R sections, each with its own header.
2712 */
2713struct t4_vpd_hdr {
2714	u8  id_tag;
2715	u8  id_len[2];
2716	u8  id_data[ID_LEN];
2717};
2718
2719struct t4_vpdr_hdr {
2720	u8  vpdr_tag;
2721	u8  vpdr_len[2];
2722};
2723
2724/*
2725 * EEPROM reads take a few tens of us while writes can take a bit over 5 ms.
2726 */
2727#define EEPROM_DELAY		10		/* 10us per poll spin */
2728#define EEPROM_MAX_POLL		5000		/* x 5000 == 50ms */
2729
2730#define EEPROM_STAT_ADDR	0x7bfc
2731#define VPD_SIZE		0x800
2732#define VPD_BASE		0x400
2733#define VPD_BASE_OLD		0
2734#define VPD_LEN			1024
2735#define VPD_INFO_FLD_HDR_SIZE	3
2736#define CHELSIO_VPD_UNIQUE_ID	0x82
2737
2738/*
2739 * Small utility function to wait till any outstanding VPD Access is complete.
2740 * We have a per-adapter state variable "VPD Busy" to indicate when we have a
2741 * VPD Access in flight.  This allows us to handle the problem of having a
2742 * previous VPD Access time out and prevent an attempt to inject a new VPD
2743 * Request before any in-flight VPD reguest has completed.
2744 */
2745static int t4_seeprom_wait(struct adapter *adapter)
2746{
2747	unsigned int base = adapter->params.pci.vpd_cap_addr;
2748	int max_poll;
2749
2750	/*
2751	 * If no VPD Access is in flight, we can just return success right
2752	 * away.
2753	 */
2754	if (!adapter->vpd_busy)
2755		return 0;
2756
2757	/*
2758	 * Poll the VPD Capability Address/Flag register waiting for it
2759	 * to indicate that the operation is complete.
2760	 */
2761	max_poll = EEPROM_MAX_POLL;
2762	do {
2763		u16 val;
2764
2765		udelay(EEPROM_DELAY);
2766		t4_os_pci_read_cfg2(adapter, base + PCI_VPD_ADDR, &val);
2767
2768		/*
2769		 * If the operation is complete, mark the VPD as no longer
2770		 * busy and return success.
2771		 */
2772		if ((val & PCI_VPD_ADDR_F) == adapter->vpd_flag) {
2773			adapter->vpd_busy = 0;
2774			return 0;
2775		}
2776	} while (--max_poll);
2777
2778	/*
2779	 * Failure!  Note that we leave the VPD Busy status set in order to
2780	 * avoid pushing a new VPD Access request into the VPD Capability till
2781	 * the current operation eventually succeeds.  It's a bug to issue a
2782	 * new request when an existing request is in flight and will result
2783	 * in corrupt hardware state.
2784	 */
2785	return -ETIMEDOUT;
2786}
2787
2788/**
2789 *	t4_seeprom_read - read a serial EEPROM location
2790 *	@adapter: adapter to read
2791 *	@addr: EEPROM virtual address
2792 *	@data: where to store the read data
2793 *
2794 *	Read a 32-bit word from a location in serial EEPROM using the card's PCI
2795 *	VPD capability.  Note that this function must be called with a virtual
2796 *	address.
2797 */
2798int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data)
2799{
2800	unsigned int base = adapter->params.pci.vpd_cap_addr;
2801	int ret;
2802
2803	/*
2804	 * VPD Accesses must alway be 4-byte aligned!
2805	 */
2806	if (addr >= EEPROMVSIZE || (addr & 3))
2807		return -EINVAL;
2808
2809	/*
2810	 * Wait for any previous operation which may still be in flight to
2811	 * complete.
2812	 */
2813	ret = t4_seeprom_wait(adapter);
2814	if (ret) {
2815		CH_ERR(adapter, "VPD still busy from previous operation\n");
2816		return ret;
2817	}
2818
2819	/*
2820	 * Issue our new VPD Read request, mark the VPD as being busy and wait
2821	 * for our request to complete.  If it doesn't complete, note the
2822	 * error and return it to our caller.  Note that we do not reset the
2823	 * VPD Busy status!
2824	 */
2825	t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR, (u16)addr);
2826	adapter->vpd_busy = 1;
2827	adapter->vpd_flag = PCI_VPD_ADDR_F;
2828	ret = t4_seeprom_wait(adapter);
2829	if (ret) {
2830		CH_ERR(adapter, "VPD read of address %#x failed\n", addr);
2831		return ret;
2832	}
2833
2834	/*
2835	 * Grab the returned data, swizzle it into our endianness and
2836	 * return success.
2837	 */
2838	t4_os_pci_read_cfg4(adapter, base + PCI_VPD_DATA, data);
2839	*data = le32_to_cpu(*data);
2840	return 0;
2841}
2842
2843/**
2844 *	t4_seeprom_write - write a serial EEPROM location
2845 *	@adapter: adapter to write
2846 *	@addr: virtual EEPROM address
2847 *	@data: value to write
2848 *
2849 *	Write a 32-bit word to a location in serial EEPROM using the card's PCI
2850 *	VPD capability.  Note that this function must be called with a virtual
2851 *	address.
2852 */
2853int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data)
2854{
2855	unsigned int base = adapter->params.pci.vpd_cap_addr;
2856	int ret;
2857	u32 stats_reg;
2858	int max_poll;
2859
2860	/*
2861	 * VPD Accesses must alway be 4-byte aligned!
2862	 */
2863	if (addr >= EEPROMVSIZE || (addr & 3))
2864		return -EINVAL;
2865
2866	/*
2867	 * Wait for any previous operation which may still be in flight to
2868	 * complete.
2869	 */
2870	ret = t4_seeprom_wait(adapter);
2871	if (ret) {
2872		CH_ERR(adapter, "VPD still busy from previous operation\n");
2873		return ret;
2874	}
2875
2876	/*
2877	 * Issue our new VPD Read request, mark the VPD as being busy and wait
2878	 * for our request to complete.  If it doesn't complete, note the
2879	 * error and return it to our caller.  Note that we do not reset the
2880	 * VPD Busy status!
2881	 */
2882	t4_os_pci_write_cfg4(adapter, base + PCI_VPD_DATA,
2883				 cpu_to_le32(data));
2884	t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR,
2885				 (u16)addr | PCI_VPD_ADDR_F);
2886	adapter->vpd_busy = 1;
2887	adapter->vpd_flag = 0;
2888	ret = t4_seeprom_wait(adapter);
2889	if (ret) {
2890		CH_ERR(adapter, "VPD write of address %#x failed\n", addr);
2891		return ret;
2892	}
2893
2894	/*
2895	 * Reset PCI_VPD_DATA register after a transaction and wait for our
2896	 * request to complete. If it doesn't complete, return error.
2897	 */
2898	t4_os_pci_write_cfg4(adapter, base + PCI_VPD_DATA, 0);
2899	max_poll = EEPROM_MAX_POLL;
2900	do {
2901		udelay(EEPROM_DELAY);
2902		t4_seeprom_read(adapter, EEPROM_STAT_ADDR, &stats_reg);
2903	} while ((stats_reg & 0x1) && --max_poll);
2904	if (!max_poll)
2905		return -ETIMEDOUT;
2906
2907	/* Return success! */
2908	return 0;
2909}
2910
2911/**
2912 *	t4_eeprom_ptov - translate a physical EEPROM address to virtual
2913 *	@phys_addr: the physical EEPROM address
2914 *	@fn: the PCI function number
2915 *	@sz: size of function-specific area
2916 *
2917 *	Translate a physical EEPROM address to virtual.  The first 1K is
2918 *	accessed through virtual addresses starting at 31K, the rest is
2919 *	accessed through virtual addresses starting at 0.
2920 *
2921 *	The mapping is as follows:
2922 *	[0..1K) -> [31K..32K)
2923 *	[1K..1K+A) -> [ES-A..ES)
2924 *	[1K+A..ES) -> [0..ES-A-1K)
2925 *
2926 *	where A = @fn * @sz, and ES = EEPROM size.
2927 */
2928int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2929{
2930	fn *= sz;
2931	if (phys_addr < 1024)
2932		return phys_addr + (31 << 10);
2933	if (phys_addr < 1024 + fn)
2934		return EEPROMSIZE - fn + phys_addr - 1024;
2935	if (phys_addr < EEPROMSIZE)
2936		return phys_addr - 1024 - fn;
2937	return -EINVAL;
2938}
2939
2940/**
2941 *	t4_seeprom_wp - enable/disable EEPROM write protection
2942 *	@adapter: the adapter
2943 *	@enable: whether to enable or disable write protection
2944 *
2945 *	Enables or disables write protection on the serial EEPROM.
2946 */
2947int t4_seeprom_wp(struct adapter *adapter, int enable)
2948{
2949	return t4_seeprom_write(adapter, EEPROM_STAT_ADDR, enable ? 0xc : 0);
2950}
2951
2952/**
2953 *	get_vpd_keyword_val - Locates an information field keyword in the VPD
2954 *	@vpd: Pointer to buffered vpd data structure
2955 *	@kw: The keyword to search for
2956 *	@region: VPD region to search (starting from 0)
2957 *
2958 *	Returns the value of the information field keyword or
2959 *	-ENOENT otherwise.
2960 */
2961static int get_vpd_keyword_val(const u8 *vpd, const char *kw, int region)
2962{
2963	int i, tag;
2964	unsigned int offset, len;
2965	const struct t4_vpdr_hdr *vpdr;
2966
2967	offset = sizeof(struct t4_vpd_hdr);
2968	vpdr = (const void *)(vpd + offset);
2969	tag = vpdr->vpdr_tag;
2970	len = (u16)vpdr->vpdr_len[0] + ((u16)vpdr->vpdr_len[1] << 8);
2971	while (region--) {
2972		offset += sizeof(struct t4_vpdr_hdr) + len;
2973		vpdr = (const void *)(vpd + offset);
2974		if (++tag != vpdr->vpdr_tag)
2975			return -ENOENT;
2976		len = (u16)vpdr->vpdr_len[0] + ((u16)vpdr->vpdr_len[1] << 8);
2977	}
2978	offset += sizeof(struct t4_vpdr_hdr);
2979
2980	if (offset + len > VPD_LEN) {
2981		return -ENOENT;
2982	}
2983
2984	for (i = offset; i + VPD_INFO_FLD_HDR_SIZE <= offset + len;) {
2985		if (memcmp(vpd + i , kw , 2) == 0){
2986			i += VPD_INFO_FLD_HDR_SIZE;
2987			return i;
2988		}
2989
2990		i += VPD_INFO_FLD_HDR_SIZE + vpd[i+2];
2991	}
2992
2993	return -ENOENT;
2994}
2995
2996
2997/**
2998 *	get_vpd_params - read VPD parameters from VPD EEPROM
2999 *	@adapter: adapter to read
3000 *	@p: where to store the parameters
3001 *	@vpd: caller provided temporary space to read the VPD into
3002 *
3003 *	Reads card parameters stored in VPD EEPROM.
3004 */
3005static int get_vpd_params(struct adapter *adapter, struct vpd_params *p,
3006    uint16_t device_id, u32 *buf)
3007{
3008	int i, ret, addr;
3009	int ec, sn, pn, na, md;
3010	u8 csum;
3011	const u8 *vpd = (const u8 *)buf;
3012
3013	/*
3014	 * Card information normally starts at VPD_BASE but early cards had
3015	 * it at 0.
3016	 */
3017	ret = t4_seeprom_read(adapter, VPD_BASE, buf);
3018	if (ret)
3019		return (ret);
3020
3021	/*
3022	 * The VPD shall have a unique identifier specified by the PCI SIG.
3023	 * For chelsio adapters, the identifier is 0x82. The first byte of a VPD
3024	 * shall be CHELSIO_VPD_UNIQUE_ID (0x82). The VPD programming software
3025	 * is expected to automatically put this entry at the
3026	 * beginning of the VPD.
3027	 */
3028	addr = *vpd == CHELSIO_VPD_UNIQUE_ID ? VPD_BASE : VPD_BASE_OLD;
3029
3030	for (i = 0; i < VPD_LEN; i += 4) {
3031		ret = t4_seeprom_read(adapter, addr + i, buf++);
3032		if (ret)
3033			return ret;
3034	}
3035
3036#define FIND_VPD_KW(var,name) do { \
3037	var = get_vpd_keyword_val(vpd, name, 0); \
3038	if (var < 0) { \
3039		CH_ERR(adapter, "missing VPD keyword " name "\n"); \
3040		return -EINVAL; \
3041	} \
3042} while (0)
3043
3044	FIND_VPD_KW(i, "RV");
3045	for (csum = 0; i >= 0; i--)
3046		csum += vpd[i];
3047
3048	if (csum) {
3049		CH_ERR(adapter,
3050			"corrupted VPD EEPROM, actual csum %u\n", csum);
3051		return -EINVAL;
3052	}
3053
3054	FIND_VPD_KW(ec, "EC");
3055	FIND_VPD_KW(sn, "SN");
3056	FIND_VPD_KW(pn, "PN");
3057	FIND_VPD_KW(na, "NA");
3058#undef FIND_VPD_KW
3059
3060	memcpy(p->id, vpd + offsetof(struct t4_vpd_hdr, id_data), ID_LEN);
3061	strstrip(p->id);
3062	memcpy(p->ec, vpd + ec, EC_LEN);
3063	strstrip(p->ec);
3064	i = vpd[sn - VPD_INFO_FLD_HDR_SIZE + 2];
3065	memcpy(p->sn, vpd + sn, min(i, SERNUM_LEN));
3066	strstrip(p->sn);
3067	i = vpd[pn - VPD_INFO_FLD_HDR_SIZE + 2];
3068	memcpy(p->pn, vpd + pn, min(i, PN_LEN));
3069	strstrip((char *)p->pn);
3070	i = vpd[na - VPD_INFO_FLD_HDR_SIZE + 2];
3071	memcpy(p->na, vpd + na, min(i, MACADDR_LEN));
3072	strstrip((char *)p->na);
3073
3074	if (device_id & 0x80)
3075		return 0;	/* Custom card */
3076
3077	md = get_vpd_keyword_val(vpd, "VF", 1);
3078	if (md < 0) {
3079		snprintf(p->md, sizeof(p->md), "unknown");
3080	} else {
3081		i = vpd[md - VPD_INFO_FLD_HDR_SIZE + 2];
3082		memcpy(p->md, vpd + md, min(i, MD_LEN));
3083		strstrip((char *)p->md);
3084	}
3085
3086	return 0;
3087}
3088
3089/* serial flash and firmware constants and flash config file constants */
3090enum {
3091	SF_ATTEMPTS = 10,	/* max retries for SF operations */
3092
3093	/* flash command opcodes */
3094	SF_PROG_PAGE    = 2,	/* program 256B page */
3095	SF_WR_DISABLE   = 4,	/* disable writes */
3096	SF_RD_STATUS    = 5,	/* read status register */
3097	SF_WR_ENABLE    = 6,	/* enable writes */
3098	SF_RD_DATA_FAST = 0xb,	/* read flash */
3099	SF_RD_ID	= 0x9f,	/* read ID */
3100	SF_ERASE_SECTOR = 0xd8,	/* erase 64KB sector */
3101};
3102
3103/**
3104 *	sf1_read - read data from the serial flash
3105 *	@adapter: the adapter
3106 *	@byte_cnt: number of bytes to read
3107 *	@cont: whether another operation will be chained
3108 *	@lock: whether to lock SF for PL access only
3109 *	@valp: where to store the read data
3110 *
3111 *	Reads up to 4 bytes of data from the serial flash.  The location of
3112 *	the read needs to be specified prior to calling this by issuing the
3113 *	appropriate commands to the serial flash.
3114 */
3115static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont,
3116		    int lock, u32 *valp)
3117{
3118	int ret;
3119
3120	if (!byte_cnt || byte_cnt > 4)
3121		return -EINVAL;
3122	if (t4_read_reg(adapter, A_SF_OP) & F_BUSY)
3123		return -EBUSY;
3124	t4_write_reg(adapter, A_SF_OP,
3125		     V_SF_LOCK(lock) | V_CONT(cont) | V_BYTECNT(byte_cnt - 1));
3126	ret = t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5);
3127	if (!ret)
3128		*valp = t4_read_reg(adapter, A_SF_DATA);
3129	return ret;
3130}
3131
3132/**
3133 *	sf1_write - write data to the serial flash
3134 *	@adapter: the adapter
3135 *	@byte_cnt: number of bytes to write
3136 *	@cont: whether another operation will be chained
3137 *	@lock: whether to lock SF for PL access only
3138 *	@val: value to write
3139 *
3140 *	Writes up to 4 bytes of data to the serial flash.  The location of
3141 *	the write needs to be specified prior to calling this by issuing the
3142 *	appropriate commands to the serial flash.
3143 */
3144static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont,
3145		     int lock, u32 val)
3146{
3147	if (!byte_cnt || byte_cnt > 4)
3148		return -EINVAL;
3149	if (t4_read_reg(adapter, A_SF_OP) & F_BUSY)
3150		return -EBUSY;
3151	t4_write_reg(adapter, A_SF_DATA, val);
3152	t4_write_reg(adapter, A_SF_OP, V_SF_LOCK(lock) |
3153		     V_CONT(cont) | V_BYTECNT(byte_cnt - 1) | V_OP(1));
3154	return t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5);
3155}
3156
3157/**
3158 *	flash_wait_op - wait for a flash operation to complete
3159 *	@adapter: the adapter
3160 *	@attempts: max number of polls of the status register
3161 *	@delay: delay between polls in ms
3162 *
3163 *	Wait for a flash operation to complete by polling the status register.
3164 */
3165static int flash_wait_op(struct adapter *adapter, int attempts, int delay)
3166{
3167	int ret;
3168	u32 status;
3169
3170	while (1) {
3171		if ((ret = sf1_write(adapter, 1, 1, 1, SF_RD_STATUS)) != 0 ||
3172		    (ret = sf1_read(adapter, 1, 0, 1, &status)) != 0)
3173			return ret;
3174		if (!(status & 1))
3175			return 0;
3176		if (--attempts == 0)
3177			return -EAGAIN;
3178		if (delay)
3179			msleep(delay);
3180	}
3181}
3182
3183/**
3184 *	t4_read_flash - read words from serial flash
3185 *	@adapter: the adapter
3186 *	@addr: the start address for the read
3187 *	@nwords: how many 32-bit words to read
3188 *	@data: where to store the read data
3189 *	@byte_oriented: whether to store data as bytes or as words
3190 *
3191 *	Read the specified number of 32-bit words from the serial flash.
3192 *	If @byte_oriented is set the read data is stored as a byte array
3193 *	(i.e., big-endian), otherwise as 32-bit words in the platform's
3194 *	natural endianness.
3195 */
3196int t4_read_flash(struct adapter *adapter, unsigned int addr,
3197		  unsigned int nwords, u32 *data, int byte_oriented)
3198{
3199	int ret;
3200
3201	if (addr + nwords * sizeof(u32) > adapter->params.sf_size || (addr & 3))
3202		return -EINVAL;
3203
3204	addr = swab32(addr) | SF_RD_DATA_FAST;
3205
3206	if ((ret = sf1_write(adapter, 4, 1, 0, addr)) != 0 ||
3207	    (ret = sf1_read(adapter, 1, 1, 0, data)) != 0)
3208		return ret;
3209
3210	for ( ; nwords; nwords--, data++) {
3211		ret = sf1_read(adapter, 4, nwords > 1, nwords == 1, data);
3212		if (nwords == 1)
3213			t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
3214		if (ret)
3215			return ret;
3216		if (byte_oriented)
3217			*data = (__force __u32)(cpu_to_be32(*data));
3218	}
3219	return 0;
3220}
3221
3222/**
3223 *	t4_write_flash - write up to a page of data to the serial flash
3224 *	@adapter: the adapter
3225 *	@addr: the start address to write
3226 *	@n: length of data to write in bytes
3227 *	@data: the data to write
3228 *	@byte_oriented: whether to store data as bytes or as words
3229 *
3230 *	Writes up to a page of data (256 bytes) to the serial flash starting
3231 *	at the given address.  All the data must be written to the same page.
3232 *	If @byte_oriented is set the write data is stored as byte stream
3233 *	(i.e. matches what on disk), otherwise in big-endian.
3234 */
3235int t4_write_flash(struct adapter *adapter, unsigned int addr,
3236			  unsigned int n, const u8 *data, int byte_oriented)
3237{
3238	int ret;
3239	u32 buf[SF_PAGE_SIZE / 4];
3240	unsigned int i, c, left, val, offset = addr & 0xff;
3241
3242	if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE)
3243		return -EINVAL;
3244
3245	val = swab32(addr) | SF_PROG_PAGE;
3246
3247	if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
3248	    (ret = sf1_write(adapter, 4, 1, 1, val)) != 0)
3249		goto unlock;
3250
3251	for (left = n; left; left -= c) {
3252		c = min(left, 4U);
3253		for (val = 0, i = 0; i < c; ++i)
3254			val = (val << 8) + *data++;
3255
3256		if (!byte_oriented)
3257			val = cpu_to_be32(val);
3258
3259		ret = sf1_write(adapter, c, c != left, 1, val);
3260		if (ret)
3261			goto unlock;
3262	}
3263	ret = flash_wait_op(adapter, 8, 1);
3264	if (ret)
3265		goto unlock;
3266
3267	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
3268
3269	/* Read the page to verify the write succeeded */
3270	ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf,
3271			    byte_oriented);
3272	if (ret)
3273		return ret;
3274
3275	if (memcmp(data - n, (u8 *)buf + offset, n)) {
3276		CH_ERR(adapter,
3277			"failed to correctly write the flash page at %#x\n",
3278			addr);
3279		return -EIO;
3280	}
3281	return 0;
3282
3283unlock:
3284	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
3285	return ret;
3286}
3287
3288/**
3289 *	t4_get_fw_version - read the firmware version
3290 *	@adapter: the adapter
3291 *	@vers: where to place the version
3292 *
3293 *	Reads the FW version from flash.
3294 */
3295int t4_get_fw_version(struct adapter *adapter, u32 *vers)
3296{
3297	return t4_read_flash(adapter, FLASH_FW_START +
3298			     offsetof(struct fw_hdr, fw_ver), 1,
3299			     vers, 0);
3300}
3301
3302/**
3303 *	t4_get_fw_hdr - read the firmware header
3304 *	@adapter: the adapter
3305 *	@hdr: where to place the version
3306 *
3307 *	Reads the FW header from flash into caller provided buffer.
3308 */
3309int t4_get_fw_hdr(struct adapter *adapter, struct fw_hdr *hdr)
3310{
3311	return t4_read_flash(adapter, FLASH_FW_START,
3312	    sizeof (*hdr) / sizeof (uint32_t), (uint32_t *)hdr, 1);
3313}
3314
3315/**
3316 *	t4_get_bs_version - read the firmware bootstrap version
3317 *	@adapter: the adapter
3318 *	@vers: where to place the version
3319 *
3320 *	Reads the FW Bootstrap version from flash.
3321 */
3322int t4_get_bs_version(struct adapter *adapter, u32 *vers)
3323{
3324	return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START +
3325			     offsetof(struct fw_hdr, fw_ver), 1,
3326			     vers, 0);
3327}
3328
3329/**
3330 *	t4_get_tp_version - read the TP microcode version
3331 *	@adapter: the adapter
3332 *	@vers: where to place the version
3333 *
3334 *	Reads the TP microcode version from flash.
3335 */
3336int t4_get_tp_version(struct adapter *adapter, u32 *vers)
3337{
3338	return t4_read_flash(adapter, FLASH_FW_START +
3339			     offsetof(struct fw_hdr, tp_microcode_ver),
3340			     1, vers, 0);
3341}
3342
3343/**
3344 *	t4_get_exprom_version - return the Expansion ROM version (if any)
3345 *	@adapter: the adapter
3346 *	@vers: where to place the version
3347 *
3348 *	Reads the Expansion ROM header from FLASH and returns the version
3349 *	number (if present) through the @vers return value pointer.  We return
3350 *	this in the Firmware Version Format since it's convenient.  Return
3351 *	0 on success, -ENOENT if no Expansion ROM is present.
3352 */
3353int t4_get_exprom_version(struct adapter *adapter, u32 *vers)
3354{
3355	struct exprom_header {
3356		unsigned char hdr_arr[16];	/* must start with 0x55aa */
3357		unsigned char hdr_ver[4];	/* Expansion ROM version */
3358	} *hdr;
3359	u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header),
3360					   sizeof(u32))];
3361	int ret;
3362
3363	ret = t4_read_flash(adapter, FLASH_EXP_ROM_START,
3364			    ARRAY_SIZE(exprom_header_buf), exprom_header_buf,
3365			    0);
3366	if (ret)
3367		return ret;
3368
3369	hdr = (struct exprom_header *)exprom_header_buf;
3370	if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa)
3371		return -ENOENT;
3372
3373	*vers = (V_FW_HDR_FW_VER_MAJOR(hdr->hdr_ver[0]) |
3374		 V_FW_HDR_FW_VER_MINOR(hdr->hdr_ver[1]) |
3375		 V_FW_HDR_FW_VER_MICRO(hdr->hdr_ver[2]) |
3376		 V_FW_HDR_FW_VER_BUILD(hdr->hdr_ver[3]));
3377	return 0;
3378}
3379
3380/**
3381 *	t4_get_scfg_version - return the Serial Configuration version
3382 *	@adapter: the adapter
3383 *	@vers: where to place the version
3384 *
3385 *	Reads the Serial Configuration Version via the Firmware interface
3386 *	(thus this can only be called once we're ready to issue Firmware
3387 *	commands).  The format of the Serial Configuration version is
3388 *	adapter specific.  Returns 0 on success, an error on failure.
3389 *
3390 *	Note that early versions of the Firmware didn't include the ability
3391 *	to retrieve the Serial Configuration version, so we zero-out the
3392 *	return-value parameter in that case to avoid leaving it with
3393 *	garbage in it.
3394 *
3395 *	Also note that the Firmware will return its cached copy of the Serial
3396 *	Initialization Revision ID, not the actual Revision ID as written in
3397 *	the Serial EEPROM.  This is only an issue if a new VPD has been written
3398 *	and the Firmware/Chip haven't yet gone through a RESET sequence.  So
3399 *	it's best to defer calling this routine till after a FW_RESET_CMD has
3400 *	been issued if the Host Driver will be performing a full adapter
3401 *	initialization.
3402 */
3403int t4_get_scfg_version(struct adapter *adapter, u32 *vers)
3404{
3405	u32 scfgrev_param;
3406	int ret;
3407
3408	scfgrev_param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3409			 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_SCFGREV));
3410	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3411			      1, &scfgrev_param, vers);
3412	if (ret)
3413		*vers = 0;
3414	return ret;
3415}
3416
3417/**
3418 *	t4_get_vpd_version - return the VPD version
3419 *	@adapter: the adapter
3420 *	@vers: where to place the version
3421 *
3422 *	Reads the VPD via the Firmware interface (thus this can only be called
3423 *	once we're ready to issue Firmware commands).  The format of the
3424 *	VPD version is adapter specific.  Returns 0 on success, an error on
3425 *	failure.
3426 *
3427 *	Note that early versions of the Firmware didn't include the ability
3428 *	to retrieve the VPD version, so we zero-out the return-value parameter
3429 *	in that case to avoid leaving it with garbage in it.
3430 *
3431 *	Also note that the Firmware will return its cached copy of the VPD
3432 *	Revision ID, not the actual Revision ID as written in the Serial
3433 *	EEPROM.  This is only an issue if a new VPD has been written and the
3434 *	Firmware/Chip haven't yet gone through a RESET sequence.  So it's best
3435 *	to defer calling this routine till after a FW_RESET_CMD has been issued
3436 *	if the Host Driver will be performing a full adapter initialization.
3437 */
3438int t4_get_vpd_version(struct adapter *adapter, u32 *vers)
3439{
3440	u32 vpdrev_param;
3441	int ret;
3442
3443	vpdrev_param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3444			V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_VPDREV));
3445	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3446			      1, &vpdrev_param, vers);
3447	if (ret)
3448		*vers = 0;
3449	return ret;
3450}
3451
3452/**
3453 *	t4_get_version_info - extract various chip/firmware version information
3454 *	@adapter: the adapter
3455 *
3456 *	Reads various chip/firmware version numbers and stores them into the
3457 *	adapter Adapter Parameters structure.  If any of the efforts fails
3458 *	the first failure will be returned, but all of the version numbers
3459 *	will be read.
3460 */
3461int t4_get_version_info(struct adapter *adapter)
3462{
3463	int ret = 0;
3464
3465	#define FIRST_RET(__getvinfo) \
3466	do { \
3467		int __ret = __getvinfo; \
3468		if (__ret && !ret) \
3469			ret = __ret; \
3470	} while (0)
3471
3472	FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers));
3473	FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers));
3474	FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers));
3475	FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers));
3476	FIRST_RET(t4_get_scfg_version(adapter, &adapter->params.scfg_vers));
3477	FIRST_RET(t4_get_vpd_version(adapter, &adapter->params.vpd_vers));
3478
3479	#undef FIRST_RET
3480
3481	return ret;
3482}
3483
3484/**
3485 *	t4_flash_erase_sectors - erase a range of flash sectors
3486 *	@adapter: the adapter
3487 *	@start: the first sector to erase
3488 *	@end: the last sector to erase
3489 *
3490 *	Erases the sectors in the given inclusive range.
3491 */
3492int t4_flash_erase_sectors(struct adapter *adapter, int start, int end)
3493{
3494	int ret = 0;
3495
3496	if (end >= adapter->params.sf_nsec)
3497		return -EINVAL;
3498
3499	while (start <= end) {
3500		if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
3501		    (ret = sf1_write(adapter, 4, 0, 1,
3502				     SF_ERASE_SECTOR | (start << 8))) != 0 ||
3503		    (ret = flash_wait_op(adapter, 14, 500)) != 0) {
3504			CH_ERR(adapter,
3505				"erase of flash sector %d failed, error %d\n",
3506				start, ret);
3507			break;
3508		}
3509		start++;
3510	}
3511	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
3512	return ret;
3513}
3514
3515/**
3516 *	t4_flash_cfg_addr - return the address of the flash configuration file
3517 *	@adapter: the adapter
3518 *
3519 *	Return the address within the flash where the Firmware Configuration
3520 *	File is stored, or an error if the device FLASH is too small to contain
3521 *	a Firmware Configuration File.
3522 */
3523int t4_flash_cfg_addr(struct adapter *adapter)
3524{
3525	/*
3526	 * If the device FLASH isn't large enough to hold a Firmware
3527	 * Configuration File, return an error.
3528	 */
3529	if (adapter->params.sf_size < FLASH_CFG_START + FLASH_CFG_MAX_SIZE)
3530		return -ENOSPC;
3531
3532	return FLASH_CFG_START;
3533}
3534
3535/*
3536 * Return TRUE if the specified firmware matches the adapter.  I.e. T4
3537 * firmware for T4 adapters, T5 firmware for T5 adapters, etc.  We go ahead
3538 * and emit an error message for mismatched firmware to save our caller the
3539 * effort ...
3540 */
3541static int t4_fw_matches_chip(struct adapter *adap,
3542			      const struct fw_hdr *hdr)
3543{
3544	/*
3545	 * The expression below will return FALSE for any unsupported adapter
3546	 * which will keep us "honest" in the future ...
3547	 */
3548	if ((is_t4(adap) && hdr->chip == FW_HDR_CHIP_T4) ||
3549	    (is_t5(adap) && hdr->chip == FW_HDR_CHIP_T5) ||
3550	    (is_t6(adap) && hdr->chip == FW_HDR_CHIP_T6))
3551		return 1;
3552
3553	CH_ERR(adap,
3554		"FW image (%d) is not suitable for this adapter (%d)\n",
3555		hdr->chip, chip_id(adap));
3556	return 0;
3557}
3558
3559/**
3560 *	t4_load_fw - download firmware
3561 *	@adap: the adapter
3562 *	@fw_data: the firmware image to write
3563 *	@size: image size
3564 *
3565 *	Write the supplied firmware image to the card's serial flash.
3566 */
3567int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size)
3568{
3569	u32 csum;
3570	int ret, addr;
3571	unsigned int i;
3572	u8 first_page[SF_PAGE_SIZE];
3573	const u32 *p = (const u32 *)fw_data;
3574	const struct fw_hdr *hdr = (const struct fw_hdr *)fw_data;
3575	unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
3576	unsigned int fw_start_sec;
3577	unsigned int fw_start;
3578	unsigned int fw_size;
3579
3580	if (ntohl(hdr->magic) == FW_HDR_MAGIC_BOOTSTRAP) {
3581		fw_start_sec = FLASH_FWBOOTSTRAP_START_SEC;
3582		fw_start = FLASH_FWBOOTSTRAP_START;
3583		fw_size = FLASH_FWBOOTSTRAP_MAX_SIZE;
3584	} else {
3585		fw_start_sec = FLASH_FW_START_SEC;
3586 		fw_start = FLASH_FW_START;
3587		fw_size = FLASH_FW_MAX_SIZE;
3588	}
3589
3590	if (!size) {
3591		CH_ERR(adap, "FW image has no data\n");
3592		return -EINVAL;
3593	}
3594	if (size & 511) {
3595		CH_ERR(adap,
3596			"FW image size not multiple of 512 bytes\n");
3597		return -EINVAL;
3598	}
3599	if ((unsigned int) be16_to_cpu(hdr->len512) * 512 != size) {
3600		CH_ERR(adap,
3601			"FW image size differs from size in FW header\n");
3602		return -EINVAL;
3603	}
3604	if (size > fw_size) {
3605		CH_ERR(adap, "FW image too large, max is %u bytes\n",
3606			fw_size);
3607		return -EFBIG;
3608	}
3609	if (!t4_fw_matches_chip(adap, hdr))
3610		return -EINVAL;
3611
3612	for (csum = 0, i = 0; i < size / sizeof(csum); i++)
3613		csum += be32_to_cpu(p[i]);
3614
3615	if (csum != 0xffffffff) {
3616		CH_ERR(adap,
3617			"corrupted firmware image, checksum %#x\n", csum);
3618		return -EINVAL;
3619	}
3620
3621	i = DIV_ROUND_UP(size, sf_sec_size);	/* # of sectors spanned */
3622	ret = t4_flash_erase_sectors(adap, fw_start_sec, fw_start_sec + i - 1);
3623	if (ret)
3624		goto out;
3625
3626	/*
3627	 * We write the correct version at the end so the driver can see a bad
3628	 * version if the FW write fails.  Start by writing a copy of the
3629	 * first page with a bad version.
3630	 */
3631	memcpy(first_page, fw_data, SF_PAGE_SIZE);
3632	((struct fw_hdr *)first_page)->fw_ver = cpu_to_be32(0xffffffff);
3633	ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page, 1);
3634	if (ret)
3635		goto out;
3636
3637	addr = fw_start;
3638	for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
3639		addr += SF_PAGE_SIZE;
3640		fw_data += SF_PAGE_SIZE;
3641		ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data, 1);
3642		if (ret)
3643			goto out;
3644	}
3645
3646	ret = t4_write_flash(adap,
3647			     fw_start + offsetof(struct fw_hdr, fw_ver),
3648			     sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver, 1);
3649out:
3650	if (ret)
3651		CH_ERR(adap, "firmware download failed, error %d\n",
3652			ret);
3653	return ret;
3654}
3655
3656/**
3657 *	t4_fwcache - firmware cache operation
3658 *	@adap: the adapter
3659 *	@op  : the operation (flush or flush and invalidate)
3660 */
3661int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op)
3662{
3663	struct fw_params_cmd c;
3664
3665	memset(&c, 0, sizeof(c));
3666	c.op_to_vfn =
3667	    cpu_to_be32(V_FW_CMD_OP(FW_PARAMS_CMD) |
3668			    F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
3669				V_FW_PARAMS_CMD_PFN(adap->pf) |
3670				V_FW_PARAMS_CMD_VFN(0));
3671	c.retval_len16 = cpu_to_be32(FW_LEN16(c));
3672	c.param[0].mnem =
3673	    cpu_to_be32(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3674			    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_FWCACHE));
3675	c.param[0].val = (__force __be32)op;
3676
3677	return t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), NULL);
3678}
3679
3680void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp,
3681			unsigned int *pif_req_wrptr,
3682			unsigned int *pif_rsp_wrptr)
3683{
3684	int i, j;
3685	u32 cfg, val, req, rsp;
3686
3687	cfg = t4_read_reg(adap, A_CIM_DEBUGCFG);
3688	if (cfg & F_LADBGEN)
3689		t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN);
3690
3691	val = t4_read_reg(adap, A_CIM_DEBUGSTS);
3692	req = G_POLADBGWRPTR(val);
3693	rsp = G_PILADBGWRPTR(val);
3694	if (pif_req_wrptr)
3695		*pif_req_wrptr = req;
3696	if (pif_rsp_wrptr)
3697		*pif_rsp_wrptr = rsp;
3698
3699	for (i = 0; i < CIM_PIFLA_SIZE; i++) {
3700		for (j = 0; j < 6; j++) {
3701			t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(req) |
3702				     V_PILADBGRDPTR(rsp));
3703			*pif_req++ = t4_read_reg(adap, A_CIM_PO_LA_DEBUGDATA);
3704			*pif_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_DEBUGDATA);
3705			req++;
3706			rsp++;
3707		}
3708		req = (req + 2) & M_POLADBGRDPTR;
3709		rsp = (rsp + 2) & M_PILADBGRDPTR;
3710	}
3711	t4_write_reg(adap, A_CIM_DEBUGCFG, cfg);
3712}
3713
3714void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp)
3715{
3716	u32 cfg;
3717	int i, j, idx;
3718
3719	cfg = t4_read_reg(adap, A_CIM_DEBUGCFG);
3720	if (cfg & F_LADBGEN)
3721		t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN);
3722
3723	for (i = 0; i < CIM_MALA_SIZE; i++) {
3724		for (j = 0; j < 5; j++) {
3725			idx = 8 * i + j;
3726			t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(idx) |
3727				     V_PILADBGRDPTR(idx));
3728			*ma_req++ = t4_read_reg(adap, A_CIM_PO_LA_MADEBUGDATA);
3729			*ma_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_MADEBUGDATA);
3730		}
3731	}
3732	t4_write_reg(adap, A_CIM_DEBUGCFG, cfg);
3733}
3734
3735void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
3736{
3737	unsigned int i, j;
3738
3739	for (i = 0; i < 8; i++) {
3740		u32 *p = la_buf + i;
3741
3742		t4_write_reg(adap, A_ULP_RX_LA_CTL, i);
3743		j = t4_read_reg(adap, A_ULP_RX_LA_WRPTR);
3744		t4_write_reg(adap, A_ULP_RX_LA_RDPTR, j);
3745		for (j = 0; j < ULPRX_LA_SIZE; j++, p += 8)
3746			*p = t4_read_reg(adap, A_ULP_RX_LA_RDDATA);
3747	}
3748}
3749
3750/**
3751 *	fwcaps16_to_caps32 - convert 16-bit Port Capabilities to 32-bits
3752 *	@caps16: a 16-bit Port Capabilities value
3753 *
3754 *	Returns the equivalent 32-bit Port Capabilities value.
3755 */
3756static uint32_t fwcaps16_to_caps32(uint16_t caps16)
3757{
3758	uint32_t caps32 = 0;
3759
3760	#define CAP16_TO_CAP32(__cap) \
3761		do { \
3762			if (caps16 & FW_PORT_CAP_##__cap) \
3763				caps32 |= FW_PORT_CAP32_##__cap; \
3764		} while (0)
3765
3766	CAP16_TO_CAP32(SPEED_100M);
3767	CAP16_TO_CAP32(SPEED_1G);
3768	CAP16_TO_CAP32(SPEED_25G);
3769	CAP16_TO_CAP32(SPEED_10G);
3770	CAP16_TO_CAP32(SPEED_40G);
3771	CAP16_TO_CAP32(SPEED_100G);
3772	CAP16_TO_CAP32(FC_RX);
3773	CAP16_TO_CAP32(FC_TX);
3774	CAP16_TO_CAP32(ANEG);
3775	CAP16_TO_CAP32(FORCE_PAUSE);
3776	CAP16_TO_CAP32(MDIAUTO);
3777	CAP16_TO_CAP32(MDISTRAIGHT);
3778	CAP16_TO_CAP32(FEC_RS);
3779	CAP16_TO_CAP32(FEC_BASER_RS);
3780	CAP16_TO_CAP32(802_3_PAUSE);
3781	CAP16_TO_CAP32(802_3_ASM_DIR);
3782
3783	#undef CAP16_TO_CAP32
3784
3785	return caps32;
3786}
3787
3788/**
3789 *	fwcaps32_to_caps16 - convert 32-bit Port Capabilities to 16-bits
3790 *	@caps32: a 32-bit Port Capabilities value
3791 *
3792 *	Returns the equivalent 16-bit Port Capabilities value.  Note that
3793 *	not all 32-bit Port Capabilities can be represented in the 16-bit
3794 *	Port Capabilities and some fields/values may not make it.
3795 */
3796static uint16_t fwcaps32_to_caps16(uint32_t caps32)
3797{
3798	uint16_t caps16 = 0;
3799
3800	#define CAP32_TO_CAP16(__cap) \
3801		do { \
3802			if (caps32 & FW_PORT_CAP32_##__cap) \
3803				caps16 |= FW_PORT_CAP_##__cap; \
3804		} while (0)
3805
3806	CAP32_TO_CAP16(SPEED_100M);
3807	CAP32_TO_CAP16(SPEED_1G);
3808	CAP32_TO_CAP16(SPEED_10G);
3809	CAP32_TO_CAP16(SPEED_25G);
3810	CAP32_TO_CAP16(SPEED_40G);
3811	CAP32_TO_CAP16(SPEED_100G);
3812	CAP32_TO_CAP16(FC_RX);
3813	CAP32_TO_CAP16(FC_TX);
3814	CAP32_TO_CAP16(802_3_PAUSE);
3815	CAP32_TO_CAP16(802_3_ASM_DIR);
3816	CAP32_TO_CAP16(ANEG);
3817	CAP32_TO_CAP16(FORCE_PAUSE);
3818	CAP32_TO_CAP16(MDIAUTO);
3819	CAP32_TO_CAP16(MDISTRAIGHT);
3820	CAP32_TO_CAP16(FEC_RS);
3821	CAP32_TO_CAP16(FEC_BASER_RS);
3822
3823	#undef CAP32_TO_CAP16
3824
3825	return caps16;
3826}
3827
3828static bool
3829is_bt(struct port_info *pi)
3830{
3831
3832	return (pi->port_type == FW_PORT_TYPE_BT_SGMII ||
3833	    pi->port_type == FW_PORT_TYPE_BT_XFI ||
3834	    pi->port_type == FW_PORT_TYPE_BT_XAUI);
3835}
3836
3837static int8_t fwcap_to_fec(uint32_t caps, bool unset_means_none)
3838{
3839	int8_t fec = 0;
3840
3841	if ((caps & V_FW_PORT_CAP32_FEC(M_FW_PORT_CAP32_FEC)) == 0)
3842		return (unset_means_none ? FEC_NONE : 0);
3843
3844	if (caps & FW_PORT_CAP32_FEC_RS)
3845		fec |= FEC_RS;
3846	if (caps & FW_PORT_CAP32_FEC_BASER_RS)
3847		fec |= FEC_BASER_RS;
3848	if (caps & FW_PORT_CAP32_FEC_NO_FEC)
3849		fec |= FEC_NONE;
3850
3851	return (fec);
3852}
3853
3854/*
3855 * Note that 0 is not translated to NO_FEC.
3856 */
3857static uint32_t fec_to_fwcap(int8_t fec)
3858{
3859	uint32_t caps = 0;
3860
3861	/* Only real FECs allowed. */
3862	MPASS((fec & ~M_FW_PORT_CAP32_FEC) == 0);
3863
3864	if (fec & FEC_RS)
3865		caps |= FW_PORT_CAP32_FEC_RS;
3866	if (fec & FEC_BASER_RS)
3867		caps |= FW_PORT_CAP32_FEC_BASER_RS;
3868	if (fec & FEC_NONE)
3869		caps |= FW_PORT_CAP32_FEC_NO_FEC;
3870
3871	return (caps);
3872}
3873
3874/**
3875 *	t4_link_l1cfg - apply link configuration to MAC/PHY
3876 *	@phy: the PHY to setup
3877 *	@mac: the MAC to setup
3878 *	@lc: the requested link configuration
3879 *
3880 *	Set up a port's MAC and PHY according to a desired link configuration.
3881 *	- If the PHY can auto-negotiate first decide what to advertise, then
3882 *	  enable/disable auto-negotiation as desired, and reset.
3883 *	- If the PHY does not auto-negotiate just reset it.
3884 *	- If auto-negotiation is off set the MAC to the proper speed/duplex/FC,
3885 *	  otherwise do it later based on the outcome of auto-negotiation.
3886 */
3887int t4_link_l1cfg(struct adapter *adap, unsigned int mbox, unsigned int port,
3888		  struct link_config *lc)
3889{
3890	struct fw_port_cmd c;
3891	unsigned int mdi = V_FW_PORT_CAP32_MDI(FW_PORT_CAP32_MDI_AUTO);
3892	unsigned int aneg, fc, fec, speed, rcap;
3893
3894	fc = 0;
3895	if (lc->requested_fc & PAUSE_RX)
3896		fc |= FW_PORT_CAP32_FC_RX;
3897	if (lc->requested_fc & PAUSE_TX)
3898		fc |= FW_PORT_CAP32_FC_TX;
3899	if (!(lc->requested_fc & PAUSE_AUTONEG))
3900		fc |= FW_PORT_CAP32_FORCE_PAUSE;
3901
3902	if (lc->requested_aneg == AUTONEG_DISABLE)
3903		aneg = 0;
3904	else if (lc->requested_aneg == AUTONEG_ENABLE)
3905		aneg = FW_PORT_CAP32_ANEG;
3906	else
3907		aneg = lc->pcaps & FW_PORT_CAP32_ANEG;
3908
3909	if (aneg) {
3910		speed = lc->pcaps &
3911		    V_FW_PORT_CAP32_SPEED(M_FW_PORT_CAP32_SPEED);
3912	} else if (lc->requested_speed != 0)
3913		speed = speed_to_fwcap(lc->requested_speed);
3914	else
3915		speed = fwcap_top_speed(lc->pcaps);
3916
3917	fec = 0;
3918	if (fec_supported(lc->pcaps)) {
3919		if (lc->requested_fec == FEC_AUTO) {
3920			if (lc->pcaps & FW_PORT_CAP32_FORCE_FEC) {
3921				if (speed & FW_PORT_CAP32_SPEED_100G) {
3922					fec |= FW_PORT_CAP32_FEC_RS;
3923					fec |= FW_PORT_CAP32_FEC_NO_FEC;
3924				} else {
3925					fec |= FW_PORT_CAP32_FEC_RS;
3926					fec |= FW_PORT_CAP32_FEC_BASER_RS;
3927					fec |= FW_PORT_CAP32_FEC_NO_FEC;
3928				}
3929			} else {
3930				/* Set only 1b with old firmwares. */
3931				fec |= fec_to_fwcap(lc->fec_hint);
3932			}
3933		} else {
3934			fec |= fec_to_fwcap(lc->requested_fec &
3935			    M_FW_PORT_CAP32_FEC);
3936			if (lc->requested_fec & FEC_MODULE)
3937				fec |= fec_to_fwcap(lc->fec_hint);
3938		}
3939
3940		if (lc->pcaps & FW_PORT_CAP32_FORCE_FEC)
3941			fec |= FW_PORT_CAP32_FORCE_FEC;
3942		else if (fec == FW_PORT_CAP32_FEC_NO_FEC)
3943			fec = 0;
3944	}
3945
3946	/* Force AN on for BT cards. */
3947	if (is_bt(adap->port[adap->chan_map[port]]))
3948		aneg = lc->pcaps & FW_PORT_CAP32_ANEG;
3949
3950	rcap = aneg | speed | fc | fec;
3951	if ((rcap | lc->pcaps) != lc->pcaps) {
3952#ifdef INVARIANTS
3953		CH_WARN(adap, "rcap 0x%08x, pcap 0x%08x, removed 0x%x\n", rcap,
3954		    lc->pcaps, rcap & (rcap ^ lc->pcaps));
3955#endif
3956		rcap &= lc->pcaps;
3957	}
3958	rcap |= mdi;
3959
3960	memset(&c, 0, sizeof(c));
3961	c.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) |
3962				     F_FW_CMD_REQUEST | F_FW_CMD_EXEC |
3963				     V_FW_PORT_CMD_PORTID(port));
3964	if (adap->params.port_caps32) {
3965		c.action_to_len16 =
3966		    cpu_to_be32(V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_L1_CFG32) |
3967			FW_LEN16(c));
3968		c.u.l1cfg32.rcap32 = cpu_to_be32(rcap);
3969	} else {
3970		c.action_to_len16 =
3971		    cpu_to_be32(V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_L1_CFG) |
3972			    FW_LEN16(c));
3973		c.u.l1cfg.rcap = cpu_to_be32(fwcaps32_to_caps16(rcap));
3974	}
3975
3976	return t4_wr_mbox_ns(adap, mbox, &c, sizeof(c), NULL);
3977}
3978
3979/**
3980 *	t4_restart_aneg - restart autonegotiation
3981 *	@adap: the adapter
3982 *	@mbox: mbox to use for the FW command
3983 *	@port: the port id
3984 *
3985 *	Restarts autonegotiation for the selected port.
3986 */
3987int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port)
3988{
3989	struct fw_port_cmd c;
3990
3991	memset(&c, 0, sizeof(c));
3992	c.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) |
3993				     F_FW_CMD_REQUEST | F_FW_CMD_EXEC |
3994				     V_FW_PORT_CMD_PORTID(port));
3995	c.action_to_len16 =
3996		cpu_to_be32(V_FW_PORT_CMD_ACTION(FW_PORT_ACTION_L1_CFG) |
3997			    FW_LEN16(c));
3998	c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP_ANEG);
3999	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
4000}
4001
4002struct intr_details {
4003	u32 mask;
4004	const char *msg;
4005};
4006
4007struct intr_action {
4008	u32 mask;
4009	int arg;
4010	bool (*action)(struct adapter *, int, bool);
4011};
4012
4013#define NONFATAL_IF_DISABLED 1
4014struct intr_info {
4015	const char *name;	/* name of the INT_CAUSE register */
4016	int cause_reg;		/* INT_CAUSE register */
4017	int enable_reg;		/* INT_ENABLE register */
4018	u32 fatal;		/* bits that are fatal */
4019	int flags;		/* hints */
4020	const struct intr_details *details;
4021	const struct intr_action *actions;
4022};
4023
4024static inline char
4025intr_alert_char(u32 cause, u32 enable, u32 fatal)
4026{
4027
4028	if (cause & fatal)
4029		return ('!');
4030	if (cause & enable)
4031		return ('*');
4032	return ('-');
4033}
4034
4035static void
4036t4_show_intr_info(struct adapter *adap, const struct intr_info *ii, u32 cause)
4037{
4038	u32 enable, fatal, leftover;
4039	const struct intr_details *details;
4040	char alert;
4041
4042	enable = t4_read_reg(adap, ii->enable_reg);
4043	if (ii->flags & NONFATAL_IF_DISABLED)
4044		fatal = ii->fatal & t4_read_reg(adap, ii->enable_reg);
4045	else
4046		fatal = ii->fatal;
4047	alert = intr_alert_char(cause, enable, fatal);
4048	CH_ALERT(adap, "%c %s 0x%x = 0x%08x, E 0x%08x, F 0x%08x\n",
4049	    alert, ii->name, ii->cause_reg, cause, enable, fatal);
4050
4051	leftover = cause;
4052	for (details = ii->details; details && details->mask != 0; details++) {
4053		u32 msgbits = details->mask & cause;
4054		if (msgbits == 0)
4055			continue;
4056		alert = intr_alert_char(msgbits, enable, ii->fatal);
4057		CH_ALERT(adap, "  %c [0x%08x] %s\n", alert, msgbits,
4058		    details->msg);
4059		leftover &= ~msgbits;
4060	}
4061	if (leftover != 0 && leftover != cause)
4062		CH_ALERT(adap, "  ? [0x%08x]\n", leftover);
4063}
4064
4065/*
4066 * Returns true for fatal error.
4067 */
4068static bool
4069t4_handle_intr(struct adapter *adap, const struct intr_info *ii,
4070    u32 additional_cause, bool verbose)
4071{
4072	u32 cause, fatal;
4073	bool rc;
4074	const struct intr_action *action;
4075
4076	/*
4077	 * Read and display cause.  Note that the top level PL_INT_CAUSE is a
4078	 * bit special and we need to completely ignore the bits that are not in
4079	 * PL_INT_ENABLE.
4080	 */
4081	cause = t4_read_reg(adap, ii->cause_reg);
4082	if (ii->cause_reg == A_PL_INT_CAUSE)
4083		cause &= t4_read_reg(adap, ii->enable_reg);
4084	if (verbose || cause != 0)
4085		t4_show_intr_info(adap, ii, cause);
4086	fatal = cause & ii->fatal;
4087	if (fatal != 0 && ii->flags & NONFATAL_IF_DISABLED)
4088		fatal &= t4_read_reg(adap, ii->enable_reg);
4089	cause |= additional_cause;
4090	if (cause == 0)
4091		return (false);
4092
4093	rc = fatal != 0;
4094	for (action = ii->actions; action && action->mask != 0; action++) {
4095		if (!(action->mask & cause))
4096			continue;
4097		rc |= (action->action)(adap, action->arg, verbose);
4098	}
4099
4100	/* clear */
4101	t4_write_reg(adap, ii->cause_reg, cause);
4102	(void)t4_read_reg(adap, ii->cause_reg);
4103
4104	return (rc);
4105}
4106
4107/*
4108 * Interrupt handler for the PCIE module.
4109 */
4110static bool pcie_intr_handler(struct adapter *adap, int arg, bool verbose)
4111{
4112	static const struct intr_details sysbus_intr_details[] = {
4113		{ F_RNPP, "RXNP array parity error" },
4114		{ F_RPCP, "RXPC array parity error" },
4115		{ F_RCIP, "RXCIF array parity error" },
4116		{ F_RCCP, "Rx completions control array parity error" },
4117		{ F_RFTP, "RXFT array parity error" },
4118		{ 0 }
4119	};
4120	static const struct intr_info sysbus_intr_info = {
4121		.name = "PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS",
4122		.cause_reg = A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS,
4123		.enable_reg = A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_INTERRUPT_ENABLE,
4124		.fatal = F_RFTP | F_RCCP | F_RCIP | F_RPCP | F_RNPP,
4125		.flags = 0,
4126		.details = sysbus_intr_details,
4127		.actions = NULL,
4128	};
4129	static const struct intr_details pcie_port_intr_details[] = {
4130		{ F_TPCP, "TXPC array parity error" },
4131		{ F_TNPP, "TXNP array parity error" },
4132		{ F_TFTP, "TXFT array parity error" },
4133		{ F_TCAP, "TXCA array parity error" },
4134		{ F_TCIP, "TXCIF array parity error" },
4135		{ F_RCAP, "RXCA array parity error" },
4136		{ F_OTDD, "outbound request TLP discarded" },
4137		{ F_RDPE, "Rx data parity error" },
4138		{ F_TDUE, "Tx uncorrectable data error" },
4139		{ 0 }
4140	};
4141	static const struct intr_info pcie_port_intr_info = {
4142		.name = "PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS",
4143		.cause_reg = A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS,
4144		.enable_reg = A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_INTERRUPT_ENABLE,
4145		.fatal = F_TPCP | F_TNPP | F_TFTP | F_TCAP | F_TCIP | F_RCAP |
4146		    F_OTDD | F_RDPE | F_TDUE,
4147		.flags = 0,
4148		.details = pcie_port_intr_details,
4149		.actions = NULL,
4150	};
4151	static const struct intr_details pcie_intr_details[] = {
4152		{ F_MSIADDRLPERR, "MSI AddrL parity error" },
4153		{ F_MSIADDRHPERR, "MSI AddrH parity error" },
4154		{ F_MSIDATAPERR, "MSI data parity error" },
4155		{ F_MSIXADDRLPERR, "MSI-X AddrL parity error" },
4156		{ F_MSIXADDRHPERR, "MSI-X AddrH parity error" },
4157		{ F_MSIXDATAPERR, "MSI-X data parity error" },
4158		{ F_MSIXDIPERR, "MSI-X DI parity error" },
4159		{ F_PIOCPLPERR, "PCIe PIO completion FIFO parity error" },
4160		{ F_PIOREQPERR, "PCIe PIO request FIFO parity error" },
4161		{ F_TARTAGPERR, "PCIe target tag FIFO parity error" },
4162		{ F_CCNTPERR, "PCIe CMD channel count parity error" },
4163		{ F_CREQPERR, "PCIe CMD channel request parity error" },
4164		{ F_CRSPPERR, "PCIe CMD channel response parity error" },
4165		{ F_DCNTPERR, "PCIe DMA channel count parity error" },
4166		{ F_DREQPERR, "PCIe DMA channel request parity error" },
4167		{ F_DRSPPERR, "PCIe DMA channel response parity error" },
4168		{ F_HCNTPERR, "PCIe HMA channel count parity error" },
4169		{ F_HREQPERR, "PCIe HMA channel request parity error" },
4170		{ F_HRSPPERR, "PCIe HMA channel response parity error" },
4171		{ F_CFGSNPPERR, "PCIe config snoop FIFO parity error" },
4172		{ F_FIDPERR, "PCIe FID parity error" },
4173		{ F_INTXCLRPERR, "PCIe INTx clear parity error" },
4174		{ F_MATAGPERR, "PCIe MA tag parity error" },
4175		{ F_PIOTAGPERR, "PCIe PIO tag parity error" },
4176		{ F_RXCPLPERR, "PCIe Rx completion parity error" },
4177		{ F_RXWRPERR, "PCIe Rx write parity error" },
4178		{ F_RPLPERR, "PCIe replay buffer parity error" },
4179		{ F_PCIESINT, "PCIe core secondary fault" },
4180		{ F_PCIEPINT, "PCIe core primary fault" },
4181		{ F_UNXSPLCPLERR, "PCIe unexpected split completion error" },
4182		{ 0 }
4183	};
4184	static const struct intr_details t5_pcie_intr_details[] = {
4185		{ F_IPGRPPERR, "Parity errors observed by IP" },
4186		{ F_NONFATALERR, "PCIe non-fatal error" },
4187		{ F_READRSPERR, "Outbound read error" },
4188		{ F_TRGT1GRPPERR, "PCIe TRGT1 group FIFOs parity error" },
4189		{ F_IPSOTPERR, "PCIe IP SOT buffer SRAM parity error" },
4190		{ F_IPRETRYPERR, "PCIe IP replay buffer parity error" },
4191		{ F_IPRXDATAGRPPERR, "PCIe IP Rx data group SRAMs parity error" },
4192		{ F_IPRXHDRGRPPERR, "PCIe IP Rx header group SRAMs parity error" },
4193		{ F_PIOTAGQPERR, "PIO tag queue FIFO parity error" },
4194		{ F_MAGRPPERR, "MA group FIFO parity error" },
4195		{ F_VFIDPERR, "VFID SRAM parity error" },
4196		{ F_FIDPERR, "FID SRAM parity error" },
4197		{ F_CFGSNPPERR, "config snoop FIFO parity error" },
4198		{ F_HRSPPERR, "HMA channel response data SRAM parity error" },
4199		{ F_HREQRDPERR, "HMA channel read request SRAM parity error" },
4200		{ F_HREQWRPERR, "HMA channel write request SRAM parity error" },
4201		{ F_DRSPPERR, "DMA channel response data SRAM parity error" },
4202		{ F_DREQRDPERR, "DMA channel write request SRAM parity error" },
4203		{ F_CRSPPERR, "CMD channel response data SRAM parity error" },
4204		{ F_CREQRDPERR, "CMD channel read request SRAM parity error" },
4205		{ F_MSTTAGQPERR, "PCIe master tag queue SRAM parity error" },
4206		{ F_TGTTAGQPERR, "PCIe target tag queue FIFO parity error" },
4207		{ F_PIOREQGRPPERR, "PIO request group FIFOs parity error" },
4208		{ F_PIOCPLGRPPERR, "PIO completion group FIFOs parity error" },
4209		{ F_MSIXDIPERR, "MSI-X DI SRAM parity error" },
4210		{ F_MSIXDATAPERR, "MSI-X data SRAM parity error" },
4211		{ F_MSIXADDRHPERR, "MSI-X AddrH SRAM parity error" },
4212		{ F_MSIXADDRLPERR, "MSI-X AddrL SRAM parity error" },
4213		{ F_MSIXSTIPERR, "MSI-X STI SRAM parity error" },
4214		{ F_MSTTIMEOUTPERR, "Master timeout FIFO parity error" },
4215		{ F_MSTGRPPERR, "Master response read queue SRAM parity error" },
4216		{ 0 }
4217	};
4218	struct intr_info pcie_intr_info = {
4219		.name = "PCIE_INT_CAUSE",
4220		.cause_reg = A_PCIE_INT_CAUSE,
4221		.enable_reg = A_PCIE_INT_ENABLE,
4222		.fatal = 0xffffffff,
4223		.flags = NONFATAL_IF_DISABLED,
4224		.details = NULL,
4225		.actions = NULL,
4226	};
4227	bool fatal = false;
4228
4229	if (is_t4(adap)) {
4230		fatal |= t4_handle_intr(adap, &sysbus_intr_info, 0, verbose);
4231		fatal |= t4_handle_intr(adap, &pcie_port_intr_info, 0, verbose);
4232
4233		pcie_intr_info.details = pcie_intr_details;
4234	} else {
4235		pcie_intr_info.details = t5_pcie_intr_details;
4236	}
4237	fatal |= t4_handle_intr(adap, &pcie_intr_info, 0, verbose);
4238
4239	return (fatal);
4240}
4241
4242/*
4243 * TP interrupt handler.
4244 */
4245static bool tp_intr_handler(struct adapter *adap, int arg, bool verbose)
4246{
4247	static const struct intr_details tp_intr_details[] = {
4248		{ 0x3fffffff, "TP parity error" },
4249		{ F_FLMTXFLSTEMPTY, "TP out of Tx pages" },
4250		{ 0 }
4251	};
4252	static const struct intr_info tp_intr_info = {
4253		.name = "TP_INT_CAUSE",
4254		.cause_reg = A_TP_INT_CAUSE,
4255		.enable_reg = A_TP_INT_ENABLE,
4256		.fatal = 0x7fffffff,
4257		.flags = NONFATAL_IF_DISABLED,
4258		.details = tp_intr_details,
4259		.actions = NULL,
4260	};
4261
4262	return (t4_handle_intr(adap, &tp_intr_info, 0, verbose));
4263}
4264
4265/*
4266 * SGE interrupt handler.
4267 */
4268static bool sge_intr_handler(struct adapter *adap, int arg, bool verbose)
4269{
4270	static const struct intr_info sge_int1_info = {
4271		.name = "SGE_INT_CAUSE1",
4272		.cause_reg = A_SGE_INT_CAUSE1,
4273		.enable_reg = A_SGE_INT_ENABLE1,
4274		.fatal = 0xffffffff,
4275		.flags = NONFATAL_IF_DISABLED,
4276		.details = NULL,
4277		.actions = NULL,
4278	};
4279	static const struct intr_info sge_int2_info = {
4280		.name = "SGE_INT_CAUSE2",
4281		.cause_reg = A_SGE_INT_CAUSE2,
4282		.enable_reg = A_SGE_INT_ENABLE2,
4283		.fatal = 0xffffffff,
4284		.flags = NONFATAL_IF_DISABLED,
4285		.details = NULL,
4286		.actions = NULL,
4287	};
4288	static const struct intr_details sge_int3_details[] = {
4289		{ F_ERR_FLM_DBP,
4290			"DBP pointer delivery for invalid context or QID" },
4291		{ F_ERR_FLM_IDMA1 | F_ERR_FLM_IDMA0,
4292			"Invalid QID or header request by IDMA" },
4293		{ F_ERR_FLM_HINT, "FLM hint is for invalid context or QID" },
4294		{ F_ERR_PCIE_ERROR3, "SGE PCIe error for DBP thread 3" },
4295		{ F_ERR_PCIE_ERROR2, "SGE PCIe error for DBP thread 2" },
4296		{ F_ERR_PCIE_ERROR1, "SGE PCIe error for DBP thread 1" },
4297		{ F_ERR_PCIE_ERROR0, "SGE PCIe error for DBP thread 0" },
4298		{ F_ERR_TIMER_ABOVE_MAX_QID,
4299			"SGE GTS with timer 0-5 for IQID > 1023" },
4300		{ F_ERR_CPL_EXCEED_IQE_SIZE,
4301			"SGE received CPL exceeding IQE size" },
4302		{ F_ERR_INVALID_CIDX_INC, "SGE GTS CIDX increment too large" },
4303		{ F_ERR_ITP_TIME_PAUSED, "SGE ITP error" },
4304		{ F_ERR_CPL_OPCODE_0, "SGE received 0-length CPL" },
4305		{ F_ERR_DROPPED_DB, "SGE DB dropped" },
4306		{ F_ERR_DATA_CPL_ON_HIGH_QID1 | F_ERR_DATA_CPL_ON_HIGH_QID0,
4307		  "SGE IQID > 1023 received CPL for FL" },
4308		{ F_ERR_BAD_DB_PIDX3 | F_ERR_BAD_DB_PIDX2 | F_ERR_BAD_DB_PIDX1 |
4309			F_ERR_BAD_DB_PIDX0, "SGE DBP pidx increment too large" },
4310		{ F_ERR_ING_PCIE_CHAN, "SGE Ingress PCIe channel mismatch" },
4311		{ F_ERR_ING_CTXT_PRIO,
4312			"Ingress context manager priority user error" },
4313		{ F_ERR_EGR_CTXT_PRIO,
4314			"Egress context manager priority user error" },
4315		{ F_DBFIFO_HP_INT, "High priority DB FIFO threshold reached" },
4316		{ F_DBFIFO_LP_INT, "Low priority DB FIFO threshold reached" },
4317		{ F_REG_ADDRESS_ERR, "Undefined SGE register accessed" },
4318		{ F_INGRESS_SIZE_ERR, "SGE illegal ingress QID" },
4319		{ F_EGRESS_SIZE_ERR, "SGE illegal egress QID" },
4320		{ 0x0000000f, "SGE context access for invalid queue" },
4321		{ 0 }
4322	};
4323	static const struct intr_details t6_sge_int3_details[] = {
4324		{ F_ERR_FLM_DBP,
4325			"DBP pointer delivery for invalid context or QID" },
4326		{ F_ERR_FLM_IDMA1 | F_ERR_FLM_IDMA0,
4327			"Invalid QID or header request by IDMA" },
4328		{ F_ERR_FLM_HINT, "FLM hint is for invalid context or QID" },
4329		{ F_ERR_PCIE_ERROR3, "SGE PCIe error for DBP thread 3" },
4330		{ F_ERR_PCIE_ERROR2, "SGE PCIe error for DBP thread 2" },
4331		{ F_ERR_PCIE_ERROR1, "SGE PCIe error for DBP thread 1" },
4332		{ F_ERR_PCIE_ERROR0, "SGE PCIe error for DBP thread 0" },
4333		{ F_ERR_TIMER_ABOVE_MAX_QID,
4334			"SGE GTS with timer 0-5 for IQID > 1023" },
4335		{ F_ERR_CPL_EXCEED_IQE_SIZE,
4336			"SGE received CPL exceeding IQE size" },
4337		{ F_ERR_INVALID_CIDX_INC, "SGE GTS CIDX increment too large" },
4338		{ F_ERR_ITP_TIME_PAUSED, "SGE ITP error" },
4339		{ F_ERR_CPL_OPCODE_0, "SGE received 0-length CPL" },
4340		{ F_ERR_DROPPED_DB, "SGE DB dropped" },
4341		{ F_ERR_DATA_CPL_ON_HIGH_QID1 | F_ERR_DATA_CPL_ON_HIGH_QID0,
4342			"SGE IQID > 1023 received CPL for FL" },
4343		{ F_ERR_BAD_DB_PIDX3 | F_ERR_BAD_DB_PIDX2 | F_ERR_BAD_DB_PIDX1 |
4344			F_ERR_BAD_DB_PIDX0, "SGE DBP pidx increment too large" },
4345		{ F_ERR_ING_PCIE_CHAN, "SGE Ingress PCIe channel mismatch" },
4346		{ F_ERR_ING_CTXT_PRIO,
4347			"Ingress context manager priority user error" },
4348		{ F_ERR_EGR_CTXT_PRIO,
4349			"Egress context manager priority user error" },
4350		{ F_DBP_TBUF_FULL, "SGE DBP tbuf full" },
4351		{ F_FATAL_WRE_LEN,
4352			"SGE WRE packet less than advertized length" },
4353		{ F_REG_ADDRESS_ERR, "Undefined SGE register accessed" },
4354		{ F_INGRESS_SIZE_ERR, "SGE illegal ingress QID" },
4355		{ F_EGRESS_SIZE_ERR, "SGE illegal egress QID" },
4356		{ 0x0000000f, "SGE context access for invalid queue" },
4357		{ 0 }
4358	};
4359	struct intr_info sge_int3_info = {
4360		.name = "SGE_INT_CAUSE3",
4361		.cause_reg = A_SGE_INT_CAUSE3,
4362		.enable_reg = A_SGE_INT_ENABLE3,
4363		.fatal = F_ERR_CPL_EXCEED_IQE_SIZE,
4364		.flags = 0,
4365		.details = NULL,
4366		.actions = NULL,
4367	};
4368	static const struct intr_info sge_int4_info = {
4369		.name = "SGE_INT_CAUSE4",
4370		.cause_reg = A_SGE_INT_CAUSE4,
4371		.enable_reg = A_SGE_INT_ENABLE4,
4372		.fatal = 0,
4373		.flags = 0,
4374		.details = NULL,
4375		.actions = NULL,
4376	};
4377	static const struct intr_info sge_int5_info = {
4378		.name = "SGE_INT_CAUSE5",
4379		.cause_reg = A_SGE_INT_CAUSE5,
4380		.enable_reg = A_SGE_INT_ENABLE5,
4381		.fatal = 0xffffffff,
4382		.flags = NONFATAL_IF_DISABLED,
4383		.details = NULL,
4384		.actions = NULL,
4385	};
4386	static const struct intr_info sge_int6_info = {
4387		.name = "SGE_INT_CAUSE6",
4388		.cause_reg = A_SGE_INT_CAUSE6,
4389		.enable_reg = A_SGE_INT_ENABLE6,
4390		.fatal = 0,
4391		.flags = 0,
4392		.details = NULL,
4393		.actions = NULL,
4394	};
4395
4396	bool fatal;
4397	u32 v;
4398
4399	if (chip_id(adap) <= CHELSIO_T5) {
4400		sge_int3_info.details = sge_int3_details;
4401	} else {
4402		sge_int3_info.details = t6_sge_int3_details;
4403	}
4404
4405	fatal = false;
4406	fatal |= t4_handle_intr(adap, &sge_int1_info, 0, verbose);
4407	fatal |= t4_handle_intr(adap, &sge_int2_info, 0, verbose);
4408	fatal |= t4_handle_intr(adap, &sge_int3_info, 0, verbose);
4409	fatal |= t4_handle_intr(adap, &sge_int4_info, 0, verbose);
4410	if (chip_id(adap) >= CHELSIO_T5)
4411		fatal |= t4_handle_intr(adap, &sge_int5_info, 0, verbose);
4412	if (chip_id(adap) >= CHELSIO_T6)
4413		fatal |= t4_handle_intr(adap, &sge_int6_info, 0, verbose);
4414
4415	v = t4_read_reg(adap, A_SGE_ERROR_STATS);
4416	if (v & F_ERROR_QID_VALID) {
4417		CH_ERR(adap, "SGE error for QID %u\n", G_ERROR_QID(v));
4418		if (v & F_UNCAPTURED_ERROR)
4419			CH_ERR(adap, "SGE UNCAPTURED_ERROR set (clearing)\n");
4420		t4_write_reg(adap, A_SGE_ERROR_STATS,
4421		    F_ERROR_QID_VALID | F_UNCAPTURED_ERROR);
4422	}
4423
4424	return (fatal);
4425}
4426
4427/*
4428 * CIM interrupt handler.
4429 */
4430static bool cim_intr_handler(struct adapter *adap, int arg, bool verbose)
4431{
4432	static const struct intr_action cim_host_intr_actions[] = {
4433		{ F_TIMER0INT, 0, t4_os_dump_cimla },
4434		{ 0 },
4435	};
4436	static const struct intr_details cim_host_intr_details[] = {
4437		/* T6+ */
4438		{ F_PCIE2CIMINTFPARERR, "CIM IBQ PCIe interface parity error" },
4439
4440		/* T5+ */
4441		{ F_MA_CIM_INTFPERR, "MA2CIM interface parity error" },
4442		{ F_PLCIM_MSTRSPDATAPARERR,
4443			"PL2CIM master response data parity error" },
4444		{ F_NCSI2CIMINTFPARERR, "CIM IBQ NC-SI interface parity error" },
4445		{ F_SGE2CIMINTFPARERR, "CIM IBQ SGE interface parity error" },
4446		{ F_ULP2CIMINTFPARERR, "CIM IBQ ULP_TX interface parity error" },
4447		{ F_TP2CIMINTFPARERR, "CIM IBQ TP interface parity error" },
4448		{ F_OBQSGERX1PARERR, "CIM OBQ SGE1_RX parity error" },
4449		{ F_OBQSGERX0PARERR, "CIM OBQ SGE0_RX parity error" },
4450
4451		/* T4+ */
4452		{ F_TIEQOUTPARERRINT, "CIM TIEQ outgoing FIFO parity error" },
4453		{ F_TIEQINPARERRINT, "CIM TIEQ incoming FIFO parity error" },
4454		{ F_MBHOSTPARERR, "CIM mailbox host read parity error" },
4455		{ F_MBUPPARERR, "CIM mailbox uP parity error" },
4456		{ F_IBQTP0PARERR, "CIM IBQ TP0 parity error" },
4457		{ F_IBQTP1PARERR, "CIM IBQ TP1 parity error" },
4458		{ F_IBQULPPARERR, "CIM IBQ ULP parity error" },
4459		{ F_IBQSGELOPARERR, "CIM IBQ SGE_LO parity error" },
4460		{ F_IBQSGEHIPARERR | F_IBQPCIEPARERR,	/* same bit */
4461			"CIM IBQ PCIe/SGE_HI parity error" },
4462		{ F_IBQNCSIPARERR, "CIM IBQ NC-SI parity error" },
4463		{ F_OBQULP0PARERR, "CIM OBQ ULP0 parity error" },
4464		{ F_OBQULP1PARERR, "CIM OBQ ULP1 parity error" },
4465		{ F_OBQULP2PARERR, "CIM OBQ ULP2 parity error" },
4466		{ F_OBQULP3PARERR, "CIM OBQ ULP3 parity error" },
4467		{ F_OBQSGEPARERR, "CIM OBQ SGE parity error" },
4468		{ F_OBQNCSIPARERR, "CIM OBQ NC-SI parity error" },
4469		{ F_TIMER1INT, "CIM TIMER0 interrupt" },
4470		{ F_TIMER0INT, "CIM TIMER0 interrupt" },
4471		{ F_PREFDROPINT, "CIM control register prefetch drop" },
4472		{ 0}
4473	};
4474	static const struct intr_info cim_host_intr_info = {
4475		.name = "CIM_HOST_INT_CAUSE",
4476		.cause_reg = A_CIM_HOST_INT_CAUSE,
4477		.enable_reg = A_CIM_HOST_INT_ENABLE,
4478		.fatal = 0x007fffe6,
4479		.flags = NONFATAL_IF_DISABLED,
4480		.details = cim_host_intr_details,
4481		.actions = cim_host_intr_actions,
4482	};
4483	static const struct intr_details cim_host_upacc_intr_details[] = {
4484		{ F_EEPROMWRINT, "CIM EEPROM came out of busy state" },
4485		{ F_TIMEOUTMAINT, "CIM PIF MA timeout" },
4486		{ F_TIMEOUTINT, "CIM PIF timeout" },
4487		{ F_RSPOVRLOOKUPINT, "CIM response FIFO overwrite" },
4488		{ F_REQOVRLOOKUPINT, "CIM request FIFO overwrite" },
4489		{ F_BLKWRPLINT, "CIM block write to PL space" },
4490		{ F_BLKRDPLINT, "CIM block read from PL space" },
4491		{ F_SGLWRPLINT,
4492			"CIM single write to PL space with illegal BEs" },
4493		{ F_SGLRDPLINT,
4494			"CIM single read from PL space with illegal BEs" },
4495		{ F_BLKWRCTLINT, "CIM block write to CTL space" },
4496		{ F_BLKRDCTLINT, "CIM block read from CTL space" },
4497		{ F_SGLWRCTLINT,
4498			"CIM single write to CTL space with illegal BEs" },
4499		{ F_SGLRDCTLINT,
4500			"CIM single read from CTL space with illegal BEs" },
4501		{ F_BLKWREEPROMINT, "CIM block write to EEPROM space" },
4502		{ F_BLKRDEEPROMINT, "CIM block read from EEPROM space" },
4503		{ F_SGLWREEPROMINT,
4504			"CIM single write to EEPROM space with illegal BEs" },
4505		{ F_SGLRDEEPROMINT,
4506			"CIM single read from EEPROM space with illegal BEs" },
4507		{ F_BLKWRFLASHINT, "CIM block write to flash space" },
4508		{ F_BLKRDFLASHINT, "CIM block read from flash space" },
4509		{ F_SGLWRFLASHINT, "CIM single write to flash space" },
4510		{ F_SGLRDFLASHINT,
4511			"CIM single read from flash space with illegal BEs" },
4512		{ F_BLKWRBOOTINT, "CIM block write to boot space" },
4513		{ F_BLKRDBOOTINT, "CIM block read from boot space" },
4514		{ F_SGLWRBOOTINT, "CIM single write to boot space" },
4515		{ F_SGLRDBOOTINT,
4516			"CIM single read from boot space with illegal BEs" },
4517		{ F_ILLWRBEINT, "CIM illegal write BEs" },
4518		{ F_ILLRDBEINT, "CIM illegal read BEs" },
4519		{ F_ILLRDINT, "CIM illegal read" },
4520		{ F_ILLWRINT, "CIM illegal write" },
4521		{ F_ILLTRANSINT, "CIM illegal transaction" },
4522		{ F_RSVDSPACEINT, "CIM reserved space access" },
4523		{0}
4524	};
4525	static const struct intr_info cim_host_upacc_intr_info = {
4526		.name = "CIM_HOST_UPACC_INT_CAUSE",
4527		.cause_reg = A_CIM_HOST_UPACC_INT_CAUSE,
4528		.enable_reg = A_CIM_HOST_UPACC_INT_ENABLE,
4529		.fatal = 0x3fffeeff,
4530		.flags = NONFATAL_IF_DISABLED,
4531		.details = cim_host_upacc_intr_details,
4532		.actions = NULL,
4533	};
4534	static const struct intr_info cim_pf_host_intr_info = {
4535		.name = "CIM_PF_HOST_INT_CAUSE",
4536		.cause_reg = MYPF_REG(A_CIM_PF_HOST_INT_CAUSE),
4537		.enable_reg = MYPF_REG(A_CIM_PF_HOST_INT_ENABLE),
4538		.fatal = 0,
4539		.flags = 0,
4540		.details = NULL,
4541		.actions = NULL,
4542	};
4543	u32 val, fw_err;
4544	bool fatal;
4545
4546	fw_err = t4_read_reg(adap, A_PCIE_FW);
4547	if (fw_err & F_PCIE_FW_ERR)
4548		t4_report_fw_error(adap);
4549
4550	/*
4551	 * When the Firmware detects an internal error which normally wouldn't
4552	 * raise a Host Interrupt, it forces a CIM Timer0 interrupt in order
4553	 * to make sure the Host sees the Firmware Crash.  So if we have a
4554	 * Timer0 interrupt and don't see a Firmware Crash, ignore the Timer0
4555	 * interrupt.
4556	 */
4557	val = t4_read_reg(adap, A_CIM_HOST_INT_CAUSE);
4558	if (val & F_TIMER0INT && (!(fw_err & F_PCIE_FW_ERR) ||
4559	    G_PCIE_FW_EVAL(fw_err) != PCIE_FW_EVAL_CRASH)) {
4560		t4_write_reg(adap, A_CIM_HOST_INT_CAUSE, F_TIMER0INT);
4561	}
4562
4563	fatal = false;
4564	fatal |= t4_handle_intr(adap, &cim_host_intr_info, 0, verbose);
4565	fatal |= t4_handle_intr(adap, &cim_host_upacc_intr_info, 0, verbose);
4566	fatal |= t4_handle_intr(adap, &cim_pf_host_intr_info, 0, verbose);
4567
4568	return (fatal);
4569}
4570
4571/*
4572 * ULP RX interrupt handler.
4573 */
4574static bool ulprx_intr_handler(struct adapter *adap, int arg, bool verbose)
4575{
4576	static const struct intr_details ulprx_intr_details[] = {
4577		/* T5+ */
4578		{ F_SE_CNT_MISMATCH_1, "ULPRX SE count mismatch in channel 1" },
4579		{ F_SE_CNT_MISMATCH_0, "ULPRX SE count mismatch in channel 0" },
4580
4581		/* T4+ */
4582		{ F_CAUSE_CTX_1, "ULPRX channel 1 context error" },
4583		{ F_CAUSE_CTX_0, "ULPRX channel 0 context error" },
4584		{ 0x007fffff, "ULPRX parity error" },
4585		{ 0 }
4586	};
4587	static const struct intr_info ulprx_intr_info = {
4588		.name = "ULP_RX_INT_CAUSE",
4589		.cause_reg = A_ULP_RX_INT_CAUSE,
4590		.enable_reg = A_ULP_RX_INT_ENABLE,
4591		.fatal = 0x07ffffff,
4592		.flags = NONFATAL_IF_DISABLED,
4593		.details = ulprx_intr_details,
4594		.actions = NULL,
4595	};
4596	static const struct intr_info ulprx_intr2_info = {
4597		.name = "ULP_RX_INT_CAUSE_2",
4598		.cause_reg = A_ULP_RX_INT_CAUSE_2,
4599		.enable_reg = A_ULP_RX_INT_ENABLE_2,
4600		.fatal = 0,
4601		.flags = 0,
4602		.details = NULL,
4603		.actions = NULL,
4604	};
4605	bool fatal = false;
4606
4607	fatal |= t4_handle_intr(adap, &ulprx_intr_info, 0, verbose);
4608	fatal |= t4_handle_intr(adap, &ulprx_intr2_info, 0, verbose);
4609
4610	return (fatal);
4611}
4612
4613/*
4614 * ULP TX interrupt handler.
4615 */
4616static bool ulptx_intr_handler(struct adapter *adap, int arg, bool verbose)
4617{
4618	static const struct intr_details ulptx_intr_details[] = {
4619		{ F_PBL_BOUND_ERR_CH3, "ULPTX channel 3 PBL out of bounds" },
4620		{ F_PBL_BOUND_ERR_CH2, "ULPTX channel 2 PBL out of bounds" },
4621		{ F_PBL_BOUND_ERR_CH1, "ULPTX channel 1 PBL out of bounds" },
4622		{ F_PBL_BOUND_ERR_CH0, "ULPTX channel 0 PBL out of bounds" },
4623		{ 0x0fffffff, "ULPTX parity error" },
4624		{ 0 }
4625	};
4626	static const struct intr_info ulptx_intr_info = {
4627		.name = "ULP_TX_INT_CAUSE",
4628		.cause_reg = A_ULP_TX_INT_CAUSE,
4629		.enable_reg = A_ULP_TX_INT_ENABLE,
4630		.fatal = 0x0fffffff,
4631		.flags = NONFATAL_IF_DISABLED,
4632		.details = ulptx_intr_details,
4633		.actions = NULL,
4634	};
4635	static const struct intr_info ulptx_intr2_info = {
4636		.name = "ULP_TX_INT_CAUSE_2",
4637		.cause_reg = A_ULP_TX_INT_CAUSE_2,
4638		.enable_reg = A_ULP_TX_INT_ENABLE_2,
4639		.fatal = 0xf0,
4640		.flags = NONFATAL_IF_DISABLED,
4641		.details = NULL,
4642		.actions = NULL,
4643	};
4644	bool fatal = false;
4645
4646	fatal |= t4_handle_intr(adap, &ulptx_intr_info, 0, verbose);
4647	fatal |= t4_handle_intr(adap, &ulptx_intr2_info, 0, verbose);
4648
4649	return (fatal);
4650}
4651
4652static bool pmtx_dump_dbg_stats(struct adapter *adap, int arg, bool verbose)
4653{
4654	int i;
4655	u32 data[17];
4656
4657	t4_read_indirect(adap, A_PM_TX_DBG_CTRL, A_PM_TX_DBG_DATA, &data[0],
4658	    ARRAY_SIZE(data), A_PM_TX_DBG_STAT0);
4659	for (i = 0; i < ARRAY_SIZE(data); i++) {
4660		CH_ALERT(adap, "  - PM_TX_DBG_STAT%u (0x%x) = 0x%08x\n", i,
4661		    A_PM_TX_DBG_STAT0 + i, data[i]);
4662	}
4663
4664	return (false);
4665}
4666
4667/*
4668 * PM TX interrupt handler.
4669 */
4670static bool pmtx_intr_handler(struct adapter *adap, int arg, bool verbose)
4671{
4672	static const struct intr_action pmtx_intr_actions[] = {
4673		{ 0xffffffff, 0, pmtx_dump_dbg_stats },
4674		{ 0 },
4675	};
4676	static const struct intr_details pmtx_intr_details[] = {
4677		{ F_PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large" },
4678		{ F_PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large" },
4679		{ F_PCMD_LEN_OVFL2, "PMTX channel 2 pcmd too large" },
4680		{ F_ZERO_C_CMD_ERROR, "PMTX 0-length pcmd" },
4681		{ 0x0f000000, "PMTX icspi FIFO2X Rx framing error" },
4682		{ 0x00f00000, "PMTX icspi FIFO Rx framing error" },
4683		{ 0x000f0000, "PMTX icspi FIFO Tx framing error" },
4684		{ 0x0000f000, "PMTX oespi FIFO Rx framing error" },
4685		{ 0x00000f00, "PMTX oespi FIFO Tx framing error" },
4686		{ 0x000000f0, "PMTX oespi FIFO2X Tx framing error" },
4687		{ F_OESPI_PAR_ERROR, "PMTX oespi parity error" },
4688		{ F_DB_OPTIONS_PAR_ERROR, "PMTX db_options parity error" },
4689		{ F_ICSPI_PAR_ERROR, "PMTX icspi parity error" },
4690		{ F_C_PCMD_PAR_ERROR, "PMTX c_pcmd parity error" },
4691		{ 0 }
4692	};
4693	static const struct intr_info pmtx_intr_info = {
4694		.name = "PM_TX_INT_CAUSE",
4695		.cause_reg = A_PM_TX_INT_CAUSE,
4696		.enable_reg = A_PM_TX_INT_ENABLE,
4697		.fatal = 0xffffffff,
4698		.flags = 0,
4699		.details = pmtx_intr_details,
4700		.actions = pmtx_intr_actions,
4701	};
4702
4703	return (t4_handle_intr(adap, &pmtx_intr_info, 0, verbose));
4704}
4705
4706/*
4707 * PM RX interrupt handler.
4708 */
4709static bool pmrx_intr_handler(struct adapter *adap, int arg, bool verbose)
4710{
4711	static const struct intr_details pmrx_intr_details[] = {
4712		/* T6+ */
4713		{ 0x18000000, "PMRX ospi overflow" },
4714		{ F_MA_INTF_SDC_ERR, "PMRX MA interface SDC parity error" },
4715		{ F_BUNDLE_LEN_PARERR, "PMRX bundle len FIFO parity error" },
4716		{ F_BUNDLE_LEN_OVFL, "PMRX bundle len FIFO overflow" },
4717		{ F_SDC_ERR, "PMRX SDC error" },
4718
4719		/* T4+ */
4720		{ F_ZERO_E_CMD_ERROR, "PMRX 0-length pcmd" },
4721		{ 0x003c0000, "PMRX iespi FIFO2X Rx framing error" },
4722		{ 0x0003c000, "PMRX iespi Rx framing error" },
4723		{ 0x00003c00, "PMRX iespi Tx framing error" },
4724		{ 0x00000300, "PMRX ocspi Rx framing error" },
4725		{ 0x000000c0, "PMRX ocspi Tx framing error" },
4726		{ 0x00000030, "PMRX ocspi FIFO2X Tx framing error" },
4727		{ F_OCSPI_PAR_ERROR, "PMRX ocspi parity error" },
4728		{ F_DB_OPTIONS_PAR_ERROR, "PMRX db_options parity error" },
4729		{ F_IESPI_PAR_ERROR, "PMRX iespi parity error" },
4730		{ F_E_PCMD_PAR_ERROR, "PMRX e_pcmd parity error"},
4731		{ 0 }
4732	};
4733	static const struct intr_info pmrx_intr_info = {
4734		.name = "PM_RX_INT_CAUSE",
4735		.cause_reg = A_PM_RX_INT_CAUSE,
4736		.enable_reg = A_PM_RX_INT_ENABLE,
4737		.fatal = 0x1fffffff,
4738		.flags = NONFATAL_IF_DISABLED,
4739		.details = pmrx_intr_details,
4740		.actions = NULL,
4741	};
4742
4743	return (t4_handle_intr(adap, &pmrx_intr_info, 0, verbose));
4744}
4745
4746/*
4747 * CPL switch interrupt handler.
4748 */
4749static bool cplsw_intr_handler(struct adapter *adap, int arg, bool verbose)
4750{
4751	static const struct intr_details cplsw_intr_details[] = {
4752		/* T5+ */
4753		{ F_PERR_CPL_128TO128_1, "CPLSW 128TO128 FIFO1 parity error" },
4754		{ F_PERR_CPL_128TO128_0, "CPLSW 128TO128 FIFO0 parity error" },
4755
4756		/* T4+ */
4757		{ F_CIM_OP_MAP_PERR, "CPLSW CIM op_map parity error" },
4758		{ F_CIM_OVFL_ERROR, "CPLSW CIM overflow" },
4759		{ F_TP_FRAMING_ERROR, "CPLSW TP framing error" },
4760		{ F_SGE_FRAMING_ERROR, "CPLSW SGE framing error" },
4761		{ F_CIM_FRAMING_ERROR, "CPLSW CIM framing error" },
4762		{ F_ZERO_SWITCH_ERROR, "CPLSW no-switch error" },
4763		{ 0 }
4764	};
4765	static const struct intr_info cplsw_intr_info = {
4766		.name = "CPL_INTR_CAUSE",
4767		.cause_reg = A_CPL_INTR_CAUSE,
4768		.enable_reg = A_CPL_INTR_ENABLE,
4769		.fatal = 0xff,
4770		.flags = NONFATAL_IF_DISABLED,
4771		.details = cplsw_intr_details,
4772		.actions = NULL,
4773	};
4774
4775	return (t4_handle_intr(adap, &cplsw_intr_info, 0, verbose));
4776}
4777
4778#define T4_LE_FATAL_MASK (F_PARITYERR | F_UNKNOWNCMD | F_REQQPARERR)
4779#define T5_LE_FATAL_MASK (T4_LE_FATAL_MASK | F_VFPARERR)
4780#define T6_LE_PERRCRC_MASK (F_PIPELINEERR | F_CLIPTCAMACCFAIL | \
4781    F_SRVSRAMACCFAIL | F_CLCAMCRCPARERR | F_CLCAMINTPERR | F_SSRAMINTPERR | \
4782    F_SRVSRAMPERR | F_VFSRAMPERR | F_TCAMINTPERR | F_TCAMCRCERR | \
4783    F_HASHTBLMEMACCERR | F_MAIFWRINTPERR | F_HASHTBLMEMCRCERR)
4784#define T6_LE_FATAL_MASK (T6_LE_PERRCRC_MASK | F_T6_UNKNOWNCMD | \
4785    F_TCAMACCFAIL | F_HASHTBLACCFAIL | F_CMDTIDERR | F_CMDPRSRINTERR | \
4786    F_TOTCNTERR | F_CLCAMFIFOERR | F_CLIPSUBERR)
4787
4788/*
4789 * LE interrupt handler.
4790 */
4791static bool le_intr_handler(struct adapter *adap, int arg, bool verbose)
4792{
4793	static const struct intr_details le_intr_details[] = {
4794		{ F_REQQPARERR, "LE request queue parity error" },
4795		{ F_UNKNOWNCMD, "LE unknown command" },
4796		{ F_ACTRGNFULL, "LE active region full" },
4797		{ F_PARITYERR, "LE parity error" },
4798		{ F_LIPMISS, "LE LIP miss" },
4799		{ F_LIP0, "LE 0 LIP error" },
4800		{ 0 }
4801	};
4802	static const struct intr_details t6_le_intr_details[] = {
4803		{ F_CLIPSUBERR, "LE CLIP CAM reverse substitution error" },
4804		{ F_CLCAMFIFOERR, "LE CLIP CAM internal FIFO error" },
4805		{ F_CTCAMINVLDENT, "Invalid IPv6 CLIP TCAM entry" },
4806		{ F_TCAMINVLDENT, "Invalid IPv6 TCAM entry" },
4807		{ F_TOTCNTERR, "LE total active < TCAM count" },
4808		{ F_CMDPRSRINTERR, "LE internal error in parser" },
4809		{ F_CMDTIDERR, "Incorrect tid in LE command" },
4810		{ F_T6_ACTRGNFULL, "LE active region full" },
4811		{ F_T6_ACTCNTIPV6TZERO, "LE IPv6 active open TCAM counter -ve" },
4812		{ F_T6_ACTCNTIPV4TZERO, "LE IPv4 active open TCAM counter -ve" },
4813		{ F_T6_ACTCNTIPV6ZERO, "LE IPv6 active open counter -ve" },
4814		{ F_T6_ACTCNTIPV4ZERO, "LE IPv4 active open counter -ve" },
4815		{ F_HASHTBLACCFAIL, "Hash table read error (proto conflict)" },
4816		{ F_TCAMACCFAIL, "LE TCAM access failure" },
4817		{ F_T6_UNKNOWNCMD, "LE unknown command" },
4818		{ F_T6_LIP0, "LE found 0 LIP during CLIP substitution" },
4819		{ F_T6_LIPMISS, "LE CLIP lookup miss" },
4820		{ T6_LE_PERRCRC_MASK, "LE parity/CRC error" },
4821		{ 0 }
4822	};
4823	struct intr_info le_intr_info = {
4824		.name = "LE_DB_INT_CAUSE",
4825		.cause_reg = A_LE_DB_INT_CAUSE,
4826		.enable_reg = A_LE_DB_INT_ENABLE,
4827		.fatal = 0,
4828		.flags = NONFATAL_IF_DISABLED,
4829		.details = NULL,
4830		.actions = NULL,
4831	};
4832
4833	if (chip_id(adap) <= CHELSIO_T5) {
4834		le_intr_info.details = le_intr_details;
4835		le_intr_info.fatal = T5_LE_FATAL_MASK;
4836	} else {
4837		le_intr_info.details = t6_le_intr_details;
4838		le_intr_info.fatal = T6_LE_FATAL_MASK;
4839	}
4840
4841	return (t4_handle_intr(adap, &le_intr_info, 0, verbose));
4842}
4843
4844/*
4845 * MPS interrupt handler.
4846 */
4847static bool mps_intr_handler(struct adapter *adap, int arg, bool verbose)
4848{
4849	static const struct intr_details mps_rx_perr_intr_details[] = {
4850		{ 0xffffffff, "MPS Rx parity error" },
4851		{ 0 }
4852	};
4853	static const struct intr_info mps_rx_perr_intr_info = {
4854		.name = "MPS_RX_PERR_INT_CAUSE",
4855		.cause_reg = A_MPS_RX_PERR_INT_CAUSE,
4856		.enable_reg = A_MPS_RX_PERR_INT_ENABLE,
4857		.fatal = 0xffffffff,
4858		.flags = NONFATAL_IF_DISABLED,
4859		.details = mps_rx_perr_intr_details,
4860		.actions = NULL,
4861	};
4862	static const struct intr_details mps_tx_intr_details[] = {
4863		{ F_PORTERR, "MPS Tx destination port is disabled" },
4864		{ F_FRMERR, "MPS Tx framing error" },
4865		{ F_SECNTERR, "MPS Tx SOP/EOP error" },
4866		{ F_BUBBLE, "MPS Tx underflow" },
4867		{ V_TXDESCFIFO(M_TXDESCFIFO), "MPS Tx desc FIFO parity error" },
4868		{ V_TXDATAFIFO(M_TXDATAFIFO), "MPS Tx data FIFO parity error" },
4869		{ F_NCSIFIFO, "MPS Tx NC-SI FIFO parity error" },
4870		{ V_TPFIFO(M_TPFIFO), "MPS Tx TP FIFO parity error" },
4871		{ 0 }
4872	};
4873	static const struct intr_info mps_tx_intr_info = {
4874		.name = "MPS_TX_INT_CAUSE",
4875		.cause_reg = A_MPS_TX_INT_CAUSE,
4876		.enable_reg = A_MPS_TX_INT_ENABLE,
4877		.fatal = 0x1ffff,
4878		.flags = NONFATAL_IF_DISABLED,
4879		.details = mps_tx_intr_details,
4880		.actions = NULL,
4881	};
4882	static const struct intr_details mps_trc_intr_details[] = {
4883		{ F_MISCPERR, "MPS TRC misc parity error" },
4884		{ V_PKTFIFO(M_PKTFIFO), "MPS TRC packet FIFO parity error" },
4885		{ V_FILTMEM(M_FILTMEM), "MPS TRC filter parity error" },
4886		{ 0 }
4887	};
4888	static const struct intr_info mps_trc_intr_info = {
4889		.name = "MPS_TRC_INT_CAUSE",
4890		.cause_reg = A_MPS_TRC_INT_CAUSE,
4891		.enable_reg = A_MPS_TRC_INT_ENABLE,
4892		.fatal = F_MISCPERR | V_PKTFIFO(M_PKTFIFO) | V_FILTMEM(M_FILTMEM),
4893		.flags = 0,
4894		.details = mps_trc_intr_details,
4895		.actions = NULL,
4896	};
4897	static const struct intr_details mps_stat_sram_intr_details[] = {
4898		{ 0xffffffff, "MPS statistics SRAM parity error" },
4899		{ 0 }
4900	};
4901	static const struct intr_info mps_stat_sram_intr_info = {
4902		.name = "MPS_STAT_PERR_INT_CAUSE_SRAM",
4903		.cause_reg = A_MPS_STAT_PERR_INT_CAUSE_SRAM,
4904		.enable_reg = A_MPS_STAT_PERR_INT_ENABLE_SRAM,
4905		.fatal = 0x1fffffff,
4906		.flags = NONFATAL_IF_DISABLED,
4907		.details = mps_stat_sram_intr_details,
4908		.actions = NULL,
4909	};
4910	static const struct intr_details mps_stat_tx_intr_details[] = {
4911		{ 0xffffff, "MPS statistics Tx FIFO parity error" },
4912		{ 0 }
4913	};
4914	static const struct intr_info mps_stat_tx_intr_info = {
4915		.name = "MPS_STAT_PERR_INT_CAUSE_TX_FIFO",
4916		.cause_reg = A_MPS_STAT_PERR_INT_CAUSE_TX_FIFO,
4917		.enable_reg = A_MPS_STAT_PERR_INT_ENABLE_TX_FIFO,
4918		.fatal =  0xffffff,
4919		.flags = NONFATAL_IF_DISABLED,
4920		.details = mps_stat_tx_intr_details,
4921		.actions = NULL,
4922	};
4923	static const struct intr_details mps_stat_rx_intr_details[] = {
4924		{ 0xffffff, "MPS statistics Rx FIFO parity error" },
4925		{ 0 }
4926	};
4927	static const struct intr_info mps_stat_rx_intr_info = {
4928		.name = "MPS_STAT_PERR_INT_CAUSE_RX_FIFO",
4929		.cause_reg = A_MPS_STAT_PERR_INT_CAUSE_RX_FIFO,
4930		.enable_reg = A_MPS_STAT_PERR_INT_ENABLE_RX_FIFO,
4931		.fatal =  0xffffff,
4932		.flags = 0,
4933		.details = mps_stat_rx_intr_details,
4934		.actions = NULL,
4935	};
4936	static const struct intr_details mps_cls_intr_details[] = {
4937		{ F_HASHSRAM, "MPS hash SRAM parity error" },
4938		{ F_MATCHTCAM, "MPS match TCAM parity error" },
4939		{ F_MATCHSRAM, "MPS match SRAM parity error" },
4940		{ 0 }
4941	};
4942	static const struct intr_info mps_cls_intr_info = {
4943		.name = "MPS_CLS_INT_CAUSE",
4944		.cause_reg = A_MPS_CLS_INT_CAUSE,
4945		.enable_reg = A_MPS_CLS_INT_ENABLE,
4946		.fatal =  F_MATCHSRAM | F_MATCHTCAM | F_HASHSRAM,
4947		.flags = 0,
4948		.details = mps_cls_intr_details,
4949		.actions = NULL,
4950	};
4951	static const struct intr_details mps_stat_sram1_intr_details[] = {
4952		{ 0xff, "MPS statistics SRAM1 parity error" },
4953		{ 0 }
4954	};
4955	static const struct intr_info mps_stat_sram1_intr_info = {
4956		.name = "MPS_STAT_PERR_INT_CAUSE_SRAM1",
4957		.cause_reg = A_MPS_STAT_PERR_INT_CAUSE_SRAM1,
4958		.enable_reg = A_MPS_STAT_PERR_INT_ENABLE_SRAM1,
4959		.fatal = 0xff,
4960		.flags = 0,
4961		.details = mps_stat_sram1_intr_details,
4962		.actions = NULL,
4963	};
4964
4965	bool fatal;
4966
4967	fatal = false;
4968	fatal |= t4_handle_intr(adap, &mps_rx_perr_intr_info, 0, verbose);
4969	fatal |= t4_handle_intr(adap, &mps_tx_intr_info, 0, verbose);
4970	fatal |= t4_handle_intr(adap, &mps_trc_intr_info, 0, verbose);
4971	fatal |= t4_handle_intr(adap, &mps_stat_sram_intr_info, 0, verbose);
4972	fatal |= t4_handle_intr(adap, &mps_stat_tx_intr_info, 0, verbose);
4973	fatal |= t4_handle_intr(adap, &mps_stat_rx_intr_info, 0, verbose);
4974	fatal |= t4_handle_intr(adap, &mps_cls_intr_info, 0, verbose);
4975	if (chip_id(adap) > CHELSIO_T4) {
4976		fatal |= t4_handle_intr(adap, &mps_stat_sram1_intr_info, 0,
4977		    verbose);
4978	}
4979
4980	t4_write_reg(adap, A_MPS_INT_CAUSE, is_t4(adap) ? 0 : 0xffffffff);
4981	t4_read_reg(adap, A_MPS_INT_CAUSE);	/* flush */
4982
4983	return (fatal);
4984
4985}
4986
4987/*
4988 * EDC/MC interrupt handler.
4989 */
4990static bool mem_intr_handler(struct adapter *adap, int idx, bool verbose)
4991{
4992	static const char name[4][5] = { "EDC0", "EDC1", "MC0", "MC1" };
4993	unsigned int count_reg, v;
4994	static const struct intr_details mem_intr_details[] = {
4995		{ F_ECC_UE_INT_CAUSE, "Uncorrectable ECC data error(s)" },
4996		{ F_ECC_CE_INT_CAUSE, "Correctable ECC data error(s)" },
4997		{ F_PERR_INT_CAUSE, "FIFO parity error" },
4998		{ 0 }
4999	};
5000	struct intr_info ii = {
5001		.fatal = F_PERR_INT_CAUSE | F_ECC_UE_INT_CAUSE,
5002		.details = mem_intr_details,
5003		.flags = 0,
5004		.actions = NULL,
5005	};
5006	bool fatal;
5007
5008	switch (idx) {
5009	case MEM_EDC0:
5010		ii.name = "EDC0_INT_CAUSE";
5011		ii.cause_reg = EDC_REG(A_EDC_INT_CAUSE, 0);
5012		ii.enable_reg = EDC_REG(A_EDC_INT_ENABLE, 0);
5013		count_reg = EDC_REG(A_EDC_ECC_STATUS, 0);
5014		break;
5015	case MEM_EDC1:
5016		ii.name = "EDC1_INT_CAUSE";
5017		ii.cause_reg = EDC_REG(A_EDC_INT_CAUSE, 1);
5018		ii.enable_reg = EDC_REG(A_EDC_INT_ENABLE, 1);
5019		count_reg = EDC_REG(A_EDC_ECC_STATUS, 1);
5020		break;
5021	case MEM_MC0:
5022		ii.name = "MC0_INT_CAUSE";
5023		if (is_t4(adap)) {
5024			ii.cause_reg = A_MC_INT_CAUSE;
5025			ii.enable_reg = A_MC_INT_ENABLE;
5026			count_reg = A_MC_ECC_STATUS;
5027		} else {
5028			ii.cause_reg = A_MC_P_INT_CAUSE;
5029			ii.enable_reg = A_MC_P_INT_ENABLE;
5030			count_reg = A_MC_P_ECC_STATUS;
5031		}
5032		break;
5033	case MEM_MC1:
5034		ii.name = "MC1_INT_CAUSE";
5035		ii.cause_reg = MC_REG(A_MC_P_INT_CAUSE, 1);
5036		ii.enable_reg = MC_REG(A_MC_P_INT_ENABLE, 1);
5037		count_reg = MC_REG(A_MC_P_ECC_STATUS, 1);
5038		break;
5039	}
5040
5041	fatal = t4_handle_intr(adap, &ii, 0, verbose);
5042
5043	v = t4_read_reg(adap, count_reg);
5044	if (v != 0) {
5045		if (G_ECC_UECNT(v) != 0) {
5046			CH_ALERT(adap,
5047			    "%s: %u uncorrectable ECC data error(s)\n",
5048			    name[idx], G_ECC_UECNT(v));
5049		}
5050		if (G_ECC_CECNT(v) != 0) {
5051			if (idx <= MEM_EDC1)
5052				t4_edc_err_read(adap, idx);
5053			CH_WARN_RATELIMIT(adap,
5054			    "%s: %u correctable ECC data error(s)\n",
5055			    name[idx], G_ECC_CECNT(v));
5056		}
5057		t4_write_reg(adap, count_reg, 0xffffffff);
5058	}
5059
5060	return (fatal);
5061}
5062
5063static bool ma_wrap_status(struct adapter *adap, int arg, bool verbose)
5064{
5065	u32 v;
5066
5067	v = t4_read_reg(adap, A_MA_INT_WRAP_STATUS);
5068	CH_ALERT(adap,
5069	    "MA address wrap-around error by client %u to address %#x\n",
5070	    G_MEM_WRAP_CLIENT_NUM(v), G_MEM_WRAP_ADDRESS(v) << 4);
5071	t4_write_reg(adap, A_MA_INT_WRAP_STATUS, v);
5072
5073	return (false);
5074}
5075
5076
5077/*
5078 * MA interrupt handler.
5079 */
5080static bool ma_intr_handler(struct adapter *adap, int arg, bool verbose)
5081{
5082	static const struct intr_action ma_intr_actions[] = {
5083		{ F_MEM_WRAP_INT_CAUSE, 0, ma_wrap_status },
5084		{ 0 },
5085	};
5086	static const struct intr_info ma_intr_info = {
5087		.name = "MA_INT_CAUSE",
5088		.cause_reg = A_MA_INT_CAUSE,
5089		.enable_reg = A_MA_INT_ENABLE,
5090		.fatal = F_MEM_PERR_INT_CAUSE | F_MEM_TO_INT_CAUSE,
5091		.flags = NONFATAL_IF_DISABLED,
5092		.details = NULL,
5093		.actions = ma_intr_actions,
5094	};
5095	static const struct intr_info ma_perr_status1 = {
5096		.name = "MA_PARITY_ERROR_STATUS1",
5097		.cause_reg = A_MA_PARITY_ERROR_STATUS1,
5098		.enable_reg = A_MA_PARITY_ERROR_ENABLE1,
5099		.fatal = 0xffffffff,
5100		.flags = 0,
5101		.details = NULL,
5102		.actions = NULL,
5103	};
5104	static const struct intr_info ma_perr_status2 = {
5105		.name = "MA_PARITY_ERROR_STATUS2",
5106		.cause_reg = A_MA_PARITY_ERROR_STATUS2,
5107		.enable_reg = A_MA_PARITY_ERROR_ENABLE2,
5108		.fatal = 0xffffffff,
5109		.flags = 0,
5110		.details = NULL,
5111		.actions = NULL,
5112	};
5113	bool fatal;
5114
5115	fatal = false;
5116	fatal |= t4_handle_intr(adap, &ma_intr_info, 0, verbose);
5117	fatal |= t4_handle_intr(adap, &ma_perr_status1, 0, verbose);
5118	if (chip_id(adap) > CHELSIO_T4)
5119		fatal |= t4_handle_intr(adap, &ma_perr_status2, 0, verbose);
5120
5121	return (fatal);
5122}
5123
5124/*
5125 * SMB interrupt handler.
5126 */
5127static bool smb_intr_handler(struct adapter *adap, int arg, bool verbose)
5128{
5129	static const struct intr_details smb_intr_details[] = {
5130		{ F_MSTTXFIFOPARINT, "SMB master Tx FIFO parity error" },
5131		{ F_MSTRXFIFOPARINT, "SMB master Rx FIFO parity error" },
5132		{ F_SLVFIFOPARINT, "SMB slave FIFO parity error" },
5133		{ 0 }
5134	};
5135	static const struct intr_info smb_intr_info = {
5136		.name = "SMB_INT_CAUSE",
5137		.cause_reg = A_SMB_INT_CAUSE,
5138		.enable_reg = A_SMB_INT_ENABLE,
5139		.fatal = F_SLVFIFOPARINT | F_MSTRXFIFOPARINT | F_MSTTXFIFOPARINT,
5140		.flags = 0,
5141		.details = smb_intr_details,
5142		.actions = NULL,
5143	};
5144
5145	return (t4_handle_intr(adap, &smb_intr_info, 0, verbose));
5146}
5147
5148/*
5149 * NC-SI interrupt handler.
5150 */
5151static bool ncsi_intr_handler(struct adapter *adap, int arg, bool verbose)
5152{
5153	static const struct intr_details ncsi_intr_details[] = {
5154		{ F_CIM_DM_PRTY_ERR, "NC-SI CIM parity error" },
5155		{ F_MPS_DM_PRTY_ERR, "NC-SI MPS parity error" },
5156		{ F_TXFIFO_PRTY_ERR, "NC-SI Tx FIFO parity error" },
5157		{ F_RXFIFO_PRTY_ERR, "NC-SI Rx FIFO parity error" },
5158		{ 0 }
5159	};
5160	static const struct intr_info ncsi_intr_info = {
5161		.name = "NCSI_INT_CAUSE",
5162		.cause_reg = A_NCSI_INT_CAUSE,
5163		.enable_reg = A_NCSI_INT_ENABLE,
5164		.fatal = F_RXFIFO_PRTY_ERR | F_TXFIFO_PRTY_ERR |
5165		    F_MPS_DM_PRTY_ERR | F_CIM_DM_PRTY_ERR,
5166		.flags = 0,
5167		.details = ncsi_intr_details,
5168		.actions = NULL,
5169	};
5170
5171	return (t4_handle_intr(adap, &ncsi_intr_info, 0, verbose));
5172}
5173
5174/*
5175 * MAC interrupt handler.
5176 */
5177static bool mac_intr_handler(struct adapter *adap, int port, bool verbose)
5178{
5179	static const struct intr_details mac_intr_details[] = {
5180		{ F_TXFIFO_PRTY_ERR, "MAC Tx FIFO parity error" },
5181		{ F_RXFIFO_PRTY_ERR, "MAC Rx FIFO parity error" },
5182		{ 0 }
5183	};
5184	char name[32];
5185	struct intr_info ii;
5186	bool fatal = false;
5187
5188	if (is_t4(adap)) {
5189		snprintf(name, sizeof(name), "XGMAC_PORT%u_INT_CAUSE", port);
5190		ii.name = &name[0];
5191		ii.cause_reg = PORT_REG(port, A_XGMAC_PORT_INT_CAUSE);
5192		ii.enable_reg = PORT_REG(port, A_XGMAC_PORT_INT_EN);
5193		ii.fatal = F_TXFIFO_PRTY_ERR | F_RXFIFO_PRTY_ERR;
5194		ii.flags = 0;
5195		ii.details = mac_intr_details;
5196		ii.actions = NULL;
5197	} else {
5198		snprintf(name, sizeof(name), "MAC_PORT%u_INT_CAUSE", port);
5199		ii.name = &name[0];
5200		ii.cause_reg = T5_PORT_REG(port, A_MAC_PORT_INT_CAUSE);
5201		ii.enable_reg = T5_PORT_REG(port, A_MAC_PORT_INT_EN);
5202		ii.fatal = F_TXFIFO_PRTY_ERR | F_RXFIFO_PRTY_ERR;
5203		ii.flags = 0;
5204		ii.details = mac_intr_details;
5205		ii.actions = NULL;
5206	}
5207	fatal |= t4_handle_intr(adap, &ii, 0, verbose);
5208
5209	if (chip_id(adap) >= CHELSIO_T5) {
5210		snprintf(name, sizeof(name), "MAC_PORT%u_PERR_INT_CAUSE", port);
5211		ii.name = &name[0];
5212		ii.cause_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_CAUSE);
5213		ii.enable_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_EN);
5214		ii.fatal = 0;
5215		ii.flags = 0;
5216		ii.details = NULL;
5217		ii.actions = NULL;
5218		fatal |= t4_handle_intr(adap, &ii, 0, verbose);
5219	}
5220
5221	if (chip_id(adap) >= CHELSIO_T6) {
5222		snprintf(name, sizeof(name), "MAC_PORT%u_PERR_INT_CAUSE_100G", port);
5223		ii.name = &name[0];
5224		ii.cause_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_CAUSE_100G);
5225		ii.enable_reg = T5_PORT_REG(port, A_MAC_PORT_PERR_INT_EN_100G);
5226		ii.fatal = 0;
5227		ii.flags = 0;
5228		ii.details = NULL;
5229		ii.actions = NULL;
5230		fatal |= t4_handle_intr(adap, &ii, 0, verbose);
5231	}
5232
5233	return (fatal);
5234}
5235
5236static bool plpl_intr_handler(struct adapter *adap, int arg, bool verbose)
5237{
5238	static const struct intr_details plpl_intr_details[] = {
5239		{ F_FATALPERR, "Fatal parity error" },
5240		{ F_PERRVFID, "VFID_MAP parity error" },
5241		{ 0 }
5242	};
5243	static const struct intr_info plpl_intr_info = {
5244		.name = "PL_PL_INT_CAUSE",
5245		.cause_reg = A_PL_PL_INT_CAUSE,
5246		.enable_reg = A_PL_PL_INT_ENABLE,
5247		.fatal = F_FATALPERR | F_PERRVFID,
5248		.flags = NONFATAL_IF_DISABLED,
5249		.details = plpl_intr_details,
5250		.actions = NULL,
5251	};
5252
5253	return (t4_handle_intr(adap, &plpl_intr_info, 0, verbose));
5254}
5255
5256/**
5257 *	t4_slow_intr_handler - control path interrupt handler
5258 *	@adap: the adapter
5259 *	@verbose: increased verbosity, for debug
5260 *
5261 *	T4 interrupt handler for non-data global interrupt events, e.g., errors.
5262 *	The designation 'slow' is because it involves register reads, while
5263 *	data interrupts typically don't involve any MMIOs.
5264 */
5265int t4_slow_intr_handler(struct adapter *adap, bool verbose)
5266{
5267	static const struct intr_details pl_intr_details[] = {
5268		{ F_MC1, "MC1" },
5269		{ F_UART, "UART" },
5270		{ F_ULP_TX, "ULP TX" },
5271		{ F_SGE, "SGE" },
5272		{ F_HMA, "HMA" },
5273		{ F_CPL_SWITCH, "CPL Switch" },
5274		{ F_ULP_RX, "ULP RX" },
5275		{ F_PM_RX, "PM RX" },
5276		{ F_PM_TX, "PM TX" },
5277		{ F_MA, "MA" },
5278		{ F_TP, "TP" },
5279		{ F_LE, "LE" },
5280		{ F_EDC1, "EDC1" },
5281		{ F_EDC0, "EDC0" },
5282		{ F_MC, "MC0" },
5283		{ F_PCIE, "PCIE" },
5284		{ F_PMU, "PMU" },
5285		{ F_MAC3, "MAC3" },
5286		{ F_MAC2, "MAC2" },
5287		{ F_MAC1, "MAC1" },
5288		{ F_MAC0, "MAC0" },
5289		{ F_SMB, "SMB" },
5290		{ F_SF, "SF" },
5291		{ F_PL, "PL" },
5292		{ F_NCSI, "NC-SI" },
5293		{ F_MPS, "MPS" },
5294		{ F_MI, "MI" },
5295		{ F_DBG, "DBG" },
5296		{ F_I2CM, "I2CM" },
5297		{ F_CIM, "CIM" },
5298		{ 0 }
5299	};
5300	static const struct intr_info pl_perr_cause = {
5301		.name = "PL_PERR_CAUSE",
5302		.cause_reg = A_PL_PERR_CAUSE,
5303		.enable_reg = A_PL_PERR_ENABLE,
5304		.fatal = 0xffffffff,
5305		.flags = 0,
5306		.details = pl_intr_details,
5307		.actions = NULL,
5308	};
5309	static const struct intr_action pl_intr_action[] = {
5310		{ F_MC1, MEM_MC1, mem_intr_handler },
5311		{ F_ULP_TX, -1, ulptx_intr_handler },
5312		{ F_SGE, -1, sge_intr_handler },
5313		{ F_CPL_SWITCH, -1, cplsw_intr_handler },
5314		{ F_ULP_RX, -1, ulprx_intr_handler },
5315		{ F_PM_RX, -1, pmrx_intr_handler},
5316		{ F_PM_TX, -1, pmtx_intr_handler},
5317		{ F_MA, -1, ma_intr_handler },
5318		{ F_TP, -1, tp_intr_handler },
5319		{ F_LE, -1, le_intr_handler },
5320		{ F_EDC1, MEM_EDC1, mem_intr_handler },
5321		{ F_EDC0, MEM_EDC0, mem_intr_handler },
5322		{ F_MC0, MEM_MC0, mem_intr_handler },
5323		{ F_PCIE, -1, pcie_intr_handler },
5324		{ F_MAC3, 3, mac_intr_handler},
5325		{ F_MAC2, 2, mac_intr_handler},
5326		{ F_MAC1, 1, mac_intr_handler},
5327		{ F_MAC0, 0, mac_intr_handler},
5328		{ F_SMB, -1, smb_intr_handler},
5329		{ F_PL, -1, plpl_intr_handler },
5330		{ F_NCSI, -1, ncsi_intr_handler},
5331		{ F_MPS, -1, mps_intr_handler },
5332		{ F_CIM, -1, cim_intr_handler },
5333		{ 0 }
5334	};
5335	static const struct intr_info pl_intr_info = {
5336		.name = "PL_INT_CAUSE",
5337		.cause_reg = A_PL_INT_CAUSE,
5338		.enable_reg = A_PL_INT_ENABLE,
5339		.fatal = 0,
5340		.flags = 0,
5341		.details = pl_intr_details,
5342		.actions = pl_intr_action,
5343	};
5344	bool fatal;
5345	u32 perr;
5346
5347	perr = t4_read_reg(adap, pl_perr_cause.cause_reg);
5348	if (verbose || perr != 0) {
5349		t4_show_intr_info(adap, &pl_perr_cause, perr);
5350		if (perr != 0)
5351			t4_write_reg(adap, pl_perr_cause.cause_reg, perr);
5352		if (verbose)
5353			perr |= t4_read_reg(adap, pl_intr_info.enable_reg);
5354	}
5355	fatal = t4_handle_intr(adap, &pl_intr_info, perr, verbose);
5356	if (fatal)
5357		t4_fatal_err(adap, false);
5358
5359	return (0);
5360}
5361
5362#define PF_INTR_MASK (F_PFSW | F_PFCIM)
5363
5364/**
5365 *	t4_intr_enable - enable interrupts
5366 *	@adapter: the adapter whose interrupts should be enabled
5367 *
5368 *	Enable PF-specific interrupts for the calling function and the top-level
5369 *	interrupt concentrator for global interrupts.  Interrupts are already
5370 *	enabled at each module,	here we just enable the roots of the interrupt
5371 *	hierarchies.
5372 *
5373 *	Note: this function should be called only when the driver manages
5374 *	non PF-specific interrupts from the various HW modules.  Only one PCI
5375 *	function at a time should be doing this.
5376 */
5377void t4_intr_enable(struct adapter *adap)
5378{
5379	u32 val = 0;
5380
5381	if (chip_id(adap) <= CHELSIO_T5)
5382		val = F_ERR_DROPPED_DB | F_ERR_EGR_CTXT_PRIO | F_DBFIFO_HP_INT;
5383	else
5384		val = F_ERR_PCIE_ERROR0 | F_ERR_PCIE_ERROR1 | F_FATAL_WRE_LEN;
5385	val |= F_ERR_CPL_EXCEED_IQE_SIZE | F_ERR_INVALID_CIDX_INC |
5386	    F_ERR_CPL_OPCODE_0 | F_ERR_DATA_CPL_ON_HIGH_QID1 |
5387	    F_INGRESS_SIZE_ERR | F_ERR_DATA_CPL_ON_HIGH_QID0 |
5388	    F_ERR_BAD_DB_PIDX3 | F_ERR_BAD_DB_PIDX2 | F_ERR_BAD_DB_PIDX1 |
5389	    F_ERR_BAD_DB_PIDX0 | F_ERR_ING_CTXT_PRIO | F_DBFIFO_LP_INT |
5390	    F_EGRESS_SIZE_ERR;
5391	t4_set_reg_field(adap, A_SGE_INT_ENABLE3, val, val);
5392	t4_write_reg(adap, MYPF_REG(A_PL_PF_INT_ENABLE), PF_INTR_MASK);
5393	t4_set_reg_field(adap, A_PL_INT_ENABLE, F_SF | F_I2CM, 0);
5394	t4_set_reg_field(adap, A_PL_INT_MAP0, 0, 1 << adap->pf);
5395}
5396
5397/**
5398 *	t4_intr_disable - disable interrupts
5399 *	@adap: the adapter whose interrupts should be disabled
5400 *
5401 *	Disable interrupts.  We only disable the top-level interrupt
5402 *	concentrators.  The caller must be a PCI function managing global
5403 *	interrupts.
5404 */
5405void t4_intr_disable(struct adapter *adap)
5406{
5407
5408	t4_write_reg(adap, MYPF_REG(A_PL_PF_INT_ENABLE), 0);
5409	t4_set_reg_field(adap, A_PL_INT_MAP0, 1 << adap->pf, 0);
5410}
5411
5412/**
5413 *	t4_intr_clear - clear all interrupts
5414 *	@adap: the adapter whose interrupts should be cleared
5415 *
5416 *	Clears all interrupts.  The caller must be a PCI function managing
5417 *	global interrupts.
5418 */
5419void t4_intr_clear(struct adapter *adap)
5420{
5421	static const u32 cause_reg[] = {
5422		A_CIM_HOST_INT_CAUSE,
5423		A_CIM_HOST_UPACC_INT_CAUSE,
5424		MYPF_REG(A_CIM_PF_HOST_INT_CAUSE),
5425		A_CPL_INTR_CAUSE,
5426		EDC_REG(A_EDC_INT_CAUSE, 0), EDC_REG(A_EDC_INT_CAUSE, 1),
5427		A_LE_DB_INT_CAUSE,
5428		A_MA_INT_WRAP_STATUS,
5429		A_MA_PARITY_ERROR_STATUS1,
5430		A_MA_INT_CAUSE,
5431		A_MPS_CLS_INT_CAUSE,
5432		A_MPS_RX_PERR_INT_CAUSE,
5433		A_MPS_STAT_PERR_INT_CAUSE_RX_FIFO,
5434		A_MPS_STAT_PERR_INT_CAUSE_SRAM,
5435		A_MPS_TRC_INT_CAUSE,
5436		A_MPS_TX_INT_CAUSE,
5437		A_MPS_STAT_PERR_INT_CAUSE_TX_FIFO,
5438		A_NCSI_INT_CAUSE,
5439		A_PCIE_INT_CAUSE,
5440		A_PCIE_NONFAT_ERR,
5441		A_PL_PL_INT_CAUSE,
5442		A_PM_RX_INT_CAUSE,
5443		A_PM_TX_INT_CAUSE,
5444		A_SGE_INT_CAUSE1,
5445		A_SGE_INT_CAUSE2,
5446		A_SGE_INT_CAUSE3,
5447		A_SGE_INT_CAUSE4,
5448		A_SMB_INT_CAUSE,
5449		A_TP_INT_CAUSE,
5450		A_ULP_RX_INT_CAUSE,
5451		A_ULP_RX_INT_CAUSE_2,
5452		A_ULP_TX_INT_CAUSE,
5453		A_ULP_TX_INT_CAUSE_2,
5454
5455		MYPF_REG(A_PL_PF_INT_CAUSE),
5456	};
5457	int i;
5458	const int nchan = adap->chip_params->nchan;
5459
5460	for (i = 0; i < ARRAY_SIZE(cause_reg); i++)
5461		t4_write_reg(adap, cause_reg[i], 0xffffffff);
5462
5463	if (is_t4(adap)) {
5464		t4_write_reg(adap, A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS,
5465		    0xffffffff);
5466		t4_write_reg(adap, A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS,
5467		    0xffffffff);
5468		t4_write_reg(adap, A_MC_INT_CAUSE, 0xffffffff);
5469		for (i = 0; i < nchan; i++) {
5470			t4_write_reg(adap, PORT_REG(i, A_XGMAC_PORT_INT_CAUSE),
5471			    0xffffffff);
5472		}
5473	}
5474	if (chip_id(adap) >= CHELSIO_T5) {
5475		t4_write_reg(adap, A_MA_PARITY_ERROR_STATUS2, 0xffffffff);
5476		t4_write_reg(adap, A_MPS_STAT_PERR_INT_CAUSE_SRAM1, 0xffffffff);
5477		t4_write_reg(adap, A_SGE_INT_CAUSE5, 0xffffffff);
5478		t4_write_reg(adap, A_MC_P_INT_CAUSE, 0xffffffff);
5479		if (is_t5(adap)) {
5480			t4_write_reg(adap, MC_REG(A_MC_P_INT_CAUSE, 1),
5481			    0xffffffff);
5482		}
5483		for (i = 0; i < nchan; i++) {
5484			t4_write_reg(adap, T5_PORT_REG(i,
5485			    A_MAC_PORT_PERR_INT_CAUSE), 0xffffffff);
5486			if (chip_id(adap) > CHELSIO_T5) {
5487				t4_write_reg(adap, T5_PORT_REG(i,
5488				    A_MAC_PORT_PERR_INT_CAUSE_100G),
5489				    0xffffffff);
5490			}
5491			t4_write_reg(adap, T5_PORT_REG(i, A_MAC_PORT_INT_CAUSE),
5492			    0xffffffff);
5493		}
5494	}
5495	if (chip_id(adap) >= CHELSIO_T6) {
5496		t4_write_reg(adap, A_SGE_INT_CAUSE6, 0xffffffff);
5497	}
5498
5499	t4_write_reg(adap, A_MPS_INT_CAUSE, is_t4(adap) ? 0 : 0xffffffff);
5500	t4_write_reg(adap, A_PL_PERR_CAUSE, 0xffffffff);
5501	t4_write_reg(adap, A_PL_INT_CAUSE, 0xffffffff);
5502	(void) t4_read_reg(adap, A_PL_INT_CAUSE);          /* flush */
5503}
5504
5505/**
5506 *	hash_mac_addr - return the hash value of a MAC address
5507 *	@addr: the 48-bit Ethernet MAC address
5508 *
5509 *	Hashes a MAC address according to the hash function used by HW inexact
5510 *	(hash) address matching.
5511 */
5512static int hash_mac_addr(const u8 *addr)
5513{
5514	u32 a = ((u32)addr[0] << 16) | ((u32)addr[1] << 8) | addr[2];
5515	u32 b = ((u32)addr[3] << 16) | ((u32)addr[4] << 8) | addr[5];
5516	a ^= b;
5517	a ^= (a >> 12);
5518	a ^= (a >> 6);
5519	return a & 0x3f;
5520}
5521
5522/**
5523 *	t4_config_rss_range - configure a portion of the RSS mapping table
5524 *	@adapter: the adapter
5525 *	@mbox: mbox to use for the FW command
5526 *	@viid: virtual interface whose RSS subtable is to be written
5527 *	@start: start entry in the table to write
5528 *	@n: how many table entries to write
5529 *	@rspq: values for the "response queue" (Ingress Queue) lookup table
5530 *	@nrspq: number of values in @rspq
5531 *
5532 *	Programs the selected part of the VI's RSS mapping table with the
5533 *	provided values.  If @nrspq < @n the supplied values are used repeatedly
5534 *	until the full table range is populated.
5535 *
5536 *	The caller must ensure the values in @rspq are in the range allowed for
5537 *	@viid.
5538 */
5539int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
5540			int start, int n, const u16 *rspq, unsigned int nrspq)
5541{
5542	int ret;
5543	const u16 *rsp = rspq;
5544	const u16 *rsp_end = rspq + nrspq;
5545	struct fw_rss_ind_tbl_cmd cmd;
5546
5547	memset(&cmd, 0, sizeof(cmd));
5548	cmd.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_RSS_IND_TBL_CMD) |
5549				     F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
5550				     V_FW_RSS_IND_TBL_CMD_VIID(viid));
5551	cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
5552
5553	/*
5554	 * Each firmware RSS command can accommodate up to 32 RSS Ingress
5555	 * Queue Identifiers.  These Ingress Queue IDs are packed three to
5556	 * a 32-bit word as 10-bit values with the upper remaining 2 bits
5557	 * reserved.
5558	 */
5559	while (n > 0) {
5560		int nq = min(n, 32);
5561		int nq_packed = 0;
5562		__be32 *qp = &cmd.iq0_to_iq2;
5563
5564		/*
5565		 * Set up the firmware RSS command header to send the next
5566		 * "nq" Ingress Queue IDs to the firmware.
5567		 */
5568		cmd.niqid = cpu_to_be16(nq);
5569		cmd.startidx = cpu_to_be16(start);
5570
5571		/*
5572		 * "nq" more done for the start of the next loop.
5573		 */
5574		start += nq;
5575		n -= nq;
5576
5577		/*
5578		 * While there are still Ingress Queue IDs to stuff into the
5579		 * current firmware RSS command, retrieve them from the
5580		 * Ingress Queue ID array and insert them into the command.
5581		 */
5582		while (nq > 0) {
5583			/*
5584			 * Grab up to the next 3 Ingress Queue IDs (wrapping
5585			 * around the Ingress Queue ID array if necessary) and
5586			 * insert them into the firmware RSS command at the
5587			 * current 3-tuple position within the commad.
5588			 */
5589			u16 qbuf[3];
5590			u16 *qbp = qbuf;
5591			int nqbuf = min(3, nq);
5592
5593			nq -= nqbuf;
5594			qbuf[0] = qbuf[1] = qbuf[2] = 0;
5595			while (nqbuf && nq_packed < 32) {
5596				nqbuf--;
5597				nq_packed++;
5598				*qbp++ = *rsp++;
5599				if (rsp >= rsp_end)
5600					rsp = rspq;
5601			}
5602			*qp++ = cpu_to_be32(V_FW_RSS_IND_TBL_CMD_IQ0(qbuf[0]) |
5603					    V_FW_RSS_IND_TBL_CMD_IQ1(qbuf[1]) |
5604					    V_FW_RSS_IND_TBL_CMD_IQ2(qbuf[2]));
5605		}
5606
5607		/*
5608		 * Send this portion of the RRS table update to the firmware;
5609		 * bail out on any errors.
5610		 */
5611		ret = t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
5612		if (ret)
5613			return ret;
5614	}
5615	return 0;
5616}
5617
5618/**
5619 *	t4_config_glbl_rss - configure the global RSS mode
5620 *	@adapter: the adapter
5621 *	@mbox: mbox to use for the FW command
5622 *	@mode: global RSS mode
5623 *	@flags: mode-specific flags
5624 *
5625 *	Sets the global RSS mode.
5626 */
5627int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
5628		       unsigned int flags)
5629{
5630	struct fw_rss_glb_config_cmd c;
5631
5632	memset(&c, 0, sizeof(c));
5633	c.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_RSS_GLB_CONFIG_CMD) |
5634				    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
5635	c.retval_len16 = cpu_to_be32(FW_LEN16(c));
5636	if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL) {
5637		c.u.manual.mode_pkd =
5638			cpu_to_be32(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode));
5639	} else if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
5640		c.u.basicvirtual.mode_keymode =
5641			cpu_to_be32(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode));
5642		c.u.basicvirtual.synmapen_to_hashtoeplitz = cpu_to_be32(flags);
5643	} else
5644		return -EINVAL;
5645	return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
5646}
5647
5648/**
5649 *	t4_config_vi_rss - configure per VI RSS settings
5650 *	@adapter: the adapter
5651 *	@mbox: mbox to use for the FW command
5652 *	@viid: the VI id
5653 *	@flags: RSS flags
5654 *	@defq: id of the default RSS queue for the VI.
5655 *	@skeyidx: RSS secret key table index for non-global mode
5656 *	@skey: RSS vf_scramble key for VI.
5657 *
5658 *	Configures VI-specific RSS properties.
5659 */
5660int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
5661		     unsigned int flags, unsigned int defq, unsigned int skeyidx,
5662		     unsigned int skey)
5663{
5664	struct fw_rss_vi_config_cmd c;
5665
5666	memset(&c, 0, sizeof(c));
5667	c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_RSS_VI_CONFIG_CMD) |
5668				   F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
5669				   V_FW_RSS_VI_CONFIG_CMD_VIID(viid));
5670	c.retval_len16 = cpu_to_be32(FW_LEN16(c));
5671	c.u.basicvirtual.defaultq_to_udpen = cpu_to_be32(flags |
5672					V_FW_RSS_VI_CONFIG_CMD_DEFAULTQ(defq));
5673	c.u.basicvirtual.secretkeyidx_pkd = cpu_to_be32(
5674					V_FW_RSS_VI_CONFIG_CMD_SECRETKEYIDX(skeyidx));
5675	c.u.basicvirtual.secretkeyxor = cpu_to_be32(skey);
5676
5677	return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
5678}
5679
5680/* Read an RSS table row */
5681static int rd_rss_row(struct adapter *adap, int row, u32 *val)
5682{
5683	t4_write_reg(adap, A_TP_RSS_LKP_TABLE, 0xfff00000 | row);
5684	return t4_wait_op_done_val(adap, A_TP_RSS_LKP_TABLE, F_LKPTBLROWVLD, 1,
5685				   5, 0, val);
5686}
5687
5688/**
5689 *	t4_read_rss - read the contents of the RSS mapping table
5690 *	@adapter: the adapter
5691 *	@map: holds the contents of the RSS mapping table
5692 *
5693 *	Reads the contents of the RSS hash->queue mapping table.
5694 */
5695int t4_read_rss(struct adapter *adapter, u16 *map)
5696{
5697	u32 val;
5698	int i, ret;
5699	int rss_nentries = adapter->chip_params->rss_nentries;
5700
5701	for (i = 0; i < rss_nentries / 2; ++i) {
5702		ret = rd_rss_row(adapter, i, &val);
5703		if (ret)
5704			return ret;
5705		*map++ = G_LKPTBLQUEUE0(val);
5706		*map++ = G_LKPTBLQUEUE1(val);
5707	}
5708	return 0;
5709}
5710
5711/**
5712 * t4_tp_fw_ldst_rw - Access TP indirect register through LDST
5713 * @adap: the adapter
5714 * @cmd: TP fw ldst address space type
5715 * @vals: where the indirect register values are stored/written
5716 * @nregs: how many indirect registers to read/write
5717 * @start_idx: index of first indirect register to read/write
5718 * @rw: Read (1) or Write (0)
5719 * @sleep_ok: if true we may sleep while awaiting command completion
5720 *
5721 * Access TP indirect registers through LDST
5722 **/
5723static int t4_tp_fw_ldst_rw(struct adapter *adap, int cmd, u32 *vals,
5724			    unsigned int nregs, unsigned int start_index,
5725			    unsigned int rw, bool sleep_ok)
5726{
5727	int ret = 0;
5728	unsigned int i;
5729	struct fw_ldst_cmd c;
5730
5731	for (i = 0; i < nregs; i++) {
5732		memset(&c, 0, sizeof(c));
5733		c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
5734						F_FW_CMD_REQUEST |
5735						(rw ? F_FW_CMD_READ :
5736						      F_FW_CMD_WRITE) |
5737						V_FW_LDST_CMD_ADDRSPACE(cmd));
5738		c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
5739
5740		c.u.addrval.addr = cpu_to_be32(start_index + i);
5741		c.u.addrval.val  = rw ? 0 : cpu_to_be32(vals[i]);
5742		ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c,
5743				      sleep_ok);
5744		if (ret)
5745			return ret;
5746
5747		if (rw)
5748			vals[i] = be32_to_cpu(c.u.addrval.val);
5749	}
5750	return 0;
5751}
5752
5753/**
5754 * t4_tp_indirect_rw - Read/Write TP indirect register through LDST or backdoor
5755 * @adap: the adapter
5756 * @reg_addr: Address Register
5757 * @reg_data: Data register
5758 * @buff: where the indirect register values are stored/written
5759 * @nregs: how many indirect registers to read/write
5760 * @start_index: index of first indirect register to read/write
5761 * @rw: READ(1) or WRITE(0)
5762 * @sleep_ok: if true we may sleep while awaiting command completion
5763 *
5764 * Read/Write TP indirect registers through LDST if possible.
5765 * Else, use backdoor access
5766 **/
5767static void t4_tp_indirect_rw(struct adapter *adap, u32 reg_addr, u32 reg_data,
5768			      u32 *buff, u32 nregs, u32 start_index, int rw,
5769			      bool sleep_ok)
5770{
5771	int rc = -EINVAL;
5772	int cmd;
5773
5774	switch (reg_addr) {
5775	case A_TP_PIO_ADDR:
5776		cmd = FW_LDST_ADDRSPC_TP_PIO;
5777		break;
5778	case A_TP_TM_PIO_ADDR:
5779		cmd = FW_LDST_ADDRSPC_TP_TM_PIO;
5780		break;
5781	case A_TP_MIB_INDEX:
5782		cmd = FW_LDST_ADDRSPC_TP_MIB;
5783		break;
5784	default:
5785		goto indirect_access;
5786	}
5787
5788	if (t4_use_ldst(adap))
5789		rc = t4_tp_fw_ldst_rw(adap, cmd, buff, nregs, start_index, rw,
5790				      sleep_ok);
5791
5792indirect_access:
5793
5794	if (rc) {
5795		if (rw)
5796			t4_read_indirect(adap, reg_addr, reg_data, buff, nregs,
5797					 start_index);
5798		else
5799			t4_write_indirect(adap, reg_addr, reg_data, buff, nregs,
5800					  start_index);
5801	}
5802}
5803
5804/**
5805 * t4_tp_pio_read - Read TP PIO registers
5806 * @adap: the adapter
5807 * @buff: where the indirect register values are written
5808 * @nregs: how many indirect registers to read
5809 * @start_index: index of first indirect register to read
5810 * @sleep_ok: if true we may sleep while awaiting command completion
5811 *
5812 * Read TP PIO Registers
5813 **/
5814void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs,
5815		    u32 start_index, bool sleep_ok)
5816{
5817	t4_tp_indirect_rw(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, buff, nregs,
5818			  start_index, 1, sleep_ok);
5819}
5820
5821/**
5822 * t4_tp_pio_write - Write TP PIO registers
5823 * @adap: the adapter
5824 * @buff: where the indirect register values are stored
5825 * @nregs: how many indirect registers to write
5826 * @start_index: index of first indirect register to write
5827 * @sleep_ok: if true we may sleep while awaiting command completion
5828 *
5829 * Write TP PIO Registers
5830 **/
5831void t4_tp_pio_write(struct adapter *adap, const u32 *buff, u32 nregs,
5832		     u32 start_index, bool sleep_ok)
5833{
5834	t4_tp_indirect_rw(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA,
5835	    __DECONST(u32 *, buff), nregs, start_index, 0, sleep_ok);
5836}
5837
5838/**
5839 * t4_tp_tm_pio_read - Read TP TM PIO registers
5840 * @adap: the adapter
5841 * @buff: where the indirect register values are written
5842 * @nregs: how many indirect registers to read
5843 * @start_index: index of first indirect register to read
5844 * @sleep_ok: if true we may sleep while awaiting command completion
5845 *
5846 * Read TP TM PIO Registers
5847 **/
5848void t4_tp_tm_pio_read(struct adapter *adap, u32 *buff, u32 nregs,
5849		       u32 start_index, bool sleep_ok)
5850{
5851	t4_tp_indirect_rw(adap, A_TP_TM_PIO_ADDR, A_TP_TM_PIO_DATA, buff,
5852			  nregs, start_index, 1, sleep_ok);
5853}
5854
5855/**
5856 * t4_tp_mib_read - Read TP MIB registers
5857 * @adap: the adapter
5858 * @buff: where the indirect register values are written
5859 * @nregs: how many indirect registers to read
5860 * @start_index: index of first indirect register to read
5861 * @sleep_ok: if true we may sleep while awaiting command completion
5862 *
5863 * Read TP MIB Registers
5864 **/
5865void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32 nregs, u32 start_index,
5866		    bool sleep_ok)
5867{
5868	t4_tp_indirect_rw(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, buff, nregs,
5869			  start_index, 1, sleep_ok);
5870}
5871
5872/**
5873 *	t4_read_rss_key - read the global RSS key
5874 *	@adap: the adapter
5875 *	@key: 10-entry array holding the 320-bit RSS key
5876 * 	@sleep_ok: if true we may sleep while awaiting command completion
5877 *
5878 *	Reads the global 320-bit RSS key.
5879 */
5880void t4_read_rss_key(struct adapter *adap, u32 *key, bool sleep_ok)
5881{
5882	t4_tp_pio_read(adap, key, 10, A_TP_RSS_SECRET_KEY0, sleep_ok);
5883}
5884
5885/**
5886 *	t4_write_rss_key - program one of the RSS keys
5887 *	@adap: the adapter
5888 *	@key: 10-entry array holding the 320-bit RSS key
5889 *	@idx: which RSS key to write
5890 * 	@sleep_ok: if true we may sleep while awaiting command completion
5891 *
5892 *	Writes one of the RSS keys with the given 320-bit value.  If @idx is
5893 *	0..15 the corresponding entry in the RSS key table is written,
5894 *	otherwise the global RSS key is written.
5895 */
5896void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx,
5897		      bool sleep_ok)
5898{
5899	u8 rss_key_addr_cnt = 16;
5900	u32 vrt = t4_read_reg(adap, A_TP_RSS_CONFIG_VRT);
5901
5902	/*
5903	 * T6 and later: for KeyMode 3 (per-vf and per-vf scramble),
5904	 * allows access to key addresses 16-63 by using KeyWrAddrX
5905	 * as index[5:4](upper 2) into key table
5906	 */
5907	if ((chip_id(adap) > CHELSIO_T5) &&
5908	    (vrt & F_KEYEXTEND) && (G_KEYMODE(vrt) == 3))
5909		rss_key_addr_cnt = 32;
5910
5911	t4_tp_pio_write(adap, key, 10, A_TP_RSS_SECRET_KEY0, sleep_ok);
5912
5913	if (idx >= 0 && idx < rss_key_addr_cnt) {
5914		if (rss_key_addr_cnt > 16)
5915			t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
5916				     vrt | V_KEYWRADDRX(idx >> 4) |
5917				     V_T6_VFWRADDR(idx) | F_KEYWREN);
5918		else
5919			t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
5920				     vrt| V_KEYWRADDR(idx) | F_KEYWREN);
5921	}
5922}
5923
5924/**
5925 *	t4_read_rss_pf_config - read PF RSS Configuration Table
5926 *	@adapter: the adapter
5927 *	@index: the entry in the PF RSS table to read
5928 *	@valp: where to store the returned value
5929 * 	@sleep_ok: if true we may sleep while awaiting command completion
5930 *
5931 *	Reads the PF RSS Configuration Table at the specified index and returns
5932 *	the value found there.
5933 */
5934void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index,
5935			   u32 *valp, bool sleep_ok)
5936{
5937	t4_tp_pio_read(adapter, valp, 1, A_TP_RSS_PF0_CONFIG + index, sleep_ok);
5938}
5939
5940/**
5941 *	t4_write_rss_pf_config - write PF RSS Configuration Table
5942 *	@adapter: the adapter
5943 *	@index: the entry in the VF RSS table to read
5944 *	@val: the value to store
5945 * 	@sleep_ok: if true we may sleep while awaiting command completion
5946 *
5947 *	Writes the PF RSS Configuration Table at the specified index with the
5948 *	specified value.
5949 */
5950void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index,
5951			    u32 val, bool sleep_ok)
5952{
5953	t4_tp_pio_write(adapter, &val, 1, A_TP_RSS_PF0_CONFIG + index,
5954			sleep_ok);
5955}
5956
5957/**
5958 *	t4_read_rss_vf_config - read VF RSS Configuration Table
5959 *	@adapter: the adapter
5960 *	@index: the entry in the VF RSS table to read
5961 *	@vfl: where to store the returned VFL
5962 *	@vfh: where to store the returned VFH
5963 * 	@sleep_ok: if true we may sleep while awaiting command completion
5964 *
5965 *	Reads the VF RSS Configuration Table at the specified index and returns
5966 *	the (VFL, VFH) values found there.
5967 */
5968void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
5969			   u32 *vfl, u32 *vfh, bool sleep_ok)
5970{
5971	u32 vrt, mask, data;
5972
5973	if (chip_id(adapter) <= CHELSIO_T5) {
5974		mask = V_VFWRADDR(M_VFWRADDR);
5975		data = V_VFWRADDR(index);
5976	} else {
5977		 mask =  V_T6_VFWRADDR(M_T6_VFWRADDR);
5978		 data = V_T6_VFWRADDR(index);
5979	}
5980	/*
5981	 * Request that the index'th VF Table values be read into VFL/VFH.
5982	 */
5983	vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT);
5984	vrt &= ~(F_VFRDRG | F_VFWREN | F_KEYWREN | mask);
5985	vrt |= data | F_VFRDEN;
5986	t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt);
5987
5988	/*
5989	 * Grab the VFL/VFH values ...
5990	 */
5991	t4_tp_pio_read(adapter, vfl, 1, A_TP_RSS_VFL_CONFIG, sleep_ok);
5992	t4_tp_pio_read(adapter, vfh, 1, A_TP_RSS_VFH_CONFIG, sleep_ok);
5993}
5994
5995/**
5996 *	t4_write_rss_vf_config - write VF RSS Configuration Table
5997 *
5998 *	@adapter: the adapter
5999 *	@index: the entry in the VF RSS table to write
6000 *	@vfl: the VFL to store
6001 *	@vfh: the VFH to store
6002 *
6003 *	Writes the VF RSS Configuration Table at the specified index with the
6004 *	specified (VFL, VFH) values.
6005 */
6006void t4_write_rss_vf_config(struct adapter *adapter, unsigned int index,
6007			    u32 vfl, u32 vfh, bool sleep_ok)
6008{
6009	u32 vrt, mask, data;
6010
6011	if (chip_id(adapter) <= CHELSIO_T5) {
6012		mask = V_VFWRADDR(M_VFWRADDR);
6013		data = V_VFWRADDR(index);
6014	} else {
6015		mask =  V_T6_VFWRADDR(M_T6_VFWRADDR);
6016		data = V_T6_VFWRADDR(index);
6017	}
6018
6019	/*
6020	 * Load up VFL/VFH with the values to be written ...
6021	 */
6022	t4_tp_pio_write(adapter, &vfl, 1, A_TP_RSS_VFL_CONFIG, sleep_ok);
6023	t4_tp_pio_write(adapter, &vfh, 1, A_TP_RSS_VFH_CONFIG, sleep_ok);
6024
6025	/*
6026	 * Write the VFL/VFH into the VF Table at index'th location.
6027	 */
6028	vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT);
6029	vrt &= ~(F_VFRDRG | F_VFWREN | F_KEYWREN | mask);
6030	vrt |= data | F_VFRDEN;
6031	t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt);
6032}
6033
6034/**
6035 *	t4_read_rss_pf_map - read PF RSS Map
6036 *	@adapter: the adapter
6037 * 	@sleep_ok: if true we may sleep while awaiting command completion
6038 *
6039 *	Reads the PF RSS Map register and returns its value.
6040 */
6041u32 t4_read_rss_pf_map(struct adapter *adapter, bool sleep_ok)
6042{
6043	u32 pfmap;
6044
6045	t4_tp_pio_read(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, sleep_ok);
6046
6047	return pfmap;
6048}
6049
6050/**
6051 *	t4_write_rss_pf_map - write PF RSS Map
6052 *	@adapter: the adapter
6053 *	@pfmap: PF RSS Map value
6054 *
6055 *	Writes the specified value to the PF RSS Map register.
6056 */
6057void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap, bool sleep_ok)
6058{
6059	t4_tp_pio_write(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, sleep_ok);
6060}
6061
6062/**
6063 *	t4_read_rss_pf_mask - read PF RSS Mask
6064 *	@adapter: the adapter
6065 * 	@sleep_ok: if true we may sleep while awaiting command completion
6066 *
6067 *	Reads the PF RSS Mask register and returns its value.
6068 */
6069u32 t4_read_rss_pf_mask(struct adapter *adapter, bool sleep_ok)
6070{
6071	u32 pfmask;
6072
6073	t4_tp_pio_read(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, sleep_ok);
6074
6075	return pfmask;
6076}
6077
6078/**
6079 *	t4_write_rss_pf_mask - write PF RSS Mask
6080 *	@adapter: the adapter
6081 *	@pfmask: PF RSS Mask value
6082 *
6083 *	Writes the specified value to the PF RSS Mask register.
6084 */
6085void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask, bool sleep_ok)
6086{
6087	t4_tp_pio_write(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, sleep_ok);
6088}
6089
6090/**
6091 *	t4_tp_get_tcp_stats - read TP's TCP MIB counters
6092 *	@adap: the adapter
6093 *	@v4: holds the TCP/IP counter values
6094 *	@v6: holds the TCP/IPv6 counter values
6095 * 	@sleep_ok: if true we may sleep while awaiting command completion
6096 *
6097 *	Returns the values of TP's TCP/IP and TCP/IPv6 MIB counters.
6098 *	Either @v4 or @v6 may be %NULL to skip the corresponding stats.
6099 */
6100void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4,
6101			 struct tp_tcp_stats *v6, bool sleep_ok)
6102{
6103	u32 val[A_TP_MIB_TCP_RXT_SEG_LO - A_TP_MIB_TCP_OUT_RST + 1];
6104
6105#define STAT_IDX(x) ((A_TP_MIB_TCP_##x) - A_TP_MIB_TCP_OUT_RST)
6106#define STAT(x)     val[STAT_IDX(x)]
6107#define STAT64(x)   (((u64)STAT(x##_HI) << 32) | STAT(x##_LO))
6108
6109	if (v4) {
6110		t4_tp_mib_read(adap, val, ARRAY_SIZE(val),
6111			       A_TP_MIB_TCP_OUT_RST, sleep_ok);
6112		v4->tcp_out_rsts = STAT(OUT_RST);
6113		v4->tcp_in_segs  = STAT64(IN_SEG);
6114		v4->tcp_out_segs = STAT64(OUT_SEG);
6115		v4->tcp_retrans_segs = STAT64(RXT_SEG);
6116	}
6117	if (v6) {
6118		t4_tp_mib_read(adap, val, ARRAY_SIZE(val),
6119			       A_TP_MIB_TCP_V6OUT_RST, sleep_ok);
6120		v6->tcp_out_rsts = STAT(OUT_RST);
6121		v6->tcp_in_segs  = STAT64(IN_SEG);
6122		v6->tcp_out_segs = STAT64(OUT_SEG);
6123		v6->tcp_retrans_segs = STAT64(RXT_SEG);
6124	}
6125#undef STAT64
6126#undef STAT
6127#undef STAT_IDX
6128}
6129
6130/**
6131 *	t4_tp_get_err_stats - read TP's error MIB counters
6132 *	@adap: the adapter
6133 *	@st: holds the counter values
6134 * 	@sleep_ok: if true we may sleep while awaiting command completion
6135 *
6136 *	Returns the values of TP's error counters.
6137 */
6138void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st,
6139			 bool sleep_ok)
6140{
6141	int nchan = adap->chip_params->nchan;
6142
6143	t4_tp_mib_read(adap, st->mac_in_errs, nchan, A_TP_MIB_MAC_IN_ERR_0,
6144		       sleep_ok);
6145
6146	t4_tp_mib_read(adap, st->hdr_in_errs, nchan, A_TP_MIB_HDR_IN_ERR_0,
6147		       sleep_ok);
6148
6149	t4_tp_mib_read(adap, st->tcp_in_errs, nchan, A_TP_MIB_TCP_IN_ERR_0,
6150		       sleep_ok);
6151
6152	t4_tp_mib_read(adap, st->tnl_cong_drops, nchan,
6153		       A_TP_MIB_TNL_CNG_DROP_0, sleep_ok);
6154
6155	t4_tp_mib_read(adap, st->ofld_chan_drops, nchan,
6156		       A_TP_MIB_OFD_CHN_DROP_0, sleep_ok);
6157
6158	t4_tp_mib_read(adap, st->tnl_tx_drops, nchan, A_TP_MIB_TNL_DROP_0,
6159		       sleep_ok);
6160
6161	t4_tp_mib_read(adap, st->ofld_vlan_drops, nchan,
6162		       A_TP_MIB_OFD_VLN_DROP_0, sleep_ok);
6163
6164	t4_tp_mib_read(adap, st->tcp6_in_errs, nchan,
6165		       A_TP_MIB_TCP_V6IN_ERR_0, sleep_ok);
6166
6167	t4_tp_mib_read(adap, &st->ofld_no_neigh, 2, A_TP_MIB_OFD_ARP_DROP,
6168		       sleep_ok);
6169}
6170
6171/**
6172 *	t4_tp_get_proxy_stats - read TP's proxy MIB counters
6173 *	@adap: the adapter
6174 *	@st: holds the counter values
6175 *
6176 *	Returns the values of TP's proxy counters.
6177 */
6178void t4_tp_get_proxy_stats(struct adapter *adap, struct tp_proxy_stats *st,
6179    bool sleep_ok)
6180{
6181	int nchan = adap->chip_params->nchan;
6182
6183	t4_tp_mib_read(adap, st->proxy, nchan, A_TP_MIB_TNL_LPBK_0, sleep_ok);
6184}
6185
6186/**
6187 *	t4_tp_get_cpl_stats - read TP's CPL MIB counters
6188 *	@adap: the adapter
6189 *	@st: holds the counter values
6190 * 	@sleep_ok: if true we may sleep while awaiting command completion
6191 *
6192 *	Returns the values of TP's CPL counters.
6193 */
6194void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st,
6195			 bool sleep_ok)
6196{
6197	int nchan = adap->chip_params->nchan;
6198
6199	t4_tp_mib_read(adap, st->req, nchan, A_TP_MIB_CPL_IN_REQ_0, sleep_ok);
6200
6201	t4_tp_mib_read(adap, st->rsp, nchan, A_TP_MIB_CPL_OUT_RSP_0, sleep_ok);
6202}
6203
6204/**
6205 *	t4_tp_get_rdma_stats - read TP's RDMA MIB counters
6206 *	@adap: the adapter
6207 *	@st: holds the counter values
6208 *
6209 *	Returns the values of TP's RDMA counters.
6210 */
6211void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st,
6212			  bool sleep_ok)
6213{
6214	t4_tp_mib_read(adap, &st->rqe_dfr_pkt, 2, A_TP_MIB_RQE_DFR_PKT,
6215		       sleep_ok);
6216}
6217
6218/**
6219 *	t4_get_fcoe_stats - read TP's FCoE MIB counters for a port
6220 *	@adap: the adapter
6221 *	@idx: the port index
6222 *	@st: holds the counter values
6223 * 	@sleep_ok: if true we may sleep while awaiting command completion
6224 *
6225 *	Returns the values of TP's FCoE counters for the selected port.
6226 */
6227void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx,
6228		       struct tp_fcoe_stats *st, bool sleep_ok)
6229{
6230	u32 val[2];
6231
6232	t4_tp_mib_read(adap, &st->frames_ddp, 1, A_TP_MIB_FCOE_DDP_0 + idx,
6233		       sleep_ok);
6234
6235	t4_tp_mib_read(adap, &st->frames_drop, 1,
6236		       A_TP_MIB_FCOE_DROP_0 + idx, sleep_ok);
6237
6238	t4_tp_mib_read(adap, val, 2, A_TP_MIB_FCOE_BYTE_0_HI + 2 * idx,
6239		       sleep_ok);
6240
6241	st->octets_ddp = ((u64)val[0] << 32) | val[1];
6242}
6243
6244/**
6245 *	t4_get_usm_stats - read TP's non-TCP DDP MIB counters
6246 *	@adap: the adapter
6247 *	@st: holds the counter values
6248 * 	@sleep_ok: if true we may sleep while awaiting command completion
6249 *
6250 *	Returns the values of TP's counters for non-TCP directly-placed packets.
6251 */
6252void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st,
6253		      bool sleep_ok)
6254{
6255	u32 val[4];
6256
6257	t4_tp_mib_read(adap, val, 4, A_TP_MIB_USM_PKTS, sleep_ok);
6258
6259	st->frames = val[0];
6260	st->drops = val[1];
6261	st->octets = ((u64)val[2] << 32) | val[3];
6262}
6263
6264/**
6265 *	t4_read_mtu_tbl - returns the values in the HW path MTU table
6266 *	@adap: the adapter
6267 *	@mtus: where to store the MTU values
6268 *	@mtu_log: where to store the MTU base-2 log (may be %NULL)
6269 *
6270 *	Reads the HW path MTU table.
6271 */
6272void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log)
6273{
6274	u32 v;
6275	int i;
6276
6277	for (i = 0; i < NMTUS; ++i) {
6278		t4_write_reg(adap, A_TP_MTU_TABLE,
6279			     V_MTUINDEX(0xff) | V_MTUVALUE(i));
6280		v = t4_read_reg(adap, A_TP_MTU_TABLE);
6281		mtus[i] = G_MTUVALUE(v);
6282		if (mtu_log)
6283			mtu_log[i] = G_MTUWIDTH(v);
6284	}
6285}
6286
6287/**
6288 *	t4_read_cong_tbl - reads the congestion control table
6289 *	@adap: the adapter
6290 *	@incr: where to store the alpha values
6291 *
6292 *	Reads the additive increments programmed into the HW congestion
6293 *	control table.
6294 */
6295void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN])
6296{
6297	unsigned int mtu, w;
6298
6299	for (mtu = 0; mtu < NMTUS; ++mtu)
6300		for (w = 0; w < NCCTRL_WIN; ++w) {
6301			t4_write_reg(adap, A_TP_CCTRL_TABLE,
6302				     V_ROWINDEX(0xffff) | (mtu << 5) | w);
6303			incr[mtu][w] = (u16)t4_read_reg(adap,
6304						A_TP_CCTRL_TABLE) & 0x1fff;
6305		}
6306}
6307
6308/**
6309 *	t4_tp_wr_bits_indirect - set/clear bits in an indirect TP register
6310 *	@adap: the adapter
6311 *	@addr: the indirect TP register address
6312 *	@mask: specifies the field within the register to modify
6313 *	@val: new value for the field
6314 *
6315 *	Sets a field of an indirect TP register to the given value.
6316 */
6317void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr,
6318			    unsigned int mask, unsigned int val)
6319{
6320	t4_write_reg(adap, A_TP_PIO_ADDR, addr);
6321	val |= t4_read_reg(adap, A_TP_PIO_DATA) & ~mask;
6322	t4_write_reg(adap, A_TP_PIO_DATA, val);
6323}
6324
6325/**
6326 *	init_cong_ctrl - initialize congestion control parameters
6327 *	@a: the alpha values for congestion control
6328 *	@b: the beta values for congestion control
6329 *
6330 *	Initialize the congestion control parameters.
6331 */
6332static void init_cong_ctrl(unsigned short *a, unsigned short *b)
6333{
6334	a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1;
6335	a[9] = 2;
6336	a[10] = 3;
6337	a[11] = 4;
6338	a[12] = 5;
6339	a[13] = 6;
6340	a[14] = 7;
6341	a[15] = 8;
6342	a[16] = 9;
6343	a[17] = 10;
6344	a[18] = 14;
6345	a[19] = 17;
6346	a[20] = 21;
6347	a[21] = 25;
6348	a[22] = 30;
6349	a[23] = 35;
6350	a[24] = 45;
6351	a[25] = 60;
6352	a[26] = 80;
6353	a[27] = 100;
6354	a[28] = 200;
6355	a[29] = 300;
6356	a[30] = 400;
6357	a[31] = 500;
6358
6359	b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0;
6360	b[9] = b[10] = 1;
6361	b[11] = b[12] = 2;
6362	b[13] = b[14] = b[15] = b[16] = 3;
6363	b[17] = b[18] = b[19] = b[20] = b[21] = 4;
6364	b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5;
6365	b[28] = b[29] = 6;
6366	b[30] = b[31] = 7;
6367}
6368
6369/* The minimum additive increment value for the congestion control table */
6370#define CC_MIN_INCR 2U
6371
6372/**
6373 *	t4_load_mtus - write the MTU and congestion control HW tables
6374 *	@adap: the adapter
6375 *	@mtus: the values for the MTU table
6376 *	@alpha: the values for the congestion control alpha parameter
6377 *	@beta: the values for the congestion control beta parameter
6378 *
6379 *	Write the HW MTU table with the supplied MTUs and the high-speed
6380 *	congestion control table with the supplied alpha, beta, and MTUs.
6381 *	We write the two tables together because the additive increments
6382 *	depend on the MTUs.
6383 */
6384void t4_load_mtus(struct adapter *adap, const unsigned short *mtus,
6385		  const unsigned short *alpha, const unsigned short *beta)
6386{
6387	static const unsigned int avg_pkts[NCCTRL_WIN] = {
6388		2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640,
6389		896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480,
6390		28672, 40960, 57344, 81920, 114688, 163840, 229376
6391	};
6392
6393	unsigned int i, w;
6394
6395	for (i = 0; i < NMTUS; ++i) {
6396		unsigned int mtu = mtus[i];
6397		unsigned int log2 = fls(mtu);
6398
6399		if (!(mtu & ((1 << log2) >> 2)))     /* round */
6400			log2--;
6401		t4_write_reg(adap, A_TP_MTU_TABLE, V_MTUINDEX(i) |
6402			     V_MTUWIDTH(log2) | V_MTUVALUE(mtu));
6403
6404		for (w = 0; w < NCCTRL_WIN; ++w) {
6405			unsigned int inc;
6406
6407			inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w],
6408				  CC_MIN_INCR);
6409
6410			t4_write_reg(adap, A_TP_CCTRL_TABLE, (i << 21) |
6411				     (w << 16) | (beta[w] << 13) | inc);
6412		}
6413	}
6414}
6415
6416/**
6417 *	t4_set_pace_tbl - set the pace table
6418 *	@adap: the adapter
6419 *	@pace_vals: the pace values in microseconds
6420 *	@start: index of the first entry in the HW pace table to set
6421 *	@n: how many entries to set
6422 *
6423 *	Sets (a subset of the) HW pace table.
6424 */
6425int t4_set_pace_tbl(struct adapter *adap, const unsigned int *pace_vals,
6426		     unsigned int start, unsigned int n)
6427{
6428	unsigned int vals[NTX_SCHED], i;
6429	unsigned int tick_ns = dack_ticks_to_usec(adap, 1000);
6430
6431	if (n > NTX_SCHED)
6432	    return -ERANGE;
6433
6434	/* convert values from us to dack ticks, rounding to closest value */
6435	for (i = 0; i < n; i++, pace_vals++) {
6436		vals[i] = (1000 * *pace_vals + tick_ns / 2) / tick_ns;
6437		if (vals[i] > 0x7ff)
6438			return -ERANGE;
6439		if (*pace_vals && vals[i] == 0)
6440			return -ERANGE;
6441	}
6442	for (i = 0; i < n; i++, start++)
6443		t4_write_reg(adap, A_TP_PACE_TABLE, (start << 16) | vals[i]);
6444	return 0;
6445}
6446
6447/**
6448 *	t4_set_sched_bps - set the bit rate for a HW traffic scheduler
6449 *	@adap: the adapter
6450 *	@kbps: target rate in Kbps
6451 *	@sched: the scheduler index
6452 *
6453 *	Configure a Tx HW scheduler for the target rate.
6454 */
6455int t4_set_sched_bps(struct adapter *adap, int sched, unsigned int kbps)
6456{
6457	unsigned int v, tps, cpt, bpt, delta, mindelta = ~0;
6458	unsigned int clk = adap->params.vpd.cclk * 1000;
6459	unsigned int selected_cpt = 0, selected_bpt = 0;
6460
6461	if (kbps > 0) {
6462		kbps *= 125;     /* -> bytes */
6463		for (cpt = 1; cpt <= 255; cpt++) {
6464			tps = clk / cpt;
6465			bpt = (kbps + tps / 2) / tps;
6466			if (bpt > 0 && bpt <= 255) {
6467				v = bpt * tps;
6468				delta = v >= kbps ? v - kbps : kbps - v;
6469				if (delta < mindelta) {
6470					mindelta = delta;
6471					selected_cpt = cpt;
6472					selected_bpt = bpt;
6473				}
6474			} else if (selected_cpt)
6475				break;
6476		}
6477		if (!selected_cpt)
6478			return -EINVAL;
6479	}
6480	t4_write_reg(adap, A_TP_TM_PIO_ADDR,
6481		     A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2);
6482	v = t4_read_reg(adap, A_TP_TM_PIO_DATA);
6483	if (sched & 1)
6484		v = (v & 0xffff) | (selected_cpt << 16) | (selected_bpt << 24);
6485	else
6486		v = (v & 0xffff0000) | selected_cpt | (selected_bpt << 8);
6487	t4_write_reg(adap, A_TP_TM_PIO_DATA, v);
6488	return 0;
6489}
6490
6491/**
6492 *	t4_set_sched_ipg - set the IPG for a Tx HW packet rate scheduler
6493 *	@adap: the adapter
6494 *	@sched: the scheduler index
6495 *	@ipg: the interpacket delay in tenths of nanoseconds
6496 *
6497 *	Set the interpacket delay for a HW packet rate scheduler.
6498 */
6499int t4_set_sched_ipg(struct adapter *adap, int sched, unsigned int ipg)
6500{
6501	unsigned int v, addr = A_TP_TX_MOD_Q1_Q0_TIMER_SEPARATOR - sched / 2;
6502
6503	/* convert ipg to nearest number of core clocks */
6504	ipg *= core_ticks_per_usec(adap);
6505	ipg = (ipg + 5000) / 10000;
6506	if (ipg > M_TXTIMERSEPQ0)
6507		return -EINVAL;
6508
6509	t4_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
6510	v = t4_read_reg(adap, A_TP_TM_PIO_DATA);
6511	if (sched & 1)
6512		v = (v & V_TXTIMERSEPQ0(M_TXTIMERSEPQ0)) | V_TXTIMERSEPQ1(ipg);
6513	else
6514		v = (v & V_TXTIMERSEPQ1(M_TXTIMERSEPQ1)) | V_TXTIMERSEPQ0(ipg);
6515	t4_write_reg(adap, A_TP_TM_PIO_DATA, v);
6516	t4_read_reg(adap, A_TP_TM_PIO_DATA);
6517	return 0;
6518}
6519
6520/*
6521 * Calculates a rate in bytes/s given the number of 256-byte units per 4K core
6522 * clocks.  The formula is
6523 *
6524 * bytes/s = bytes256 * 256 * ClkFreq / 4096
6525 *
6526 * which is equivalent to
6527 *
6528 * bytes/s = 62.5 * bytes256 * ClkFreq_ms
6529 */
6530static u64 chan_rate(struct adapter *adap, unsigned int bytes256)
6531{
6532	u64 v = (u64)bytes256 * adap->params.vpd.cclk;
6533
6534	return v * 62 + v / 2;
6535}
6536
6537/**
6538 *	t4_get_chan_txrate - get the current per channel Tx rates
6539 *	@adap: the adapter
6540 *	@nic_rate: rates for NIC traffic
6541 *	@ofld_rate: rates for offloaded traffic
6542 *
6543 *	Return the current Tx rates in bytes/s for NIC and offloaded traffic
6544 *	for each channel.
6545 */
6546void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate)
6547{
6548	u32 v;
6549
6550	v = t4_read_reg(adap, A_TP_TX_TRATE);
6551	nic_rate[0] = chan_rate(adap, G_TNLRATE0(v));
6552	nic_rate[1] = chan_rate(adap, G_TNLRATE1(v));
6553	if (adap->chip_params->nchan > 2) {
6554		nic_rate[2] = chan_rate(adap, G_TNLRATE2(v));
6555		nic_rate[3] = chan_rate(adap, G_TNLRATE3(v));
6556	}
6557
6558	v = t4_read_reg(adap, A_TP_TX_ORATE);
6559	ofld_rate[0] = chan_rate(adap, G_OFDRATE0(v));
6560	ofld_rate[1] = chan_rate(adap, G_OFDRATE1(v));
6561	if (adap->chip_params->nchan > 2) {
6562		ofld_rate[2] = chan_rate(adap, G_OFDRATE2(v));
6563		ofld_rate[3] = chan_rate(adap, G_OFDRATE3(v));
6564	}
6565}
6566
6567/**
6568 *	t4_set_trace_filter - configure one of the tracing filters
6569 *	@adap: the adapter
6570 *	@tp: the desired trace filter parameters
6571 *	@idx: which filter to configure
6572 *	@enable: whether to enable or disable the filter
6573 *
6574 *	Configures one of the tracing filters available in HW.  If @tp is %NULL
6575 *	it indicates that the filter is already written in the register and it
6576 *	just needs to be enabled or disabled.
6577 */
6578int t4_set_trace_filter(struct adapter *adap, const struct trace_params *tp,
6579    int idx, int enable)
6580{
6581	int i, ofst = idx * 4;
6582	u32 data_reg, mask_reg, cfg;
6583	u32 multitrc = F_TRCMULTIFILTER;
6584	u32 en = is_t4(adap) ? F_TFEN : F_T5_TFEN;
6585
6586	if (idx < 0 || idx >= NTRACE)
6587		return -EINVAL;
6588
6589	if (tp == NULL || !enable) {
6590		t4_set_reg_field(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, en,
6591		    enable ? en : 0);
6592		return 0;
6593	}
6594
6595	/*
6596	 * TODO - After T4 data book is updated, specify the exact
6597	 * section below.
6598	 *
6599	 * See T4 data book - MPS section for a complete description
6600	 * of the below if..else handling of A_MPS_TRC_CFG register
6601	 * value.
6602	 */
6603	cfg = t4_read_reg(adap, A_MPS_TRC_CFG);
6604	if (cfg & F_TRCMULTIFILTER) {
6605		/*
6606		 * If multiple tracers are enabled, then maximum
6607		 * capture size is 2.5KB (FIFO size of a single channel)
6608		 * minus 2 flits for CPL_TRACE_PKT header.
6609		 */
6610		if (tp->snap_len > ((10 * 1024 / 4) - (2 * 8)))
6611			return -EINVAL;
6612	} else {
6613		/*
6614		 * If multiple tracers are disabled, to avoid deadlocks
6615		 * maximum packet capture size of 9600 bytes is recommended.
6616		 * Also in this mode, only trace0 can be enabled and running.
6617		 */
6618		multitrc = 0;
6619		if (tp->snap_len > 9600 || idx)
6620			return -EINVAL;
6621	}
6622
6623	if (tp->port > (is_t4(adap) ? 11 : 19) || tp->invert > 1 ||
6624	    tp->skip_len > M_TFLENGTH || tp->skip_ofst > M_TFOFFSET ||
6625	    tp->min_len > M_TFMINPKTSIZE)
6626		return -EINVAL;
6627
6628	/* stop the tracer we'll be changing */
6629	t4_set_reg_field(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, en, 0);
6630
6631	idx *= (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH);
6632	data_reg = A_MPS_TRC_FILTER0_MATCH + idx;
6633	mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + idx;
6634
6635	for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
6636		t4_write_reg(adap, data_reg, tp->data[i]);
6637		t4_write_reg(adap, mask_reg, ~tp->mask[i]);
6638	}
6639	t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst,
6640		     V_TFCAPTUREMAX(tp->snap_len) |
6641		     V_TFMINPKTSIZE(tp->min_len));
6642	t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst,
6643		     V_TFOFFSET(tp->skip_ofst) | V_TFLENGTH(tp->skip_len) | en |
6644		     (is_t4(adap) ?
6645		     V_TFPORT(tp->port) | V_TFINVERTMATCH(tp->invert) :
6646		     V_T5_TFPORT(tp->port) | V_T5_TFINVERTMATCH(tp->invert)));
6647
6648	return 0;
6649}
6650
6651/**
6652 *	t4_get_trace_filter - query one of the tracing filters
6653 *	@adap: the adapter
6654 *	@tp: the current trace filter parameters
6655 *	@idx: which trace filter to query
6656 *	@enabled: non-zero if the filter is enabled
6657 *
6658 *	Returns the current settings of one of the HW tracing filters.
6659 */
6660void t4_get_trace_filter(struct adapter *adap, struct trace_params *tp, int idx,
6661			 int *enabled)
6662{
6663	u32 ctla, ctlb;
6664	int i, ofst = idx * 4;
6665	u32 data_reg, mask_reg;
6666
6667	ctla = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst);
6668	ctlb = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst);
6669
6670	if (is_t4(adap)) {
6671		*enabled = !!(ctla & F_TFEN);
6672		tp->port =  G_TFPORT(ctla);
6673		tp->invert = !!(ctla & F_TFINVERTMATCH);
6674	} else {
6675		*enabled = !!(ctla & F_T5_TFEN);
6676		tp->port = G_T5_TFPORT(ctla);
6677		tp->invert = !!(ctla & F_T5_TFINVERTMATCH);
6678	}
6679	tp->snap_len = G_TFCAPTUREMAX(ctlb);
6680	tp->min_len = G_TFMINPKTSIZE(ctlb);
6681	tp->skip_ofst = G_TFOFFSET(ctla);
6682	tp->skip_len = G_TFLENGTH(ctla);
6683
6684	ofst = (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH) * idx;
6685	data_reg = A_MPS_TRC_FILTER0_MATCH + ofst;
6686	mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + ofst;
6687
6688	for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
6689		tp->mask[i] = ~t4_read_reg(adap, mask_reg);
6690		tp->data[i] = t4_read_reg(adap, data_reg) & tp->mask[i];
6691	}
6692}
6693
6694/**
6695 *	t4_pmtx_get_stats - returns the HW stats from PMTX
6696 *	@adap: the adapter
6697 *	@cnt: where to store the count statistics
6698 *	@cycles: where to store the cycle statistics
6699 *
6700 *	Returns performance statistics from PMTX.
6701 */
6702void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
6703{
6704	int i;
6705	u32 data[2];
6706
6707	for (i = 0; i < adap->chip_params->pm_stats_cnt; i++) {
6708		t4_write_reg(adap, A_PM_TX_STAT_CONFIG, i + 1);
6709		cnt[i] = t4_read_reg(adap, A_PM_TX_STAT_COUNT);
6710		if (is_t4(adap))
6711			cycles[i] = t4_read_reg64(adap, A_PM_TX_STAT_LSB);
6712		else {
6713			t4_read_indirect(adap, A_PM_TX_DBG_CTRL,
6714					 A_PM_TX_DBG_DATA, data, 2,
6715					 A_PM_TX_DBG_STAT_MSB);
6716			cycles[i] = (((u64)data[0] << 32) | data[1]);
6717		}
6718	}
6719}
6720
6721/**
6722 *	t4_pmrx_get_stats - returns the HW stats from PMRX
6723 *	@adap: the adapter
6724 *	@cnt: where to store the count statistics
6725 *	@cycles: where to store the cycle statistics
6726 *
6727 *	Returns performance statistics from PMRX.
6728 */
6729void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
6730{
6731	int i;
6732	u32 data[2];
6733
6734	for (i = 0; i < adap->chip_params->pm_stats_cnt; i++) {
6735		t4_write_reg(adap, A_PM_RX_STAT_CONFIG, i + 1);
6736		cnt[i] = t4_read_reg(adap, A_PM_RX_STAT_COUNT);
6737		if (is_t4(adap)) {
6738			cycles[i] = t4_read_reg64(adap, A_PM_RX_STAT_LSB);
6739		} else {
6740			t4_read_indirect(adap, A_PM_RX_DBG_CTRL,
6741					 A_PM_RX_DBG_DATA, data, 2,
6742					 A_PM_RX_DBG_STAT_MSB);
6743			cycles[i] = (((u64)data[0] << 32) | data[1]);
6744		}
6745	}
6746}
6747
6748/**
6749 *	t4_get_mps_bg_map - return the buffer groups associated with a port
6750 *	@adap: the adapter
6751 *	@idx: the port index
6752 *
6753 *	Returns a bitmap indicating which MPS buffer groups are associated
6754 *	with the given port.  Bit i is set if buffer group i is used by the
6755 *	port.
6756 */
6757static unsigned int t4_get_mps_bg_map(struct adapter *adap, int idx)
6758{
6759	u32 n;
6760
6761	if (adap->params.mps_bg_map)
6762		return ((adap->params.mps_bg_map >> (idx << 3)) & 0xff);
6763
6764	n = G_NUMPORTS(t4_read_reg(adap, A_MPS_CMN_CTL));
6765	if (n == 0)
6766		return idx == 0 ? 0xf : 0;
6767	if (n == 1 && chip_id(adap) <= CHELSIO_T5)
6768		return idx < 2 ? (3 << (2 * idx)) : 0;
6769	return 1 << idx;
6770}
6771
6772/*
6773 * TP RX e-channels associated with the port.
6774 */
6775static unsigned int t4_get_rx_e_chan_map(struct adapter *adap, int idx)
6776{
6777	u32 n = G_NUMPORTS(t4_read_reg(adap, A_MPS_CMN_CTL));
6778	const u32 all_chan = (1 << adap->chip_params->nchan) - 1;
6779
6780	if (n == 0)
6781		return idx == 0 ? all_chan : 0;
6782	if (n == 1 && chip_id(adap) <= CHELSIO_T5)
6783		return idx < 2 ? (3 << (2 * idx)) : 0;
6784	return 1 << idx;
6785}
6786
6787/**
6788 *      t4_get_port_type_description - return Port Type string description
6789 *      @port_type: firmware Port Type enumeration
6790 */
6791const char *t4_get_port_type_description(enum fw_port_type port_type)
6792{
6793	static const char *const port_type_description[] = {
6794		"Fiber_XFI",
6795		"Fiber_XAUI",
6796		"BT_SGMII",
6797		"BT_XFI",
6798		"BT_XAUI",
6799		"KX4",
6800		"CX4",
6801		"KX",
6802		"KR",
6803		"SFP",
6804		"BP_AP",
6805		"BP4_AP",
6806		"QSFP_10G",
6807		"QSA",
6808		"QSFP",
6809		"BP40_BA",
6810		"KR4_100G",
6811		"CR4_QSFP",
6812		"CR_QSFP",
6813		"CR2_QSFP",
6814		"SFP28",
6815		"KR_SFP28",
6816	};
6817
6818	if (port_type < ARRAY_SIZE(port_type_description))
6819		return port_type_description[port_type];
6820	return "UNKNOWN";
6821}
6822
6823/**
6