1/*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12/*
13 * This file is part of the Chelsio T4/T5/T6 Ethernet driver.
14 *
15 * Copyright (C) 2003-2019 Chelsio Communications.  All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23/*
24 * Copyright 2020 RackTop Systems, Inc.
25 */
26
27#include "common.h"
28#include "t4_regs.h"
29#include "t4_regs_values.h"
30#include "t4fw_interface.h"
31
32/**
33 *	t4_wait_op_done_val - wait until an operation is completed
34 *	@adapter: the adapter performing the operation
35 *	@reg: the register to check for completion
36 *	@mask: a single-bit field within @reg that indicates completion
37 *	@polarity: the value of the field when the operation is completed
38 *	@attempts: number of check iterations
39 *	@delay: delay in usecs between iterations
40 *	@valp: where to store the value of the register at completion time
41 *
42 *	Wait until an operation is completed by checking a bit in a register
43 *	up to @attempts times.  If @valp is not NULL the value of the register
44 *	at the time it indicated completion is stored there.  Returns 0 if the
45 *	operation completes and	-EAGAIN	otherwise.
46 */
47static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask,
48			       int polarity, int attempts, int delay, u32 *valp)
49{
50	while (1) {
51		u32 val = t4_read_reg(adapter, reg);
52
53		if (!!(val & mask) == polarity) {
54			if (valp)
55				*valp = val;
56			return 0;
57		}
58		if (--attempts == 0)
59			return -EAGAIN;
60		if (delay)
61			udelay(delay);
62	}
63}
64
65static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
66				  int polarity, int attempts, int delay)
67{
68	return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts,
69				   delay, NULL);
70}
71
72/**
73 *	t4_set_reg_field - set a register field to a value
74 *	@adapter: the adapter to program
75 *	@addr: the register address
76 *	@mask: specifies the portion of the register to modify
77 *	@val: the new value for the register field
78 *
79 *	Sets a register field specified by the supplied mask to the
80 *	given value.
81 */
82void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask,
83		      u32 val)
84{
85	u32 v = t4_read_reg(adapter, addr) & ~mask;
86
87	t4_write_reg(adapter, addr, v | val);
88	(void) t4_read_reg(adapter, addr);      /* flush */
89}
90
91/**
92 *	t4_read_indirect - read indirectly addressed registers
93 *	@adap: the adapter
94 *	@addr_reg: register holding the indirect address
95 *	@data_reg: register holding the value of the indirect register
96 *	@vals: where the read register values are stored
97 *	@nregs: how many indirect registers to read
98 *	@start_idx: index of first indirect register to read
99 *
100 *	Reads registers that are accessed indirectly through an address/data
101 *	register pair.
102 */
103void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
104			     unsigned int data_reg, u32 *vals,
105			     unsigned int nregs, unsigned int start_idx)
106{
107	while (nregs--) {
108		t4_write_reg(adap, addr_reg, start_idx);
109		*vals++ = t4_read_reg(adap, data_reg);
110		start_idx++;
111	}
112}
113
114/**
115 *	t4_write_indirect - write indirectly addressed registers
116 *	@adap: the adapter
117 *	@addr_reg: register holding the indirect addresses
118 *	@data_reg: register holding the value for the indirect registers
119 *	@vals: values to write
120 *	@nregs: how many indirect registers to write
121 *	@start_idx: address of first indirect register to write
122 *
123 *	Writes a sequential block of registers that are accessed indirectly
124 *	through an address/data register pair.
125 */
126void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
127		       unsigned int data_reg, const u32 *vals,
128		       unsigned int nregs, unsigned int start_idx)
129{
130	while (nregs--) {
131		t4_write_reg(adap, addr_reg, start_idx++);
132		t4_write_reg(adap, data_reg, *vals++);
133	}
134}
135
136/*
137 * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor
138 * mechanism.  This guarantees that we get the real value even if we're
139 * operating within a Virtual Machine and the Hypervisor is trapping our
140 * Configuration Space accesses.
141 *
142 * N.B. This routine should only be used as a last resort: the firmware uses
143 *      the backdoor registers on a regular basis and we can end up
144 *      conflicting with it's uses!
145 */
146void t4_hw_pci_read_cfg4(struct adapter *adap, int reg, u32 *val)
147{
148	u32 req = V_FUNCTION(adap->pf) | V_REGISTER(reg);
149
150	if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
151		req |= F_ENABLE;
152	else
153		req |= F_T6_ENABLE;
154
155	if (is_t4(adap->params.chip))
156		req |= F_LOCALCFG;
157
158	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, req);
159	*val = t4_read_reg(adap, A_PCIE_CFG_SPACE_DATA);
160
161	/* Reset F_ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a
162	 * Configuration Space read.  (None of the other fields matter when
163	 * F_ENABLE is 0 so a simple register write is easier than a
164	 * read-modify-write via t4_set_reg_field().)
165	 */
166	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, 0);
167}
168
169/*
170 * t4_report_fw_error - report firmware error
171 * @adap: the adapter
172 *
173 * The adapter firmware can indicate error conditions to the host.
174 * If the firmware has indicated an error, print out the reason for
175 * the firmware error.
176 */
177static void t4_report_fw_error(struct adapter *adap)
178{
179	static const char *const reason[] = {
180		"Crash",			/* PCIE_FW_EVAL_CRASH */
181		"During Device Preparation",	/* PCIE_FW_EVAL_PREP */
182		"During Device Configuration",	/* PCIE_FW_EVAL_CONF */
183		"During Device Initialization",	/* PCIE_FW_EVAL_INIT */
184		"Unexpected Event",		/* PCIE_FW_EVAL_UNEXPECTEDEVENT */
185		"Insufficient Airflow",		/* PCIE_FW_EVAL_OVERHEAT */
186		"Device Shutdown",		/* PCIE_FW_EVAL_DEVICESHUTDOWN */
187		"Reserved",			/* reserved */
188	};
189	u32 pcie_fw;
190
191	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
192	if (pcie_fw & F_PCIE_FW_ERR) {
193		CH_ERR(adap, "Firmware reports adapter error: %s\n",
194			reason[G_PCIE_FW_EVAL(pcie_fw)]);
195		adap->flags &= ~FW_OK;
196	}
197}
198
199/*
200 * Get the reply to a mailbox command and store it in @rpl in big-endian order.
201 */
202static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit,
203			 u32 mbox_addr)
204{
205	for ( ; nflit; nflit--, mbox_addr += 8)
206		*rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr));
207}
208
209/*
210 * Handle a FW assertion reported in a mailbox.
211 */
212static void fw_asrt(struct adapter *adap, struct fw_debug_cmd *asrt)
213{
214	CH_ALERT(adap,
215		  "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n",
216		  asrt->u.assert.filename_0_7,
217		  be32_to_cpu(asrt->u.assert.line),
218		  be32_to_cpu(asrt->u.assert.x),
219		  be32_to_cpu(asrt->u.assert.y));
220}
221
222#define X_CIM_PF_NOACCESS 0xeeeeeeee
223
224/*
225 * If the Host OS Driver needs locking arround accesses to the mailbox, this
226 * can be turned on via the T4_OS_NEEDS_MBOX_LOCKING CPP define ...
227 */
228/* makes single-statement usage a bit cleaner ... */
229#ifdef T4_OS_NEEDS_MBOX_LOCKING
230#define T4_OS_MBOX_LOCKING(x) x
231#else
232#define T4_OS_MBOX_LOCKING(x) do {} while (0)
233#endif
234
235/*
236 * If the OS Driver wants busy waits to keep a watchdog happy, tap it during
237 * busy loops which don't sleep.
238 */
239#ifdef T4_OS_NEEDS_TOUCH_NMI_WATCHDOG
240#define T4_OS_TOUCH_NMI_WATCHDOG()	t4_os_touch_nmi_watchdog()
241#else
242#define T4_OS_TOUCH_NMI_WATCHDOG()
243#endif
244
245#ifdef T4_OS_LOG_MBOX_CMDS
246/**
247 *	t4_record_mbox - record a Firmware Mailbox Command/Reply in the log
248 *	@adapter: the adapter
249 *	@cmd: the Firmware Mailbox Command or Reply
250 *	@size: command length in bytes
251 *	@access: the time (ms) needed to access the Firmware Mailbox
252 *	@execute: the time (ms) the command spent being executed
253 */
254static void t4_record_mbox(struct adapter *adapter,
255			   const __be64 *cmd, unsigned int size,
256			   int access, int execute)
257{
258	struct mbox_cmd_log *log = adapter->mbox_log;
259	struct mbox_cmd *entry;
260	int i;
261
262	entry = mbox_cmd_log_entry(log, log->cursor++);
263	if (log->cursor == log->size)
264		log->cursor = 0;
265
266	for (i = 0; i < size/8; i++)
267		entry->cmd[i] = be64_to_cpu(cmd[i]);
268	while (i < MBOX_LEN/8)
269		entry->cmd[i++] = 0;
270	entry->timestamp = t4_os_timestamp();
271	entry->seqno = log->seqno++;
272	entry->access = access;
273	entry->execute = execute;
274}
275
276#define T4_RECORD_MBOX(__adapter, __cmd, __size, __access, __execute) \
277	t4_record_mbox(__adapter, __cmd, __size, __access, __execute)
278
279#else /* !T4_OS_LOG_MBOX_CMDS */
280
281#define T4_RECORD_MBOX(__adapter, __cmd, __size, __access, __execute) \
282	/* nothing */
283
284#endif /* !T4_OS_LOG_MBOX_CMDS */
285
286/**
287 *	t4_record_mbox_marker - record a marker in the mailbox log
288 *	@adapter: the adapter
289 *	@marker: byte array marker
290 *	@size: marker size in bytes
291 *
292 *	We inject a "fake mailbox command" into the Firmware Mailbox Log
293 *	using a known command token and then the bytes of the specified
294 *	marker.  This lets debugging code inject markers into the log to
295 *	help identify which commands are in response to higher level code.
296 */
297void t4_record_mbox_marker(struct adapter *adapter,
298			   const void *marker, unsigned int size)
299{
300#ifdef T4_OS_LOG_MBOX_CMDS
301	__be64 marker_cmd[MBOX_LEN/8];
302	const unsigned int max_marker = sizeof marker_cmd - sizeof (__be64);
303	unsigned int marker_cmd_size;
304
305	if (size > max_marker)
306		size = max_marker;
307
308	marker_cmd[0] = cpu_to_be64(~0LLU);
309	memcpy(&marker_cmd[1], marker, size);
310	memset((unsigned char *)&marker_cmd[1] + size, 0, max_marker - size);
311	marker_cmd_size = sizeof (__be64) + roundup(size, sizeof (__be64));
312
313	t4_record_mbox(adapter, marker_cmd, marker_cmd_size, 0, 0);
314#endif /* T4_OS_LOG_MBOX_CMDS */
315}
316
317/*
318 * Delay time in microseconds to wait for mailbox access/fw reply
319 * to mailbox command
320 */
321#define MIN_MBOX_CMD_DELAY 900
322#define MBOX_CMD_DELAY 1000
323
324/**
325 *	t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox
326 *	@adap: the adapter
327 *	@mbox: index of the mailbox to use
328 *	@cmd: the command to write
329 *	@size: command length in bytes
330 *	@rpl: where to optionally store the reply
331 *	@sleep_ok: if true we may sleep while awaiting command completion
332 *	@timeout: time to wait for command to finish before timing out
333 *		(negative implies @sleep_ok=false)
334 *
335 *	Sends the given command to FW through the selected mailbox and waits
336 *	for the FW to execute the command.  If @rpl is not %NULL it is used to
337 *	store the FW's reply to the command.  The command and its optional
338 *	reply are of the same length.  Some FW commands like RESET and
339 *	INITIALIZE can take a considerable amount of time to execute.
340 *	@sleep_ok determines whether we may sleep while awaiting the response.
341 *	If sleeping is allowed we use progressive backoff otherwise we spin.
342 *	Note that passing in a negative @timeout is an alternate mechanism
343 *	for specifying @sleep_ok=false.  This is useful when a higher level
344 *	interface allows for specification of @timeout but not @sleep_ok ...
345 *
346 *	The return value is 0 on success or a negative errno on failure.  A
347 *	failure can happen either because we are not able to execute the
348 *	command or FW executes it but signals an error.  In the latter case
349 *	the return value is the error code indicated by FW (negated).
350 */
351int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
352			    int size, void *rpl, bool sleep_ok, int timeout)
353{
354#ifdef T4_OS_NEEDS_MBOX_LOCKING
355	u16 access = 0;
356#endif
357	u32 v;
358	u64 res;
359	int i, ret;
360	const __be64 *p = cmd;
361	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
362	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
363	u32 ctl;
364	__be64 cmd_rpl[MBOX_LEN/8];
365	T4_OS_MBOX_LOCKING(t4_os_list_t entry);
366	u32 pcie_fw;
367
368	if ((size & 15) || size > MBOX_LEN)
369		return -EINVAL;
370
371	/*
372	 * If we have a negative timeout, that implies that we can't sleep.
373	 */
374	if (timeout < 0) {
375		sleep_ok = false;
376		timeout = -timeout;
377	}
378
379#ifdef T4_OS_NEEDS_MBOX_LOCKING
380	/*
381	 * Queue ourselves onto the mailbox access list.  When our entry is at
382	 * the front of the list, we have rights to access the mailbox.  So we
383	 * wait [for a while] till we're at the front [or bail out with an
384	 * EBUSY] ...
385	 */
386	t4_os_atomic_add_tail(&entry, &adap->mbox_list, &adap->mbox_lock);
387
388	for (i = 0; ; i++) {
389		/*
390		 * If we've waited too long, return a busy indication.  This
391		 * really ought to be based on our initial position in the
392		 * mailbox access list but this is a start.  We very rarely
393		 * contend on access to the mailbox ...  Also check for a
394		 * firmware error which we'll report as a device error.
395		 */
396		pcie_fw = t4_read_reg(adap, A_PCIE_FW);
397		if (i > 4*timeout || (pcie_fw & F_PCIE_FW_ERR)) {
398			t4_os_atomic_list_del(&entry, &adap->mbox_lock);
399			t4_report_fw_error(adap);
400			ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -EBUSY;
401			T4_RECORD_MBOX(adap, cmd, size, ret, 0);
402			return ret;
403		}
404
405		/*
406		 * If we're at the head, break out and start the mailbox
407		 * protocol.
408		 */
409		if (t4_os_list_first_entry(&adap->mbox_list) == &entry)
410			break;
411
412		/*
413		 * Delay for a bit before checking again ...
414		 */
415		if (sleep_ok) {
416			usleep_range(MIN_MBOX_CMD_DELAY, MBOX_CMD_DELAY);
417		} else {
418			T4_OS_TOUCH_NMI_WATCHDOG();
419			udelay(MBOX_CMD_DELAY);
420		}
421	}
422	access = i;
423#endif /* T4_OS_NEEDS_MBOX_LOCKING */
424
425	/*
426	 * Attempt to gain access to the mailbox.
427	 */
428	for (i = 0; i < 4; i++) {
429		ctl = t4_read_reg(adap, ctl_reg);
430		v = G_MBOWNER(ctl);
431		if (v != X_MBOWNER_NONE)
432			break;
433	}
434
435	/*
436	 * If we were unable to gain access, dequeue ourselves from the
437	 * mailbox atomic access list and report the error to our caller.
438	 */
439	if (v != X_MBOWNER_PL) {
440		T4_OS_MBOX_LOCKING(t4_os_atomic_list_del(&entry,
441							 &adap->mbox_lock));
442		t4_report_fw_error(adap);
443		ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT;
444		T4_RECORD_MBOX(adap, cmd, size, access, ret);
445		return ret;
446	}
447
448	/*
449	 * If we gain ownership of the mailbox and there's a "valid" message
450	 * in it, this is likely an asynchronous error message from the
451	 * firmware.  So we'll report that and then proceed on with attempting
452	 * to issue our own command ... which may well fail if the error
453	 * presaged the firmware crashing ...
454	 */
455	if (ctl & F_MBMSGVALID) {
456		CH_ERR(adap, "found VALID command in mbox %u: "
457		       "%llx %llx %llx %llx %llx %llx %llx %llx\n", mbox,
458		       (unsigned long long)t4_read_reg64(adap, data_reg),
459		       (unsigned long long)t4_read_reg64(adap, data_reg + 8),
460		       (unsigned long long)t4_read_reg64(adap, data_reg + 16),
461		       (unsigned long long)t4_read_reg64(adap, data_reg + 24),
462		       (unsigned long long)t4_read_reg64(adap, data_reg + 32),
463		       (unsigned long long)t4_read_reg64(adap, data_reg + 40),
464		       (unsigned long long)t4_read_reg64(adap, data_reg + 48),
465		       (unsigned long long)t4_read_reg64(adap, data_reg + 56));
466	}
467
468	/*
469	 * Copy in the new mailbox command and send it on its way ...
470	 */
471	T4_RECORD_MBOX(adap, cmd, size, access, 0);
472	for (i = 0; i < size; i += 8, p++)
473		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));
474
475	/*
476	 * XXX It's not clear that we need this anymore now
477	 * XXX that we have mailbox logging ...
478	 */
479	CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
480
481	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
482	(void) t4_read_reg(adap, ctl_reg);	/* flush write */
483
484	/*
485	 * Loop waiting for the reply; bail out if we time out or the firmware
486	 * reports an error.
487	 */
488	for (i = 0;
489	     !((pcie_fw = t4_read_reg(adap, A_PCIE_FW)) & F_PCIE_FW_ERR) &&
490	     i < timeout;
491	     i++) {
492		if (sleep_ok) {
493			usleep_range(MIN_MBOX_CMD_DELAY, MBOX_CMD_DELAY);
494		} else {
495			T4_OS_TOUCH_NMI_WATCHDOG();
496			udelay(MBOX_CMD_DELAY);
497		}
498
499		v = t4_read_reg(adap, ctl_reg);
500		if (v == X_CIM_PF_NOACCESS)
501			continue;
502		if (G_MBOWNER(v) == X_MBOWNER_PL) {
503			if (!(v & F_MBMSGVALID)) {
504				t4_write_reg(adap, ctl_reg,
505					     V_MBOWNER(X_MBOWNER_NONE));
506				continue;
507			}
508
509			/*
510			 * Retrieve the command reply and release the mailbox.
511			 */
512			get_mbox_rpl(adap, cmd_rpl, size/8, data_reg);
513			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));
514			T4_OS_MBOX_LOCKING(t4_os_atomic_list_del(&entry,
515								 &adap->mbox_lock));
516
517			T4_RECORD_MBOX(adap, cmd_rpl, size, access, i + 1);
518
519			/*
520			 * XXX It's not clear that we need this anymore now
521			 * XXX that we have mailbox logging ...
522			 */
523			CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
524			CH_MSG(adap, INFO, HW,
525			       "command completed in %d ms (%ssleeping)\n",
526			       i + 1, sleep_ok ? "" : "non-");
527
528			res = be64_to_cpu(cmd_rpl[0]);
529			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
530				fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl);
531				res = V_FW_CMD_RETVAL(EIO);
532			} else if (rpl)
533				memcpy(rpl, cmd_rpl, size);
534			return -G_FW_CMD_RETVAL((int)res);
535		}
536	}
537
538	/*
539	 * We timed out waiting for a reply to our mailbox command.  Report
540	 * the error and also check to see if the firmware reported any
541	 * errors ...
542	 */
543	T4_OS_MBOX_LOCKING(t4_os_atomic_list_del(&entry, &adap->mbox_lock));
544
545	ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -ETIMEDOUT;
546	T4_RECORD_MBOX(adap, cmd, size, access, ret);
547	CH_ERR(adap, "command %#x in mailbox %d timed out\n",
548	       *(const u8 *)cmd, mbox);
549
550	t4_report_fw_error(adap);
551	t4_fatal_err(adap);
552	return ret;
553}
554
555#ifdef CONFIG_CUDBG
556/*
557 * The maximum number of times to iterate for FW reply before
558 * issuing a mailbox timeout
559 */
560#define FW_REPLY_WAIT_LOOP 6000000
561
562/**
563 *	t4_wr_mbox_meat_timeout_panic - send a command to FW through the given
564 *	mailbox. This function is a minimal version of t4_wr_mbox_meat_timeout()
565 *	and is only invoked during a kernel crash. Since this function is
566 *	called through a atomic notifier chain ,we cannot sleep awaiting a
567 *	response from FW, hence repeatedly loop until we get a reply.
568 *
569 *	@adap: the adapter
570 *	@mbox: index of the mailbox to use
571 *	@cmd: the command to write
572 *	@size: command length in bytes
573 *	@rpl: where to optionally store the reply
574 */
575
576static int t4_wr_mbox_meat_timeout_panic(struct adapter *adap, int mbox,
577			    const void *cmd, int size, void *rpl)
578{
579	u32 v;
580	u64 res;
581	int i, ret;
582	u64 cnt;
583	const __be64 *p = cmd;
584	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
585	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
586	u32 ctl;
587	__be64 cmd_rpl[MBOX_LEN/8];
588	u32 pcie_fw;
589
590	if ((size & 15) || size > MBOX_LEN)
591		return -EINVAL;
592
593	/*
594	 * Check for a firmware error which we'll report as a
595	 * device error.
596	 */
597	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
598	if (pcie_fw & F_PCIE_FW_ERR) {
599		t4_report_fw_error(adap);
600		ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -EBUSY;
601		return ret;
602	}
603
604	/*
605	 * Attempt to gain access to the mailbox.
606	 */
607	for (i = 0; i < 4; i++) {
608		ctl = t4_read_reg(adap, ctl_reg);
609		v = G_MBOWNER(ctl);
610		if (v != X_MBOWNER_NONE)
611			break;
612	}
613
614	/*
615	 * If we were unable to gain access, report the error to our caller.
616	 */
617	if (v != X_MBOWNER_PL) {
618		t4_report_fw_error(adap);
619		ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT;
620		return ret;
621	}
622
623	/*
624	 * If we gain ownership of the mailbox and there's a "valid" message
625	 * in it, this is likely an asynchronous error message from the
626	 * firmware.  So we'll report that and then proceed on with attempting
627	 * to issue our own command ... which may well fail if the error
628	 * presaged the firmware crashing ...
629	 */
630	if (ctl & F_MBMSGVALID) {
631		CH_ERR(adap, "found VALID command in mbox %u: "
632		       "%llx %llx %llx %llx %llx %llx %llx %llx\n", mbox,
633		       (unsigned long long)t4_read_reg64(adap, data_reg),
634		       (unsigned long long)t4_read_reg64(adap, data_reg + 8),
635		       (unsigned long long)t4_read_reg64(adap, data_reg + 16),
636		       (unsigned long long)t4_read_reg64(adap, data_reg + 24),
637		       (unsigned long long)t4_read_reg64(adap, data_reg + 32),
638		       (unsigned long long)t4_read_reg64(adap, data_reg + 40),
639		       (unsigned long long)t4_read_reg64(adap, data_reg + 48),
640		       (unsigned long long)t4_read_reg64(adap, data_reg + 56));
641	}
642
643	/*
644	 * Copy in the new mailbox command and send it on its way ...
645	 */
646	for (i = 0; i < size; i += 8, p++)
647		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));
648
649	CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
650
651	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
652	t4_read_reg(adap, ctl_reg);	/* flush write */
653
654	/*
655	 * Loop waiting for the reply; bail out if we time out or the firmware
656	 * reports an error.
657	 */
658	for (cnt = 0;
659	    !((pcie_fw = t4_read_reg(adap, A_PCIE_FW)) & F_PCIE_FW_ERR) &&
660	    cnt < FW_REPLY_WAIT_LOOP;
661	    cnt++) {
662		v = t4_read_reg(adap, ctl_reg);
663		if (v == X_CIM_PF_NOACCESS)
664			continue;
665		if (G_MBOWNER(v) == X_MBOWNER_PL) {
666			if (!(v & F_MBMSGVALID)) {
667				t4_write_reg(adap, ctl_reg,
668					     V_MBOWNER(X_MBOWNER_NONE));
669				continue;
670			}
671
672			/*
673			 * Retrieve the command reply and release the mailbox.
674			 */
675			get_mbox_rpl(adap, cmd_rpl, size/8, data_reg);
676			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));
677
678			CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
679
680			res = be64_to_cpu(cmd_rpl[0]);
681			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
682				fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl);
683				res = V_FW_CMD_RETVAL(EIO);
684			} else if (rpl)
685				memcpy(rpl, cmd_rpl, size);
686			return -G_FW_CMD_RETVAL((int)res);
687		}
688	}
689
690	/*
691	 * We timed out waiting for a reply to our mailbox command.  Report
692	 * the error and also check to see if the firmware reported any
693	 * errors ...
694	 */
695	ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -ETIMEDOUT;
696	CH_ERR(adap, "command %#x in mailbox %d timed out\n",
697	       *(const u8 *)cmd, mbox);
698
699	t4_report_fw_error(adap);
700	t4_fatal_err(adap);
701	return ret;
702}
703#endif
704
705int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
706		    void *rpl, bool sleep_ok)
707{
708#ifdef CONFIG_CUDBG
709	if (adap->flags & K_CRASH)
710		return t4_wr_mbox_meat_timeout_panic(adap, mbox, cmd, size,
711						     rpl);
712	else
713#endif
714		return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl,
715					       sleep_ok, FW_CMD_MAX_TIMEOUT);
716
717}
718
719static int t4_edc_err_read(struct adapter *adap, int idx)
720{
721	u32 edc_ecc_err_addr_reg;
722	u32 edc_bist_status_rdata_reg;
723
724	if (is_t4(adap->params.chip)) {
725		CH_WARN(adap, "%s: T4 NOT supported.\n", __func__);
726		return 0;
727	}
728	if (idx != MEM_EDC0 && idx != MEM_EDC1) {
729		CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx);
730		return 0;
731	}
732
733	edc_ecc_err_addr_reg = EDC_T5_REG(A_EDC_H_ECC_ERR_ADDR, idx);
734	edc_bist_status_rdata_reg = EDC_T5_REG(A_EDC_H_BIST_STATUS_RDATA, idx);
735
736	CH_WARN(adap,
737		"edc%d err addr 0x%x: 0x%x.\n",
738		idx, edc_ecc_err_addr_reg,
739		t4_read_reg(adap, edc_ecc_err_addr_reg));
740	CH_WARN(adap,
741	 	"bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
742		edc_bist_status_rdata_reg,
743		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg),
744		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 8),
745		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 16),
746		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 24),
747		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 32),
748		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 40),
749		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 48),
750		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 56),
751		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 64));
752
753	return 0;
754}
755
756/**
757 *	t4_memory_rw_addr - read/write adapter memory via PCIE memory window
758 *	@adap: the adapter
759 *	@win: PCI-E Memory Window to use
760 *	@addr: address within adapter memory
761 *	@len: amount of memory to transfer
762 *	@hbuf: host memory buffer
763 *	@dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
764 *
765 *	Reads/writes an [almost] arbitrary memory region in the firmware: the
766 *	firmware memory address and host buffer must be aligned on 32-bit
767 *	boudaries; the length may be arbitrary.
768 *
769 *	NOTES:
770 *	 1. The memory is transferred as a raw byte sequence from/to the
771 *	    firmware's memory.  If this memory contains data structures which
772 *	    contain multi-byte integers, it's the caller's responsibility to
773 *	    perform appropriate byte order conversions.
774 *
775 *	 2. It is the Caller's responsibility to ensure that no other code
776 *	    uses the specified PCI-E Memory Window while this routine is
777 *	    using it.  This is typically done via the use of OS-specific
778 *	    locks, etc.
779 */
780int t4_memory_rw_addr(struct adapter *adap, int win, u32 addr,
781		      u32 len, void *hbuf, int dir)
782{
783	u32 pos, offset, resid;
784	u32 win_pf, mem_reg, mem_aperture, mem_base;
785	u32 *buf;
786
787	/* Argument sanity checks ...
788	 */
789	if (addr & 0x3 || (uintptr_t)hbuf & 0x3)
790		return -EINVAL;
791	buf = (u32 *)hbuf;
792
793	/* It's convenient to be able to handle lengths which aren't a
794	 * multiple of 32-bits because we often end up transferring files to
795	 * the firmware.  So we'll handle that by normalizing the length here
796	 * and then handling any residual transfer at the end.
797	 */
798	resid = len & 0x3;
799	len -= resid;
800
801	/* Each PCI-E Memory Window is programmed with a window size -- or
802	 * "aperture" -- which controls the granularity of its mapping onto
803	 * adapter memory.  We need to grab that aperture in order to know
804	 * how to use the specified window.  The window is also programmed
805	 * with the base address of the Memory Window in BAR0's address
806	 * space.  For T4 this is an absolute PCI-E Bus Address.  For T5
807	 * the address is relative to BAR0.
808	 */
809	mem_reg = t4_read_reg(adap,
810			      PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN,
811						  win));
812
813	/* a dead adapter will return 0xffffffff for PIO reads */
814	if (mem_reg == 0xffffffff) {
815		CH_WARN(adap, "Unable to read PCI-E Memory Window Base[%d]\n",
816			win);
817		return -ENXIO;
818	}
819
820	mem_aperture = 1 << (G_WINDOW(mem_reg) + X_WINDOW_SHIFT);
821	mem_base = G_PCIEOFST(mem_reg) << X_PCIEOFST_SHIFT;
822	if (is_t4(adap->params.chip))
823		mem_base -= adap->t4_bar0;
824	win_pf = is_t4(adap->params.chip) ? 0 : V_PFNUM(adap->pf);
825
826	/* Calculate our initial PCI-E Memory Window Position and Offset into
827	 * that Window.
828	 */
829	pos = addr & ~(mem_aperture-1);
830	offset = addr - pos;
831
832	/* Set up initial PCI-E Memory Window to cover the start of our
833	 * transfer.  (Read it back to ensure that changes propagate before we
834	 * attempt to use the new value.)
835	 */
836	t4_write_reg(adap,
837		     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win),
838		     pos | win_pf);
839	t4_read_reg(adap,
840		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win));
841
842	/* Transfer data to/from the adapter as long as there's an integral
843	 * number of 32-bit transfers to complete.
844	 *
845	 * A note on Endianness issues:
846	 *
847	 * The "register" reads and writes below from/to the PCI-E Memory
848	 * Window invoke the standard adapter Big-Endian to PCI-E Link
849	 * Little-Endian "swizzel."  As a result, if we have the following
850	 * data in adapter memory:
851	 *
852	 *     Memory:  ... | b0 | b1 | b2 | b3 | ...
853	 *     Address:      i+0  i+1  i+2  i+3
854	 *
855	 * Then a read of the adapter memory via the PCI-E Memory Window
856	 * will yield:
857	 *
858	 *     x = readl(i)
859	 *	   31                  0
860	 *         [ b3 | b2 | b1 | b0 ]
861	 *
862	 * If this value is stored into local memory on a Little-Endian system
863	 * it will show up correctly in local memory as:
864	 *
865	 *     ( ..., b0, b1, b2, b3, ... )
866	 *
867	 * But on a Big-Endian system, the store will show up in memory
868	 * incorrectly swizzled as:
869	 *
870	 *     ( ..., b3, b2, b1, b0, ... )
871	 *
872	 * So we need to account for this in the reads and writes to the
873	 * PCI-E Memory Window below by undoing the register read/write
874	 * swizzels.
875	 */
876	while (len > 0) {
877		if (dir == T4_MEMORY_READ)
878			*buf++ = le32_to_cpu((__force __le32)t4_read_reg(adap,
879						mem_base + offset));
880		else
881			t4_write_reg(adap, mem_base + offset,
882				     (__force u32)cpu_to_le32(*buf++));
883		offset += sizeof(__be32);
884		len -= sizeof(__be32);
885
886		/* If we've reached the end of our current window aperture,
887		 * move the PCI-E Memory Window on to the next.  Note that
888		 * doing this here after "len" may be 0 allows us to set up
889		 * the PCI-E Memory Window for a possible final residual
890		 * transfer below ...
891		 */
892		if (offset == mem_aperture) {
893			pos += mem_aperture;
894			offset = 0;
895			t4_write_reg(adap,
896				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
897						    win), pos | win_pf);
898			t4_read_reg(adap,
899				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
900						    win));
901		}
902	}
903
904	/* If the original transfer had a length which wasn't a multiple of
905	 * 32-bits, now's where we need to finish off the transfer of the
906	 * residual amount.  The PCI-E Memory Window has already been moved
907	 * above (if necessary) to cover this final transfer.
908	 */
909	if (resid) {
910		union {
911			u32 word;
912			char byte[4];
913		} last;
914		unsigned char *bp;
915		int i;
916
917		if (dir == T4_MEMORY_READ) {
918			last.word = le32_to_cpu(
919					(__force __le32)t4_read_reg(adap,
920						mem_base + offset));
921			for (bp = (unsigned char *)buf, i = resid; i < 4; i++)
922				bp[i] = last.byte[i];
923		} else {
924			last.word = *buf;
925			for (i = resid; i < 4; i++)
926				last.byte[i] = 0;
927			t4_write_reg(adap, mem_base + offset,
928				     (__force u32)cpu_to_le32(last.word));
929		}
930	}
931
932	return 0;
933}
934
935/**
936 *	t4_memory_rw_mtype - read/write EDC 0, EDC 1 or MC via PCIE memory window
937 *	@adap: the adapter
938 *	@win: PCI-E Memory Window to use
939 *	@mtype: memory type: MEM_EDC0, MEM_EDC1, MEM_HMA or MEM_MC
940 *	@maddr: address within indicated memory type
941 *	@len: amount of memory to transfer
942 *	@hbuf: host memory buffer
943 *	@dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
944 *
945 *	Reads/writes adapter memory using t4_memory_rw_addr().  This routine
946 *	provides an (memory type, address withing memory type) interface.
947 */
948int t4_memory_rw_mtype(struct adapter *adap, int win, int mtype, u32 maddr,
949		       u32 len, void *hbuf, int dir)
950{
951	u32 mtype_offset;
952	u32 edc_size, mc_size;
953
954	/* Offset into the region of memory which is being accessed
955	 * MEM_EDC0 = 0
956	 * MEM_EDC1 = 1
957	 * MEM_MC   = 2 -- MEM_MC for chips with only 1 memory controller
958	 * MEM_MC1  = 3 -- for chips with 2 memory controllers (e.g. T5)
959	 * MEM_HMA  = 4
960	 */
961	edc_size  = G_EDRAM0_SIZE(t4_read_reg(adap, A_MA_EDRAM0_BAR));
962	if (mtype == MEM_HMA) {
963		mtype_offset = 2 * (edc_size * 1024 * 1024);
964	} else if (mtype != MEM_MC1)
965		mtype_offset = (mtype * (edc_size * 1024 * 1024));
966	else {
967		mc_size = G_EXT_MEM0_SIZE(t4_read_reg(adap,
968						      A_MA_EXT_MEMORY0_BAR));
969		mtype_offset = (MEM_MC0 * edc_size + mc_size) * 1024 * 1024;
970	}
971
972	return t4_memory_rw_addr(adap, win,
973				 mtype_offset + maddr, len,
974				 hbuf, dir);
975}
976
977/*
978 * Return the specified PCI-E Configuration Space register from our Physical
979 * Function.  We try first via a Firmware LDST Command (if fw_attach != 0)
980 * since we prefer to let the firmware own all of these registers, but if that
981 * fails we go for it directly ourselves.
982 */
983u32 t4_read_pcie_cfg4(struct adapter *adap, int reg, int drv_fw_attach)
984{
985	u32 val;
986
987	/*
988	 * If fw_attach != 0, construct and send the Firmware LDST Command to
989	 * retrieve the specified PCI-E Configuration Space register.
990	 */
991	if (drv_fw_attach != 0) {
992		struct fw_ldst_cmd ldst_cmd;
993		int ret;
994
995		memset(&ldst_cmd, 0, sizeof(ldst_cmd));
996		ldst_cmd.op_to_addrspace =
997			cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
998				    F_FW_CMD_REQUEST |
999				    F_FW_CMD_READ |
1000				    V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_FUNC_PCIE));
1001		ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd));
1002		ldst_cmd.u.pcie.select_naccess = V_FW_LDST_CMD_NACCESS(1);
1003		ldst_cmd.u.pcie.ctrl_to_fn =
1004			(F_FW_LDST_CMD_LC | V_FW_LDST_CMD_FN(adap->pf));
1005		ldst_cmd.u.pcie.r = reg;
1006
1007		/*
1008		 * If the LDST Command succeeds, return the result, otherwise
1009		 * fall through to reading it directly ourselves ...
1010		 */
1011		ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
1012				 &ldst_cmd);
1013		if (ret == 0)
1014			return be32_to_cpu(ldst_cmd.u.pcie.data[0]);
1015
1016		CH_WARN(adap, "Firmware failed to return "
1017			"Configuration Space register %d, err = %d\n",
1018			reg, -ret);
1019	}
1020
1021	/*
1022	 * Read the desired Configuration Space register via the PCI-E
1023	 * Backdoor mechanism.
1024	 */
1025	t4_hw_pci_read_cfg4(adap, reg, &val);
1026	return val;
1027}
1028
1029/*
1030 * Get the window based on base passed to it.
1031 * Window aperture is currently unhandled, but there is no use case for it
1032 * right now
1033 */
1034static int t4_get_window(struct adapter *adap, u64 pci_base, u64 pci_mask, u64 memwin_base, int drv_fw_attach)
1035{
1036	if (is_t4(adap->params.chip)) {
1037		u32 bar0;
1038
1039		/*
1040		 * Truncation intentional: we only read the bottom 32-bits of
1041		 * the 64-bit BAR0/BAR1 ...  We use the hardware backdoor
1042		 * mechanism to read BAR0 instead of using
1043		 * pci_resource_start() because we could be operating from
1044		 * within a Virtual Machine which is trapping our accesses to
1045		 * our Configuration Space and we need to set up the PCI-E
1046		 * Memory Window decoders with the actual addresses which will
1047		 * be coming across the PCI-E link.
1048		 */
1049		bar0 = t4_read_pcie_cfg4(adap, pci_base, drv_fw_attach);
1050		bar0 &= pci_mask;
1051		adap->t4_bar0 = bar0;
1052
1053		return bar0 + memwin_base;
1054	} else {
1055		/* For T5, only relative offset inside the PCIe BAR is passed */
1056		return memwin_base;
1057	}
1058}
1059
1060/* Get the default utility window (win0) used by everyone */
1061int t4_get_util_window(struct adapter *adap, int drv_fw_attach)
1062{
1063	return t4_get_window(adap, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_MEM_MASK, MEMWIN0_BASE, drv_fw_attach);
1064}
1065
1066/*
1067 * Set up memory window for accessing adapter memory ranges.  (Read
1068 * back MA register to ensure that changes propagate before we attempt
1069 * to use the new values.)
1070 */
1071void t4_setup_memwin(struct adapter *adap, u32 memwin_base, u32 window)
1072{
1073	t4_write_reg(adap, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, window),
1074		     memwin_base | V_BIR(0) |
1075		     V_WINDOW(ilog2(MEMWIN0_APERTURE) - X_WINDOW_SHIFT));
1076	t4_read_reg(adap, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, window));
1077}
1078
1079/**
1080 *	t4_get_regs_len - return the size of the chips register set
1081 *	@adapter: the adapter
1082 *
1083 *	Returns the size of the chip's BAR0 register space.
1084 */
1085unsigned int t4_get_regs_len(struct adapter *adapter)
1086{
1087	unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
1088
1089	switch (chip_version) {
1090	case CHELSIO_T4:
1091		return T4_REGMAP_SIZE;
1092
1093	case CHELSIO_T5:
1094	case CHELSIO_T6:
1095		return T5_REGMAP_SIZE;
1096	}
1097
1098	CH_ERR(adapter,
1099		"Unsupported chip version %d\n", chip_version);
1100	return 0;
1101}
1102
1103/**
1104 *	t4_get_regs - read chip registers into provided buffer
1105 *	@adap: the adapter
1106 *	@buf: register buffer
1107 *	@buf_size: size (in bytes) of register buffer
1108 *
1109 *	If the provided register buffer isn't large enough for the chip's
1110 *	full register range, the register dump will be truncated to the
1111 *	register buffer's size.
1112 */
1113void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
1114{
1115	static const unsigned int t4_reg_ranges[] = {
1116		0x1008, 0x1108,
1117		0x1180, 0x1184,
1118		0x1190, 0x1194,
1119		0x11a0, 0x11a4,
1120		0x11b0, 0x11b4,
1121		0x11fc, 0x123c,
1122		0x1300, 0x173c,
1123		0x1800, 0x18fc,
1124		0x3000, 0x30d8,
1125		0x30e0, 0x30e4,
1126		0x30ec, 0x5910,
1127		0x5920, 0x5924,
1128		0x5960, 0x5960,
1129		0x5968, 0x5968,
1130		0x5970, 0x5970,
1131		0x5978, 0x5978,
1132		0x5980, 0x5980,
1133		0x5988, 0x5988,
1134		0x5990, 0x5990,
1135		0x5998, 0x5998,
1136		0x59a0, 0x59d4,
1137		0x5a00, 0x5ae0,
1138		0x5ae8, 0x5ae8,
1139		0x5af0, 0x5af0,
1140		0x5af8, 0x5af8,
1141		0x6000, 0x6098,
1142		0x6100, 0x6150,
1143		0x6200, 0x6208,
1144		0x6240, 0x6248,
1145		0x6280, 0x62b0,
1146		0x62c0, 0x6338,
1147		0x6370, 0x638c,
1148		0x6400, 0x643c,
1149		0x6500, 0x6524,
1150		0x6a00, 0x6a04,
1151		0x6a14, 0x6a38,
1152		0x6a60, 0x6a70,
1153		0x6a78, 0x6a78,
1154		0x6b00, 0x6b0c,
1155		0x6b1c, 0x6b84,
1156		0x6bf0, 0x6bf8,
1157		0x6c00, 0x6c0c,
1158		0x6c1c, 0x6c84,
1159		0x6cf0, 0x6cf8,
1160		0x6d00, 0x6d0c,
1161		0x6d1c, 0x6d84,
1162		0x6df0, 0x6df8,
1163		0x6e00, 0x6e0c,
1164		0x6e1c, 0x6e84,
1165		0x6ef0, 0x6ef8,
1166		0x6f00, 0x6f0c,
1167		0x6f1c, 0x6f84,
1168		0x6ff0, 0x6ff8,
1169		0x7000, 0x700c,
1170		0x701c, 0x7084,
1171		0x70f0, 0x70f8,
1172		0x7100, 0x710c,
1173		0x711c, 0x7184,
1174		0x71f0, 0x71f8,
1175		0x7200, 0x720c,
1176		0x721c, 0x7284,
1177		0x72f0, 0x72f8,
1178		0x7300, 0x730c,
1179		0x731c, 0x7384,
1180		0x73f0, 0x73f8,
1181		0x7400, 0x7450,
1182		0x7500, 0x7530,
1183		0x7600, 0x760c,
1184		0x7614, 0x761c,
1185		0x7680, 0x76cc,
1186		0x7700, 0x7798,
1187		0x77c0, 0x77fc,
1188		0x7900, 0x79fc,
1189		0x7b00, 0x7b58,
1190		0x7b60, 0x7b84,
1191		0x7b8c, 0x7c38,
1192		0x7d00, 0x7d38,
1193		0x7d40, 0x7d80,
1194		0x7d8c, 0x7ddc,
1195		0x7de4, 0x7e04,
1196		0x7e10, 0x7e1c,
1197		0x7e24, 0x7e38,
1198		0x7e40, 0x7e44,
1199		0x7e4c, 0x7e78,
1200		0x7e80, 0x7ea4,
1201		0x7eac, 0x7edc,
1202		0x7ee8, 0x7efc,
1203		0x8dc0, 0x8e04,
1204		0x8e10, 0x8e1c,
1205		0x8e30, 0x8e78,
1206		0x8ea0, 0x8eb8,
1207		0x8ec0, 0x8f6c,
1208		0x8fc0, 0x9008,
1209		0x9010, 0x9058,
1210		0x9060, 0x9060,
1211		0x9068, 0x9074,
1212		0x90fc, 0x90fc,
1213		0x9400, 0x9408,
1214		0x9410, 0x9458,
1215		0x9600, 0x9600,
1216		0x9608, 0x9638,
1217		0x9640, 0x96bc,
1218		0x9800, 0x9808,
1219		0x9820, 0x983c,
1220		0x9850, 0x9864,
1221		0x9c00, 0x9c6c,
1222		0x9c80, 0x9cec,
1223		0x9d00, 0x9d6c,
1224		0x9d80, 0x9dec,
1225		0x9e00, 0x9e6c,
1226		0x9e80, 0x9eec,
1227		0x9f00, 0x9f6c,
1228		0x9f80, 0x9fec,
1229		0xd004, 0xd004,
1230		0xd010, 0xd03c,
1231		0xdfc0, 0xdfe0,
1232		0xe000, 0xea7c,
1233		0xf000, 0x11110,
1234		0x11118, 0x11190,
1235		0x19040, 0x1906c,
1236		0x19078, 0x19080,
1237		0x1908c, 0x190e4,
1238		0x190f0, 0x190f8,
1239		0x19100, 0x19110,
1240		0x19120, 0x19124,
1241		0x19150, 0x19194,
1242		0x1919c, 0x191b0,
1243		0x191d0, 0x191e8,
1244		0x19238, 0x1924c,
1245		0x193f8, 0x1943c,
1246		0x1944c, 0x19474,
1247		0x19490, 0x194e0,
1248		0x194f0, 0x194f8,
1249		0x19800, 0x19c08,
1250		0x19c10, 0x19c90,
1251		0x19ca0, 0x19ce4,
1252		0x19cf0, 0x19d40,
1253		0x19d50, 0x19d94,
1254		0x19da0, 0x19de8,
1255		0x19df0, 0x19e40,
1256		0x19e50, 0x19e90,
1257		0x19ea0, 0x19f4c,
1258		0x1a000, 0x1a004,
1259		0x1a010, 0x1a06c,
1260		0x1a0b0, 0x1a0e4,
1261		0x1a0ec, 0x1a0f4,
1262		0x1a100, 0x1a108,
1263		0x1a114, 0x1a120,
1264		0x1a128, 0x1a130,
1265		0x1a138, 0x1a138,
1266		0x1a190, 0x1a1c4,
1267		0x1a1fc, 0x1a1fc,
1268		0x1e040, 0x1e04c,
1269		0x1e284, 0x1e28c,
1270		0x1e2c0, 0x1e2c0,
1271		0x1e2e0, 0x1e2e0,
1272		0x1e300, 0x1e384,
1273		0x1e3c0, 0x1e3c8,
1274		0x1e440, 0x1e44c,
1275		0x1e684, 0x1e68c,
1276		0x1e6c0, 0x1e6c0,
1277		0x1e6e0, 0x1e6e0,
1278		0x1e700, 0x1e784,
1279		0x1e7c0, 0x1e7c8,
1280		0x1e840, 0x1e84c,
1281		0x1ea84, 0x1ea8c,
1282		0x1eac0, 0x1eac0,
1283		0x1eae0, 0x1eae0,
1284		0x1eb00, 0x1eb84,
1285		0x1ebc0, 0x1ebc8,
1286		0x1ec40, 0x1ec4c,
1287		0x1ee84, 0x1ee8c,
1288		0x1eec0, 0x1eec0,
1289		0x1eee0, 0x1eee0,
1290		0x1ef00, 0x1ef84,
1291		0x1efc0, 0x1efc8,
1292		0x1f040, 0x1f04c,
1293		0x1f284, 0x1f28c,
1294		0x1f2c0, 0x1f2c0,
1295		0x1f2e0, 0x1f2e0,
1296		0x1f300, 0x1f384,
1297		0x1f3c0, 0x1f3c8,
1298		0x1f440, 0x1f44c,
1299		0x1f684, 0x1f68c,
1300		0x1f6c0, 0x1f6c0,
1301		0x1f6e0, 0x1f6e0,
1302		0x1f700, 0x1f784,
1303		0x1f7c0, 0x1f7c8,
1304		0x1f840, 0x1f84c,
1305		0x1fa84, 0x1fa8c,
1306		0x1fac0, 0x1fac0,
1307		0x1fae0, 0x1fae0,
1308		0x1fb00, 0x1fb84,
1309		0x1fbc0, 0x1fbc8,
1310		0x1fc40, 0x1fc4c,
1311		0x1fe84, 0x1fe8c,
1312		0x1fec0, 0x1fec0,
1313		0x1fee0, 0x1fee0,
1314		0x1ff00, 0x1ff84,
1315		0x1ffc0, 0x1ffc8,
1316		0x20000, 0x2002c,
1317		0x20100, 0x2013c,
1318		0x20190, 0x201a0,
1319		0x201a8, 0x201b8,
1320		0x201c4, 0x201c8,
1321		0x20200, 0x20318,
1322		0x20400, 0x204b4,
1323		0x204c0, 0x20528,
1324		0x20540, 0x20614,
1325		0x21000, 0x21040,
1326		0x2104c, 0x21060,
1327		0x210c0, 0x210ec,
1328		0x21200, 0x21268,
1329		0x21270, 0x21284,
1330		0x212fc, 0x21388,
1331		0x21400, 0x21404,
1332		0x21500, 0x21500,
1333		0x21510, 0x21518,
1334		0x2152c, 0x21530,
1335		0x2153c, 0x2153c,
1336		0x21550, 0x21554,
1337		0x21600, 0x21600,
1338		0x21608, 0x2161c,
1339		0x21624, 0x21628,
1340		0x21630, 0x21634,
1341		0x2163c, 0x2163c,
1342		0x21700, 0x2171c,
1343		0x21780, 0x2178c,
1344		0x21800, 0x21818,
1345		0x21820, 0x21828,
1346		0x21830, 0x21848,
1347		0x21850, 0x21854,
1348		0x21860, 0x21868,
1349		0x21870, 0x21870,
1350		0x21878, 0x21898,
1351		0x218a0, 0x218a8,
1352		0x218b0, 0x218c8,
1353		0x218d0, 0x218d4,
1354		0x218e0, 0x218e8,
1355		0x218f0, 0x218f0,
1356		0x218f8, 0x21a18,
1357		0x21a20, 0x21a28,
1358		0x21a30, 0x21a48,
1359		0x21a50, 0x21a54,
1360		0x21a60, 0x21a68,
1361		0x21a70, 0x21a70,
1362		0x21a78, 0x21a98,
1363		0x21aa0, 0x21aa8,
1364		0x21ab0, 0x21ac8,
1365		0x21ad0, 0x21ad4,
1366		0x21ae0, 0x21ae8,
1367		0x21af0, 0x21af0,
1368		0x21af8, 0x21c18,
1369		0x21c20, 0x21c20,
1370		0x21c28, 0x21c30,
1371		0x21c38, 0x21c38,
1372		0x21c80, 0x21c98,
1373		0x21ca0, 0x21ca8,
1374		0x21cb0, 0x21cc8,
1375		0x21cd0, 0x21cd4,
1376		0x21ce0, 0x21ce8,
1377		0x21cf0, 0x21cf0,
1378		0x21cf8, 0x21d7c,
1379		0x21e00, 0x21e04,
1380		0x22000, 0x2202c,
1381		0x22100, 0x2213c,
1382		0x22190, 0x221a0,
1383		0x221a8, 0x221b8,
1384		0x221c4, 0x221c8,
1385		0x22200, 0x22318,
1386		0x22400, 0x224b4,
1387		0x224c0, 0x22528,
1388		0x22540, 0x22614,
1389		0x23000, 0x23040,
1390		0x2304c, 0x23060,
1391		0x230c0, 0x230ec,
1392		0x23200, 0x23268,
1393		0x23270, 0x23284,
1394		0x232fc, 0x23388,
1395		0x23400, 0x23404,
1396		0x23500, 0x23500,
1397		0x23510, 0x23518,
1398		0x2352c, 0x23530,
1399		0x2353c, 0x2353c,
1400		0x23550, 0x23554,
1401		0x23600, 0x23600,
1402		0x23608, 0x2361c,
1403		0x23624, 0x23628,
1404		0x23630, 0x23634,
1405		0x2363c, 0x2363c,
1406		0x23700, 0x2371c,
1407		0x23780, 0x2378c,
1408		0x23800, 0x23818,
1409		0x23820, 0x23828,
1410		0x23830, 0x23848,
1411		0x23850, 0x23854,
1412		0x23860, 0x23868,
1413		0x23870, 0x23870,
1414		0x23878, 0x23898,
1415		0x238a0, 0x238a8,
1416		0x238b0, 0x238c8,
1417		0x238d0, 0x238d4,
1418		0x238e0, 0x238e8,
1419		0x238f0, 0x238f0,
1420		0x238f8, 0x23a18,
1421		0x23a20, 0x23a28,
1422		0x23a30, 0x23a48,
1423		0x23a50, 0x23a54,
1424		0x23a60, 0x23a68,
1425		0x23a70, 0x23a70,
1426		0x23a78, 0x23a98,
1427		0x23aa0, 0x23aa8,
1428		0x23ab0, 0x23ac8,
1429		0x23ad0, 0x23ad4,
1430		0x23ae0, 0x23ae8,
1431		0x23af0, 0x23af0,
1432		0x23af8, 0x23c18,
1433		0x23c20, 0x23c20,
1434		0x23c28, 0x23c30,
1435		0x23c38, 0x23c38,
1436		0x23c80, 0x23c98,
1437		0x23ca0, 0x23ca8,
1438		0x23cb0, 0x23cc8,
1439		0x23cd0, 0x23cd4,
1440		0x23ce0, 0x23ce8,
1441		0x23cf0, 0x23cf0,
1442		0x23cf8, 0x23d7c,
1443		0x23e00, 0x23e04,
1444		0x24000, 0x2402c,
1445		0x24100, 0x2413c,
1446		0x24190, 0x241a0,
1447		0x241a8, 0x241b8,
1448		0x241c4, 0x241c8,
1449		0x24200, 0x24318,
1450		0x24400, 0x244b4,
1451		0x244c0, 0x24528,
1452		0x24540, 0x24614,
1453		0x25000, 0x25040,
1454		0x2504c, 0x25060,
1455		0x250c0, 0x250ec,
1456		0x25200, 0x25268,
1457		0x25270, 0x25284,
1458		0x252fc, 0x25388,
1459		0x25400, 0x25404,
1460		0x25500, 0x25500,
1461		0x25510, 0x25518,
1462		0x2552c, 0x25530,
1463		0x2553c, 0x2553c,
1464		0x25550, 0x25554,
1465		0x25600, 0x25600,
1466		0x25608, 0x2561c,
1467		0x25624, 0x25628,
1468		0x25630, 0x25634,
1469		0x2563c, 0x2563c,
1470		0x25700, 0x2571c,
1471		0x25780, 0x2578c,
1472		0x25800, 0x25818,
1473		0x25820, 0x25828,
1474		0x25830, 0x25848,
1475		0x25850, 0x25854,
1476		0x25860, 0x25868,
1477		0x25870, 0x25870,
1478		0x25878, 0x25898,
1479		0x258a0, 0x258a8,
1480		0x258b0, 0x258c8,
1481		0x258d0, 0x258d4,
1482		0x258e0, 0x258e8,
1483		0x258f0, 0x258f0,
1484		0x258f8, 0x25a18,
1485		0x25a20, 0x25a28,
1486		0x25a30, 0x25a48,
1487		0x25a50, 0x25a54,
1488		0x25a60, 0x25a68,
1489		0x25a70, 0x25a70,
1490		0x25a78, 0x25a98,
1491		0x25aa0, 0x25aa8,
1492		0x25ab0, 0x25ac8,
1493		0x25ad0, 0x25ad4,
1494		0x25ae0, 0x25ae8,
1495		0x25af0, 0x25af0,
1496		0x25af8, 0x25c18,
1497		0x25c20, 0x25c20,
1498		0x25c28, 0x25c30,
1499		0x25c38, 0x25c38,
1500		0x25c80, 0x25c98,
1501		0x25ca0, 0x25ca8,
1502		0x25cb0, 0x25cc8,
1503		0x25cd0, 0x25cd4,
1504		0x25ce0, 0x25ce8,
1505		0x25cf0, 0x25cf0,
1506		0x25cf8, 0x25d7c,
1507		0x25e00, 0x25e04,
1508		0x26000, 0x2602c,
1509		0x26100, 0x2613c,
1510		0x26190, 0x261a0,
1511		0x261a8, 0x261b8,
1512		0x261c4, 0x261c8,
1513		0x26200, 0x26318,
1514		0x26400, 0x264b4,
1515		0x264c0, 0x26528,
1516		0x26540, 0x26614,
1517		0x27000, 0x27040,
1518		0x2704c, 0x27060,
1519		0x270c0, 0x270ec,
1520		0x27200, 0x27268,
1521		0x27270, 0x27284,
1522		0x272fc, 0x27388,
1523		0x27400, 0x27404,
1524		0x27500, 0x27500,
1525		0x27510, 0x27518,
1526		0x2752c, 0x27530,
1527		0x2753c, 0x2753c,
1528		0x27550, 0x27554,
1529		0x27600, 0x27600,
1530		0x27608, 0x2761c,
1531		0x27624, 0x27628,
1532		0x27630, 0x27634,
1533		0x2763c, 0x2763c,
1534		0x27700, 0x2771c,
1535		0x27780, 0x2778c,
1536		0x27800, 0x27818,
1537		0x27820, 0x27828,
1538		0x27830, 0x27848,
1539		0x27850, 0x27854,
1540		0x27860, 0x27868,
1541		0x27870, 0x27870,
1542		0x27878, 0x27898,
1543		0x278a0, 0x278a8,
1544		0x278b0, 0x278c8,
1545		0x278d0, 0x278d4,
1546		0x278e0, 0x278e8,
1547		0x278f0, 0x278f0,
1548		0x278f8, 0x27a18,
1549		0x27a20, 0x27a28,
1550		0x27a30, 0x27a48,
1551		0x27a50, 0x27a54,
1552		0x27a60, 0x27a68,
1553		0x27a70, 0x27a70,
1554		0x27a78, 0x27a98,
1555		0x27aa0, 0x27aa8,
1556		0x27ab0, 0x27ac8,
1557		0x27ad0, 0x27ad4,
1558		0x27ae0, 0x27ae8,
1559		0x27af0, 0x27af0,
1560		0x27af8, 0x27c18,
1561		0x27c20, 0x27c20,
1562		0x27c28, 0x27c30,
1563		0x27c38, 0x27c38,
1564		0x27c80, 0x27c98,
1565		0x27ca0, 0x27ca8,
1566		0x27cb0, 0x27cc8,
1567		0x27cd0, 0x27cd4,
1568		0x27ce0, 0x27ce8,
1569		0x27cf0, 0x27cf0,
1570		0x27cf8, 0x27d7c,
1571		0x27e00, 0x27e04,
1572	};
1573
1574	static const unsigned int t5_reg_ranges[] = {
1575		0x1008, 0x10c0,
1576		0x10cc, 0x10f8,
1577		0x1100, 0x1100,
1578		0x110c, 0x1148,
1579		0x1180, 0x1184,
1580		0x1190, 0x1194,
1581		0x11a0, 0x11a4,
1582		0x11b0, 0x11b4,
1583		0x11fc, 0x123c,
1584		0x1280, 0x173c,
1585		0x1800, 0x18fc,
1586		0x3000, 0x3028,
1587		0x3060, 0x30b0,
1588		0x30b8, 0x30d8,
1589		0x30e0, 0x30fc,
1590		0x3140, 0x357c,
1591		0x35a8, 0x35cc,
1592		0x35ec, 0x35ec,
1593		0x3600, 0x5624,
1594		0x56cc, 0x56ec,
1595		0x56f4, 0x5720,
1596		0x5728, 0x575c,
1597		0x580c, 0x5814,
1598		0x5890, 0x589c,
1599		0x58a4, 0x58ac,
1600		0x58b8, 0x58bc,
1601		0x5940, 0x59c8,
1602		0x59d0, 0x59dc,
1603		0x59fc, 0x5a18,
1604		0x5a60, 0x5a70,
1605		0x5a80, 0x5a9c,
1606		0x5b94, 0x5bfc,
1607		0x6000, 0x6020,
1608		0x6028, 0x6040,
1609		0x6058, 0x609c,
1610		0x60a8, 0x614c,
1611		0x7700, 0x7798,
1612		0x77c0, 0x78fc,
1613		0x7b00, 0x7b58,
1614		0x7b60, 0x7b84,
1615		0x7b8c, 0x7c54,
1616		0x7d00, 0x7d38,
1617		0x7d40, 0x7d80,
1618		0x7d8c, 0x7ddc,
1619		0x7de4, 0x7e04,
1620		0x7e10, 0x7e1c,
1621		0x7e24, 0x7e38,
1622		0x7e40, 0x7e44,
1623		0x7e4c, 0x7e78,
1624		0x7e80, 0x7edc,
1625		0x7ee8, 0x7efc,
1626		0x8dc0, 0x8de0,
1627		0x8df8, 0x8e04,
1628		0x8e10, 0x8e84,
1629		0x8ea0, 0x8f84,
1630		0x8fc0, 0x9058,
1631		0x9060, 0x9060,
1632		0x9068, 0x90f8,
1633		0x9400, 0x9408,
1634		0x9410, 0x9470,
1635		0x9600, 0x9600,
1636		0x9608, 0x9638,
1637		0x9640, 0x96f4,
1638		0x9800, 0x9808,
1639		0x9820, 0x983c,
1640		0x9850, 0x9864,
1641		0x9c00, 0x9c6c,
1642		0x9c80, 0x9cec,
1643		0x9d00, 0x9d6c,
1644		0x9d80, 0x9dec,
1645		0x9e00, 0x9e6c,
1646		0x9e80, 0x9eec,
1647		0x9f00, 0x9f6c,
1648		0x9f80, 0xa020,
1649		0xd004, 0xd004,
1650		0xd010, 0xd03c,
1651		0xdfc0, 0xdfe0,
1652		0xe000, 0x1106c,
1653		0x11074, 0x11088,
1654		0x1109c, 0x1117c,
1655		0x11190, 0x11204,
1656		0x19040, 0x1906c,
1657		0x19078, 0x19080,
1658		0x1908c, 0x190e8,
1659		0x190f0, 0x190f8,
1660		0x19100, 0x19110,
1661		0x19120, 0x19124,
1662		0x19150, 0x19194,
1663		0x1919c, 0x191b0,
1664		0x191d0, 0x191e8,
1665		0x19238, 0x19290,
1666		0x193f8, 0x19428,
1667		0x19430, 0x19444,
1668		0x1944c, 0x1946c,
1669		0x19474, 0x19474,
1670		0x19490, 0x194cc,
1671		0x194f0, 0x194f8,
1672		0x19c00, 0x19c08,
1673		0x19c10, 0x19c60,
1674		0x19c94, 0x19ce4,
1675		0x19cf0, 0x19d40,
1676		0x19d50, 0x19d94,
1677		0x19da0, 0x19de8,
1678		0x19df0, 0x19e10,
1679		0x19e50, 0x19e90,
1680		0x19ea0, 0x19f24,
1681		0x19f34, 0x19f34,
1682		0x19f40, 0x19f50,
1683		0x19f90, 0x19fb4,
1684		0x19fc4, 0x19fe4,
1685		0x1a000, 0x1a004,
1686		0x1a010, 0x1a06c,
1687		0x1a0b0, 0x1a0e4,
1688		0x1a0ec, 0x1a0f8,
1689		0x1a100, 0x1a108,
1690		0x1a114, 0x1a120,
1691		0x1a128, 0x1a130,
1692		0x1a138, 0x1a138,
1693		0x1a190, 0x1a1c4,
1694		0x1a1fc, 0x1a1fc,
1695		0x1e008, 0x1e00c,
1696		0x1e040, 0x1e044,
1697		0x1e04c, 0x1e04c,
1698		0x1e284, 0x1e290,
1699		0x1e2c0, 0x1e2c0,
1700		0x1e2e0, 0x1e2e0,
1701		0x1e300, 0x1e384,
1702		0x1e3c0, 0x1e3c8,
1703		0x1e408, 0x1e40c,
1704		0x1e440, 0x1e444,
1705		0x1e44c, 0x1e44c,
1706		0x1e684, 0x1e690,
1707		0x1e6c0, 0x1e6c0,
1708		0x1e6e0, 0x1e6e0,
1709		0x1e700, 0x1e784,
1710		0x1e7c0, 0x1e7c8,
1711		0x1e808, 0x1e80c,
1712		0x1e840, 0x1e844,
1713		0x1e84c, 0x1e84c,
1714		0x1ea84, 0x1ea90,
1715		0x1eac0, 0x1eac0,
1716		0x1eae0, 0x1eae0,
1717		0x1eb00, 0x1eb84,
1718		0x1ebc0, 0x1ebc8,
1719		0x1ec08, 0x1ec0c,
1720		0x1ec40, 0x1ec44,
1721		0x1ec4c, 0x1ec4c,
1722		0x1ee84, 0x1ee90,
1723		0x1eec0, 0x1eec0,
1724		0x1eee0, 0x1eee0,
1725		0x1ef00, 0x1ef84,
1726		0x1efc0, 0x1efc8,
1727		0x1f008, 0x1f00c,
1728		0x1f040, 0x1f044,
1729		0x1f04c, 0x1f04c,
1730		0x1f284, 0x1f290,
1731		0x1f2c0, 0x1f2c0,
1732		0x1f2e0, 0x1f2e0,
1733		0x1f300, 0x1f384,
1734		0x1f3c0, 0x1f3c8,
1735		0x1f408, 0x1f40c,
1736		0x1f440, 0x1f444,
1737		0x1f44c, 0x1f44c,
1738		0x1f684, 0x1f690,
1739		0x1f6c0, 0x1f6c0,
1740		0x1f6e0, 0x1f6e0,
1741		0x1f700, 0x1f784,
1742		0x1f7c0, 0x1f7c8,
1743		0x1f808, 0x1f80c,
1744		0x1f840, 0x1f844,
1745		0x1f84c, 0x1f84c,
1746		0x1fa84, 0x1fa90,
1747		0x1fac0, 0x1fac0,
1748		0x1fae0, 0x1fae0,
1749		0x1fb00, 0x1fb84,
1750		0x1fbc0, 0x1fbc8,
1751		0x1fc08, 0x1fc0c,
1752		0x1fc40, 0x1fc44,
1753		0x1fc4c, 0x1fc4c,
1754		0x1fe84, 0x1fe90,
1755		0x1fec0, 0x1fec0,
1756		0x1fee0, 0x1fee0,
1757		0x1ff00, 0x1ff84,
1758		0x1ffc0, 0x1ffc8,
1759		0x30000, 0x30030,
1760		0x30100, 0x30144,
1761		0x30190, 0x301a0,
1762		0x301a8, 0x301b8,
1763		0x301c4, 0x301c8,
1764		0x301d0, 0x301d0,
1765		0x30200, 0x30318,
1766		0x30400, 0x304b4,
1767		0x304c0, 0x3052c,
1768		0x30540, 0x3061c,
1769		0x30800, 0x30828,
1770		0x30834, 0x30834,
1771		0x308c0, 0x30908,
1772		0x30910, 0x309ac,
1773		0x30a00, 0x30a14,
1774		0x30a1c, 0x30a2c,
1775		0x30a44, 0x30a50,
1776		0x30a74, 0x30a74,
1777		0x30a7c, 0x30afc,
1778		0x30b08, 0x30c24,
1779		0x30d00, 0x30d00,
1780		0x30d08, 0x30d14,
1781		0x30d1c, 0x30d20,
1782		0x30d3c, 0x30d3c,
1783		0x30d48, 0x30d50,
1784		0x31200, 0x3120c,
1785		0x31220, 0x31220,
1786		0x31240, 0x31240,
1787		0x31600, 0x3160c,
1788		0x31a00, 0x31a1c,
1789		0x31e00, 0x31e20,
1790		0x31e38, 0x31e3c,
1791		0x31e80, 0x31e80,
1792		0x31e88, 0x31ea8,
1793		0x31eb0, 0x31eb4,
1794		0x31ec8, 0x31ed4,
1795		0x31fb8, 0x32004,
1796		0x32200, 0x32200,
1797		0x32208, 0x32240,
1798		0x32248, 0x32280,
1799		0x32288, 0x322c0,
1800		0x322c8, 0x322fc,
1801		0x32600, 0x32630,
1802		0x32a00, 0x32abc,
1803		0x32b00, 0x32b10,
1804		0x32b20, 0x32b30,
1805		0x32b40, 0x32b50,
1806		0x32b60, 0x32b70,
1807		0x33000, 0x33028,
1808		0x33030, 0x33048,
1809		0x33060, 0x33068,
1810		0x33070, 0x3309c,
1811		0x330f0, 0x33128,
1812		0x33130, 0x33148,
1813		0x33160, 0x33168,
1814		0x33170, 0x3319c,
1815		0x331f0, 0x33238,
1816		0x33240, 0x33240,
1817		0x33248, 0x33250,
1818		0x3325c, 0x33264,
1819		0x33270, 0x332b8,
1820		0x332c0, 0x332e4,
1821		0x332f8, 0x33338,
1822		0x33340, 0x33340,
1823		0x33348, 0x33350,
1824		0x3335c, 0x33364,
1825		0x33370, 0x333b8,
1826		0x333c0, 0x333e4,
1827		0x333f8, 0x33428,
1828		0x33430, 0x33448,
1829		0x33460, 0x33468,
1830		0x33470, 0x3349c,
1831		0x334f0, 0x33528,
1832		0x33530, 0x33548,
1833		0x33560, 0x33568,
1834		0x33570, 0x3359c,
1835		0x335f0, 0x33638,
1836		0x33640, 0x33640,
1837		0x33648, 0x33650,
1838		0x3365c, 0x33664,
1839		0x33670, 0x336b8,
1840		0x336c0, 0x336e4,
1841		0x336f8, 0x33738,
1842		0x33740, 0x33740,
1843		0x33748, 0x33750,
1844		0x3375c, 0x33764,
1845		0x33770, 0x337b8,
1846		0x337c0, 0x337e4,
1847		0x337f8, 0x337fc,
1848		0x33814, 0x33814,
1849		0x3382c, 0x3382c,
1850		0x33880, 0x3388c,
1851		0x338e8, 0x338ec,
1852		0x33900, 0x33928,
1853		0x33930, 0x33948,
1854		0x33960, 0x33968,
1855		0x33970, 0x3399c,
1856		0x339f0, 0x33a38,
1857		0x33a40, 0x33a40,
1858		0x33a48, 0x33a50,
1859		0x33a5c, 0x33a64,
1860		0x33a70, 0x33ab8,
1861		0x33ac0, 0x33ae4,
1862		0x33af8, 0x33b10,
1863		0x33b28, 0x33b28,
1864		0x33b3c, 0x33b50,
1865		0x33bf0, 0x33c10,
1866		0x33c28, 0x33c28,
1867		0x33c3c, 0x33c50,
1868		0x33cf0, 0x33cfc,
1869		0x34000, 0x34030,
1870		0x34100, 0x34144,
1871		0x34190, 0x341a0,
1872		0x341a8, 0x341b8,
1873		0x341c4, 0x341c8,
1874		0x341d0, 0x341d0,
1875		0x34200, 0x34318,
1876		0x34400, 0x344b4,
1877		0x344c0, 0x3452c,
1878		0x34540, 0x3461c,
1879		0x34800, 0x34828,
1880		0x34834, 0x34834,
1881		0x348c0, 0x34908,
1882		0x34910, 0x349ac,
1883		0x34a00, 0x34a14,
1884		0x34a1c, 0x34a2c,
1885		0x34a44, 0x34a50,
1886		0x34a74, 0x34a74,
1887		0x34a7c, 0x34afc,
1888		0x34b08, 0x34c24,
1889		0x34d00, 0x34d00,
1890		0x34d08, 0x34d14,
1891		0x34d1c, 0x34d20,
1892		0x34d3c, 0x34d3c,
1893		0x34d48, 0x34d50,
1894		0x35200, 0x3520c,
1895		0x35220, 0x35220,
1896		0x35240, 0x35240,
1897		0x35600, 0x3560c,
1898		0x35a00, 0x35a1c,
1899		0x35e00, 0x35e20,
1900		0x35e38, 0x35e3c,
1901		0x35e80, 0x35e80,
1902		0x35e88, 0x35ea8,
1903		0x35eb0, 0x35eb4,
1904		0x35ec8, 0x35ed4,
1905		0x35fb8, 0x36004,
1906		0x36200, 0x36200,
1907		0x36208, 0x36240,
1908		0x36248, 0x36280,
1909		0x36288, 0x362c0,
1910		0x362c8, 0x362fc,
1911		0x36600, 0x36630,
1912		0x36a00, 0x36abc,
1913		0x36b00, 0x36b10,
1914		0x36b20, 0x36b30,
1915		0x36b40, 0x36b50,
1916		0x36b60, 0x36b70,
1917		0x37000, 0x37028,
1918		0x37030, 0x37048,
1919		0x37060, 0x37068,
1920		0x37070, 0x3709c,
1921		0x370f0, 0x37128,
1922		0x37130, 0x37148,
1923		0x37160, 0x37168,
1924		0x37170, 0x3719c,
1925		0x371f0, 0x37238,
1926		0x37240, 0x37240,
1927		0x37248, 0x37250,
1928		0x3725c, 0x37264,
1929		0x37270, 0x372b8,
1930		0x372c0, 0x372e4,
1931		0x372f8, 0x37338,
1932		0x37340, 0x37340,
1933		0x37348, 0x37350,
1934		0x3735c, 0x37364,
1935		0x37370, 0x373b8,
1936		0x373c0, 0x373e4,
1937		0x373f8, 0x37428,
1938		0x37430, 0x37448,
1939		0x37460, 0x37468,
1940		0x37470, 0x3749c,
1941		0x374f0, 0x37528,
1942		0x37530, 0x37548,
1943		0x37560, 0x37568,
1944		0x37570, 0x3759c,
1945		0x375f0, 0x37638,
1946		0x37640, 0x37640,
1947		0x37648, 0x37650,
1948		0x3765c, 0x37664,
1949		0x37670, 0x376b8,
1950		0x376c0, 0x376e4,
1951		0x376f8, 0x37738,
1952		0x37740, 0x37740,
1953		0x37748, 0x37750,
1954		0x3775c, 0x37764,
1955		0x37770, 0x377b8,
1956		0x377c0, 0x377e4,
1957		0x377f8, 0x377fc,
1958		0x37814, 0x37814,
1959		0x3782c, 0x3782c,
1960		0x37880, 0x3788c,
1961		0x378e8, 0x378ec,
1962		0x37900, 0x37928,
1963		0x37930, 0x37948,
1964		0x37960, 0x37968,
1965		0x37970, 0x3799c,
1966		0x379f0, 0x37a38,
1967		0x37a40, 0x37a40,
1968		0x37a48, 0x37a50,
1969		0x37a5c, 0x37a64,
1970		0x37a70, 0x37ab8,
1971		0x37ac0, 0x37ae4,
1972		0x37af8, 0x37b10,
1973		0x37b28, 0x37b28,
1974		0x37b3c, 0x37b50,
1975		0x37bf0, 0x37c10,
1976		0x37c28, 0x37c28,
1977		0x37c3c, 0x37c50,
1978		0x37cf0, 0x37cfc,
1979		0x38000, 0x38030,
1980		0x38100, 0x38144,
1981		0x38190, 0x381a0,
1982		0x381a8, 0x381b8,
1983		0x381c4, 0x381c8,
1984		0x381d0, 0x381d0,
1985		0x38200, 0x38318,
1986		0x38400, 0x384b4,
1987		0x384c0, 0x3852c,
1988		0x38540, 0x3861c,
1989		0x38800, 0x38828,
1990		0x38834, 0x38834,
1991		0x388c0, 0x38908,
1992		0x38910, 0x389ac,
1993		0x38a00, 0x38a14,
1994		0x38a1c, 0x38a2c,
1995		0x38a44, 0x38a50,
1996		0x38a74, 0x38a74,
1997		0x38a7c, 0x38afc,
1998		0x38b08, 0x38c24,
1999		0x38d00, 0x38d00,
2000		0x38d08, 0x38d14,
2001		0x38d1c, 0x38d20,
2002		0x38d3c, 0x38d3c,
2003		0x38d48, 0x38d50,
2004		0x39200, 0x3920c,
2005		0x39220, 0x39220,
2006		0x39240, 0x39240,
2007		0x39600, 0x3960c,
2008		0x39a00, 0x39a1c,
2009		0x39e00, 0x39e20,
2010		0x39e38, 0x39e3c,
2011		0x39e80, 0x39e80,
2012		0x39e88, 0x39ea8,
2013		0x39eb0, 0x39eb4,
2014		0x39ec8, 0x39ed4,
2015		0x39fb8, 0x3a004,
2016		0x3a200, 0x3a200,
2017		0x3a208, 0x3a240,
2018		0x3a248, 0x3a280,
2019		0x3a288, 0x3a2c0,
2020		0x3a2c8, 0x3a2fc,
2021		0x3a600, 0x3a630,
2022		0x3aa00, 0x3aabc,
2023		0x3ab00, 0x3ab10,
2024		0x3ab20, 0x3ab30,
2025		0x3ab40, 0x3ab50,
2026		0x3ab60, 0x3ab70,
2027		0x3b000, 0x3b028,
2028		0x3b030, 0x3b048,
2029		0x3b060, 0x3b068,
2030		0x3b070, 0x3b09c,
2031		0x3b0f0, 0x3b128,
2032		0x3b130, 0x3b148,
2033		0x3b160, 0x3b168,
2034		0x3b170, 0x3b19c,
2035		0x3b1f0, 0x3b238,
2036		0x3b240, 0x3b240,
2037		0x3b248, 0x3b250,
2038		0x3b25c, 0x3b264,
2039		0x3b270, 0x3b2b8,
2040		0x3b2c0, 0x3b2e4,
2041		0x3b2f8, 0x3b338,
2042		0x3b340, 0x3b340,
2043		0x3b348, 0x3b350,
2044		0x3b35c, 0x3b364,
2045		0x3b370, 0x3b3b8,
2046		0x3b3c0, 0x3b3e4,
2047		0x3b3f8, 0x3b428,
2048		0x3b430, 0x3b448,
2049		0x3b460, 0x3b468,
2050		0x3b470, 0x3b49c,
2051		0x3b4f0, 0x3b528,
2052		0x3b530, 0x3b548,
2053		0x3b560, 0x3b568,
2054		0x3b570, 0x3b59c,
2055		0x3b5f0, 0x3b638,
2056		0x3b640, 0x3b640,
2057		0x3b648, 0x3b650,
2058		0x3b65c, 0x3b664,
2059		0x3b670, 0x3b6b8,
2060		0x3b6c0, 0x3b6e4,
2061		0x3b6f8, 0x3b738,
2062		0x3b740, 0x3b740,
2063		0x3b748, 0x3b750,
2064		0x3b75c, 0x3b764,
2065		0x3b770, 0x3b7b8,
2066		0x3b7c0, 0x3b7e4,
2067		0x3b7f8, 0x3b7fc,
2068		0x3b814, 0x3b814,
2069		0x3b82c, 0x3b82c,
2070		0x3b880, 0x3b88c,
2071		0x3b8e8, 0x3b8ec,
2072		0x3b900, 0x3b928,
2073		0x3b930, 0x3b948,
2074		0x3b960, 0x3b968,
2075		0x3b970, 0x3b99c,
2076		0x3b9f0, 0x3ba38,
2077		0x3ba40, 0x3ba40,
2078		0x3ba48, 0x3ba50,
2079		0x3ba5c, 0x3ba64,
2080		0x3ba70, 0x3bab8,
2081		0x3bac0, 0x3bae4,
2082		0x3baf8, 0x3bb10,
2083		0x3bb28, 0x3bb28,
2084		0x3bb3c, 0x3bb50,
2085		0x3bbf0, 0x3bc10,
2086		0x3bc28, 0x3bc28,
2087		0x3bc3c, 0x3bc50,
2088		0x3bcf0, 0x3bcfc,
2089		0x3c000, 0x3c030,
2090		0x3c100, 0x3c144,
2091		0x3c190, 0x3c1a0,
2092		0x3c1a8, 0x3c1b8,
2093		0x3c1c4, 0x3c1c8,
2094		0x3c1d0, 0x3c1d0,
2095		0x3c200, 0x3c318,
2096		0x3c400, 0x3c4b4,
2097		0x3c4c0, 0x3c52c,
2098		0x3c540, 0x3c61c,
2099		0x3c800, 0x3c828,
2100		0x3c834, 0x3c834,
2101		0x3c8c0, 0x3c908,
2102		0x3c910, 0x3c9ac,
2103		0x3ca00, 0x3ca14,
2104		0x3ca1c, 0x3ca2c,
2105		0x3ca44, 0x3ca50,
2106		0x3ca74, 0x3ca74,
2107		0x3ca7c, 0x3cafc,
2108		0x3cb08, 0x3cc24,
2109		0x3cd00, 0x3cd00,
2110		0x3cd08, 0x3cd14,
2111		0x3cd1c, 0x3cd20,
2112		0x3cd3c, 0x3cd3c,
2113		0x3cd48, 0x3cd50,
2114		0x3d200, 0x3d20c,
2115		0x3d220, 0x3d220,
2116		0x3d240, 0x3d240,
2117		0x3d600, 0x3d60c,
2118		0x3da00, 0x3da1c,
2119		0x3de00, 0x3de20,
2120		0x3de38, 0x3de3c,
2121		0x3de80, 0x3de80,
2122		0x3de88, 0x3dea8,
2123		0x3deb0, 0x3deb4,
2124		0x3dec8, 0x3ded4,
2125		0x3dfb8, 0x3e004,
2126		0x3e200, 0x3e200,
2127		0x3e208, 0x3e240,
2128		0x3e248, 0x3e280,
2129		0x3e288, 0x3e2c0,
2130		0x3e2c8, 0x3e2fc,
2131		0x3e600, 0x3e630,
2132		0x3ea00, 0x3eabc,
2133		0x3eb00, 0x3eb10,
2134		0x3eb20, 0x3eb30,
2135		0x3eb40, 0x3eb50,
2136		0x3eb60, 0x3eb70,
2137		0x3f000, 0x3f028,
2138		0x3f030, 0x3f048,
2139		0x3f060, 0x3f068,
2140		0x3f070, 0x3f09c,
2141		0x3f0f0, 0x3f128,
2142		0x3f130, 0x3f148,
2143		0x3f160, 0x3f168,
2144		0x3f170, 0x3f19c,
2145		0x3f1f0, 0x3f238,
2146		0x3f240, 0x3f240,
2147		0x3f248, 0x3f250,
2148		0x3f25c, 0x3f264,
2149		0x3f270, 0x3f2b8,
2150		0x3f2c0, 0x3f2e4,
2151		0x3f2f8, 0x3f338,
2152		0x3f340, 0x3f340,
2153		0x3f348, 0x3f350,
2154		0x3f35c, 0x3f364,
2155		0x3f370, 0x3f3b8,
2156		0x3f3c0, 0x3f3e4,
2157		0x3f3f8, 0x3f428,
2158		0x3f430, 0x3f448,
2159		0x3f460, 0x3f468,
2160		0x3f470, 0x3f49c,
2161		0x3f4f0, 0x3f528,
2162		0x3f530, 0x3f548,
2163		0x3f560, 0x3f568,
2164		0x3f570, 0x3f59c,
2165		0x3f5f0, 0x3f638,
2166		0x3f640, 0x3f640,
2167		0x3f648, 0x3f650,
2168		0x3f65c, 0x3f664,
2169		0x3f670, 0x3f6b8,
2170		0x3f6c0, 0x3f6e4,
2171		0x3f6f8, 0x3f738,
2172		0x3f740, 0x3f740,
2173		0x3f748, 0x3f750,
2174		0x3f75c, 0x3f764,
2175		0x3f770, 0x3f7b8,
2176		0x3f7c0, 0x3f7e4,
2177		0x3f7f8, 0x3f7fc,
2178		0x3f814, 0x3f814,
2179		0x3f82c, 0x3f82c,
2180		0x3f880, 0x3f88c,
2181		0x3f8e8, 0x3f8ec,
2182		0x3f900, 0x3f928,
2183		0x3f930, 0x3f948,
2184		0x3f960, 0x3f968,
2185		0x3f970, 0x3f99c,
2186		0x3f9f0, 0x3fa38,
2187		0x3fa40, 0x3fa40,
2188		0x3fa48, 0x3fa50,
2189		0x3fa5c, 0x3fa64,
2190		0x3fa70, 0x3fab8,
2191		0x3fac0, 0x3fae4,
2192		0x3faf8, 0x3fb10,
2193		0x3fb28, 0x3fb28,
2194		0x3fb3c, 0x3fb50,
2195		0x3fbf0, 0x3fc10,
2196		0x3fc28, 0x3fc28,
2197		0x3fc3c, 0x3fc50,
2198		0x3fcf0, 0x3fcfc,
2199		0x40000, 0x4000c,
2200		0x40040, 0x40050,
2201		0x40060, 0x40068,
2202		0x4007c, 0x4008c,
2203		0x40094, 0x400b0,
2204		0x400c0, 0x40144,
2205		0x40180, 0x4018c,
2206		0x40200, 0x40254,
2207		0x40260, 0x40264,
2208		0x40270, 0x40288,
2209		0x40290, 0x40298,
2210		0x402ac, 0x402c8,
2211		0x402d0, 0x402e0,
2212		0x402f0, 0x402f0,
2213		0x40300, 0x4033c,
2214		0x403f8, 0x403fc,
2215		0x41304, 0x413c4,
2216		0x41400, 0x4140c,
2217		0x41414, 0x4141c,
2218		0x41480, 0x414d0,
2219		0x44000, 0x44054,
2220		0x4405c, 0x44078,
2221		0x440c0, 0x44174,
2222		0x44180, 0x441ac,
2223		0x441b4, 0x441b8,
2224		0x441c0, 0x44254,
2225		0x4425c, 0x44278,
2226		0x442c0, 0x44374,
2227		0x44380, 0x443ac,
2228		0x443b4, 0x443b8,
2229		0x443c0, 0x44454,
2230		0x4445c, 0x44478,
2231		0x444c0, 0x44574,
2232		0x44580, 0x445ac,
2233		0x445b4, 0x445b8,
2234		0x445c0, 0x44654,
2235		0x4465c, 0x44678,
2236		0x446c0, 0x44774,
2237		0x44780, 0x447ac,
2238		0x447b4, 0x447b8,
2239		0x447c0, 0x44854,
2240		0x4485c, 0x44878,
2241		0x448c0, 0x44974,
2242		0x44980, 0x449ac,
2243		0x449b4, 0x449b8,
2244		0x449c0, 0x449fc,
2245		0x45000, 0x45004,
2246		0x45010, 0x45030,
2247		0x45040, 0x45060,
2248		0x45068, 0x45068,
2249		0x45080, 0x45084,
2250		0x450a0, 0x450b0,
2251		0x45200, 0x45204,
2252		0x45210, 0x45230,
2253		0x45240, 0x45260,
2254		0x45268, 0x45268,
2255		0x45280, 0x45284,
2256		0x452a0, 0x452b0,
2257		0x460c0, 0x460e4,
2258		0x47000, 0x4703c,
2259		0x47044, 0x4708c,
2260		0x47200, 0x47250,
2261		0x47400, 0x47408,
2262		0x47414, 0x47420,
2263		0x47600, 0x47618,
2264		0x47800, 0x47814,
2265		0x48000, 0x4800c,
2266		0x48040, 0x48050,
2267		0x48060, 0x48068,
2268		0x4807c, 0x4808c,
2269		0x48094, 0x480b0,
2270		0x480c0, 0x48144,
2271		0x48180, 0x4818c,
2272		0x48200, 0x48254,
2273		0x48260, 0x48264,
2274		0x48270, 0x48288,
2275		0x48290, 0x48298,
2276		0x482ac, 0x482c8,
2277		0x482d0, 0x482e0,
2278		0x482f0, 0x482f0,
2279		0x48300, 0x4833c,
2280		0x483f8, 0x483fc,
2281		0x49304, 0x493c4,
2282		0x49400, 0x4940c,
2283		0x49414, 0x4941c,
2284		0x49480, 0x494d0,
2285		0x4c000, 0x4c054,
2286		0x4c05c, 0x4c078,
2287		0x4c0c0, 0x4c174,
2288		0x4c180, 0x4c1ac,
2289		0x4c1b4, 0x4c1b8,
2290		0x4c1c0, 0x4c254,
2291		0x4c25c, 0x4c278,
2292		0x4c2c0, 0x4c374,
2293		0x4c380, 0x4c3ac,
2294		0x4c3b4, 0x4c3b8,
2295		0x4c3c0, 0x4c454,
2296		0x4c45c, 0x4c478,
2297		0x4c4c0, 0x4c574,
2298		0x4c580, 0x4c5ac,
2299		0x4c5b4, 0x4c5b8,
2300		0x4c5c0, 0x4c654,
2301		0x4c65c, 0x4c678,
2302		0x4c6c0, 0x4c774,
2303		0x4c780, 0x4c7ac,
2304		0x4c7b4, 0x4c7b8,
2305		0x4c7c0, 0x4c854,
2306		0x4c85c, 0x4c878,
2307		0x4c8c0, 0x4c974,
2308		0x4c980, 0x4c9ac,
2309		0x4c9b4, 0x4c9b8,
2310		0x4c9c0, 0x4c9fc,
2311		0x4d000, 0x4d004,
2312		0x4d010, 0x4d030,
2313		0x4d040, 0x4d060,
2314		0x4d068, 0x4d068,
2315		0x4d080, 0x4d084,
2316		0x4d0a0, 0x4d0b0,
2317		0x4d200, 0x4d204,
2318		0x4d210, 0x4d230,
2319		0x4d240, 0x4d260,
2320		0x4d268, 0x4d268,
2321		0x4d280, 0x4d284,
2322		0x4d2a0, 0x4d2b0,
2323		0x4e0c0, 0x4e0e4,
2324		0x4f000, 0x4f03c,
2325		0x4f044, 0x4f08c,
2326		0x4f200, 0x4f250,
2327		0x4f400, 0x4f408,
2328		0x4f414, 0x4f420,
2329		0x4f600, 0x4f618,
2330		0x4f800, 0x4f814,
2331		0x50000, 0x50084,
2332		0x50090, 0x500cc,
2333		0x50400, 0x50400,
2334		0x50800, 0x50884,
2335		0x50890, 0x508cc,
2336		0x50c00, 0x50c00,
2337		0x51000, 0x5101c,
2338		0x51300, 0x51308,
2339	};
2340
2341	static const unsigned int t6_reg_ranges[] = {
2342		0x1008, 0x101c,
2343		0x1024, 0x10a8,
2344		0x10b4, 0x10f8,
2345		0x1100, 0x1114,
2346		0x111c, 0x112c,
2347		0x1138, 0x113c,
2348		0x1144, 0x114c,
2349		0x1180, 0x1184,
2350		0x1190, 0x1194,
2351		0x11a0, 0x11a4,
2352		0x11b0, 0x11c4,
2353		0x11fc, 0x1274,
2354		0x1280, 0x133c,
2355		0x1800, 0x18fc,
2356		0x3000, 0x302c,
2357		0x3060, 0x30b0,
2358		0x30b8, 0x30d8,
2359		0x30e0, 0x30fc,
2360		0x3140, 0x357c,
2361		0x35a8, 0x35cc,
2362		0x35ec, 0x35ec,
2363		0x3600, 0x5624,
2364		0x56cc, 0x56ec,
2365		0x56f4, 0x5720,
2366		0x5728, 0x575c,
2367		0x580c, 0x5814,
2368		0x5890, 0x589c,
2369		0x58a4, 0x58ac,
2370		0x58b8, 0x58bc,
2371		0x5940, 0x595c,
2372		0x5980, 0x598c,
2373		0x59b0, 0x59c8,
2374		0x59d0, 0x59dc,
2375		0x59fc, 0x5a18,
2376		0x5a60, 0x5a6c,
2377		0x5a80, 0x5a8c,
2378		0x5a94, 0x5a9c,
2379		0x5b94, 0x5bfc,
2380		0x5c10, 0x5e48,
2381		0x5e50, 0x5e94,
2382		0x5ea0, 0x5eb0,
2383		0x5ec0, 0x5ec0,
2384		0x5ec8, 0x5ed0,
2385		0x5ee0, 0x5ee0,
2386		0x5ef0, 0x5ef0,
2387		0x5f00, 0x5f00,
2388		0x6000, 0x6020,
2389		0x6028, 0x6040,
2390		0x6058, 0x609c,
2391		0x60a8, 0x619c,
2392		0x7700, 0x7798,
2393		0x77c0, 0x7880,
2394		0x78cc, 0x78fc,
2395		0x7b00, 0x7b58,
2396		0x7b60, 0x7b84,
2397		0x7b8c, 0x7c54,
2398		0x7d00, 0x7d38,
2399		0x7d40, 0x7d84,
2400		0x7d8c, 0x7ddc,
2401		0x7de4, 0x7e04,
2402		0x7e10, 0x7e1c,
2403		0x7e24, 0x7e38,
2404		0x7e40, 0x7e44,
2405		0x7e4c, 0x7e78,
2406		0x7e80, 0x7edc,
2407		0x7ee8, 0x7efc,
2408		0x8dc0, 0x8de0,
2409		0x8df8, 0x8e04,
2410		0x8e10, 0x8e84,
2411		0x8ea0, 0x8f88,
2412		0x8fb8, 0x9058,
2413		0x9060, 0x9060,
2414		0x9068, 0x90f8,
2415		0x9100, 0x9124,
2416		0x9400, 0x9470,
2417		0x9600, 0x9600,
2418		0x9608, 0x9638,
2419		0x9640, 0x9704,
2420		0x9710, 0x971c,
2421		0x9800, 0x9808,
2422		0x9820, 0x983c,
2423		0x9850, 0x9864,
2424		0x9c00, 0x9c6c,
2425		0x9c80, 0x9cec,
2426		0x9d00, 0x9d6c,
2427		0x9d80, 0x9dec,
2428		0x9e00, 0x9e6c,
2429		0x9e80, 0x9eec,
2430		0x9f00, 0x9f6c,
2431		0x9f80, 0xa020,
2432		0xd004, 0xd03c,
2433		0xd100, 0xd118,
2434		0xd200, 0xd214,
2435		0xd220, 0xd234,
2436		0xd240, 0xd254,
2437		0xd260, 0xd274,
2438		0xd280, 0xd294,
2439		0xd2a0, 0xd2b4,
2440		0xd2c0, 0xd2d4,
2441		0xd2e0, 0xd2f4,
2442		0xd300, 0xd31c,
2443		0xdfc0, 0xdfe0,
2444		0xe000, 0xf008,
2445		0xf010, 0xf018,
2446		0xf020, 0xf028,
2447		0x11000, 0x11014,
2448		0x11048, 0x1106c,
2449		0x11074, 0x11088,
2450		0x11098, 0x11120,
2451		0x1112c, 0x1117c,
2452		0x11190, 0x112e0,
2453		0x11300, 0x1130c,
2454		0x12000, 0x1206c,
2455		0x19040, 0x1906c,
2456		0x19078, 0x19080,
2457		0x1908c, 0x190e8,
2458		0x190f0, 0x190f8,
2459		0x19100, 0x19110,
2460		0x19120, 0x19124,
2461		0x19150, 0x19194,
2462		0x1919c, 0x191b0,
2463		0x191d0, 0x191e8,
2464		0x19238, 0x19290,
2465		0x192a4, 0x192b0,
2466		0x19348, 0x1934c,
2467		0x193f8, 0x19418,
2468		0x19420, 0x19428,
2469		0x19430, 0x19444,
2470		0x1944c, 0x1946c,
2471		0x19474, 0x19474,
2472		0x19490, 0x194cc,
2473		0x194f0, 0x194f8,
2474		0x19c00, 0x19c48,
2475		0x19c50, 0x19c80,
2476		0x19c94, 0x19c98,
2477		0x19ca0, 0x19cbc,
2478		0x19ce4, 0x19ce4,
2479		0x19cf0, 0x19cf8,
2480		0x19d00, 0x19d28,
2481		0x19d50, 0x19d78,
2482		0x19d94, 0x19d98,
2483		0x19da0, 0x19de0,
2484		0x19df0, 0x19e10,
2485		0x19e50, 0x19e6c,
2486		0x19ea0, 0x19ebc,
2487		0x19ec4, 0x19ef4,
2488		0x19f04, 0x19f2c,
2489		0x19f34, 0x19f34,
2490		0x19f40, 0x19f50,
2491		0x19f90, 0x19fac,
2492		0x19fc4, 0x19fc8,
2493		0x19fd0, 0x19fe4,
2494		0x1a000, 0x1a004,
2495		0x1a010, 0x1a06c,
2496		0x1a0b0, 0x1a0e4,
2497		0x1a0ec, 0x1a0f8,
2498		0x1a100, 0x1a108,
2499		0x1a114, 0x1a120,
2500		0x1a128, 0x1a130,
2501		0x1a138, 0x1a138,
2502		0x1a190, 0x1a1c4,
2503		0x1a1fc, 0x1a1fc,
2504		0x1e008, 0x1e00c,
2505		0x1e040, 0x1e044,
2506		0x1e04c, 0x1e04c,
2507		0x1e284, 0x1e290,
2508		0x1e2c0, 0x1e2c0,
2509		0x1e2e0, 0x1e2e0,
2510		0x1e300, 0x1e384,
2511		0x1e3c0, 0x1e3c8,
2512		0x1e408, 0x1e40c,
2513		0x1e440, 0x1e444,
2514		0x1e44c, 0x1e44c,
2515		0x1e684, 0x1e690,
2516		0x1e6c0, 0x1e6c0,
2517		0x1e6e0, 0x1e6e0,
2518		0x1e700, 0x1e784,
2519		0x1e7c0, 0x1e7c8,
2520		0x1e808, 0x1e80c,
2521		0x1e840, 0x1e844,
2522		0x1e84c, 0x1e84c,
2523		0x1ea84, 0x1ea90,
2524		0x1eac0, 0x1eac0,
2525		0x1eae0, 0x1eae0,
2526		0x1eb00, 0x1eb84,
2527		0x1ebc0, 0x1ebc8,
2528		0x1ec08, 0x1ec0c,
2529		0x1ec40, 0x1ec44,
2530		0x1ec4c, 0x1ec4c,
2531		0x1ee84, 0x1ee90,
2532		0x1eec0, 0x1eec0,
2533		0x1eee0, 0x1eee0,
2534		0x1ef00, 0x1ef84,
2535		0x1efc0, 0x1efc8,
2536		0x1f008, 0x1f00c,
2537		0x1f040, 0x1f044,
2538		0x1f04c, 0x1f04c,
2539		0x1f284, 0x1f290,
2540		0x1f2c0, 0x1f2c0,
2541		0x1f2e0, 0x1f2e0,
2542		0x1f300, 0x1f384,
2543		0x1f3c0, 0x1f3c8,
2544		0x1f408, 0x1f40c,
2545		0x1f440, 0x1f444,
2546		0x1f44c, 0x1f44c,
2547		0x1f684, 0x1f690,
2548		0x1f6c0, 0x1f6c0,
2549		0x1f6e0, 0x1f6e0,
2550		0x1f700, 0x1f784,
2551		0x1f7c0, 0x1f7c8,
2552		0x1f808, 0x1f80c,
2553		0x1f840, 0x1f844,
2554		0x1f84c, 0x1f84c,
2555		0x1fa84, 0x1fa90,
2556		0x1fac0, 0x1fac0,
2557		0x1fae0, 0x1fae0,
2558		0x1fb00, 0x1fb84,
2559		0x1fbc0, 0x1fbc8,
2560		0x1fc08, 0x1fc0c,
2561		0x1fc40, 0x1fc44,
2562		0x1fc4c, 0x1fc4c,
2563		0x1fe84, 0x1fe90,
2564		0x1fec0, 0x1fec0,
2565		0x1fee0, 0x1fee0,
2566		0x1ff00, 0x1ff84,
2567		0x1ffc0, 0x1ffc8,
2568		0x30000, 0x30030,
2569		0x30100, 0x30168,
2570		0x30190, 0x301a0,
2571		0x301a8, 0x301b8,
2572		0x301c4, 0x301c8,
2573		0x301d0, 0x301d0,
2574		0x30200, 0x30320,
2575		0x30400, 0x304b4,
2576		0x304c0, 0x3052c,
2577		0x30540, 0x3061c,
2578		0x30800, 0x308a0,
2579		0x308c0, 0x30908,
2580		0x30910, 0x309b8,
2581		0x30a00, 0x30a04,
2582		0x30a0c, 0x30a14,
2583		0x30a1c, 0x30a2c,
2584		0x30a44, 0x30a50,
2585		0x30a74, 0x30a74,
2586		0x30a7c, 0x30afc,
2587		0x30b08, 0x30c24,
2588		0x30d00, 0x30d14,
2589		0x30d1c, 0x30d3c,
2590		0x30d44, 0x30d4c,
2591		0x30d54, 0x30d74,
2592		0x30d7c, 0x30d7c,
2593		0x30de0, 0x30de0,
2594		0x30e00, 0x30ed4,
2595		0x30f00, 0x30fa4,
2596		0x30fc0, 0x30fc4,
2597		0x31000, 0x31004,
2598		0x31080, 0x310fc,
2599		0x31208, 0x31220,
2600		0x3123c, 0x31254,
2601		0x31300, 0x31300,
2602		0x31308, 0x3131c,
2603		0x31338, 0x3133c,
2604		0x31380, 0x31380,
2605		0x31388, 0x313a8,
2606		0x313b4, 0x313b4,
2607		0x31400, 0x31420,
2608		0x31438, 0x3143c,
2609		0x31480, 0x31480,
2610		0x314a8, 0x314a8,
2611		0x314b0, 0x314b4,
2612		0x314c8, 0x314d4,
2613		0x31a40, 0x31a4c,
2614		0x31af0, 0x31b20,
2615		0x31b38, 0x31b3c,
2616		0x31b80, 0x31b80,
2617		0x31ba8, 0x31ba8,
2618		0x31bb0, 0x31bb4,
2619		0x31bc8, 0x31bd4,
2620		0x32140, 0x3218c,
2621		0x321f0, 0x321f4,
2622		0x32200, 0x32200,
2623		0x32218, 0x32218,
2624		0x32400, 0x32400,
2625		0x32408, 0x3241c,
2626		0x32618, 0x32620,
2627		0x32664, 0x32664,
2628		0x326a8, 0x326a8,
2629		0x326ec, 0x326ec,
2630		0x32a00, 0x32abc,
2631		0x32b00, 0x32b18,
2632		0x32b20, 0x32b38,
2633		0x32b40, 0x32b58,
2634		0x32b60, 0x32b78,
2635		0x32c00, 0x32c00,
2636		0x32c08, 0x32c3c,
2637		0x33000, 0x3302c,
2638		0x33034, 0x33050,
2639		0x33058, 0x33058,
2640		0x33060, 0x3308c,
2641		0x3309c, 0x330ac,
2642		0x330c0, 0x330c0,
2643		0x330c8, 0x330d0,
2644		0x330d8, 0x330e0,
2645		0x330ec, 0x3312c,
2646		0x33134, 0x33150,
2647		0x33158, 0x33158,
2648		0x33160, 0x3318c,
2649		0x3319c, 0x331ac,
2650		0x331c0, 0x331c0,
2651		0x331c8, 0x331d0,
2652		0x331d8, 0x331e0,
2653		0x331ec, 0x33290,
2654		0x33298, 0x332c4,
2655		0x332e4, 0x33390,
2656		0x33398, 0x333c4,
2657		0x333e4, 0x3342c,
2658		0x33434, 0x33450,
2659		0x33458, 0x33458,
2660		0x33460, 0x3348c,
2661		0x3349c, 0x334ac,
2662		0x334c0, 0x334c0,
2663		0x334c8, 0x334d0,
2664		0x334d8, 0x334e0,
2665		0x334ec, 0x3352c,
2666		0x33534, 0x33550,
2667		0x33558, 0x33558,
2668		0x33560, 0x3358c,
2669		0x3359c, 0x335ac,
2670		0x335c0, 0x335c0,
2671		0x335c8, 0x335d0,
2672		0x335d8, 0x335e0,
2673		0x335ec, 0x33690,
2674		0x33698, 0x336c4,
2675		0x336e4, 0x33790,
2676		0x33798, 0x337c4,
2677		0x337e4, 0x337fc,
2678		0x33814, 0x33814,
2679		0x33854, 0x33868,
2680		0x33880, 0x3388c,
2681		0x338c0, 0x338d0,
2682		0x338e8, 0x338ec,
2683		0x33900, 0x3392c,
2684		0x33934, 0x33950,
2685		0x33958, 0x33958,
2686		0x33960, 0x3398c,
2687		0x3399c, 0x339ac,
2688		0x339c0, 0x339c0,
2689		0x339c8, 0x339d0,
2690		0x339d8, 0x339e0,
2691		0x339ec, 0x33a90,
2692		0x33a98, 0x33ac4,
2693		0x33ae4, 0x33b10,
2694		0x33b24, 0x33b28,
2695		0x33b38, 0x33b50,
2696		0x33bf0, 0x33c10,
2697		0x33c24, 0x33c28,
2698		0x33c38, 0x33c50,
2699		0x33cf0, 0x33cfc,
2700		0x34000, 0x34030,
2701		0x34100, 0x34168,
2702		0x34190, 0x341a0,
2703		0x341a8, 0x341b8,
2704		0x341c4, 0x341c8,
2705		0x341d0, 0x341d0,
2706		0x34200, 0x34320,
2707		0x34400, 0x344b4,
2708		0x344c0, 0x3452c,
2709		0x34540, 0x3461c,
2710		0x34800, 0x348a0,
2711		0x348c0, 0x34908,
2712		0x34910, 0x349b8,
2713		0x34a00, 0x34a04,
2714		0x34a0c, 0x34a14,
2715		0x34a1c, 0x34a2c,
2716		0x34a44, 0x34a50,
2717		0x34a74, 0x34a74,
2718		0x34a7c, 0x34afc,
2719		0x34b08, 0x34c24,
2720		0x34d00, 0x34d14,
2721		0x34d1c, 0x34d3c,
2722		0x34d44, 0x34d4c,
2723		0x34d54, 0x34d74,
2724		0x34d7c, 0x34d7c,
2725		0x34de0, 0x34de0,
2726		0x34e00, 0x34ed4,
2727		0x34f00, 0x34fa4,
2728		0x34fc0, 0x34fc4,
2729		0x35000, 0x35004,
2730		0x35080, 0x350fc,
2731		0x35208, 0x35220,
2732		0x3523c, 0x35254,
2733		0x35300, 0x35300,
2734		0x35308, 0x3531c,
2735		0x35338, 0x3533c,
2736		0x35380, 0x35380,
2737		0x35388, 0x353a8,
2738		0x353b4, 0x353b4,
2739		0x35400, 0x35420,
2740		0x35438, 0x3543c,
2741		0x35480, 0x35480,
2742		0x354a8, 0x354a8,
2743		0x354b0, 0x354b4,
2744		0x354c8, 0x354d4,
2745		0x35a40, 0x35a4c,
2746		0x35af0, 0x35b20,
2747		0x35b38, 0x35b3c,
2748		0x35b80, 0x35b80,
2749		0x35ba8, 0x35ba8,
2750		0x35bb0, 0x35bb4,
2751		0x35bc8, 0x35bd4,
2752		0x36140, 0x3618c,
2753		0x361f0, 0x361f4,
2754		0x36200, 0x36200,
2755		0x36218, 0x36218,
2756		0x36400, 0x36400,
2757		0x36408, 0x3641c,
2758		0x36618, 0x36620,
2759		0x36664, 0x36664,
2760		0x366a8, 0x366a8,
2761		0x366ec, 0x366ec,
2762		0x36a00, 0x36abc,
2763		0x36b00, 0x36b18,
2764		0x36b20, 0x36b38,
2765		0x36b40, 0x36b58,
2766		0x36b60, 0x36b78,
2767		0x36c00, 0x36c00,
2768		0x36c08, 0x36c3c,
2769		0x37000, 0x3702c,
2770		0x37034, 0x37050,
2771		0x37058, 0x37058,
2772		0x37060, 0x3708c,
2773		0x3709c, 0x370ac,
2774		0x370c0, 0x370c0,
2775		0x370c8, 0x370d0,
2776		0x370d8, 0x370e0,
2777		0x370ec, 0x3712c,
2778		0x37134, 0x37150,
2779		0x37158, 0x37158,
2780		0x37160, 0x3718c,
2781		0x3719c, 0x371ac,
2782		0x371c0, 0x371c0,
2783		0x371c8, 0x371d0,
2784		0x371d8, 0x371e0,
2785		0x371ec, 0x37290,
2786		0x37298, 0x372c4,
2787		0x372e4, 0x37390,
2788		0x37398, 0x373c4,
2789		0x373e4, 0x3742c,
2790		0x37434, 0x37450,
2791		0x37458, 0x37458,
2792		0x37460, 0x3748c,
2793		0x3749c, 0x374ac,
2794		0x374c0, 0x374c0,
2795		0x374c8, 0x374d0,
2796		0x374d8, 0x374e0,
2797		0x374ec, 0x3752c,
2798		0x37534, 0x37550,
2799		0x37558, 0x37558,
2800		0x37560, 0x3758c,
2801		0x3759c, 0x375ac,
2802		0x375c0, 0x375c0,
2803		0x375c8, 0x375d0,
2804		0x375d8, 0x375e0,
2805		0x375ec, 0x37690,
2806		0x37698, 0x376c4,
2807		0x376e4, 0x37790,
2808		0x37798, 0x377c4,
2809		0x377e4, 0x377fc,
2810		0x37814, 0x37814,
2811		0x37854, 0x37868,
2812		0x37880, 0x3788c,
2813		0x378c0, 0x378d0,
2814		0x378e8, 0x378ec,
2815		0x37900, 0x3792c,
2816		0x37934, 0x37950,
2817		0x37958, 0x37958,
2818		0x37960, 0x3798c,
2819		0x3799c, 0x379ac,
2820		0x379c0, 0x379c0,
2821		0x379c8, 0x379d0,
2822		0x379d8, 0x379e0,
2823		0x379ec, 0x37a90,
2824		0x37a98, 0x37ac4,
2825		0x37ae4, 0x37b10,
2826		0x37b24, 0x37b28,
2827		0x37b38, 0x37b50,
2828		0x37bf0, 0x37c10,
2829		0x37c24, 0x37c28,
2830		0x37c38, 0x37c50,
2831		0x37cf0, 0x37cfc,
2832		0x40040, 0x40040,
2833		0x40080, 0x40084,
2834		0x40100, 0x40100,
2835		0x40140, 0x401bc,
2836		0x40200, 0x40214,
2837		0x40228, 0x40228,
2838		0x40240, 0x40258,
2839		0x40280, 0x40280,
2840		0x40304, 0x40304,
2841		0x40330, 0x4033c,
2842		0x41304, 0x413c8,
2843		0x413d0, 0x413dc,
2844		0x413f0, 0x413f0,
2845		0x41400, 0x4140c,
2846		0x41414, 0x4141c,
2847		0x41480, 0x414d0,
2848		0x44000, 0x4407c,
2849		0x440c0, 0x441ac,
2850		0x441b4, 0x4427c,
2851		0x442c0, 0x443ac,
2852		0x443b4, 0x4447c,
2853		0x444c0, 0x445ac,
2854		0x445b4, 0x4467c,
2855		0x446c0, 0x447ac,
2856		0x447b4, 0x4487c,
2857		0x448c0, 0x449ac,
2858		0x449b4, 0x44a7c,
2859		0x44ac0, 0x44bac,
2860		0x44bb4, 0x44c7c,
2861		0x44cc0, 0x44dac,
2862		0x44db4, 0x44e7c,
2863		0x44ec0, 0x44fac,
2864		0x44fb4, 0x4507c,
2865		0x450c0, 0x451ac,
2866		0x451b4, 0x451fc,
2867		0x45800, 0x45804,
2868		0x45810, 0x45830,
2869		0x45840, 0x45860,
2870		0x45868, 0x45868,
2871		0x45880, 0x45884,
2872		0x458a0, 0x458b0,
2873		0x45a00, 0x45a04,
2874		0x45a10, 0x45a30,
2875		0x45a40, 0x45a60,
2876		0x45a68, 0x45a68,
2877		0x45a80, 0x45a84,
2878		0x45aa0, 0x45ab0,
2879		0x460c0, 0x460e4,
2880		0x47000, 0x4703c,
2881		0x47044, 0x4708c,
2882		0x47200, 0x47250,
2883		0x47400, 0x47408,
2884		0x47414, 0x47420,
2885		0x47600, 0x47618,
2886		0x47800, 0x47814,
2887		0x47820, 0x4782c,
2888		0x50000, 0x50084,
2889		0x50090, 0x500cc,
2890		0x50300, 0x50384,
2891		0x50400, 0x50400,
2892		0x50800, 0x50884,
2893		0x50890, 0x508cc,
2894		0x50b00, 0x50b84,
2895		0x50c00, 0x50c00,
2896		0x51000, 0x51020,
2897		0x51028, 0x510b0,
2898		0x51300, 0x51324,
2899	};
2900
2901	u32 *buf_end = (u32 *)((char *)buf + buf_size);
2902	const unsigned int *reg_ranges;
2903	int reg_ranges_size, range;
2904	unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
2905
2906	/* Select the right set of register ranges to dump depending on the
2907	 * adapter chip type.
2908	 */
2909	switch (chip_version) {
2910	case CHELSIO_T4:
2911		reg_ranges = t4_reg_ranges;
2912		reg_ranges_size = ARRAY_SIZE(t4_reg_ranges);
2913		break;
2914
2915	case CHELSIO_T5:
2916		reg_ranges = t5_reg_ranges;
2917		reg_ranges_size = ARRAY_SIZE(t5_reg_ranges);
2918		break;
2919
2920	case CHELSIO_T6:
2921		reg_ranges = t6_reg_ranges;
2922		reg_ranges_size = ARRAY_SIZE(t6_reg_ranges);
2923		break;
2924
2925	default:
2926		CH_ERR(adap,
2927			"Unsupported chip version %d\n", chip_version);
2928		return;
2929	}
2930
2931	/* Clear the register buffer and insert the appropriate register
2932	 * values selected by the above register ranges.
2933	 */
2934	memset(buf, 0, buf_size);
2935	for (range = 0; range < reg_ranges_size; range += 2) {
2936		unsigned int reg = reg_ranges[range];
2937		unsigned int last_reg = reg_ranges[range + 1];
2938		u32 *bufp = (u32 *)((char *)buf + reg);
2939
2940		/* Iterate across the register range filling in the register
2941		 * buffer but don't write past the end of the register buffer.
2942		 */
2943		while (reg <= last_reg && bufp < buf_end) {
2944			*bufp++ = t4_read_reg(adap, reg);
2945			reg += sizeof(u32);
2946		}
2947	}
2948}
2949
2950/*
2951 * EEPROM reads take a few tens of us while writes can take a bit over 5 ms.
2952 */
2953#define EEPROM_DELAY		10		// 10us per poll spin
2954#define EEPROM_MAX_POLL		5000		// x 5000 == 50ms
2955
2956#define EEPROM_STAT_ADDR	0x7bfc
2957#define VPD_SIZE		0x800
2958#define VPD_BASE		0x400
2959#define VPD_BASE_OLD		0
2960#define VPD_LEN			1024
2961#define VPD_INFO_FLD_HDR_SIZE	3
2962#define CHELSIO_VPD_UNIQUE_ID	0x82
2963
2964/*
2965 * Small utility function to wait till any outstanding VPD Access is complete.
2966 * We have a per-adapter state variable "VPD Busy" to indicate when we have a
2967 * VPD Access in flight.  This allows us to handle the problem of having a
2968 * previous VPD Access time out and prevent an attempt to inject a new VPD
2969 * Request before any in-flight VPD reguest has completed.
2970 */
2971static int t4_seeprom_wait(struct adapter *adapter)
2972{
2973	unsigned int base = adapter->params.pci.vpd_cap_addr;
2974	int max_poll;
2975
2976	/*
2977	 * If no VPD Access is in flight, we can just return success right
2978	 * away.
2979	 */
2980	if (!adapter->vpd_busy)
2981		return 0;
2982
2983	/*
2984	 * Poll the VPD Capability Address/Flag register waiting for it
2985	 * to indicate that the operation is complete.
2986	 */
2987	max_poll = EEPROM_MAX_POLL;
2988	do {
2989		u16 val;
2990
2991		udelay(EEPROM_DELAY);
2992		t4_os_pci_read_cfg2(adapter, base + PCI_VPD_ADDR, &val);
2993
2994		/*
2995		 * If the operation is complete, mark the VPD as no longer
2996		 * busy and return success.
2997		 */
2998		if ((val & PCI_VPD_ADDR_F) == adapter->vpd_flag) {
2999			adapter->vpd_busy = 0;
3000			return 0;
3001		}
3002	} while (--max_poll);
3003
3004	/*
3005	 * Failure!  Note that we leave the VPD Busy status set in order to
3006	 * avoid pushing a new VPD Access request into the VPD Capability till
3007	 * the current operation eventually succeeds.  It's a bug to issue a
3008	 * new request when an existing request is in flight and will result
3009	 * in corrupt hardware state.
3010	 */
3011	return -ETIMEDOUT;
3012}
3013
3014/**
3015 *	t4_seeprom_read - read a serial EEPROM location
3016 *	@adapter: adapter to read
3017 *	@addr: EEPROM virtual address
3018 *	@data: where to store the read data
3019 *
3020 *	Read a 32-bit word from a location in serial EEPROM using the card's PCI
3021 *	VPD capability.  Note that this function must be called with a virtual
3022 *	address.
3023 */
3024int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data)
3025{
3026	unsigned int base = adapter->params.pci.vpd_cap_addr;
3027	int ret;
3028
3029	/*
3030	 * VPD Accesses must alway be 4-byte aligned!
3031	 */
3032	if (addr >= EEPROMVSIZE || (addr & 3))
3033		return -EINVAL;
3034
3035	/*
3036	 * Wait for any previous operation which may still be in flight to
3037	 * complete.
3038	 */
3039	ret = t4_seeprom_wait(adapter);
3040	if (ret) {
3041		CH_ERR(adapter, "VPD still busy from previous operation\n");
3042		return ret;
3043	}
3044
3045	/*
3046	 * Issue our new VPD Read request, mark the VPD as being busy and wait
3047	 * for our request to complete.  If it doesn't complete, note the
3048	 * error and return it to our caller.  Note that we do not reset the
3049	 * VPD Busy status!
3050	 */
3051	t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR, (u16)addr);
3052	adapter->vpd_busy = 1;
3053	adapter->vpd_flag = PCI_VPD_ADDR_F;
3054	ret = t4_seeprom_wait(adapter);
3055	if (ret) {
3056		CH_ERR(adapter, "VPD read of address %#x failed\n", addr);
3057		return ret;
3058	}
3059
3060	/*
3061	 * Grab the returned data, swizzle it into our endianess and
3062	 * return success.
3063	 */
3064	t4_os_pci_read_cfg4(adapter, base + PCI_VPD_DATA, data);
3065	*data = le32_to_cpu(*data);
3066	return 0;
3067}
3068
3069/**
3070 *	t4_seeprom_write - write a serial EEPROM location
3071 *	@adapter: adapter to write
3072 *	@addr: virtual EEPROM address
3073 *	@data: value to write
3074 *
3075 *	Write a 32-bit word to a location in serial EEPROM using the card's PCI
3076 *	VPD capability.  Note that this function must be called with a virtual
3077 *	address.
3078 */
3079int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data)
3080{
3081	unsigned int base = adapter->params.pci.vpd_cap_addr;
3082	int ret;
3083	u32 stats_reg;
3084	int max_poll;
3085
3086	/*
3087	 * VPD Accesses must alway be 4-byte aligned!
3088	 */
3089	if (addr >= EEPROMVSIZE || (addr & 3))
3090		return -EINVAL;
3091
3092	/*
3093	 * Wait for any previous operation which may still be in flight to
3094	 * complete.
3095	 */
3096	ret = t4_seeprom_wait(adapter);
3097	if (ret) {
3098		CH_ERR(adapter, "VPD still busy from previous operation\n");
3099		return ret;
3100	}
3101
3102	/*
3103	 * Issue our new VPD Read request, mark the VPD as being busy and wait
3104	 * for our request to complete.  If it doesn't complete, note the
3105	 * error and return it to our caller.  Note that we do not reset the
3106	 * VPD Busy status!
3107	 */
3108	t4_os_pci_write_cfg4(adapter, base + PCI_VPD_DATA,
3109				 cpu_to_le32(data));
3110	t4_os_pci_write_cfg2(adapter, base + PCI_VPD_ADDR,
3111				 (u16)addr | PCI_VPD_ADDR_F);
3112	adapter->vpd_busy = 1;
3113	adapter->vpd_flag = 0;
3114	ret = t4_seeprom_wait(adapter);
3115	if (ret) {
3116		CH_ERR(adapter, "VPD write of address %#x failed\n", addr);
3117		return ret;
3118	}
3119
3120	/*
3121	 * Reset PCI_VPD_DATA register after a transaction and wait for our
3122	 * request to complete. If it doesn't complete, return error.
3123	 */
3124	t4_os_pci_write_cfg4(adapter, base + PCI_VPD_DATA, 0);
3125	max_poll = EEPROM_MAX_POLL;
3126	do {
3127		udelay(EEPROM_DELAY);
3128		t4_seeprom_read(adapter, EEPROM_STAT_ADDR, &stats_reg);
3129	} while ((stats_reg & 0x1) && --max_poll);
3130	if (!max_poll)
3131		return -ETIMEDOUT;
3132
3133	/* Return success! */
3134	return 0;
3135}
3136
3137/**
3138 *	t4_eeprom_ptov - translate a physical EEPROM address to virtual
3139 *	@phys_addr: the physical EEPROM address
3140 *	@fn: the PCI function number
3141 *	@sz: size of function-specific area
3142 *
3143 *	Translate a physical EEPROM address to virtual.  The first 1K is
3144 *	accessed through virtual addresses starting at 31K, the rest is
3145 *	accessed through virtual addresses starting at 0.
3146 *
3147 *	The mapping is as follows:
3148 *	[0..1K) -> [31K..32K)
3149 *	[1K..1K+A) -> [ES-A..ES)
3150 *	[1K+A..ES) -> [0..ES-A-1K)
3151 *
3152 *	where A = @fn * @sz, and ES = EEPROM size.
3153 */
3154int t4_eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
3155{
3156	fn *= sz;
3157	if (phys_addr < 1024)
3158		return phys_addr + (31 << 10);
3159	if (phys_addr < 1024 + fn)
3160		return EEPROMSIZE - fn + phys_addr - 1024;
3161	if (phys_addr < EEPROMSIZE)
3162		return phys_addr - 1024 - fn;
3163	return -EINVAL;
3164}
3165
3166/**
3167 *	t4_seeprom_wp - enable/disable EEPROM write protection
3168 *	@adapter: the adapter
3169 *	@enable: whether to enable or disable write protection
3170 *
3171 *	Enables or disables write protection on the serial EEPROM.
3172 */
3173int t4_seeprom_wp(struct adapter *adapter, int enable)
3174{
3175	return t4_os_pci_write_seeprom(adapter, EEPROM_STAT_ADDR, enable ? 0xc : 0);
3176}
3177
3178/**
3179 *	get_vpd_keyword_val - Locates an information field keyword in the VPD
3180 *	@v: Pointer to buffered vpd data structure
3181 *	@kw: The keyword to search for
3182 *
3183 *	Returns the value of the information field keyword or
3184 *	-ENOENT otherwise.
3185 */
3186int get_vpd_keyword_val(const struct t4_vpd_hdr *v, const char *kw)
3187{
3188	int i;
3189	unsigned int offset , len;
3190	const u8 *buf = (const u8 *)v;
3191	const u8 *vpdr_len = &v->vpdr_len[0];
3192	offset = sizeof(struct t4_vpd_hdr);
3193	len =  (u16)vpdr_len[0] + ((u16)vpdr_len[1] << 8);
3194
3195	if (len + sizeof(struct t4_vpd_hdr) > VPD_LEN) {
3196		return -ENOENT;
3197	}
3198
3199	for (i = offset; i + VPD_INFO_FLD_HDR_SIZE <= offset + len;) {
3200		if(memcmp(buf + i , kw , 2) == 0){
3201			i += VPD_INFO_FLD_HDR_SIZE;
3202			return i;
3203		}
3204
3205		i += VPD_INFO_FLD_HDR_SIZE + buf[i+2];
3206	}
3207
3208	return -ENOENT;
3209}
3210
3211/*
3212 * str_strip
3213 * Removes trailing whitespaces from string "s"
3214 * Based on strstrip() implementation in string.c
3215 */
3216static void str_strip(char *s)
3217{
3218	size_t size;
3219	char *end;
3220
3221	size = strlen(s);
3222	if (!size)
3223		return;
3224
3225	end = s + size - 1;
3226	while (end >= s && isspace(*end))
3227		end--;
3228	*(end + 1) = '\0';
3229}
3230
3231/**
3232 *	t4_get_raw_vpd_params - read VPD parameters from VPD EEPROM
3233 *	@adapter: adapter to read
3234 *	@p: where to store the parameters
3235 *
3236 *	Reads card parameters stored in VPD EEPROM.
3237 */
3238int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p)
3239{
3240	int i, ret = 0, addr;
3241	int ec, sn, pn, na;
3242	u8 *vpd, csum;
3243	const struct t4_vpd_hdr *v;
3244
3245	vpd = (u8 *)t4_os_alloc(sizeof(u8) * VPD_LEN);
3246	if (!vpd)
3247		return -ENOMEM;
3248
3249	/* We have two VPD data structures stored in the adapter VPD area.
3250	 * By default, Linux calculates the size of the VPD area by traversing
3251	 * the first VPD area at offset 0x0, so we need to tell the OS what
3252	 * our real VPD size is.
3253	 */
3254	ret = t4_os_pci_set_vpd_size(adapter, VPD_SIZE);
3255	if (ret < 0)
3256		goto out;
3257
3258	/* Card information normally starts at VPD_BASE but early cards had
3259	 * it at 0.
3260	 */
3261	ret = t4_os_pci_read_seeprom(adapter, VPD_BASE, (u32 *)(vpd));
3262	if (ret)
3263		goto out;
3264
3265	/* The VPD shall have a unique identifier specified by the PCI SIG.
3266	 * For chelsio adapters, the identifier is 0x82. The first byte of a VPD
3267	 * shall be CHELSIO_VPD_UNIQUE_ID (0x82). The VPD programming software
3268	 * is expected to automatically put this entry at the
3269	 * beginning of the VPD.
3270	 */
3271	addr = *vpd == CHELSIO_VPD_UNIQUE_ID ? VPD_BASE : VPD_BASE_OLD;
3272
3273	for (i = 0; i < VPD_LEN; i += 4) {
3274		ret = t4_os_pci_read_seeprom(adapter, addr+i, (u32 *)(vpd+i));
3275		if (ret)
3276			goto out;
3277	}
3278 	v = (const struct t4_vpd_hdr *)vpd;
3279
3280#define FIND_VPD_KW(var,name) do { \
3281	var = get_vpd_keyword_val(v , name); \
3282	if (var < 0) { \
3283		CH_ERR(adapter, "missing VPD keyword " name "\n"); \
3284		ret = -EINVAL; \
3285		goto out;      \
3286	} \
3287} while (0)
3288
3289	FIND_VPD_KW(i, "RV");
3290	for (csum = 0; i >= 0; i--)
3291		csum += vpd[i];
3292
3293	if (csum) {
3294		CH_ERR(adapter,
3295			"corrupted VPD EEPROM, actual csum %u\n", csum);
3296		ret = -EINVAL;
3297		goto out;
3298	}
3299
3300	FIND_VPD_KW(ec, "EC");
3301	FIND_VPD_KW(sn, "SN");
3302	FIND_VPD_KW(pn, "PN");
3303	FIND_VPD_KW(na, "NA");
3304#undef FIND_VPD_KW
3305
3306	memcpy(p->id, v->id_data, ID_LEN);
3307	str_strip((char *)p->id);
3308	memcpy(p->ec, vpd + ec, EC_LEN);
3309	str_strip((char *)p->ec);
3310	i = vpd[sn - VPD_INFO_FLD_HDR_SIZE + 2];
3311	memcpy(p->sn, vpd + sn, min(i, SERNUM_LEN));
3312	str_strip((char *)p->sn);
3313	i = vpd[pn - VPD_INFO_FLD_HDR_SIZE + 2];
3314	memcpy(p->pn, vpd + pn, min(i, PN_LEN));
3315	str_strip((char *)p->pn);
3316	i = vpd[na - VPD_INFO_FLD_HDR_SIZE + 2];
3317	memcpy(p->na, vpd + na, min(i, MACADDR_LEN));
3318	str_strip((char *)p->na);
3319
3320out:
3321	kmem_free(vpd, sizeof(u8) * VPD_LEN);
3322	return ret < 0 ? ret : 0;
3323}
3324
3325/**
3326 *	t4_get_vpd_params - read VPD parameters & retrieve Core Clock
3327 *	@adapter: adapter to read
3328 *	@p: where to store the parameters
3329 *
3330 *	Reads card parameters stored in VPD EEPROM and retrieves the Core
3331 *	Clock.  This can only be called after a connection to the firmware
3332 *	is established.
3333 */
3334int t4_get_vpd_params(struct adapter *adapter, struct vpd_params *p)
3335{
3336	u32 cclk_param, cclk_val;
3337	int ret;
3338
3339	/*
3340	 * Grab the raw VPD parameters.
3341	 */
3342	ret = t4_get_raw_vpd_params(adapter, p);
3343	if (ret)
3344		return ret;
3345
3346	/*
3347	 * Ask firmware for the Core Clock since it knows how to translate the
3348	 * Reference Clock ('V2') VPD field into a Core Clock value ...
3349	 */
3350	cclk_param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3351		      V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CCLK));
3352	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3353			      1, &cclk_param, &cclk_val);
3354
3355	if (ret)
3356		return ret;
3357	p->cclk = cclk_val;
3358
3359	return 0;
3360}
3361
3362/**
3363 *	t4_get_pfres - retrieve VF resource limits
3364 *	@adapter: the adapter
3365 *
3366 *	Retrieves configured resource limits and capabilities for a physical
3367 *	function.  The results are stored in @adapter->pfres.
3368 */
3369int t4_get_pfres(struct adapter *adapter)
3370{
3371	struct pf_resources *pfres = &adapter->params.pfres;
3372	struct fw_pfvf_cmd cmd, rpl;
3373	int v;
3374	u32 word;
3375
3376	/*
3377	 * Execute PFVF Read command to get VF resource limits; bail out early
3378	 * with error on command failure.
3379	 */
3380	memset(&cmd, 0, sizeof(cmd));
3381	cmd.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_PFVF_CMD) |
3382				    F_FW_CMD_REQUEST |
3383				    F_FW_CMD_READ |
3384				    V_FW_PFVF_CMD_PFN(adapter->pf) |
3385				    V_FW_PFVF_CMD_VFN(0));
3386	cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
3387	v = t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &rpl);
3388	if (v != FW_SUCCESS)
3389		return v;
3390
3391	/*
3392	 * Extract PF resource limits and return success.
3393	 */
3394	word = be32_to_cpu(rpl.niqflint_niq);
3395	pfres->niqflint = G_FW_PFVF_CMD_NIQFLINT(word);
3396	pfres->niq = G_FW_PFVF_CMD_NIQ(word);
3397
3398	word = be32_to_cpu(rpl.type_to_neq);
3399	pfres->neq = G_FW_PFVF_CMD_NEQ(word);
3400	pfres->pmask = G_FW_PFVF_CMD_PMASK(word);
3401
3402	word = be32_to_cpu(rpl.tc_to_nexactf);
3403	pfres->tc = G_FW_PFVF_CMD_TC(word);
3404	pfres->nvi = G_FW_PFVF_CMD_NVI(word);
3405	pfres->nexactf = G_FW_PFVF_CMD_NEXACTF(word);
3406
3407	word = be32_to_cpu(rpl.r_caps_to_nethctrl);
3408	pfres->r_caps = G_FW_PFVF_CMD_R_CAPS(word);
3409	pfres->wx_caps = G_FW_PFVF_CMD_WX_CAPS(word);
3410	pfres->nethctrl = G_FW_PFVF_CMD_NETHCTRL(word);
3411
3412	return 0;
3413}
3414
3415/* serial flash and firmware constants and flash config file constants */
3416enum {
3417	SF_ATTEMPTS = 10,	/* max retries for SF operations */
3418
3419	/* flash command opcodes */
3420	SF_PROG_PAGE    = 2,	/* program page */
3421	SF_WR_DISABLE   = 4,	/* disable writes */
3422	SF_RD_STATUS    = 5,	/* read status register */
3423	SF_WR_ENABLE    = 6,	/* enable writes */
3424	SF_RD_DATA_FAST = 0xb,	/* read flash */
3425	SF_RD_ID	= 0x9f,	/* read ID */
3426	SF_ERASE_SECTOR = 0xd8,	/* erase sector */
3427};
3428
3429/**
3430 *	sf1_read - read data from the serial flash
3431 *	@adapter: the adapter
3432 *	@byte_cnt: number of bytes to read
3433 *	@cont: whether another operation will be chained
3434 *	@lock: whether to lock SF for PL access only
3435 *	@valp: where to store the read data
3436 *
3437 *	Reads up to 4 bytes of data from the serial flash.  The location of
3438 *	the read needs to be specified prior to calling this by issuing the
3439 *	appropriate commands to the serial flash.
3440 */
3441static int sf1_read(struct adapter *adapter, unsigned int byte_cnt, int cont,
3442		    int lock, u32 *valp)
3443{
3444	int ret;
3445
3446	if (!byte_cnt || byte_cnt > 4)
3447		return -EINVAL;
3448	if (t4_read_reg(adapter, A_SF_OP) & F_BUSY)
3449		return -EBUSY;
3450	t4_write_reg(adapter, A_SF_OP,
3451		     V_SF_LOCK(lock) | V_CONT(cont) | V_BYTECNT(byte_cnt - 1));
3452	ret = t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5);
3453	if (!ret)
3454		*valp = t4_read_reg(adapter, A_SF_DATA);
3455	return ret;
3456}
3457
3458/**
3459 *	sf1_write - write data to the serial flash
3460 *	@adapter: the adapter
3461 *	@byte_cnt: number of bytes to write
3462 *	@cont: whether another operation will be chained
3463 *	@lock: whether to lock SF for PL access only
3464 *	@val: value to write
3465 *
3466 *	Writes up to 4 bytes of data to the serial flash.  The location of
3467 *	the write needs to be specified prior to calling this by issuing the
3468 *	appropriate commands to the serial flash.
3469 */
3470static int sf1_write(struct adapter *adapter, unsigned int byte_cnt, int cont,
3471		     int lock, u32 val)
3472{
3473	if (!byte_cnt || byte_cnt > 4)
3474		return -EINVAL;
3475	if (t4_read_reg(adapter, A_SF_OP) & F_BUSY)
3476		return -EBUSY;
3477	t4_write_reg(adapter, A_SF_DATA, val);
3478	t4_write_reg(adapter, A_SF_OP, V_SF_LOCK(lock) |
3479		     V_CONT(cont) | V_BYTECNT(byte_cnt - 1) | V_OP(1));
3480	return t4_wait_op_done(adapter, A_SF_OP, F_BUSY, 0, SF_ATTEMPTS, 5);
3481}
3482
3483/**
3484 *	flash_wait_op - wait for a flash operation to complete
3485 *	@adapter: the adapter
3486 *	@attempts: max number of polls of the status register
3487 *	@delay: delay between polls in ms
3488 *
3489 *	Wait for a flash operation to complete by polling the status register.
3490 */
3491static int flash_wait_op(struct adapter *adapter, int attempts, int ch_delay)
3492{
3493	int ret;
3494	u32 status;
3495
3496	while (1) {
3497		if ((ret = sf1_write(adapter, 1, 1, 1, SF_RD_STATUS)) != 0 ||
3498		    (ret = sf1_read(adapter, 1, 0, 1, &status)) != 0)
3499			return ret;
3500		if (!(status & 1))
3501			return 0;
3502		if (--attempts == 0)
3503			return -EAGAIN;
3504		if (ch_delay) {
3505#ifdef CONFIG_CUDBG
3506			if (adapter->flags & K_CRASH)
3507				mdelay(ch_delay);
3508			else
3509#endif
3510				msleep(ch_delay);
3511		}
3512	}
3513}
3514
3515/**
3516 *	t4_read_flash - read words from serial flash
3517 *	@adapter: the adapter
3518 *	@addr: the start address for the read
3519 *	@nwords: how many 32-bit words to read
3520 *	@data: where to store the read data
3521 *	@byte_oriented: whether to store data as bytes or as words
3522 *
3523 *	Read the specified number of 32-bit words from the serial flash.
3524 *	If @byte_oriented is set the read data is stored as a byte array
3525 *	(i.e., big-endian), otherwise as 32-bit words in the platform's
3526 *	natural endianness.
3527 */
3528int t4_read_flash(struct adapter *adapter, unsigned int addr,
3529		  unsigned int nwords, u32 *data, int byte_oriented)
3530{
3531	int ret;
3532
3533	if (addr + nwords * sizeof(u32) > adapter->params.sf_size || (addr & 3))
3534		return -EINVAL;
3535
3536	addr = swab32(addr) | SF_RD_DATA_FAST;
3537
3538	if ((ret = sf1_write(adapter, 4, 1, 0, addr)) != 0 ||
3539	    (ret = sf1_read(adapter, 1, 1, 0, data)) != 0)
3540		return ret;
3541
3542	for ( ; nwords; nwords--, data++) {
3543		ret = sf1_read(adapter, 4, nwords > 1, nwords == 1, data);
3544		if (nwords == 1)
3545			t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
3546		if (ret)
3547			return ret;
3548		if (byte_oriented)
3549			*data = (__force __u32)(cpu_to_be32(*data));
3550	}
3551	return 0;
3552}
3553
3554/**
3555 *	t4_write_flash - write up to a page of data to the serial flash
3556 *	@adapter: the adapter
3557 *	@addr: the start address to write
3558 *	@n: length of data to write in bytes
3559 *	@data: the data to write
3560 *	@byte_oriented: whether to store data as bytes or as words
3561 *
3562 *	Writes up to a page of data (256 bytes) to the serial flash starting
3563 *	at the given address.  All the data must be written to the same page.
3564 *	If @byte_oriented is set the write data is stored as byte stream
3565 *	(i.e. matches what on disk), otherwise in big-endian.
3566 */
3567int t4_write_flash(struct adapter *adapter, unsigned int addr,
3568			  unsigned int n, const u8 *data, int byte_oriented)
3569{
3570	int ret;
3571	u32 buf[64];
3572	unsigned int i, c, left, val, offset = addr & 0xff;
3573
3574	if (addr >= adapter->params.sf_size || offset + n > SF_PAGE_SIZE)
3575		return -EINVAL;
3576
3577	val = swab32(addr) | SF_PROG_PAGE;
3578
3579	if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
3580	    (ret = sf1_write(adapter, 4, 1, 1, val)) != 0)
3581		goto unlock;
3582
3583	for (left = n; left; left -= c) {
3584		c = min(left, 4U);
3585		for (val = 0, i = 0; i < c; ++i)
3586			val = (val << 8) + *data++;
3587
3588		if (!byte_oriented)
3589			val = cpu_to_be32(val);
3590
3591		ret = sf1_write(adapter, c, c != left, 1, val);
3592		if (ret)
3593			goto unlock;
3594	}
3595	ret = flash_wait_op(adapter, 8, 1);
3596	if (ret)
3597		goto unlock;
3598
3599	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
3600
3601	/* Read the page to verify the write succeeded */
3602	ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf,
3603			    byte_oriented);
3604	if (ret)
3605		return ret;
3606
3607	if (memcmp(data - n, (u8 *)buf + offset, n)) {
3608		CH_ERR(adapter,
3609			"failed to correctly write the flash page at %#x\n",
3610			addr);
3611		return -EIO;
3612	}
3613	return 0;
3614
3615unlock:
3616	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
3617	return ret;
3618}
3619
3620/**
3621 *	t4_get_fw_version - read the firmware version
3622 *	@adapter: the adapter
3623 *	@vers: where to place the version
3624 *
3625 *	Reads the FW version from flash.
3626 */
3627int t4_get_fw_version(struct adapter *adapter, u32 *vers)
3628{
3629	return t4_read_flash(adapter, FLASH_FW_START +
3630			     offsetof(struct fw_hdr, fw_ver), 1,
3631			     vers, 0);
3632}
3633
3634/**
3635 *	t4_get_bs_version - read the firmware bootstrap version
3636 *	@adapter: the adapter
3637 *	@vers: where to place the version
3638 *
3639 *	Reads the FW Bootstrap version from flash.
3640 */
3641int t4_get_bs_version(struct adapter *adapter, u32 *vers)
3642{
3643	return t4_read_flash(adapter, FLASH_FWBOOTSTRAP_START +
3644			     offsetof(struct fw_hdr, fw_ver), 1,
3645			     vers, 0);
3646}
3647
3648/**
3649 *	t4_get_tp_version - read the TP microcode version
3650 *	@adapter: the adapter
3651 *	@vers: where to place the version
3652 *
3653 *	Reads the TP microcode version from flash.
3654 */
3655int t4_get_tp_version(struct adapter *adapter, u32 *vers)
3656{
3657	return t4_read_flash(adapter, FLASH_FW_START +
3658			     offsetof(struct fw_hdr, tp_microcode_ver),
3659			     1, vers, 0);
3660}
3661
3662/**
3663 *	t4_get_exprom_version - return the Expansion ROM version (if any)
3664 *	@adapter: the adapter
3665 *	@vers: where to place the version
3666 *
3667 *	Reads the Expansion ROM header from FLASH and returns the version
3668 *	number (if present) through the @vers return value pointer.  We return
3669 *	this in the Firmware Version Format since it's convenient.  Return
3670 *	0 on success, -ENOENT if no Expansion ROM is present.
3671 */
3672int t4_get_exprom_version(struct adapter *adapter, u32 *vers)
3673{
3674	struct exprom_header {
3675		unsigned char hdr_arr[16];	/* must start with 0x55aa */
3676		unsigned char hdr_ver[4];	/* Expansion ROM version */
3677	} *hdr;
3678	u32 exprom_header_buf[DIV_ROUND_UP(sizeof(struct exprom_header),
3679					   sizeof(u32))];
3680	int ret;
3681
3682	ret = t4_read_flash(adapter, FLASH_EXP_ROM_START,
3683			    ARRAY_SIZE(exprom_header_buf), exprom_header_buf,
3684			    0);
3685	if (ret)
3686		return ret;
3687
3688	hdr = (struct exprom_header *)exprom_header_buf;
3689	if (hdr->hdr_arr[0] != 0x55 || hdr->hdr_arr[1] != 0xaa)
3690		return -ENOENT;
3691
3692	*vers = (V_FW_HDR_FW_VER_MAJOR(hdr->hdr_ver[0]) |
3693		 V_FW_HDR_FW_VER_MINOR(hdr->hdr_ver[1]) |
3694		 V_FW_HDR_FW_VER_MICRO(hdr->hdr_ver[2]) |
3695		 V_FW_HDR_FW_VER_BUILD(hdr->hdr_ver[3]));
3696	return 0;
3697}
3698
3699/**
3700 *	t4_get_scfg_version - return the Serial Configuration version
3701 *	@adapter: the adapter
3702 *	@vers: where to place the version
3703 *
3704 *	Reads the Serial Configuration Version via the Firmware interface
3705 *	(thus this can only be called once we're ready to issue Firmware
3706 *	commands).  The format of the Serial Configuration version is
3707 *	adapter specific.  Returns 0 on success, an error on failure.
3708 *
3709 *	Note that early versions of the Firmware didn't include the ability
3710 *	to retrieve the Serial Configuration version, so we zero-out the
3711 *	return-value parameter in that case to avoid leaving it with
3712 *	garbage in it.
3713 *
3714 *	Also note that the Firmware will return its cached copy of the Serial
3715 *	Initialization Revision ID, not the actual Revision ID as written in
3716 *	the Serial EEPROM.  This is only an issue if a new VPD has been written
3717 *	and the Firmware/Chip haven't yet gone through a RESET sequence.  So
3718 *	it's best to defer calling this routine till after a FW_RESET_CMD has
3719 *	been issued if the Host Driver will be performing a full adapter
3720 *	initialization.
3721 */
3722int t4_get_scfg_version(struct adapter *adapter, u32 *vers)
3723{
3724	u32 scfgrev_param;
3725	int ret;
3726
3727	scfgrev_param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3728			 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_SCFGREV));
3729	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3730			      1, &scfgrev_param, vers);
3731	if (ret)
3732		*vers = 0;
3733	return ret;
3734}
3735
3736/**
3737 *	t4_get_vpd_version - return the VPD version
3738 *	@adapter: the adapter
3739 *	@vers: where to place the version
3740 *
3741 *	Reads the VPD via the Firmware interface (thus this can only be called
3742 *	once we're ready to issue Firmware commands).  The format of the
3743 *	VPD version is adapter specific.  Returns 0 on success, an error on
3744 *	failure.
3745 *
3746 *	Note that early versions of the Firmware didn't include the ability
3747 *	to retrieve the VPD version, so we zero-out the return-value parameter
3748 *	in that case to avoid leaving it with garbage in it.
3749 *
3750 *	Also note that the Firmware will return its cached copy of the VPD
3751 *	Revision ID, not the actual Revision ID as written in the Serial
3752 *	EEPROM.  This is only an issue if a new VPD has been written and the
3753 *	Firmware/Chip haven't yet gone through a RESET sequence.  So it's best
3754 *	to defer calling this routine till after a FW_RESET_CMD has been issued
3755 *	if the Host Driver will be performing a full adapter initialization.
3756 */
3757int t4_get_vpd_version(struct adapter *adapter, u32 *vers)
3758{
3759	u32 vpdrev_param;
3760	int ret;
3761
3762	vpdrev_param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3763			V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_VPDREV));
3764	ret = t4_query_params(adapter, adapter->mbox, adapter->pf, 0,
3765			      1, &vpdrev_param, vers);
3766	if (ret)
3767		*vers = 0;
3768	return ret;
3769}
3770
3771/**
3772 *	t4_get_version_info - extract various chip/firmware version information
3773 *	@adapter: the adapter
3774 *
3775 *	Reads various chip/firmware version numbers and stores them into the
3776 *	adapter Adapter Parameters structure.  If any of the efforts fails
3777 *	the first failure will be returned, but all of the version numbers
3778 *	will be read.
3779 */
3780int t4_get_version_info(struct adapter *adapter)
3781{
3782	int ret = 0;
3783
3784	#define FIRST_RET(__getvinfo) \
3785	do { \
3786		int __ret = __getvinfo; \
3787		if (__ret && !ret) \
3788			ret = __ret; \
3789	} while (0)
3790
3791	FIRST_RET(t4_get_fw_version(adapter, &adapter->params.fw_vers));
3792	FIRST_RET(t4_get_bs_version(adapter, &adapter->params.bs_vers));
3793	FIRST_RET(t4_get_tp_version(adapter, &adapter->params.tp_vers));
3794	FIRST_RET(t4_get_exprom_version(adapter, &adapter->params.er_vers));
3795	FIRST_RET(t4_get_scfg_version(adapter, &adapter->params.scfg_vers));
3796	FIRST_RET(t4_get_vpd_version(adapter, &adapter->params.vpd_vers));
3797
3798	#undef FIRST_RET
3799
3800	return ret;
3801}
3802
3803/**
3804 *	t4_dump_version_info - dump all of the adapter configuration IDs
3805 *	@adapter: the adapter
3806 *
3807 *	Dumps all of the various bits of adapter configuration version/revision
3808 *	IDs information.  This is typically called at some point after
3809 *	t4_get_version_info() has been called.
3810 */
3811void t4_dump_version_info(struct adapter *adapter)
3812{
3813	/*
3814	 * Device information.
3815	 */
3816	CH_INFO(adapter, "Chelsio %s rev %d\n",
3817		adapter->params.vpd.id,
3818		CHELSIO_CHIP_RELEASE(adapter->params.chip));
3819	CH_INFO(adapter, "S/N: %s, P/N: %s\n",
3820		adapter->params.vpd.sn,
3821		adapter->params.vpd.pn);
3822
3823	/*
3824	 * Firmware Version.
3825	 */
3826	if (!adapter->params.fw_vers)
3827		CH_WARN(adapter, "No firmware loaded\n");
3828	else
3829		CH_INFO(adapter, "Firmware version: %u.%u.%u.%u\n",
3830			G_FW_HDR_FW_VER_MAJOR(adapter->params.fw_vers),
3831			G_FW_HDR_FW_VER_MINOR(adapter->params.fw_vers),
3832			G_FW_HDR_FW_VER_MICRO(adapter->params.fw_vers),
3833			G_FW_HDR_FW_VER_BUILD(adapter->params.fw_vers));
3834
3835	/*
3836	 * Bootstrap Firmware Version.  (Some adapters don't have Bootstrap
3837	 * Firmware, so dev_info() is more appropriate here.)
3838	 */
3839	if (!adapter->params.bs_vers)
3840		CH_INFO(adapter, "No bootstrap loaded\n");
3841	else
3842		CH_INFO(adapter, "Bootstrap version: %u.%u.%u.%u\n",
3843			G_FW_HDR_FW_VER_MAJOR(adapter->params.bs_vers),
3844			G_FW_HDR_FW_VER_MINOR(adapter->params.bs_vers),
3845			G_FW_HDR_FW_VER_MICRO(adapter->params.bs_vers),
3846			G_FW_HDR_FW_VER_BUILD(adapter->params.bs_vers));
3847
3848	/*
3849	 * TP Microcode Version.
3850	 */
3851	if (!adapter->params.tp_vers)
3852		CH_WARN(adapter, "No TP Microcode loaded\n");
3853	else
3854		CH_INFO(adapter, "TP Microcode version: %u.%u.%u.%u\n",
3855			G_FW_HDR_FW_VER_MAJOR(adapter->params.tp_vers),
3856			G_FW_HDR_FW_VER_MINOR(adapter->params.tp_vers),
3857			G_FW_HDR_FW_VER_MICRO(adapter->params.tp_vers),
3858			G_FW_HDR_FW_VER_BUILD(adapter->params.tp_vers));
3859
3860	/*
3861	 * Expansion ROM version.
3862	 */
3863	if (!adapter->params.er_vers)
3864		CH_INFO(adapter, "No Expansion ROM loaded\n");
3865	else
3866		CH_INFO(adapter, "Expansion ROM version: %u.%u.%u.%u\n",
3867			G_FW_HDR_FW_VER_MAJOR(adapter->params.er_vers),
3868			G_FW_HDR_FW_VER_MINOR(adapter->params.er_vers),
3869			G_FW_HDR_FW_VER_MICRO(adapter->params.er_vers),
3870			G_FW_HDR_FW_VER_BUILD(adapter->params.er_vers));
3871
3872
3873	/*
3874	 * Serial Configuration version.
3875	 */
3876	CH_INFO(adapter, "Serial Configuration version: %x\n",
3877		adapter->params.scfg_vers);
3878
3879	/*
3880	 * VPD  version.
3881	 */
3882	CH_INFO(adapter, "VPD version: %x\n",
3883		adapter->params.vpd_vers);
3884}
3885
3886/**
3887 *	t4_check_fw_version - check if the FW is supported with this driver
3888 *	@adap: the adapter
3889 *
3890 *	Checks if an adapter's FW is compatible with the driver.  Returns 0
3891 *	if there's exact match, a negative error if the version could not be
3892 *	read or there's a major version mismatch
3893 */
3894int t4_check_fw_version(struct adapter *adap)
3895{
3896	int ret, major, minor, micro;
3897	int exp_major, exp_minor, exp_micro;
3898	unsigned int chip_version = CHELSIO_CHIP_VERSION(adap->params.chip);
3899
3900	ret = t4_get_fw_version(adap, &adap->params.fw_vers);
3901	if (ret)
3902		return ret;
3903
3904	major = G_FW_HDR_FW_VER_MAJOR(adap->params.fw_vers);
3905	minor = G_FW_HDR_FW_VER_MINOR(adap->params.fw_vers);
3906	micro = G_FW_HDR_FW_VER_MICRO(adap->params.fw_vers);
3907
3908	switch (chip_version) {
3909	case CHELSIO_T4:
3910		exp_major = T4FW_MIN_VERSION_MAJOR;
3911		exp_minor = T4FW_MIN_VERSION_MINOR;
3912		exp_micro = T4FW_MIN_VERSION_MICRO;
3913		break;
3914	case CHELSIO_T5:
3915		exp_major = T5FW_MIN_VERSION_MAJOR;
3916		exp_minor = T5FW_MIN_VERSION_MINOR;
3917		exp_micro = T5FW_MIN_VERSION_MICRO;
3918		break;
3919	case CHELSIO_T6:
3920		exp_major = T6FW_MIN_VERSION_MAJOR;
3921		exp_minor = T6FW_MIN_VERSION_MINOR;
3922		exp_micro = T6FW_MIN_VERSION_MICRO;
3923		break;
3924	default:
3925		CH_ERR(adap, "Unsupported chip type, %x\n",
3926			adap->params.chip);
3927		return -EINVAL;
3928	}
3929
3930	if (major < exp_major || (major == exp_major && minor < exp_minor) ||
3931	    (major == exp_major && minor == exp_minor && micro < exp_micro)) {
3932		CH_ERR(adap, "Card has firmware version %u.%u.%u, minimum "
3933			"supported firmware is %u.%u.%u.\n", major, minor,
3934			micro, exp_major, exp_minor, exp_micro);
3935		return -EFAULT;
3936	}
3937	return 0;
3938}
3939
3940/* Is the given firmware API compatible with the one the driver was compiled
3941 * with?
3942 */
3943static int fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2)
3944{
3945
3946	/* short circuit if it's the exact same firmware version */
3947	if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
3948		return 1;
3949
3950	/*
3951	 * XXX: Is this too conservative?  Perhaps I should limit this to the
3952	 * features that are supported in the driver.
3953	 */
3954#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
3955	if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
3956	    SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
3957	    SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
3958		return 1;
3959#undef SAME_INTF
3960
3961	return 0;
3962}
3963
3964/* The firmware in the filesystem is usable, but should it be installed?
3965 * This routine explains itself in detail if it indicates the filesystem
3966 * firmware should be installed.
3967 */
3968static int should_install_fs_fw(struct adapter *adap, int card_fw_usable,
3969				int k, int c, int t4_fw_install)
3970{
3971	const char *reason;
3972
3973	if (!card_fw_usable) {
3974		reason = "incompatible or unusable";
3975		goto install;
3976	}
3977
3978	if (k > c) {
3979		reason = "older than the version bundled with this driver";
3980		goto install;
3981	}
3982
3983	if (t4_fw_install == 2 && k != c) {
3984		reason = "different than the version bundled with this driver";
3985		goto install;
3986	}
3987
3988	return 0;
3989
3990install:
3991	if (t4_fw_install == 0) {
3992		CH_ERR(adap, "firmware on card (%u.%u.%u.%u) is %s, "
3993		       "but the driver is prohibited from installing a "
3994		       "different firmware on the card.\n",
3995		       G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
3996		       G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
3997		       reason);
3998
3999		return (0);
4000	}
4001
4002	CH_ERR(adap, "firmware on card (%u.%u.%u.%u) is %s, "
4003	       "installing firmware %u.%u.%u.%u on card.\n",
4004	       G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
4005	       G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
4006	       G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
4007	       G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
4008
4009	return 1;
4010}
4011
4012int t4_prep_fw(struct adapter *adap, struct fw_info *fw_info,
4013	       const u8 *fw_data, unsigned int fw_size,
4014	       struct fw_hdr *card_fw, const int t4_fw_install,
4015	       enum dev_state state, int *reset)
4016{
4017	int ret, card_fw_usable, fs_fw_usable;
4018	const struct fw_hdr *fs_fw;
4019	const struct fw_hdr *drv_fw;
4020
4021	drv_fw = &fw_info->fw_hdr;
4022
4023	/* Read the header of the firmware on the card */
4024	ret = -t4_read_flash(adap, FLASH_FW_START,
4025			    sizeof(*card_fw) / sizeof(uint32_t),
4026			    (uint32_t *)card_fw, 1);
4027	if (ret == 0) {
4028		card_fw_usable = fw_compatible(drv_fw, (const void *)card_fw);
4029	} else {
4030		CH_ERR(adap,
4031			"Unable to read card's firmware header: %d\n", ret);
4032		card_fw_usable = 0;
4033	}
4034
4035	if (fw_data != NULL) {
4036		fs_fw = (const void *)fw_data;
4037		fs_fw_usable = fw_compatible(drv_fw, fs_fw);
4038	} else {
4039		fs_fw = NULL;
4040		fs_fw_usable = 0;
4041	}
4042
4043	if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver &&
4044	    (!fs_fw_usable || fs_fw->fw_ver == drv_fw->fw_ver)) {
4045		/* Common case: the firmware on the card is an exact match and
4046		 * the filesystem one is an exact match too, or the filesystem
4047		 * one is absent/incompatible.  Note that t4_fw_install = 2
4048		 * is ignored here -- use cxgbtool loadfw if you want to
4049		 * reinstall the same firmware as the one on the card.
4050		 */
4051	} else if (fs_fw_usable && state == DEV_STATE_UNINIT &&
4052		   should_install_fs_fw(adap, card_fw_usable,
4053					be32_to_cpu(fs_fw->fw_ver),
4054					be32_to_cpu(card_fw->fw_ver),
4055					 t4_fw_install)) {
4056
4057		ret = -t4_fw_upgrade(adap, adap->mbox, fw_data,
4058				     fw_size, 0);
4059		if (ret != 0) {
4060			CH_ERR(adap,
4061				"failed to install firmware: %d\n", ret);
4062			goto bye;
4063		}
4064
4065		/* Installed successfully, update cached information */
4066		memcpy(card_fw, fs_fw, sizeof(*card_fw));
4067		(void)t4_init_devlog_params(adap, 1);
4068		card_fw_usable = 1;
4069		*reset = 0;	/* already reset as part of load_fw */
4070	}
4071
4072	if (!card_fw_usable) {
4073		uint32_t d, c, k;
4074
4075		d = be32_to_cpu(drv_fw->fw_ver);
4076		c = be32_to_cpu(card_fw->fw_ver);
4077		k = fs_fw ? be32_to_cpu(fs_fw->fw_ver) : 0;
4078
4079		CH_ERR(adap, "Cannot find a usable firmware: "
4080			"fw_install %d, chip state %d, "
4081			"driver compiled with %d.%d.%d.%d, "
4082			"card has %d.%d.%d.%d, filesystem has %d.%d.%d.%d\n",
4083			t4_fw_install, state,
4084			G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
4085			G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d),
4086			G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
4087			G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c),
4088			G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
4089			G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k));
4090		ret = EINVAL;
4091		goto bye;
4092	}
4093
4094	/* We're using whatever's on the card and it's known to be good. */
4095	adap->params.fw_vers = be32_to_cpu(card_fw->fw_ver);
4096	adap->params.tp_vers = be32_to_cpu(card_fw->tp_microcode_ver);
4097
4098bye:
4099	return ret;
4100
4101}
4102
4103/**
4104 *	t4_flash_erase_sectors - erase a range of flash sectors
4105 *	@adapter: the adapter
4106 *	@start: the first sector to erase
4107 *	@end: the last sector to erase
4108 *
4109 *	Erases the sectors in the given inclusive range.
4110 */
4111int t4_flash_erase_sectors(struct adapter *adapter, int start, int end)
4112{
4113	int ret = 0;
4114
4115	if (end >= adapter->params.sf_nsec)
4116		return -EINVAL;
4117
4118	while (start <= end) {
4119		if ((ret = sf1_write(adapter, 1, 0, 1, SF_WR_ENABLE)) != 0 ||
4120		    (ret = sf1_write(adapter, 4, 0, 1,
4121				     SF_ERASE_SECTOR | (start << 8))) != 0 ||
4122		    (ret = flash_wait_op(adapter, 14, 500)) != 0) {
4123			CH_ERR(adapter,
4124				"erase of flash sector %d failed, error %d\n",
4125				start, ret);
4126			break;
4127		}
4128		start++;
4129	}
4130	t4_write_reg(adapter, A_SF_OP, 0);    /* unlock SF */
4131	return ret;
4132}
4133
4134/**
4135 *	t4_flash_cfg_addr - return the address of the flash configuration file
4136 *	@adapter: the adapter
4137 *
4138 *	Return the address within the flash where the Firmware Configuration
4139 *	File is stored, or an error if the device FLASH is too small to contain
4140 *	a Firmware Configuration File.
4141 */
4142int t4_flash_cfg_addr(struct adapter *adapter)
4143{
4144	/*
4145	 * If the device FLASH isn't large enough to hold a Firmware
4146	 * Configuration File, return an error.
4147	 */
4148	if (adapter->params.sf_size < FLASH_CFG_START + FLASH_CFG_MAX_SIZE)
4149		return -ENOSPC;
4150
4151	return FLASH_CFG_START;
4152}
4153
4154/* Return TRUE if the specified firmware matches the adapter.  I.e. T4
4155 * firmware for T4 adapters, T5 firmware for T5 adapters, etc.  We go ahead
4156 * and emit an error message for mismatched firmware to save our caller the
4157 * effort ...
4158 */
4159static int t4_fw_matches_chip(const struct adapter *adap,
4160			      const struct fw_hdr *hdr)
4161{
4162	/*
4163	 * The expression below will return FALSE for any unsupported adapter
4164	 * which will keep us "honest" in the future ...
4165	 */
4166	if ((is_t4(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T4) ||
4167	    (is_t5(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T5) ||
4168	    (is_t6(adap->params.chip) && hdr->chip == FW_HDR_CHIP_T6))
4169		return 1;
4170
4171	CH_ERR(adap,
4172		"FW image (%d) is not suitable for this adapter (%d)\n",
4173		hdr->chip, CHELSIO_CHIP_VERSION(adap->params.chip));
4174	return 0;
4175}
4176
4177/**
4178 *	t4_load_fw - download firmware
4179 *	@adap: the adapter
4180 *	@fw_data: the firmware image to write
4181 *	@size: image size
4182 *	@bootstrap: indicates if the binary is a bootstrap fw
4183 *
4184 *	Write the supplied firmware image to the card's serial flash.
4185 */
4186int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size,
4187	       unsigned int bootstrap)
4188{
4189	u32 csum;
4190	int ret, addr;
4191	unsigned int i;
4192	u8 first_page[SF_PAGE_SIZE];
4193	const __be32 *p = (const __be32 *)fw_data;
4194	const struct fw_hdr *hdr = (const struct fw_hdr *)fw_data;
4195	unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
4196	unsigned int fw_start_sec;
4197	unsigned int fw_start;
4198	unsigned int fw_size;
4199
4200	if (bootstrap) {
4201		fw_start_sec = FLASH_FWBOOTSTRAP_START_SEC;
4202		fw_start = FLASH_FWBOOTSTRAP_START;
4203		fw_size = FLASH_FWBOOTSTRAP_MAX_SIZE;
4204	} else {
4205		fw_start_sec = FLASH_FW_START_SEC;
4206 		fw_start = FLASH_FW_START;
4207		fw_size = FLASH_FW_MAX_SIZE;
4208	}
4209
4210	if (!size) {
4211		CH_ERR(adap, "FW image has no data\n");
4212		return -EINVAL;
4213	}
4214	if (size & 511) {
4215		CH_ERR(adap,
4216			"FW image size not multiple of 512 bytes\n");
4217		return -EINVAL;
4218	}
4219	if ((unsigned int) be16_to_cpu(hdr->len512) * 512 != size) {
4220		CH_ERR(adap,
4221			"FW image size differs from size in FW header\n");
4222		return -EINVAL;
4223	}
4224	if (size > fw_size) {
4225		CH_ERR(adap, "FW image too large, max is %u bytes\n",
4226			fw_size);
4227		return -EFBIG;
4228	}
4229	if (!t4_fw_matches_chip(adap, hdr))
4230		return -EINVAL;
4231
4232	for (csum = 0, i = 0; i < size / sizeof(csum); i++)
4233		csum += be32_to_cpu(p[i]);
4234
4235	if (csum != 0xffffffff) {
4236		CH_ERR(adap,
4237			"corrupted firmware image, checksum %#x\n", csum);
4238		return -EINVAL;
4239	}
4240
4241	i = DIV_ROUND_UP(size, sf_sec_size);	/* # of sectors spanned */
4242	ret = t4_flash_erase_sectors(adap, fw_start_sec, fw_start_sec + i - 1);
4243	if (ret)
4244		goto out;
4245
4246	/*
4247	 * We write the correct version at the end so the driver can see a bad
4248	 * version if the FW write fails.  Start by writing a copy of the
4249	 * first page with a bad version.
4250	 */
4251	memcpy(first_page, fw_data, SF_PAGE_SIZE);
4252	((struct fw_hdr *)first_page)->fw_ver = cpu_to_be32(0xffffffff);
4253	ret = t4_write_flash(adap, fw_start, SF_PAGE_SIZE, first_page, 1);
4254	if (ret)
4255		goto out;
4256
4257	addr = fw_start;
4258	for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
4259		addr += SF_PAGE_SIZE;
4260		fw_data += SF_PAGE_SIZE;
4261		ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data, 1);
4262		if (ret)
4263			goto out;
4264	}
4265
4266	ret = t4_write_flash(adap,
4267			     fw_start + offsetof(struct fw_hdr, fw_ver),
4268			     sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver, 1);
4269out:
4270	if (ret)
4271		CH_ERR(adap, "firmware download failed, error %d\n",
4272			ret);
4273	else {
4274		if (bootstrap)
4275			ret = t4_get_bs_version(adap, &adap->params.bs_vers);
4276		else
4277			ret = t4_get_fw_version(adap, &adap->params.fw_vers);
4278	}
4279	return ret;
4280}
4281
4282/**
4283 *	t4_phy_fw_ver - return current PHY firmware version
4284 *	@adap: the adapter
4285 *	@phy_fw_ver: return value buffer for PHY firmware version
4286 *
4287 *	Returns the current version of external PHY firmware on the
4288 *	adapter.
4289 */
4290int t4_phy_fw_ver(struct adapter *adap, int *phy_fw_ver)
4291{
4292	u32 param, val;
4293	int ret;
4294
4295	param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4296		 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PHYFW) |
4297		 V_FW_PARAMS_PARAM_Y(adap->params.portvec) |
4298		 V_FW_PARAMS_PARAM_Z(FW_PARAMS_PARAM_DEV_PHYFW_VERSION));
4299	ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1,
4300			      &param, &val);
4301	if (ret < 0)
4302		return ret;
4303	*phy_fw_ver = val;
4304	return 0;
4305}
4306
4307/**
4308 *	t4_load_phy_fw - download port PHY firmware
4309 *	@adap: the adapter
4310 *	@win: the PCI-E Memory Window index to use for t4_memory_rw()
4311 *	@lock: the lock to use to guard the memory copy
4312 *	@phy_fw_version: function to check PHY firmware versions
4313 *	@phy_fw_data: the PHY firmware image to write
4314 *	@phy_fw_size: image size
4315 *
4316 *	Transfer the specified PHY firmware to the adapter.  If a non-NULL
4317 *	@phy_fw_version is supplied, then it will be used to determine if
4318 *	it's necessary to perform the transfer by comparing the version
4319 *	of any existing adapter PHY firmware with that of the passed in
4320 *	PHY firmware image.  If @lock is non-NULL then it will be used
4321 *	around the call to t4_memory_rw() which transfers the PHY firmware
4322 *	to the adapter.
4323 *
4324 *	A negative error number will be returned if an error occurs.  If
4325 *	version number support is available and there's no need to upgrade
4326 *	the firmware, 0 will be returned.  If firmware is successfully
4327 *	transferred to the adapter, 1 will be retured.
4328 *
4329 *	NOTE: some adapters only have local RAM to store the PHY firmware.  As
4330 *	a result, a RESET of the adapter would cause that RAM to lose its
4331 *	contents.  Thus, loading PHY firmware on such adapters must happen after any
4332 *	FW_RESET_CMDs ...
4333 */
4334int t4_load_phy_fw(struct adapter *adap,
4335		   int win, t4_os_lock_t *lock,
4336		   int (*phy_fw_version)(const u8 *, size_t),
4337		   const u8 *phy_fw_data, size_t phy_fw_size)
4338{
4339	unsigned long mtype = 0, maddr = 0;
4340	u32 param, val;
4341	int cur_phy_fw_ver = 0, new_phy_fw_vers = 0;
4342	int ret;
4343
4344	/*
4345	 * If we have version number support, then check to see if the adapter
4346	 * already has up-to-date PHY firmware loaded.
4347	 */
4348	 if (phy_fw_version) {
4349		new_phy_fw_vers = phy_fw_version(phy_fw_data, phy_fw_size);
4350		ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver);
4351		if (ret < 0)
4352			return ret;;
4353
4354		if (cur_phy_fw_ver >= new_phy_fw_vers) {
4355			CH_WARN(adap, "PHY Firmware already up-to-date, "
4356				"version %#x\n", cur_phy_fw_ver);
4357			return 0;
4358		}
4359	}
4360
4361	/*
4362	 * Ask the firmware where it wants us to copy the PHY firmware image.
4363	 * The size of the file requires a special version of the READ coommand
4364	 * which will pass the file size via the values field in PARAMS_CMD and
4365	 * retreive the return value from firmware and place it in the same
4366	 * buffer values
4367	 */
4368	param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4369		 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PHYFW) |
4370		 V_FW_PARAMS_PARAM_Y(adap->params.portvec) |
4371		 V_FW_PARAMS_PARAM_Z(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD));
4372	val = phy_fw_size;
4373	ret = t4_query_params_rw(adap, adap->mbox, adap->pf, 0, 1,
4374			      &param, &val, 1, true);
4375	if (ret < 0)
4376		return ret;
4377	mtype = val >> 8;
4378	maddr = (val & 0xff) << 16;
4379
4380	/*
4381	 * Copy the supplied PHY Firmware image to the adapter memory location
4382	 * allocated by the adapter firmware.
4383	 */
4384	if (lock)
4385		t4_os_lock(lock);
4386	ret = t4_memory_rw(adap, win, mtype, maddr,
4387			   phy_fw_size, (__be32*)phy_fw_data,
4388			   T4_MEMORY_WRITE);
4389	if (lock)
4390		t4_os_unlock(lock);
4391	if (ret)
4392		return ret;
4393
4394	/*
4395	 * Tell the firmware that the PHY firmware image has been written to
4396	 * RAM and it can now start copying it over to the PHYs.  The chip
4397	 * firmware will RESET the affected PHYs as part of this operation
4398	 * leaving them running the new PHY firmware image.
4399	 */
4400	param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4401		 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PHYFW) |
4402		 V_FW_PARAMS_PARAM_Y(adap->params.portvec) |
4403		 V_FW_PARAMS_PARAM_Z(FW_PARAMS_PARAM_DEV_PHYFW_DOWNLOAD));
4404	ret = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1,
4405				    &param, &val, 30000);
4406
4407	/*
4408	 * If we have version number support, then check to see that the new
4409	 * firmware got loaded properly.
4410	 */
4411	if (phy_fw_version) {
4412		ret = t4_phy_fw_ver(adap, &cur_phy_fw_ver);
4413		if (ret < 0)
4414			return ret;
4415
4416		if (cur_phy_fw_ver != new_phy_fw_vers) {
4417			CH_WARN(adap, "PHY Firmware did not update: "
4418				"version on adapter %#x, "
4419				"version flashed %#x\n",
4420				cur_phy_fw_ver, new_phy_fw_vers);
4421			return -ENXIO;
4422		}
4423	}
4424
4425	return 1;
4426}
4427
4428/**
4429 *	t4_fwcache - firmware cache operation
4430 *	@adap: the adapter
4431 *	@op  : the operation (flush or flush and invalidate)
4432 */
4433int t4_fwcache(struct adapter *adap, enum fw_params_param_dev_fwcache op)
4434{
4435	struct fw_params_cmd c;
4436
4437	memset(&c, 0, sizeof(c));
4438	c.op_to_vfn =
4439	    cpu_to_be32(V_FW_CMD_OP(FW_PARAMS_CMD) |
4440			    F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
4441				V_FW_PARAMS_CMD_PFN(adap->pf) |
4442				V_FW_PARAMS_CMD_VFN(0));
4443	c.retval_len16 = cpu_to_be32(FW_LEN16(c));
4444	c.param[0].mnem =
4445	    cpu_to_be32(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4446			    V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_FWCACHE));
4447	c.param[0].val = (__force __be32)op;
4448
4449	return t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), NULL);
4450}
4451
4452void t4_cim_read_pif_la(struct adapter *adap, u32 *pif_req, u32 *pif_rsp,
4453			unsigned int *pif_req_wrptr,
4454			unsigned int *pif_rsp_wrptr)
4455{
4456	int i, j;
4457	u32 cfg, val, req, rsp;
4458
4459	cfg = t4_read_reg(adap, A_CIM_DEBUGCFG);
4460	if (cfg & F_LADBGEN)
4461		t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN);
4462
4463	val = t4_read_reg(adap, A_CIM_DEBUGSTS);
4464	req = G_POLADBGWRPTR(val);
4465	rsp = G_PILADBGWRPTR(val);
4466	if (pif_req_wrptr)
4467		*pif_req_wrptr = req;
4468	if (pif_rsp_wrptr)
4469		*pif_rsp_wrptr = rsp;
4470
4471	for (i = 0; i < CIM_PIFLA_SIZE; i++) {
4472		for (j = 0; j < 6; j++) {
4473			t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(req) |
4474				     V_PILADBGRDPTR(rsp));
4475			*pif_req++ = t4_read_reg(adap, A_CIM_PO_LA_DEBUGDATA);
4476			*pif_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_DEBUGDATA);
4477			req++;
4478			rsp++;
4479		}
4480		req = (req + 2) & M_POLADBGRDPTR;
4481		rsp = (rsp + 2) & M_PILADBGRDPTR;
4482	}
4483	t4_write_reg(adap, A_CIM_DEBUGCFG, cfg);
4484}
4485
4486void t4_cim_read_ma_la(struct adapter *adap, u32 *ma_req, u32 *ma_rsp)
4487{
4488	u32 cfg;
4489	int i, j, idx;
4490
4491	cfg = t4_read_reg(adap, A_CIM_DEBUGCFG);
4492	if (cfg & F_LADBGEN)
4493		t4_write_reg(adap, A_CIM_DEBUGCFG, cfg ^ F_LADBGEN);
4494
4495	for (i = 0; i < CIM_MALA_SIZE; i++) {
4496		for (j = 0; j < 5; j++) {
4497			idx = 8 * i + j;
4498			t4_write_reg(adap, A_CIM_DEBUGCFG, V_POLADBGRDPTR(idx) |
4499				     V_PILADBGRDPTR(idx));
4500			*ma_req++ = t4_read_reg(adap, A_CIM_PO_LA_MADEBUGDATA);
4501			*ma_rsp++ = t4_read_reg(adap, A_CIM_PI_LA_MADEBUGDATA);
4502		}
4503	}
4504	t4_write_reg(adap, A_CIM_DEBUGCFG, cfg);
4505}
4506
4507void t4_ulprx_read_la(struct adapter *adap, u32 *la_buf)
4508{
4509	unsigned int i, j;
4510
4511	for (i = 0; i < 8; i++) {
4512		u32 *p = la_buf + i;
4513
4514		t4_write_reg(adap, A_ULP_RX_LA_CTL, i);
4515		j = t4_read_reg(adap, A_ULP_RX_LA_WRPTR);
4516		t4_write_reg(adap, A_ULP_RX_LA_RDPTR, j);
4517		for (j = 0; j < ULPRX_LA_SIZE; j++, p += 8)
4518			*p = t4_read_reg(adap, A_ULP_RX_LA_RDDATA);
4519	}
4520}
4521
4522/* The ADVERT_MASK is used to mask out all of the Advertised Firmware Port
4523 * Capabilities which we control with separate controls -- see, for instance,
4524 * Pause Frames and Forward Error Correction.  In order to determine what the
4525 * full set of Advertised Port Capabilities are, the base Advertised Port
4526 * Capabilities (masked by ADVERT_MASK) must be combined with the Advertised
4527 * Port Capabilities associated with those other controls.  See
4528 * t4_link_acaps() for how this is done.
4529 */
4530#define ADVERT_MASK (V_FW_PORT_CAP32_SPEED(M_FW_PORT_CAP32_SPEED) | \
4531		     FW_PORT_CAP32_ANEG)
4532
4533/**
4534 *	fwcaps16_to_caps32 - convert 16-bit Port Capabilities to 32-bits
4535 *	@caps16: a 16-bit Port Capabilities value
4536 *
4537 *	Returns the equivalent 32-bit Port Capabilities value.
4538 */
4539static fw_port_cap32_t fwcaps16_to_caps32(fw_port_cap16_t caps16)
4540{
4541	fw_port_cap32_t caps32 = 0;
4542
4543	#define CAP16_TO_CAP32(__cap) \
4544		do { \
4545			if (caps16 & FW_PORT_CAP_##__cap) \
4546				caps32 |= FW_PORT_CAP32_##__cap; \
4547		} while (0)
4548
4549	CAP16_TO_CAP32(SPEED_100M);
4550	CAP16_TO_CAP32(SPEED_1G);
4551	CAP16_TO_CAP32(SPEED_25G);
4552	CAP16_TO_CAP32(SPEED_10G);
4553	CAP16_TO_CAP32(SPEED_40G);
4554	CAP16_TO_CAP32(SPEED_100G);
4555	CAP16_TO_CAP32(FC_RX);
4556	CAP16_TO_CAP32(FC_TX);
4557	CAP16_TO_CAP32(ANEG);
4558	CAP16_TO_CAP32(FORCE_PAUSE);
4559	CAP16_TO_CAP32(MDIAUTO);
4560	CAP16_TO_CAP32(MDISTRAIGHT);
4561	CAP16_TO_CAP32(FEC_RS);
4562	CAP16_TO_CAP32(FEC_BASER_RS);
4563	CAP16_TO_CAP32(802_3_PAUSE);
4564	CAP16_TO_CAP32(802_3_ASM_DIR);
4565
4566	#undef CAP16_TO_CAP32
4567
4568	return caps32;
4569}
4570
4571/**
4572 *	fwcaps32_to_caps16 - convert 32-bit Port Capabilities to 16-bits
4573 *	@caps32: a 32-bit Port Capabilities value
4574 *
4575 *	Returns the equivalent 16-bit Port Capabilities value.  Note that
4576 *	not all 32-bit Port Capabilities can be represented in the 16-bit
4577 *	Port Capabilities and some fields/values may not make it.
4578 */
4579static fw_port_cap16_t fwcaps32_to_caps16(fw_port_cap32_t caps32)
4580{
4581	fw_port_cap16_t caps16 = 0;
4582
4583	#define CAP32_TO_CAP16(__cap) \
4584		do { \
4585			if (caps32 & FW_PORT_CAP32_##__cap) \
4586				caps16 |= FW_PORT_CAP_##__cap; \
4587		} while (0)
4588
4589	CAP32_TO_CAP16(SPEED_100M);
4590	CAP32_TO_CAP16(SPEED_1G);
4591	CAP32_TO_CAP16(SPEED_10G);
4592	CAP32_TO_CAP16(SPEED_25G);
4593	CAP32_TO_CAP16(SPEED_40G);
4594	CAP32_TO_CAP16(SPEED_100G);
4595	CAP32_TO_CAP16(FC_RX);
4596	CAP32_TO_CAP16(FC_TX);
4597	CAP32_TO_CAP16(802_3_PAUSE);
4598	CAP32_TO_CAP16(802_3_ASM_DIR);
4599	CAP32_TO_CAP16(ANEG);
4600	CAP32_TO_CAP16(FORCE_PAUSE);
4601	CAP32_TO_CAP16(MDIAUTO);
4602	CAP32_TO_CAP16(MDISTRAIGHT);
4603	CAP32_TO_CAP16(FEC_RS);
4604	CAP32_TO_CAP16(FEC_BASER_RS);
4605
4606	#undef CAP32_TO_CAP16
4607
4608	return caps16;
4609}
4610
4611/* Translate Firmware Port Capabilities Pause specification to Common Code */
4612static inline cc_pause_t fwcap_to_cc_pause(fw_port_cap32_t fw_pause)
4613{
4614	cc_pause_t cc_pause = 0;
4615
4616	if (fw_pause & FW_PORT_CAP32_FC_RX)
4617		cc_pause |= PAUSE_RX;
4618	if (fw_pause & FW_PORT_CAP32_FC_TX)
4619		cc_pause |= PAUSE_TX;
4620
4621	return cc_pause;
4622}
4623
4624/* Translate Common Code Pause specification into Firmware Port Capabilities */
4625static inline fw_port_cap32_t cc_to_fwcap_pause(cc_pause_t cc_pause)
4626{
4627	fw_port_cap32_t fw_pause = 0;
4628
4629	/* Translate orthogonal RX/TX Pause Controls for L1 Configure
4630	 * commands, etc.
4631	 */
4632	if (cc_pause & PAUSE_RX)
4633		fw_pause |= FW_PORT_CAP32_FC_RX;
4634	if (cc_pause & PAUSE_TX)
4635		fw_pause |= FW_PORT_CAP32_FC_TX;
4636	if (!(cc_pause & PAUSE_AUTONEG))
4637		fw_pause |= FW_PORT_CAP32_FORCE_PAUSE;
4638
4639	return fw_pause;
4640}
4641
4642/* Translate Firmware Forward Error Correction specification to Common Code */
4643static inline cc_fec_t fwcap_to_cc_fec(fw_port_cap32_t fw_fec)
4644{
4645	cc_fec_t cc_fec = 0;
4646
4647	if (fw_fec & FW_PORT_CAP32_FEC_RS)
4648		cc_fec |= FEC_RS;
4649	if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
4650		cc_fec |= FEC_BASER_RS;
4651
4652	if (cc_fec == 0)
4653		cc_fec = FEC_NONE;
4654
4655	return (cc_fec);
4656}
4657
4658/* Translate Common Code Forward Error Correction specification to Firmware */
4659static inline boolean_t
4660cc_to_fwcap_fec(fw_port_cap32_t *fw_fecp, cc_fec_t cc_fec,
4661    struct link_config *lc)
4662{
4663	fw_port_cap32_t fw_fec = 0;
4664
4665	if ((cc_fec & FEC_AUTO) != 0) {
4666		if ((lc->pcaps & FW_PORT_CAP32_SPEED_100G) == 0)
4667			fw_fec |= FW_PORT_CAP32_FEC_BASER_RS;
4668
4669		if ((lc->pcaps & FW_PORT_CAP32_FORCE_FEC) != 0)
4670			fw_fec |= FW_PORT_CAP32_FEC_NO_FEC;
4671
4672		fw_fec |= FW_PORT_CAP32_FEC_RS;
4673
4674		*fw_fecp = fw_fec;
4675		return (B_TRUE);
4676	}
4677
4678	if ((cc_fec & FEC_RS) != 0)
4679		fw_fec |= FW_PORT_CAP32_FEC_RS;
4680
4681	if ((cc_fec & FEC_BASER_RS) != 0 &&
4682	    (lc->pcaps & FW_PORT_CAP32_SPEED_100G) == 0)
4683		fw_fec |= FW_PORT_CAP32_FEC_BASER_RS;
4684
4685	if ((cc_fec & FEC_NONE) != 0) {
4686		if ((lc->pcaps & FW_PORT_CAP32_FORCE_FEC) != 0) {
4687			fw_fec |= FW_PORT_CAP32_FORCE_FEC;
4688			fw_fec |= FW_PORT_CAP32_FEC_NO_FEC;
4689		}
4690
4691		*fw_fecp = fw_fec;
4692		return (B_TRUE);
4693	}
4694
4695	if (fw_fec == 0)
4696		return (B_FALSE);
4697
4698	if ((lc->pcaps & FW_PORT_CAP32_FORCE_FEC) != 0)
4699		fw_fec |= FW_PORT_CAP32_FORCE_FEC;
4700
4701	*fw_fecp = fw_fec;
4702	return (B_TRUE);
4703}
4704
4705/**
4706 *	t4_link_acaps - compute Link Advertised Port Capabilities
4707 *	@adapter: the adapter
4708 *	@port: the Port ID
4709 *	@lc: the Port's Link Configuration
4710 *
4711 *	Synthesize the Advertised Port Capabilities we'll be using based on
4712 *	the base Advertised Port Capabilities (which have been filtered by
4713 *	ADVERT_MASK) plus the individual controls for things like Pause
4714 *	Frames, Forward Error Correction, MDI, etc.
4715 */
4716fw_port_cap32_t t4_link_acaps(struct adapter *adapter, unsigned int port,
4717			      struct link_config *lc)
4718{
4719	unsigned int fw_mdi =
4720		(V_FW_PORT_CAP32_MDI(FW_PORT_CAP32_MDI_AUTO) & lc->pcaps);
4721	fw_port_cap32_t fw_fc, fw_fec, acaps;
4722	cc_fec_t cc_fec;
4723
4724	/* Convert driver coding of Pause Frame Flow Control settings into the
4725	 * Firmware's API.
4726	 */
4727	fw_fc = cc_to_fwcap_pause(lc->requested_fc);
4728
4729	/* Convert Common Code Forward Error Control settings into the
4730	 * Firmware's API.  If the current Requested FEC has "Automatic"
4731	 * (IEEE 802.3) specified, then we use whatever the Firmware
4732	 * sent us as part of it's IEEE 802.3-based interpratation of
4733	 * the Transceiver Module EPROM FEC parameters.  Otherwise we
4734	 * use whatever is in the current Requested FEC settings.
4735	 */
4736	if (fec_supported(lc->pcaps)) {
4737		if (lc->requested_fec & FEC_AUTO)
4738			cc_fec = fwcap_to_cc_fec(lc->def_acaps);
4739		else
4740			cc_fec = lc->requested_fec;
4741
4742		if (!cc_to_fwcap_fec(&fw_fec, cc_fec, lc))
4743			return (0);
4744	} else {
4745		fw_fec = 0;
4746		cc_fec = FEC_NONE;
4747	}
4748
4749	/* Figure out what our Requested Port Capabilities are going to be.
4750	 * Note parallel structure in t4_handle_get_port_info() and
4751	 * init_link_config().
4752	 */
4753	if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
4754		acaps = lc->acaps | fw_fc | fw_fec;
4755		lc->fc = lc->requested_fc & ~PAUSE_AUTONEG;
4756		lc->fec = cc_fec;
4757	} else if (lc->autoneg == AUTONEG_DISABLE) {
4758		acaps = lc->speed_caps | fw_fc | fw_fec | fw_mdi;
4759		lc->fc = lc->requested_fc & ~PAUSE_AUTONEG;
4760		lc->fec = cc_fec;
4761	} else
4762		acaps = lc->acaps | fw_fc | fw_fec | fw_mdi;
4763
4764	/* Some Requested Port Capabilities are trivially wrong if they exceed
4765	 * the Physical Port Capabilities.  We can check that here and provide
4766	 * moderately useful feedback in the system log.
4767	 *
4768	 * Note that older Firmware doesn't have FW_PORT_CAP32_FORCE_PAUSE, so
4769	 * we need to exclude this from this check in order to maintain
4770	 * compatibility ...
4771	 */
4772	if ((acaps & ~lc->pcaps) & ~FW_PORT_CAP32_FORCE_PAUSE) {
4773		CH_ERR(adapter,
4774		       "Requested Port Capabilities %#x exceed Physical Port Capabilities %#x\n",
4775		       acaps, lc->pcaps);
4776		return 0;
4777	}
4778
4779	return acaps;
4780}
4781
4782/**
4783 *	t4_link_l1cfg_core - apply link configuration to MAC/PHY
4784 *	@adapter: the adapter
4785 *	@mbox: the Firmware Mailbox to use
4786 *	@port: the Port ID
4787 *	@lc: the Port's Link Configuration
4788 *	@sleep_ok: if true we may sleep while awaiting command completion
4789 *	@timeout: time to wait for command to finish before timing out
4790 *		(negative implies @sleep_ok=false)
4791 *
4792 *	Set up a port's MAC and PHY according to a desired link configuration.
4793 *	- If the PHY can auto-negotiate first decide what to advertise, then
4794 *	  enable/disable auto-negotiation as desired, and reset.
4795 *	- If the PHY does not auto-negotiate just reset it.
4796 *	- If auto-negotiation is off set the MAC to the proper speed/duplex/FC,
4797 *	  otherwise do it later based on the outcome of auto-negotiation.
4798 */
4799int t4_link_l1cfg_core(struct adapter *adapter, unsigned int mbox,
4800		       unsigned int port, struct link_config *lc,
4801		       bool sleep_ok, int timeout)
4802{
4803	unsigned int fw_caps = adapter->params.fw_caps_support;
4804	fw_port_cap32_t rcap;
4805	struct fw_port_cmd cmd;
4806	int ret;
4807
4808	/* Filter out nonsense.
4809	 */
4810	if (!(lc->pcaps & FW_PORT_CAP32_ANEG) &&
4811	    lc->autoneg == AUTONEG_ENABLE)
4812		return -EINVAL;
4813
4814	/* Compute our Requested Port Capabilities and send that on to the
4815	 * Firmware.
4816	 */
4817	rcap = t4_link_acaps(adapter, port, lc);
4818	if(!rcap)
4819		return -EINVAL;
4820	memset(&cmd, 0, sizeof(cmd));
4821	cmd.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) |
4822				       F_FW_CMD_REQUEST | F_FW_CMD_EXEC |
4823				       V_FW_PORT_CMD_PORTID(port));
4824	cmd.action_to_len16 =
4825		cpu_to_be32(V_FW_PORT_CMD_ACTION(fw_caps == FW_CAPS16
4826						 ? FW_PORT_ACTION_L1_CFG
4827						 : FW_PORT_ACTION_L1_CFG32) |
4828			    FW_LEN16(cmd));
4829	if (fw_caps == FW_CAPS16)
4830		cmd.u.l1cfg.rcap = cpu_to_be32(fwcaps32_to_caps16(rcap));
4831	else
4832		cmd.u.l1cfg32.rcap32 = cpu_to_be32(rcap);
4833	ret = t4_wr_mbox_meat_timeout(adapter, mbox, &cmd, sizeof(cmd), NULL,
4834				      sleep_ok, timeout);
4835
4836	/* Unfortunately, even if the Requested Port Capabilities "fit" within
4837	 * the Physical Port Capabilities, some combinations of features may
4838	 * still not be legal.  For example, 40Gb/s and Reed-Solomon Forward
4839	 * Error Correction.  So if the Firmware rejects the L1 Configure
4840	 * request, flag that here.
4841	 */
4842	if (ret) {
4843		CH_ERR(adapter,
4844		       "Requested Port Capabilities %#x rejected, error %d\n",
4845		       rcap, -ret);
4846		return ret;
4847	}
4848	return 0;
4849}
4850
4851/**
4852 *	t4_restart_aneg - restart autonegotiation
4853 *	@adap: the adapter
4854 *	@mbox: mbox to use for the FW command
4855 *	@port: the port id
4856 *
4857 *	Restarts autonegotiation for the selected port.
4858 */
4859int t4_restart_aneg(struct adapter *adap, unsigned int mbox, unsigned int port)
4860{
4861	unsigned int fw_caps = adap->params.fw_caps_support;
4862	struct fw_port_cmd c;
4863
4864	memset(&c, 0, sizeof(c));
4865	c.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) |
4866				     F_FW_CMD_REQUEST | F_FW_CMD_EXEC |
4867				     V_FW_PORT_CMD_PORTID(port));
4868	c.action_to_len16 =
4869		cpu_to_be32(V_FW_PORT_CMD_ACTION(fw_caps == FW_CAPS16
4870						 ? FW_PORT_ACTION_L1_CFG
4871						 : FW_PORT_ACTION_L1_CFG32) |
4872			    FW_LEN16(c));
4873	if (fw_caps == FW_CAPS16)
4874		c.u.l1cfg.rcap = cpu_to_be32(FW_PORT_CAP_ANEG);
4875	else
4876		c.u.l1cfg32.rcap32 = cpu_to_be32(FW_PORT_CAP32_ANEG);
4877	return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
4878}
4879
4880typedef void (*int_handler_t)(struct adapter *adap);
4881
4882struct intr_info {
4883	unsigned int mask;	/* bits to check in interrupt status */
4884	const char *msg;	/* message to print or NULL */
4885	short stat_idx;		/* stat counter to increment or -1 */
4886	unsigned short fatal;	/* whether the condition reported is fatal */
4887	int_handler_t int_handler;	/* platform-specific int handler */
4888};
4889
4890/**
4891 *	t4_handle_intr_status - table driven interrupt handler
4892 *	@adapter: the adapter that generated the interrupt
4893 *	@reg: the interrupt status register to process
4894 *	@acts: table of interrupt actions
4895 *
4896 *	A table driven interrupt handler that applies a set of masks to an
4897 *	interrupt status word and performs the corresponding actions if the
4898 *	interrupts described by the mask have occurred.  The actions include
4899 *	optionally emitting a warning or alert message.  The table is terminated
4900 *	by an entry specifying mask 0.  Returns the number of fatal interrupt
4901 *	conditions.
4902 */
4903static int t4_handle_intr_status(struct adapter *adapter, unsigned int reg,
4904				 const struct intr_info *acts)
4905{
4906	int fatal = 0;
4907	unsigned int mask = 0;
4908	unsigned int status = t4_read_reg(adapter, reg);
4909
4910	for ( ; acts->mask; ++acts) {
4911		if (!(status & acts->mask))
4912			continue;
4913		if (acts->fatal) {
4914			fatal++;
4915			CH_ALERT(adapter, "%s (0x%x)\n", acts->msg,
4916				  status & acts->mask);
4917		} else if (acts->msg)
4918			CH_WARN_RATELIMIT(adapter, "%s (0x%x)\n", acts->msg,
4919				 status & acts->mask);
4920		if (acts->int_handler)
4921			acts->int_handler(adapter);
4922		mask |= acts->mask;
4923	}
4924	status &= mask;
4925	if (status)	/* clear processed interrupts */
4926		t4_write_reg(adapter, reg, status);
4927	return fatal;
4928}
4929
4930/*
4931 * Interrupt handler for the PCIE module.
4932 */
4933static void pcie_intr_handler(struct adapter *adapter)
4934{
4935	static const struct intr_info sysbus_intr_info[] = {
4936		{ F_RNPP, "RXNP array parity error", -1, 1 },
4937		{ F_RPCP, "RXPC array parity error", -1, 1 },
4938		{ F_RCIP, "RXCIF array parity error", -1, 1 },
4939		{ F_RCCP, "Rx completions control array parity error", -1, 1 },
4940		{ F_RFTP, "RXFT array parity error", -1, 1 },
4941		{ 0 }
4942	};
4943	static const struct intr_info pcie_port_intr_info[] = {
4944		{ F_TPCP, "TXPC array parity error", -1, 1 },
4945		{ F_TNPP, "TXNP array parity error", -1, 1 },
4946		{ F_TFTP, "TXFT array parity error", -1, 1 },
4947		{ F_TCAP, "TXCA array parity error", -1, 1 },
4948		{ F_TCIP, "TXCIF array parity error", -1, 1 },
4949		{ F_RCAP, "RXCA array parity error", -1, 1 },
4950		{ F_OTDD, "outbound request TLP discarded", -1, 1 },
4951		{ F_RDPE, "Rx data parity error", -1, 1 },
4952		{ F_TDUE, "Tx uncorrectable data error", -1, 1 },
4953		{ 0 }
4954	};
4955	static const struct intr_info pcie_intr_info[] = {
4956		{ F_MSIADDRLPERR, "MSI AddrL parity error", -1, 1 },
4957		{ F_MSIADDRHPERR, "MSI AddrH parity error", -1, 1 },
4958		{ F_MSIDATAPERR, "MSI data parity error", -1, 1 },
4959		{ F_MSIXADDRLPERR, "MSI-X AddrL parity error", -1, 1 },
4960		{ F_MSIXADDRHPERR, "MSI-X AddrH parity error", -1, 1 },
4961		{ F_MSIXDATAPERR, "MSI-X data parity error", -1, 1 },
4962		{ F_MSIXDIPERR, "MSI-X DI parity error", -1, 1 },
4963		{ F_PIOCPLPERR, "PCI PIO completion FIFO parity error", -1, 1 },
4964		{ F_PIOREQPERR, "PCI PIO request FIFO parity error", -1, 1 },
4965		{ F_TARTAGPERR, "PCI PCI target tag FIFO parity error", -1, 1 },
4966		{ F_CCNTPERR, "PCI CMD channel count parity error", -1, 1 },
4967		{ F_CREQPERR, "PCI CMD channel request parity error", -1, 1 },
4968		{ F_CRSPPERR, "PCI CMD channel response parity error", -1, 1 },
4969		{ F_DCNTPERR, "PCI DMA channel count parity error", -1, 1 },
4970		{ F_DREQPERR, "PCI DMA channel request parity error", -1, 1 },
4971		{ F_DRSPPERR, "PCI DMA channel response parity error", -1, 1 },
4972		{ F_HCNTPERR, "PCI HMA channel count parity error", -1, 1 },
4973		{ F_HREQPERR, "PCI HMA channel request parity error", -1, 1 },
4974		{ F_HRSPPERR, "PCI HMA channel response parity error", -1, 1 },
4975		{ F_CFGSNPPERR, "PCI config snoop FIFO parity error", -1, 1 },
4976		{ F_FIDPERR, "PCI FID parity error", -1, 1 },
4977		{ F_INTXCLRPERR, "PCI INTx clear parity error", -1, 1 },
4978		{ F_MATAGPERR, "PCI MA tag parity error", -1, 1 },
4979		{ F_PIOTAGPERR, "PCI PIO tag parity error", -1, 1 },
4980		{ F_RXCPLPERR, "PCI Rx completion parity error", -1, 1 },
4981		{ F_RXWRPERR, "PCI Rx write parity error", -1, 1 },
4982		{ F_RPLPERR, "PCI replay buffer parity error", -1, 1 },
4983		{ F_PCIESINT, "PCI core secondary fault", -1, 1 },
4984		{ F_PCIEPINT, "PCI core primary fault", -1, 1 },
4985		{ F_UNXSPLCPLERR, "PCI unexpected split completion error", -1,
4986		  0 },
4987		{ 0 }
4988	};
4989
4990	static struct intr_info t5_pcie_intr_info[] = {
4991		{ F_MSTGRPPERR, "Master Response Read Queue parity error",
4992		  -1, 1 },
4993		{ F_MSTTIMEOUTPERR, "Master Timeout FIFO parity error", -1, 1 },
4994		{ F_MSIXSTIPERR, "MSI-X STI SRAM parity error", -1, 1 },
4995		{ F_MSIXADDRLPERR, "MSI-X AddrL parity error", -1, 1 },
4996		{ F_MSIXADDRHPERR, "MSI-X AddrH parity error", -1, 1 },
4997		{ F_MSIXDATAPERR, "MSI-X data parity error", -1, 1 },
4998		{ F_MSIXDIPERR, "MSI-X DI parity error", -1, 1 },
4999		{ F_PIOCPLGRPPERR, "PCI PIO completion Group FIFO parity error",
5000		  -1, 1 },
5001		{ F_PIOREQGRPPERR, "PCI PIO request Group FIFO parity error",
5002		  -1, 1 },
5003		{ F_TARTAGPERR, "PCI PCI target tag FIFO parity error", -1, 1 },
5004		{ F_MSTTAGQPERR, "PCI master tag queue parity error", -1, 1 },
5005		{ F_CREQPERR, "PCI CMD channel request parity error", -1, 1 },
5006		{ F_CRSPPERR, "PCI CMD channel response parity error", -1, 1 },
5007		{ F_DREQWRPERR, "PCI DMA channel write request parity error",
5008		  -1, 1 },
5009		{ F_DREQPERR, "PCI DMA channel request parity error", -1, 1 },
5010		{ F_DRSPPERR, "PCI DMA channel response parity error", -1, 1 },
5011		{ F_HREQWRPERR, "PCI HMA channel count parity error", -1, 1 },
5012		{ F_HREQPERR, "PCI HMA channel request parity error", -1, 1 },
5013		{ F_HRSPPERR, "PCI HMA channel response parity error", -1, 1 },
5014		{ F_CFGSNPPERR, "PCI config snoop FIFO parity error", -1, 1 },
5015		{ F_FIDPERR, "PCI FID parity error", -1, 1 },
5016		{ F_VFIDPERR, "PCI INTx clear parity error", -1, 1 },
5017		{ F_MAGRPPERR, "PCI MA group FIFO parity error", -1, 1 },
5018		{ F_PIOTAGPERR, "PCI PIO tag parity error", -1, 1 },
5019		{ F_IPRXHDRGRPPERR, "PCI IP Rx header group parity error",
5020		  -1, 1 },
5021		{ F_IPRXDATAGRPPERR, "PCI IP Rx data group parity error",
5022		  -1, 1 },
5023		{ F_RPLPERR, "PCI IP replay buffer parity error", -1, 1 },
5024		{ F_IPSOTPERR, "PCI IP SOT buffer parity error", -1, 1 },
5025		{ F_TRGT1GRPPERR, "PCI TRGT1 group FIFOs parity error", -1, 1 },
5026		{ F_READRSPERR, "Outbound read error", -1,
5027		  0 },
5028		{ 0 }
5029	};
5030
5031	int fat;
5032
5033	if (is_t4(adapter->params.chip))
5034		fat = t4_handle_intr_status(adapter,
5035				A_PCIE_CORE_UTL_SYSTEM_BUS_AGENT_STATUS,
5036				sysbus_intr_info) +
5037			t4_handle_intr_status(adapter,
5038					A_PCIE_CORE_UTL_PCI_EXPRESS_PORT_STATUS,
5039					pcie_port_intr_info) +
5040			t4_handle_intr_status(adapter, A_PCIE_INT_CAUSE,
5041					      pcie_intr_info);
5042	else
5043		fat = t4_handle_intr_status(adapter, A_PCIE_INT_CAUSE,
5044					    t5_pcie_intr_info);
5045	if (fat)
5046		t4_fatal_err(adapter);
5047}
5048
5049/*
5050 * TP interrupt handler.
5051 */
5052static void tp_intr_handler(struct adapter *adapter)
5053{
5054	static const struct intr_info tp_intr_info[] = {
5055		{ 0x3fffffff, "TP parity error", -1, 1 },
5056		{ F_FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1 },
5057		{ 0 }
5058	};
5059
5060	if (t4_handle_intr_status(adapter, A_TP_INT_CAUSE, tp_intr_info))
5061		t4_fatal_err(adapter);
5062}
5063
5064/*
5065 * SGE interrupt handler.
5066 */
5067static void sge_intr_handler(struct adapter *adapter)
5068{
5069	u32 v = 0, perr;
5070	u32 err;
5071
5072	static const struct intr_info sge_intr_info[] = {
5073		{ F_ERR_CPL_EXCEED_IQE_SIZE,
5074		  "SGE received CPL exceeding IQE size", -1, 1 },
5075		{ F_ERR_INVALID_CIDX_INC,
5076		  "SGE GTS CIDX increment too large", -1, 0 },
5077		{ F_ERR_CPL_OPCODE_0, "SGE received 0-length CPL", -1, 0 },
5078		{ F_DBFIFO_LP_INT, NULL, -1, 0, t4_db_full },
5079		{ F_ERR_DATA_CPL_ON_HIGH_QID1 | F_ERR_DATA_CPL_ON_HIGH_QID0,
5080		  "SGE IQID > 1023 received CPL for FL", -1, 0 },
5081		{ F_ERR_BAD_DB_PIDX3, "SGE DBP 3 pidx increment too large", -1,
5082		  0 },
5083		{ F_ERR_BAD_DB_PIDX2, "SGE DBP 2 pidx increment too large", -1,
5084		  0 },
5085		{ F_ERR_BAD_DB_PIDX1, "SGE DBP 1 pidx increment too large", -1,
5086		  0 },
5087		{ F_ERR_BAD_DB_PIDX0, "SGE DBP 0 pidx increment too large", -1,
5088		  0 },
5089		{ F_ERR_ING_CTXT_PRIO,
5090		  "SGE too many priority ingress contexts", -1, 0 },
5091		{ F_INGRESS_SIZE_ERR, "SGE illegal ingress QID", -1, 0 },
5092		{ F_EGRESS_SIZE_ERR, "SGE illegal egress QID", -1, 0 },
5093		{ F_ERR_PCIE_ERROR0 | F_ERR_PCIE_ERROR1 |
5094		  F_ERR_PCIE_ERROR2 | F_ERR_PCIE_ERROR3,
5095		  "SGE PCIe error for a DBP thread", -1, 0 },
5096		{ 0 }
5097	};
5098
5099	static struct intr_info t4t5_sge_intr_info[] = {
5100		{ F_ERR_DROPPED_DB, NULL, -1, 0, t4_db_dropped },
5101		{ F_DBFIFO_HP_INT, NULL, -1, 0, t4_db_full },
5102		{ F_ERR_EGR_CTXT_PRIO,
5103		  "SGE too many priority egress contexts", -1, 0 },
5104		{ 0 }
5105	};
5106
5107	/*
5108 	* For now, treat below interrupts as fatal so that we disable SGE and
5109 	* get better debug */
5110	static struct intr_info t6_sge_intr_info[] = {
5111		{ F_FATAL_WRE_LEN,
5112		  "SGE Actual WRE packet is less than advertized length",
5113		  -1, 1 },
5114		{ 0 }
5115	};
5116
5117	perr = t4_read_reg(adapter, A_SGE_INT_CAUSE1);
5118	if (perr) {
5119		v |= perr;
5120		CH_ALERT(adapter, "SGE Cause1 Parity Error %#x\n", perr);
5121	}
5122	perr = t4_read_reg(adapter, A_SGE_INT_CAUSE2);
5123	if (perr) {
5124		v |= perr;
5125		CH_ALERT(adapter, "SGE Cause2 Parity Error %#x\n", perr);
5126	}
5127	if (CHELSIO_CHIP_VERSION(adapter->params.chip) >= CHELSIO_T5) {
5128		perr = t4_read_reg(adapter, A_SGE_INT_CAUSE5);
5129		if (perr) {
5130			v |= perr;
5131			CH_ALERT(adapter, "SGE Cause5 Parity Error %#x\n", perr);
5132		}
5133	}
5134
5135	v |= t4_handle_intr_status(adapter, A_SGE_INT_CAUSE3, sge_intr_info);
5136	if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
5137		v |= t4_handle_intr_status(adapter, A_SGE_INT_CAUSE3,
5138					   t4t5_sge_intr_info);
5139	else
5140		v |= t4_handle_intr_status(adapter, A_SGE_INT_CAUSE3,
5141					   t6_sge_intr_info);
5142
5143	err = t4_read_reg(adapter, A_SGE_ERROR_STATS);
5144	if (err & F_ERROR_QID_VALID) {
5145		CH_ERR(adapter, "SGE error for queue %u\n", G_ERROR_QID(err));
5146		if (err & F_UNCAPTURED_ERROR)
5147			CH_ERR(adapter, "SGE UNCAPTURED_ERROR set (clearing)\n");
5148		t4_write_reg(adapter, A_SGE_ERROR_STATS, F_ERROR_QID_VALID |
5149			     F_UNCAPTURED_ERROR);
5150	}
5151
5152	if (v != 0)
5153		t4_fatal_err(adapter);
5154}
5155
5156#define CIM_OBQ_INTR (F_OBQULP0PARERR | F_OBQULP1PARERR | F_OBQULP2PARERR |\
5157		      F_OBQULP3PARERR | F_OBQSGEPARERR | F_OBQNCSIPARERR)
5158#define CIM_IBQ_INTR (F_IBQTP0PARERR | F_IBQTP1PARERR | F_IBQULPPARERR |\
5159		      F_IBQSGEHIPARERR | F_IBQSGELOPARERR | F_IBQNCSIPARERR)
5160
5161/*
5162 * CIM interrupt handler.
5163 */
5164static void cim_intr_handler(struct adapter *adapter)
5165{
5166	static const struct intr_info cim_intr_info[] = {
5167		{ F_PREFDROPINT, "CIM control register prefetch drop", -1, 1 },
5168		{ CIM_OBQ_INTR, "CIM OBQ parity error", -1, 1 },
5169		{ CIM_IBQ_INTR, "CIM IBQ parity error", -1, 1 },
5170		{ F_MBUPPARERR, "CIM mailbox uP parity error", -1, 1 },
5171		{ F_MBHOSTPARERR, "CIM mailbox host parity error", -1, 1 },
5172		{ F_TIEQINPARERRINT, "CIM TIEQ outgoing parity error", -1, 1 },
5173		{ F_TIEQOUTPARERRINT, "CIM TIEQ incoming parity error", -1, 1 },
5174		{ F_TIMER0INT, "CIM TIMER0 interrupt", -1, 1 },
5175		{ 0 }
5176	};
5177	static const struct intr_info cim_upintr_info[] = {
5178		{ F_RSVDSPACEINT, "CIM reserved space access", -1, 1 },
5179		{ F_ILLTRANSINT, "CIM illegal transaction", -1, 1 },
5180		{ F_ILLWRINT, "CIM illegal write", -1, 1 },
5181		{ F_ILLRDINT, "CIM illegal read", -1, 1 },
5182		{ F_ILLRDBEINT, "CIM illegal read BE", -1, 1 },
5183		{ F_ILLWRBEINT, "CIM illegal write BE", -1, 1 },
5184		{ F_SGLRDBOOTINT, "CIM single read from boot space", -1, 1 },
5185		{ F_SGLWRBOOTINT, "CIM single write to boot space", -1, 1 },
5186		{ F_BLKWRBOOTINT, "CIM block write to boot space", -1, 1 },
5187		{ F_SGLRDFLASHINT, "CIM single read from flash space", -1, 1 },
5188		{ F_SGLWRFLASHINT, "CIM single write to flash space", -1, 1 },
5189		{ F_BLKWRFLASHINT, "CIM block write to flash space", -1, 1 },
5190		{ F_SGLRDEEPROMINT, "CIM single EEPROM read", -1, 1 },
5191		{ F_SGLWREEPROMINT, "CIM single EEPROM write", -1, 1 },
5192		{ F_BLKRDEEPROMINT, "CIM block EEPROM read", -1, 1 },
5193		{ F_BLKWREEPROMINT, "CIM block EEPROM write", -1, 1 },
5194		{ F_SGLRDCTLINT , "CIM single read from CTL space", -1, 1 },
5195		{ F_SGLWRCTLINT , "CIM single write to CTL space", -1, 1 },
5196		{ F_BLKRDCTLINT , "CIM block read from CTL space", -1, 1 },
5197		{ F_BLKWRCTLINT , "CIM block write to CTL space", -1, 1 },
5198		{ F_SGLRDPLINT , "CIM single read from PL space", -1, 1 },
5199		{ F_SGLWRPLINT , "CIM single write to PL space", -1, 1 },
5200		{ F_BLKRDPLINT , "CIM block read from PL space", -1, 1 },
5201		{ F_BLKWRPLINT , "CIM block write to PL space", -1, 1 },
5202		{ F_REQOVRLOOKUPINT , "CIM request FIFO overwrite", -1, 1 },
5203		{ F_RSPOVRLOOKUPINT , "CIM response FIFO overwrite", -1, 1 },
5204		{ F_TIMEOUTINT , "CIM PIF timeout", -1, 1 },
5205		{ F_TIMEOUTMAINT , "CIM PIF MA timeout", -1, 1 },
5206		{ 0 }
5207	};
5208	u32 val, fw_err;
5209	int fat;
5210
5211	fw_err = t4_read_reg(adapter, A_PCIE_FW);
5212	if (fw_err & F_PCIE_FW_ERR)
5213		t4_report_fw_error(adapter);
5214
5215	/* When the Firmware detects an internal error which normally wouldn't
5216	 * raise a Host Interrupt, it forces a CIM Timer0 interrupt in order
5217	 * to make sure the Host sees the Firmware Crash.  So if we have a
5218	 * Timer0 interrupt and don't see a Firmware Crash, ignore the Timer0
5219	 * interrupt.
5220	 */
5221	val = t4_read_reg(adapter, A_CIM_HOST_INT_CAUSE);
5222	if (val & F_TIMER0INT)
5223		if (!(fw_err & F_PCIE_FW_ERR) ||
5224		    (G_PCIE_FW_EVAL(fw_err) != PCIE_FW_EVAL_CRASH))
5225			t4_write_reg(adapter, A_CIM_HOST_INT_CAUSE,
5226				     F_TIMER0INT);
5227
5228	fat = t4_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE,
5229				    cim_intr_info) +
5230	      t4_handle_intr_status(adapter, A_CIM_HOST_UPACC_INT_CAUSE,
5231				    cim_upintr_info);
5232	if (fat)
5233		t4_fatal_err(adapter);
5234}
5235
5236/*
5237 * ULP RX interrupt handler.
5238 */
5239static void ulprx_intr_handler(struct adapter *adapter)
5240{
5241	static const struct intr_info ulprx_intr_info[] = {
5242		{ F_CAUSE_CTX_1, "ULPRX channel 1 context error", -1, 1 },
5243		{ F_CAUSE_CTX_0, "ULPRX channel 0 context error", -1, 1 },
5244		{ 0x7fffff, "ULPRX parity error", -1, 1 },
5245		{ 0 }
5246	};
5247
5248	if (t4_handle_intr_status(adapter, A_ULP_RX_INT_CAUSE, ulprx_intr_info))
5249		t4_fatal_err(adapter);
5250}
5251
5252/*
5253 * ULP TX interrupt handler.
5254 */
5255static void ulptx_intr_handler(struct adapter *adapter)
5256{
5257	static const struct intr_info ulptx_intr_info[] = {
5258		{ F_PBL_BOUND_ERR_CH3, "ULPTX channel 3 PBL out of bounds", -1,
5259		  0 },
5260		{ F_PBL_BOUND_ERR_CH2, "ULPTX channel 2 PBL out of bounds", -1,
5261		  0 },
5262		{ F_PBL_BOUND_ERR_CH1, "ULPTX channel 1 PBL out of bounds", -1,
5263		  0 },
5264		{ F_PBL_BOUND_ERR_CH0, "ULPTX channel 0 PBL out of bounds", -1,
5265		  0 },
5266		{ 0xfffffff, "ULPTX parity error", -1, 1 },
5267		{ 0 }
5268	};
5269
5270	if (t4_handle_intr_status(adapter, A_ULP_TX_INT_CAUSE, ulptx_intr_info))
5271		t4_fatal_err(adapter);
5272}
5273
5274/*
5275 * PM TX interrupt handler.
5276 */
5277static void pmtx_intr_handler(struct adapter *adapter)
5278{
5279	static const struct intr_info pmtx_intr_info[] = {
5280		{ F_PCMD_LEN_OVFL0, "PMTX channel 0 pcmd too large", -1, 1 },
5281		{ F_PCMD_LEN_OVFL1, "PMTX channel 1 pcmd too large", -1, 1 },
5282		{ F_PCMD_LEN_OVFL2, "PMTX channel 2 pcmd too large", -1, 1 },
5283		{ F_ZERO_C_CMD_ERROR, "PMTX 0-length pcmd", -1, 1 },
5284		{ 0xffffff0, "PMTX framing error", -1, 1 },
5285		{ F_OESPI_PAR_ERROR, "PMTX oespi parity error", -1, 1 },
5286		{ F_DB_OPTIONS_PAR_ERROR, "PMTX db_options parity error", -1,
5287		  1 },
5288		{ F_ICSPI_PAR_ERROR, "PMTX icspi parity error", -1, 1 },
5289		{ F_C_PCMD_PAR_ERROR, "PMTX c_pcmd parity error", -1, 1},
5290		{ 0 }
5291	};
5292
5293	if (t4_handle_intr_status(adapter, A_PM_TX_INT_CAUSE, pmtx_intr_info))
5294		t4_fatal_err(adapter);
5295}
5296
5297/*
5298 * PM RX interrupt handler.
5299 */
5300static void pmrx_intr_handler(struct adapter *adapter)
5301{
5302	static const struct intr_info pmrx_intr_info[] = {
5303		{ F_ZERO_E_CMD_ERROR, "PMRX 0-length pcmd", -1, 1 },
5304		{ 0x3ffff0, "PMRX framing error", -1, 1 },
5305		{ F_OCSPI_PAR_ERROR, "PMRX ocspi parity error", -1, 1 },
5306		{ F_DB_OPTIONS_PAR_ERROR, "PMRX db_options parity error", -1,
5307		  1 },
5308		{ F_IESPI_PAR_ERROR, "PMRX iespi parity error", -1, 1 },
5309		{ F_E_PCMD_PAR_ERROR, "PMRX e_pcmd parity error", -1, 1},
5310		{ 0 }
5311	};
5312
5313	if (t4_handle_intr_status(adapter, A_PM_RX_INT_CAUSE, pmrx_intr_info))
5314		t4_fatal_err(adapter);
5315}
5316
5317/*
5318 * CPL switch interrupt handler.
5319 */
5320static void cplsw_intr_handler(struct adapter *adapter)
5321{
5322	static const struct intr_info cplsw_intr_info[] = {
5323		{ F_CIM_OP_MAP_PERR, "CPLSW CIM op_map parity error", -1, 1 },
5324		{ F_CIM_OVFL_ERROR, "CPLSW CIM overflow", -1, 1 },
5325		{ F_TP_FRAMING_ERROR, "CPLSW TP framing error", -1, 1 },
5326		{ F_SGE_FRAMING_ERROR, "CPLSW SGE framing error", -1, 1 },
5327		{ F_CIM_FRAMING_ERROR, "CPLSW CIM framing error", -1, 1 },
5328		{ F_ZERO_SWITCH_ERROR, "CPLSW no-switch error", -1, 1 },
5329		{ 0 }
5330	};
5331
5332	if (t4_handle_intr_status(adapter, A_CPL_INTR_CAUSE, cplsw_intr_info))
5333		t4_fatal_err(adapter);
5334}
5335
5336/*
5337 * LE interrupt handler.
5338 */
5339static void le_intr_handler(struct adapter *adap)
5340{
5341	unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
5342	static const struct intr_info le_intr_info[] = {
5343		{ F_LIPMISS, "LE LIP miss", -1, 0 },
5344		{ F_LIP0, "LE 0 LIP error", -1, 0 },
5345		{ F_PARITYERR, "LE parity error", -1, 1 },
5346		{ F_UNKNOWNCMD, "LE unknown command", -1, 1 },
5347		{ F_REQQPARERR, "LE request queue parity error", -1, 1 },
5348		{ 0 }
5349	};
5350
5351	static struct intr_info t6_le_intr_info[] = {
5352		/* log an error for HASHTBLMEMCRCERR and clear the bit */
5353		{ F_T6_HASHTBLMEMCRCERR, "LE hash table mem crc error", -1, 0 },
5354		{ F_T6_LIPMISS, "LE LIP miss", -1, 0 },
5355		{ F_T6_LIP0, "LE 0 LIP error", -1, 0 },
5356		{ F_TCAMINTPERR, "LE parity error", -1, 1 },
5357		{ F_T6_UNKNOWNCMD, "LE unknown command", -1, 1 },
5358		{ F_SSRAMINTPERR, "LE request queue parity error", -1, 1 },
5359		{ 0 }
5360	};
5361
5362	if (t4_handle_intr_status(adap, A_LE_DB_INT_CAUSE,
5363				  (chip_ver <= CHELSIO_T5) ?
5364				  le_intr_info : t6_le_intr_info))
5365		t4_fatal_err(adap);
5366}
5367
5368/*
5369 * MPS interrupt handler.
5370 */
5371static void mps_intr_handler(struct adapter *adapter)
5372{
5373	static const struct intr_info mps_rx_intr_info[] = {
5374		{ 0xffffff, "MPS Rx parity error", -1, 1 },
5375		{ 0 }
5376	};
5377	static const struct intr_info mps_tx_intr_info[] = {
5378		{ V_TPFIFO(M_TPFIFO), "MPS Tx TP FIFO parity error", -1, 1 },
5379		{ F_NCSIFIFO, "MPS Tx NC-SI FIFO parity error", -1, 1 },
5380		{ V_TXDATAFIFO(M_TXDATAFIFO), "MPS Tx data FIFO parity error",
5381		  -1, 1 },
5382		{ V_TXDESCFIFO(M_TXDESCFIFO), "MPS Tx desc FIFO parity error",
5383		  -1, 1 },
5384		{ F_BUBBLE, "MPS Tx underflow", -1, 1 },
5385		{ F_SECNTERR, "MPS Tx SOP/EOP error", -1, 1 },
5386		{ F_FRMERR, "MPS Tx framing error", -1, 1 },
5387		{ 0 }
5388	};
5389	static const struct intr_info t6_mps_tx_intr_info[] = {
5390		{ V_TPFIFO(M_TPFIFO), "MPS Tx TP FIFO parity error", -1, 1 },
5391		{ F_NCSIFIFO, "MPS Tx NC-SI FIFO parity error", -1, 1 },
5392		{ V_TXDATAFIFO(M_TXDATAFIFO), "MPS Tx data FIFO parity error",
5393		  -1, 1 },
5394		{ V_TXDESCFIFO(M_TXDESCFIFO), "MPS Tx desc FIFO parity error",
5395		  -1, 1 },
5396		/* MPS Tx Bubble is normal for T6 */
5397		{ F_SECNTERR, "MPS Tx SOP/EOP error", -1, 1 },
5398		{ F_FRMERR, "MPS Tx framing error", -1, 1 },
5399		{ 0 }
5400	};
5401	static const struct intr_info mps_trc_intr_info[] = {
5402		{ V_FILTMEM(M_FILTMEM), "MPS TRC filter parity error", -1, 1 },
5403		{ V_PKTFIFO(M_PKTFIFO), "MPS TRC packet FIFO parity error", -1,
5404		  1 },
5405		{ F_MISCPERR, "MPS TRC misc parity error", -1, 1 },
5406		{ 0 }
5407	};
5408	static const struct intr_info mps_stat_sram_intr_info[] = {
5409		{ 0x1fffff, "MPS statistics SRAM parity error", -1, 1 },
5410		{ 0 }
5411	};
5412	static const struct intr_info mps_stat_tx_intr_info[] = {
5413		{ 0xfffff, "MPS statistics Tx FIFO parity error", -1, 1 },
5414		{ 0 }
5415	};
5416	static const struct intr_info mps_stat_rx_intr_info[] = {
5417		{ 0xffffff, "MPS statistics Rx FIFO parity error", -1, 1 },
5418		{ 0 }
5419	};
5420	static const struct intr_info mps_cls_intr_info[] = {
5421		{ F_MATCHSRAM, "MPS match SRAM parity error", -1, 1 },
5422		{ F_MATCHTCAM, "MPS match TCAM parity error", -1, 1 },
5423		{ F_HASHSRAM, "MPS hash SRAM parity error", -1, 1 },
5424		{ 0 }
5425	};
5426
5427	int fat;
5428
5429	fat = t4_handle_intr_status(adapter, A_MPS_RX_PERR_INT_CAUSE,
5430				    mps_rx_intr_info) +
5431	      t4_handle_intr_status(adapter, A_MPS_TX_INT_CAUSE,
5432				    is_t6(adapter->params.chip)
5433				    ? t6_mps_tx_intr_info
5434				    : mps_tx_intr_info) +
5435	      t4_handle_intr_status(adapter, A_MPS_TRC_INT_CAUSE,
5436				    mps_trc_intr_info) +
5437	      t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_SRAM,
5438				    mps_stat_sram_intr_info) +
5439	      t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_TX_FIFO,
5440				    mps_stat_tx_intr_info) +
5441	      t4_handle_intr_status(adapter, A_MPS_STAT_PERR_INT_CAUSE_RX_FIFO,
5442				    mps_stat_rx_intr_info) +
5443	      t4_handle_intr_status(adapter, A_MPS_CLS_INT_CAUSE,
5444				    mps_cls_intr_info);
5445
5446	t4_write_reg(adapter, A_MPS_INT_CAUSE, 0);
5447	t4_read_reg(adapter, A_MPS_INT_CAUSE);	/* flush */
5448	if (fat)
5449		t4_fatal_err(adapter);
5450}
5451
5452#define MEM_INT_MASK (F_PERR_INT_CAUSE | F_ECC_CE_INT_CAUSE | \
5453		      F_ECC_UE_INT_CAUSE)
5454
5455/*
5456 * EDC/MC interrupt handler.
5457 */
5458static void mem_intr_handler(struct adapter *adapter, int idx)
5459{
5460	static const char name[4][7] = { "EDC0", "EDC1", "MC/MC0", "MC1" };
5461
5462	unsigned int addr, cnt_addr, v;
5463
5464	if (idx <= MEM_EDC1) {
5465		addr = EDC_REG(A_EDC_INT_CAUSE, idx);
5466		cnt_addr = EDC_REG(A_EDC_ECC_STATUS, idx);
5467	} else if (idx == MEM_MC) {
5468		if (is_t4(adapter->params.chip)) {
5469			addr = A_MC_INT_CAUSE;
5470			cnt_addr = A_MC_ECC_STATUS;
5471		} else {
5472			addr = A_MC_P_INT_CAUSE;
5473			cnt_addr = A_MC_P_ECC_STATUS;
5474		}
5475	} else {
5476		addr = MC_REG(A_MC_P_INT_CAUSE, 1);
5477		cnt_addr = MC_REG(A_MC_P_ECC_STATUS, 1);
5478	}
5479
5480	v = t4_read_reg(adapter, addr) & MEM_INT_MASK;
5481	if (v & F_PERR_INT_CAUSE)
5482		CH_ALERT(adapter, "%s FIFO parity error\n",
5483			  name[idx]);
5484	if (v & F_ECC_CE_INT_CAUSE) {
5485		u32 cnt = G_ECC_CECNT(t4_read_reg(adapter, cnt_addr));
5486
5487		if (idx <= MEM_EDC1)
5488			t4_edc_err_read(adapter, idx);
5489
5490		t4_write_reg(adapter, cnt_addr, V_ECC_CECNT(M_ECC_CECNT));
5491		CH_WARN_RATELIMIT(adapter,
5492				  "%u %s correctable ECC data error%s\n",
5493				  cnt, name[idx], cnt > 1 ? "s" : "");
5494	}
5495	if (v & F_ECC_UE_INT_CAUSE)
5496		CH_ALERT(adapter,
5497			 "%s uncorrectable ECC data error\n", name[idx]);
5498
5499	t4_write_reg(adapter, addr, v);
5500	if (v & (F_PERR_INT_CAUSE | F_ECC_UE_INT_CAUSE))
5501		t4_fatal_err(adapter);
5502}
5503
5504/*
5505 * MA interrupt handler.
5506 */
5507static void ma_intr_handler(struct adapter *adapter)
5508{
5509	u32 v, status = t4_read_reg(adapter, A_MA_INT_CAUSE);
5510
5511	if (status & F_MEM_PERR_INT_CAUSE) {
5512		CH_ALERT(adapter,
5513			  "MA parity error, parity status %#x\n",
5514			  t4_read_reg(adapter, A_MA_PARITY_ERROR_STATUS1));
5515		if (is_t5(adapter->params.chip))
5516			CH_ALERT(adapter,
5517				  "MA parity error, parity status %#x\n",
5518				  t4_read_reg(adapter,
5519					      A_MA_PARITY_ERROR_STATUS2));
5520	}
5521	if (status & F_MEM_WRAP_INT_CAUSE) {
5522		v = t4_read_reg(adapter, A_MA_INT_WRAP_STATUS);
5523		CH_ALERT(adapter, "MA address wrap-around error by "
5524			  "client %u to address %#x\n",
5525			  G_MEM_WRAP_CLIENT_NUM(v),
5526			  G_MEM_WRAP_ADDRESS(v) << 4);
5527	}
5528	t4_write_reg(adapter, A_MA_INT_CAUSE, status);
5529	t4_fatal_err(adapter);
5530}
5531
5532/*
5533 * SMB interrupt handler.
5534 */
5535static void smb_intr_handler(struct adapter *adap)
5536{
5537	static const struct intr_info smb_intr_info[] = {
5538		{ F_MSTTXFIFOPARINT, "SMB master Tx FIFO parity error", -1, 1 },
5539		{ F_MSTRXFIFOPARINT, "SMB master Rx FIFO parity error", -1, 1 },
5540		{ F_SLVFIFOPARINT, "SMB slave FIFO parity error", -1, 1 },
5541		{ 0 }
5542	};
5543
5544	if (t4_handle_intr_status(adap, A_SMB_INT_CAUSE, smb_intr_info))
5545		t4_fatal_err(adap);
5546}
5547
5548/*
5549 * NC-SI interrupt handler.
5550 */
5551static void ncsi_intr_handler(struct adapter *adap)
5552{
5553	static const struct intr_info ncsi_intr_info[] = {
5554		{ F_CIM_DM_PRTY_ERR, "NC-SI CIM parity error", -1, 1 },
5555		{ F_MPS_DM_PRTY_ERR, "NC-SI MPS parity error", -1, 1 },
5556		{ F_TXFIFO_PRTY_ERR, "NC-SI Tx FIFO parity error", -1, 1 },
5557		{ F_RXFIFO_PRTY_ERR, "NC-SI Rx FIFO parity error", -1, 1 },
5558		{ 0 }
5559	};
5560
5561	if (t4_handle_intr_status(adap, A_NCSI_INT_CAUSE, ncsi_intr_info))
5562		t4_fatal_err(adap);
5563}
5564
5565/*
5566 * XGMAC interrupt handler.
5567 */
5568static void xgmac_intr_handler(struct adapter *adap, int port)
5569{
5570	u32 v, int_cause_reg;
5571
5572	if (is_t4(adap->params.chip))
5573		int_cause_reg = PORT_REG(port, A_XGMAC_PORT_INT_CAUSE);
5574	else
5575		int_cause_reg = T5_PORT_REG(port, A_MAC_PORT_INT_CAUSE);
5576
5577	v = t4_read_reg(adap, int_cause_reg);
5578
5579	v &= (F_TXFIFO_PRTY_ERR | F_RXFIFO_PRTY_ERR);
5580	if (!v)
5581		return;
5582
5583	if (v & F_TXFIFO_PRTY_ERR)
5584		CH_ALERT(adap, "XGMAC %d Tx FIFO parity error\n",
5585			  port);
5586	if (v & F_RXFIFO_PRTY_ERR)
5587		CH_ALERT(adap, "XGMAC %d Rx FIFO parity error\n",
5588			  port);
5589	t4_write_reg(adap, int_cause_reg, v);
5590	t4_fatal_err(adap);
5591}
5592
5593/*
5594 * PL Parity Error interrupt handler.
5595 */
5596static void pl_perr_intr_handler(struct adapter *adap)
5597{
5598	static const struct intr_info pl_perr_info[] = {
5599		{ F_UART, "UART Parity Error", -1, },
5600		{ F_ULP_TX, "ULP TX Parity Error", -1 },
5601		{ F_SGE, "SGE Parity Error", -1 },
5602		{ F_HMA, "HMA Parity Error", -1 },
5603		{ F_CPL_SWITCH, "CPL Switch Parity Error", -1 },
5604		{ F_ULP_RX, "ULP RX Parity Error", -1 },
5605		{ F_PM_RX, "PM RX Parity Error", -1 },
5606		{ F_PM_TX, "PM TX Parity Error", -1 },
5607		{ F_MA, "MA Parity Error", -1 },
5608		{ F_TP, "TP Parity Error", -1 },
5609		{ F_LE, "LE Parity Error", -1 },
5610		{ F_EDC1, "EDC1 Parity Error", -1 },
5611		{ F_EDC0, "EDC0 Parity Error", -1 },
5612		{ F_MC, "MC Parity Error", -1 },
5613		{ F_PCIE, "PCIE Parity Error", -1 },
5614		{ F_PMU, "PMU Parity Error", -1 },
5615		{ F_XGMAC_KR1, "XGMAC_KR1 Parity Error", -1 },
5616		{ F_XGMAC_KR0, "XGMAC_KR0 Parity Error", -1 },
5617		{ F_XGMAC1, "XGMAC1 Parity Error", -1 },
5618		{ F_XGMAC0, "XGMAC0 Parity Error", -1 },
5619		{ F_SMB, "SMB Parity Error", -1 },
5620		{ F_SF, "SF Parity Error", -1 },
5621		{ F_PL, "PL Parity Error", -1 },
5622		{ F_NCSI, "NCSI Parity Error", -1 },
5623		{ F_MPS, "MPS Parity Error", -1 },
5624		{ F_MI, "MI Parity Error", -1 },
5625		{ F_DBG, "DBG Parity Error", -1 },
5626		{ F_I2CM, "I2CM Parity Error", -1 },
5627		{ F_CIM, "CIM Parity Error", -1 },
5628	};
5629
5630	t4_handle_intr_status(adap, A_PL_PERR_CAUSE, pl_perr_info);
5631	/* pl_intr_handler() will do the t4_fatal_err(adap) */
5632}
5633
5634/*
5635 * PL interrupt handler.
5636 */
5637static void pl_intr_handler(struct adapter *adap)
5638{
5639	static const struct intr_info pl_intr_info[] = {
5640		{ F_FATALPERR, "Fatal parity error", -1, 1,
5641		  pl_perr_intr_handler },
5642		{ F_PERRVFID, "PL VFID_MAP parity error", -1, 1 },
5643		{ 0 }
5644	};
5645
5646	static struct intr_info t5_pl_intr_info[] = {
5647		{ F_FATALPERR, "Fatal parity error", -1, 1,
5648		  pl_perr_intr_handler },
5649		{ 0 }
5650	};
5651
5652	if (t4_handle_intr_status(adap, A_PL_PL_INT_CAUSE,
5653				  is_t4(adap->params.chip) ?
5654				  pl_intr_info : t5_pl_intr_info))
5655		t4_fatal_err(adap);
5656}
5657
5658#define PF_INTR_MASK (F_PFSW | F_PFCIM)
5659
5660/**
5661 *	t4_slow_intr_handler - control path interrupt handler
5662 *	@adapter: the adapter
5663 *
5664 *	T4 interrupt handler for non-data global interrupt events, e.g., errors.
5665 *	The designation 'slow' is because it involves register reads, while
5666 *	data interrupts typically don't involve any MMIOs.
5667 */
5668int t4_slow_intr_handler(struct adapter *adapter)
5669{
5670	/* There are rare cases where a PL_INT_CAUSE bit may end up getting
5671	 * set when the corresponding PL_INT_ENABLE bit isn't set.  It's
5672	 * easiest just to mask that case here.
5673	 */
5674	u32 raw_cause = t4_read_reg(adapter, A_PL_INT_CAUSE);
5675	u32 enable = t4_read_reg(adapter, A_PL_INT_ENABLE);
5676	u32 cause = raw_cause & enable;
5677
5678	if (!(cause & GLBL_INTR_MASK))
5679		return 0;
5680
5681	/* Disable all the interrupt(bits) in PL_INT_ENABLE */
5682	t4_write_reg(adapter, A_PL_INT_ENABLE, 0);
5683	(void)t4_read_reg(adapter, A_PL_INT_ENABLE); /* flush */
5684
5685	if (cause & F_CIM)
5686		cim_intr_handler(adapter);
5687	if (cause & F_MPS)
5688		mps_intr_handler(adapter);
5689	if (cause & F_NCSI)
5690		ncsi_intr_handler(adapter);
5691	if (cause & F_PL)
5692		pl_intr_handler(adapter);
5693	if (cause & F_SMB)
5694		smb_intr_handler(adapter);
5695	if (cause & F_MAC0)
5696		xgmac_intr_handler(adapter, 0);
5697	if (cause & F_MAC1)
5698		xgmac_intr_handler(adapter, 1);
5699	if (cause & F_MAC2)
5700		xgmac_intr_handler(adapter, 2);
5701	if (cause & F_MAC3)
5702		xgmac_intr_handler(adapter, 3);
5703	if (cause & F_PCIE)
5704		pcie_intr_handler(adapter);
5705	if (cause & F_MC0)
5706		mem_intr_handler(adapter, MEM_MC);
5707	if (is_t5(adapter->params.chip) && (cause & F_MC1))
5708		mem_intr_handler(adapter, MEM_MC1);
5709	if (cause & F_EDC0)
5710		mem_intr_handler(adapter, MEM_EDC0);
5711	if (cause & F_EDC1)
5712		mem_intr_handler(adapter, MEM_EDC1);
5713	if (cause & F_LE)
5714		le_intr_handler(adapter);
5715	if (cause & F_TP)
5716		tp_intr_handler(adapter);
5717	if (cause & F_MA)
5718		ma_intr_handler(adapter);
5719	if (cause & F_PM_TX)
5720		pmtx_intr_handler(adapter);
5721	if (cause & F_PM_RX)
5722		pmrx_intr_handler(adapter);
5723	if (cause & F_ULP_RX)
5724		ulprx_intr_handler(adapter);
5725	if (cause & F_CPL_SWITCH)
5726		cplsw_intr_handler(adapter);
5727	if (cause & F_SGE)
5728		sge_intr_handler(adapter);
5729	if (cause & F_ULP_TX)
5730		ulptx_intr_handler(adapter);
5731
5732	/* Clear the interrupts just processed for which we are the master. */
5733	t4_write_reg(adapter, A_PL_INT_CAUSE, raw_cause & GLBL_INTR_MASK);
5734
5735	/* re-enable the interrupts (bits that were disabled
5736	 * earlier in PL_INT_ENABLE)
5737	 */
5738	t4_write_reg(adapter, A_PL_INT_ENABLE, enable);
5739	(void)t4_read_reg(adapter, A_PL_INT_ENABLE); /* flush */
5740	return 1;
5741}
5742
5743/**
5744 *	t4_intr_enable - enable interrupts
5745 *	@adapter: the adapter whose interrupts should be enabled
5746 *
5747 *	Enable PF-specific interrupts for the calling function and the top-level
5748 *	interrupt concentrator for global interrupts.  Interrupts are already
5749 *	enabled at each module,	here we just enable the roots of the interrupt
5750 *	hierarchies.
5751 *
5752 *	Note: this function should be called only when the driver manages
5753 *	non PF-specific interrupts from the various HW modules.  Only one PCI
5754 *	function at a time should be doing this.
5755 */
5756void t4_intr_enable(struct adapter *adapter)
5757{
5758	u32 val = 0;
5759	u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI);
5760	u32 pf = (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5
5761		  ? G_SOURCEPF(whoami)
5762		  : G_T6_SOURCEPF(whoami));
5763
5764	if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5)
5765		val = F_ERR_DROPPED_DB | F_ERR_EGR_CTXT_PRIO | F_DBFIFO_HP_INT;
5766	else
5767		val = F_ERR_PCIE_ERROR0 | F_ERR_PCIE_ERROR1 | F_FATAL_WRE_LEN;
5768	t4_write_reg(adapter, A_SGE_INT_ENABLE3, F_ERR_CPL_EXCEED_IQE_SIZE |
5769		     F_ERR_INVALID_CIDX_INC | F_ERR_CPL_OPCODE_0 |
5770		     F_ERR_DATA_CPL_ON_HIGH_QID1 | F_INGRESS_SIZE_ERR |
5771		     F_ERR_DATA_CPL_ON_HIGH_QID0 | F_ERR_BAD_DB_PIDX3 |
5772		     F_ERR_BAD_DB_PIDX2 | F_ERR_BAD_DB_PIDX1 |
5773		     F_ERR_BAD_DB_PIDX0 | F_ERR_ING_CTXT_PRIO |
5774		     F_DBFIFO_LP_INT | F_EGRESS_SIZE_ERR | val);
5775	t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), PF_INTR_MASK);
5776	t4_set_reg_field(adapter, A_PL_INT_MAP0, 0, 1 << pf);
5777}
5778
5779/**
5780 *	t4_intr_disable - disable interrupts
5781 *	@adapter: the adapter whose interrupts should be disabled
5782 *
5783 *	Disable interrupts.  We only disable the top-level interrupt
5784 *	concentrators.  The caller must be a PCI function managing global
5785 *	interrupts.
5786 */
5787void t4_intr_disable(struct adapter *adapter)
5788{
5789	u32 whoami = t4_read_reg(adapter, A_PL_WHOAMI);
5790	u32 pf = (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5
5791		  ? G_SOURCEPF(whoami)
5792		  : G_T6_SOURCEPF(whoami));
5793
5794	t4_write_reg(adapter, MYPF_REG(A_PL_PF_INT_ENABLE), 0);
5795	t4_set_reg_field(adapter, A_PL_INT_MAP0, 1 << pf, 0);
5796}
5797
5798unsigned int t4_chip_rss_size(struct adapter *adap)
5799{
5800	if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
5801		return RSS_NENTRIES;
5802	else
5803		return T6_RSS_NENTRIES;
5804}
5805
5806/**
5807 *	t4_config_rss_range - configure a portion of the RSS mapping table
5808 *	@adapter: the adapter
5809 *	@mbox: mbox to use for the FW command
5810 *	@viid: virtual interface whose RSS subtable is to be written
5811 *	@start: start entry in the table to write
5812 *	@n: how many table entries to write
5813 *	@rspq: values for the "response queue" (Ingress Queue) lookup table
5814 *	@nrspq: number of values in @rspq
5815 *
5816 *	Programs the selected part of the VI's RSS mapping table with the
5817 *	provided values.  If @nrspq < @n the supplied values are used repeatedly
5818 *	until the full table range is populated.
5819 *
5820 *	The caller must ensure the values in @rspq are in the range allowed for
5821 *	@viid.
5822 */
5823int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
5824			int start, int n, const u16 *rspq, unsigned int nrspq)
5825{
5826	int ret;
5827	const u16 *rsp = rspq;
5828	const u16 *rsp_end = rspq + nrspq;
5829	struct fw_rss_ind_tbl_cmd cmd;
5830
5831	memset(&cmd, 0, sizeof(cmd));
5832	cmd.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_RSS_IND_TBL_CMD) |
5833				     F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
5834				     V_FW_RSS_IND_TBL_CMD_VIID(viid));
5835	cmd.retval_len16 = cpu_to_be32(FW_LEN16(cmd));
5836
5837	/* Each firmware RSS command can accommodate up to 32 RSS Ingress
5838	 * Queue Identifiers.  These Ingress Queue IDs are packed three to
5839	 * a 32-bit word as 10-bit values with the upper remaining 2 bits
5840	 * reserved.
5841	 */
5842	while (n > 0) {
5843		int nq = min(n, 32);
5844		int nq_packed = 0;
5845		__be32 *qp = &cmd.iq0_to_iq2;
5846
5847		/* Set up the firmware RSS command header to send the next
5848		 * "nq" Ingress Queue IDs to the firmware.
5849		 */
5850		cmd.niqid = cpu_to_be16(nq);
5851		cmd.startidx = cpu_to_be16(start);
5852
5853		/* "nq" more done for the start of the next loop.
5854		 */
5855		start += nq;
5856		n -= nq;
5857
5858		/* While there are still Ingress Queue IDs to stuff into the
5859		 * current firmware RSS command, retrieve them from the
5860		 * Ingress Queue ID array and insert them into the command.
5861		 */
5862		while (nq > 0) {
5863			/* Grab up to the next 3 Ingress Queue IDs (wrapping
5864			 * around the Ingress Queue ID array if necessary) and
5865			 * insert them into the firmware RSS command at the
5866			 * current 3-tuple position within the commad.
5867			 */
5868			u16 qbuf[3];
5869			u16 *qbp = qbuf;
5870			int nqbuf = min(3, nq);
5871
5872			nq -= nqbuf;
5873			qbuf[0] = qbuf[1] = qbuf[2] = 0;
5874			while (nqbuf && nq_packed < 32) {
5875				nqbuf--;
5876				nq_packed++;
5877				*qbp++ = *rsp++;
5878				if (rsp >= rsp_end)
5879					rsp = rspq;
5880			}
5881			*qp++ = cpu_to_be32(V_FW_RSS_IND_TBL_CMD_IQ0(qbuf[0]) |
5882					    V_FW_RSS_IND_TBL_CMD_IQ1(qbuf[1]) |
5883					    V_FW_RSS_IND_TBL_CMD_IQ2(qbuf[2]));
5884		}
5885
5886		/* Send this portion of the RRS table update to the firmware;
5887		 * bail out on any errors.
5888		 */
5889		ret = t4_wr_mbox(adapter, mbox, &cmd, sizeof(cmd), NULL);
5890		if (ret)
5891			return ret;
5892	}
5893	return 0;
5894}
5895
5896/**
5897 *	t4_config_glbl_rss - configure the global RSS mode
5898 *	@adapter: the adapter
5899 *	@mbox: mbox to use for the FW command
5900 *	@mode: global RSS mode
5901 *	@flags: mode-specific flags
5902 *
5903 *	Sets the global RSS mode.
5904 */
5905int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
5906		       unsigned int flags)
5907{
5908	struct fw_rss_glb_config_cmd c;
5909
5910	memset(&c, 0, sizeof(c));
5911	c.op_to_write = cpu_to_be32(V_FW_CMD_OP(FW_RSS_GLB_CONFIG_CMD) |
5912				    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
5913	c.retval_len16 = cpu_to_be32(FW_LEN16(c));
5914	if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_MANUAL) {
5915		c.u.manual.mode_pkd =
5916			cpu_to_be32(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode));
5917	} else if (mode == FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
5918		c.u.basicvirtual.mode_keymode =
5919			cpu_to_be32(V_FW_RSS_GLB_CONFIG_CMD_MODE(mode));
5920		c.u.basicvirtual.synmapen_to_hashtoeplitz = cpu_to_be32(flags);
5921	} else
5922		return -EINVAL;
5923	return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
5924}
5925
5926/**
5927 *	t4_config_vi_rss - configure per VI RSS settings
5928 *	@adapter: the adapter
5929 *	@mbox: mbox to use for the FW command
5930 *	@viid: the VI id
5931 *	@flags: RSS flags
5932 *	@defq: id of the default RSS queue for the VI.
5933 *	@skeyidx: RSS secret key table index for non-global mode
5934 *	@skey: RSS vf_scramble key for VI.
5935 *
5936 *	Configures VI-specific RSS properties.
5937 */
5938int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
5939		     unsigned int flags, unsigned int defq, unsigned int skeyidx,
5940		     unsigned int skey)
5941{
5942	struct fw_rss_vi_config_cmd c;
5943
5944	memset(&c, 0, sizeof(c));
5945	c.op_to_viid = cpu_to_be32(V_FW_CMD_OP(FW_RSS_VI_CONFIG_CMD) |
5946				   F_FW_CMD_REQUEST | F_FW_CMD_WRITE |
5947				   V_FW_RSS_VI_CONFIG_CMD_VIID(viid));
5948	c.retval_len16 = cpu_to_be32(FW_LEN16(c));
5949	c.u.basicvirtual.defaultq_to_udpen = cpu_to_be32(flags |
5950					V_FW_RSS_VI_CONFIG_CMD_DEFAULTQ(defq));
5951	c.u.basicvirtual.secretkeyidx_pkd = cpu_to_be32(
5952					V_FW_RSS_VI_CONFIG_CMD_SECRETKEYIDX(skeyidx));
5953	c.u.basicvirtual.secretkeyxor = cpu_to_be32(skey);
5954
5955	return t4_wr_mbox(adapter, mbox, &c, sizeof(c), NULL);
5956}
5957
5958/* Read an RSS table row */
5959static int rd_rss_row(struct adapter *adap, int row, u32 *val)
5960{
5961	t4_write_reg(adap, A_TP_RSS_LKP_TABLE, 0xfff00000 | row);
5962	return t4_wait_op_done_val(adap, A_TP_RSS_LKP_TABLE, F_LKPTBLROWVLD, 1,
5963				   5, 0, val);
5964}
5965
5966/**
5967 *	t4_read_rss - read the contents of the RSS mapping table
5968 *	@adapter: the adapter
5969 *	@map: holds the contents of the RSS mapping table
5970 *
5971 *	Reads the contents of the RSS hash->queue mapping table.
5972 */
5973int t4_read_rss(struct adapter *adapter, u16 *map)
5974{
5975	u32 val;
5976	int i, ret, nentries;
5977
5978	nentries = t4_chip_rss_size(adapter);
5979	for (i = 0; i < nentries / 2; ++i) {
5980		ret = rd_rss_row(adapter, i, &val);
5981		if (ret)
5982			return ret;
5983		*map++ = G_LKPTBLQUEUE0(val);
5984		*map++ = G_LKPTBLQUEUE1(val);
5985	}
5986	return 0;
5987}
5988
5989/**
5990 * t4_tp_fw_ldst_rw - Access TP indirect register through LDST
5991 * @adap: the adapter
5992 * @cmd: TP fw ldst address space type
5993 * @vals: where the indirect register values are stored/written
5994 * @nregs: how many indirect registers to read/write
5995 * @start_idx: index of first indirect register to read/write
5996 * @rw: Read (1) or Write (0)
5997 * @sleep_ok: if true we may sleep while awaiting command completion
5998 *
5999 * Access TP indirect registers through LDST
6000 **/
6001static int t4_tp_fw_ldst_rw(struct adapter *adap, int cmd, u32 *vals,
6002			    unsigned int nregs, unsigned int start_index,
6003			    unsigned int rw, bool sleep_ok)
6004{
6005	int ret = 0;
6006	unsigned int i;
6007	struct fw_ldst_cmd c;
6008
6009	for (i = 0; i < nregs; i++) {
6010		memset(&c, 0, sizeof(c));
6011		c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
6012						F_FW_CMD_REQUEST |
6013						(rw ? F_FW_CMD_READ :
6014						      F_FW_CMD_WRITE) |
6015						V_FW_LDST_CMD_ADDRSPACE(cmd));
6016		c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
6017
6018		c.u.addrval.addr = cpu_to_be32(start_index + i);
6019		c.u.addrval.val  = rw ? 0 : cpu_to_be32(vals[i]);
6020		ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c,
6021				      sleep_ok);
6022		if (ret)
6023			return ret;
6024
6025		if (rw)
6026			vals[i] = be32_to_cpu(c.u.addrval.val);
6027	}
6028	return 0;
6029}
6030
6031/**
6032 * t4_tp_indirect_rw - Read/Write TP indirect register through LDST or backdoor
6033 * @adap: the adapter
6034 * @reg_addr: Address Register
6035 * @reg_data: Data register
6036 * @buff: where the indirect register values are stored/written
6037 * @nregs: how many indirect registers to read/write
6038 * @start_index: index of first indirect register to read/write
6039 * @rw: READ(1) or WRITE(0)
6040 * @sleep_ok: if true we may sleep while awaiting command completion
6041 *
6042 * Read/Write TP indirect registers through LDST if possible.
6043 * Else, use backdoor access
6044 **/
6045static void t4_tp_indirect_rw(struct adapter *adap, u32 reg_addr, u32 reg_data,
6046			      u32 *buff, u32 nregs, u32 start_index, int rw,
6047			      bool sleep_ok)
6048{
6049	int rc = -EINVAL;
6050	int cmd;
6051
6052	switch (reg_addr) {
6053	case A_TP_PIO_ADDR:
6054		cmd = FW_LDST_ADDRSPC_TP_PIO;
6055		break;
6056	case A_TP_TM_PIO_ADDR:
6057		cmd = FW_LDST_ADDRSPC_TP_TM_PIO;
6058		break;
6059	case A_TP_MIB_INDEX:
6060		cmd = FW_LDST_ADDRSPC_TP_MIB;
6061		break;
6062	default:
6063		goto indirect_access;
6064	}
6065
6066	if (t4_use_ldst(adap))
6067		rc = t4_tp_fw_ldst_rw(adap, cmd, buff, nregs, start_index, rw,
6068				      sleep_ok);
6069
6070indirect_access:
6071
6072	if (rc) {
6073		if (rw)
6074			t4_read_indirect(adap, reg_addr, reg_data, buff, nregs,
6075					 start_index);
6076		else
6077			t4_write_indirect(adap, reg_addr, reg_data, buff, nregs,
6078					  start_index);
6079	}
6080}
6081
6082/**
6083 * t4_tp_pio_read - Read TP PIO registers
6084 * @adap: the adapter
6085 * @buff: where the indirect register values are written
6086 * @nregs: how many indirect registers to read
6087 * @start_index: index of first indirect register to read
6088 * @sleep_ok: if true we may sleep while awaiting command completion
6089 *
6090 * Read TP PIO Registers
6091 **/
6092void t4_tp_pio_read(struct adapter *adap, u32 *buff, u32 nregs,
6093		    u32 start_index, bool sleep_ok)
6094{
6095	t4_tp_indirect_rw(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, buff, nregs,
6096			  start_index, 1, sleep_ok);
6097}
6098
6099/**
6100 * t4_tp_pio_write - Write TP PIO registers
6101 * @adap: the adapter
6102 * @buff: where the indirect register values are stored
6103 * @nregs: how many indirect registers to write
6104 * @start_index: index of first indirect register to write
6105 * @sleep_ok: if true we may sleep while awaiting command completion
6106 *
6107 * Write TP PIO Registers
6108 **/
6109void t4_tp_pio_write(struct adapter *adap, u32 *buff, u32 nregs,
6110		     u32 start_index, bool sleep_ok)
6111{
6112	t4_tp_indirect_rw(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, buff, nregs,
6113			  start_index, 0, sleep_ok);
6114}
6115
6116/**
6117 * t4_tp_tm_pio_read - Read TP TM PIO registers
6118 * @adap: the adapter
6119 * @buff: where the indirect register values are written
6120 * @nregs: how many indirect registers to read
6121 * @start_index: index of first indirect register to read
6122 * @sleep_ok: if true we may sleep while awaiting command completion
6123 *
6124 * Read TP TM PIO Registers
6125 **/
6126void t4_tp_tm_pio_read(struct adapter *adap, u32 *buff, u32 nregs,
6127		       u32 start_index, bool sleep_ok)
6128{
6129	t4_tp_indirect_rw(adap, A_TP_TM_PIO_ADDR, A_TP_TM_PIO_DATA, buff,
6130			  nregs, start_index, 1, sleep_ok);
6131}
6132
6133/**
6134 * t4_tp_mib_read - Read TP MIB registers
6135 * @adap: the adapter
6136 * @buff: where the indirect register values are written
6137 * @nregs: how many indirect registers to read
6138 * @start_index: index of first indirect register to read
6139 * @sleep_ok: if true we may sleep while awaiting command completion
6140 *
6141 * Read TP MIB Registers
6142 **/
6143void t4_tp_mib_read(struct adapter *adap, u32 *buff, u32 nregs, u32 start_index,
6144		    bool sleep_ok)
6145{
6146	t4_tp_indirect_rw(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, buff, nregs,
6147			  start_index, 1, sleep_ok);
6148}
6149
6150/**
6151 *	t4_read_rss_key - read the global RSS key
6152 *	@adap: the adapter
6153 *	@key: 10-entry array holding the 320-bit RSS key
6154 * 	@sleep_ok: if true we may sleep while awaiting command completion
6155 *
6156 *	Reads the global 320-bit RSS key.
6157 */
6158void t4_read_rss_key(struct adapter *adap, u32 *key, bool sleep_ok)
6159{
6160	t4_tp_pio_read(adap, key, 10, A_TP_RSS_SECRET_KEY0, sleep_ok);
6161}
6162
6163/**
6164 *	t4_write_rss_key - program one of the RSS keys
6165 *	@adap: the adapter
6166 *	@key: 10-entry array holding the 320-bit RSS key
6167 *	@idx: which RSS key to write
6168 * 	@sleep_ok: if true we may sleep while awaiting command completion
6169 *
6170 *	Writes one of the RSS keys with the given 320-bit value.  If @idx is
6171 *	0..15 the corresponding entry in the RSS key table is written,
6172 *	otherwise the global RSS key is written.
6173 */
6174void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx,
6175		      bool sleep_ok)
6176{
6177	u8 rss_key_addr_cnt = 16;
6178	u32 vrt = t4_read_reg(adap, A_TP_RSS_CONFIG_VRT);
6179
6180	/* T6 and later: for KeyMode 3 (per-vf and per-vf scramble),
6181	 * allows access to key addresses 16-63 by using KeyWrAddrX
6182	 * as index[5:4](upper 2) into key table
6183	 */
6184	if ((CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) &&
6185	    (vrt & F_KEYEXTEND) && (G_KEYMODE(vrt) == 3))
6186		rss_key_addr_cnt = 32;
6187
6188	t4_tp_pio_write(adap, (void *)key, 10, A_TP_RSS_SECRET_KEY0, sleep_ok);
6189
6190	if (idx >= 0 && idx < rss_key_addr_cnt) {
6191		if (rss_key_addr_cnt > 16)
6192			t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
6193				     vrt | V_KEYWRADDRX(idx >> 4) |
6194				     V_T6_VFWRADDR(idx) | F_KEYWREN);
6195		else
6196			t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
6197				     vrt| V_KEYWRADDR(idx) | F_KEYWREN);
6198	}
6199}
6200
6201/**
6202 *	t4_read_rss_pf_config - read PF RSS Configuration Table
6203 *	@adapter: the adapter
6204 *	@index: the entry in the PF RSS table to read
6205 *	@valp: where to store the returned value
6206 * 	@sleep_ok: if true we may sleep while awaiting command completion
6207 *
6208 *	Reads the PF RSS Configuration Table at the specified index and returns
6209 *	the value found there.
6210 */
6211void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index,
6212			   u32 *valp, bool sleep_ok)
6213{
6214	t4_tp_pio_read(adapter, valp, 1, A_TP_RSS_PF0_CONFIG + index, sleep_ok);
6215}
6216
6217/**
6218 *	t4_write_rss_pf_config - write PF RSS Configuration Table
6219 *	@adapter: the adapter
6220 *	@index: the entry in the VF RSS table to read
6221 *	@val: the value to store
6222 * 	@sleep_ok: if true we may sleep while awaiting command completion
6223 *
6224 *	Writes the PF RSS Configuration Table at the specified index with the
6225 *	specified value.
6226 */
6227void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index,
6228			    u32 val, bool sleep_ok)
6229{
6230	t4_tp_pio_write(adapter, &val, 1, A_TP_RSS_PF0_CONFIG + index,
6231			sleep_ok);
6232}
6233
6234/**
6235 *	t4_read_rss_vf_config - read VF RSS Configuration Table
6236 *	@adapter: the adapter
6237 *	@index: the entry in the VF RSS table to read
6238 *	@vfl: where to store the returned VFL
6239 *	@vfh: where to store the returned VFH
6240 * 	@sleep_ok: if true we may sleep while awaiting command completion
6241 *
6242 *	Reads the VF RSS Configuration Table at the specified index and returns
6243 *	the (VFL, VFH) values found there.
6244 */
6245void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
6246			   u32 *vfl, u32 *vfh, bool sleep_ok)
6247{
6248	u32 vrt, mask, data;
6249
6250	if (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) {
6251		mask = V_VFWRADDR(M_VFWRADDR);
6252		data = V_VFWRADDR(index);
6253	} else {
6254		 mask =  V_T6_VFWRADDR(M_T6_VFWRADDR);
6255		 data = V_T6_VFWRADDR(index);
6256	}
6257	/*
6258	 * Request that the index'th VF Table values be read into VFL/VFH.
6259	 */
6260	vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT);
6261	vrt &= ~(F_VFRDRG | F_VFWREN | F_KEYWREN | mask);
6262	vrt |= data | F_VFRDEN;
6263	t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt);
6264
6265	/*
6266	 * Grab the VFL/VFH values ...
6267	 */
6268	t4_tp_pio_read(adapter, vfl, 1, A_TP_RSS_VFL_CONFIG, sleep_ok);
6269	t4_tp_pio_read(adapter, vfh, 1, A_TP_RSS_VFH_CONFIG, sleep_ok);
6270}
6271
6272/**
6273 *	t4_read_rss_pf_map - read PF RSS Map
6274 *	@adapter: the adapter
6275 * 	@sleep_ok: if true we may sleep while awaiting command completion
6276 *
6277 *	Reads the PF RSS Map register and returns its value.
6278 */
6279u32 t4_read_rss_pf_map(struct adapter *adapter, bool sleep_ok)
6280{
6281	u32 pfmap;
6282
6283	t4_tp_pio_read(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, sleep_ok);
6284
6285	return pfmap;
6286}
6287
6288/**
6289 *	t4_read_rss_pf_mask - read PF RSS Mask
6290 *	@adapter: the adapter
6291 * 	@sleep_ok: if true we may sleep while awaiting command completion
6292 *
6293 *	Reads the PF RSS Mask register and returns its value.
6294 */
6295u32 t4_read_rss_pf_mask(struct adapter *adapter, bool sleep_ok)
6296{
6297	u32 pfmask;
6298
6299	t4_tp_pio_read(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, sleep_ok);
6300
6301	return pfmask;
6302}
6303
6304/**
6305 *	t4_tp_get_tcp_stats - read TP's TCP MIB counters
6306 *	@adap: the adapter
6307 *	@v4: holds the TCP/IP counter values
6308 *	@v6: holds the TCP/IPv6 counter values
6309 * 	@sleep_ok: if true we may sleep while awaiting command completion
6310 *
6311 *	Returns the values of TP's TCP/IP and TCP/IPv6 MIB counters.
6312 *	Either @v4 or @v6 may be %NULL to skip the corresponding stats.
6313 */
6314void t4_tp_get_tcp_stats(struct adapter *adap, struct tp_tcp_stats *v4,
6315			 struct tp_tcp_stats *v6, bool sleep_ok)
6316{
6317	u32 val[A_TP_MIB_TCP_RXT_SEG_LO - A_TP_MIB_TCP_OUT_RST + 1];
6318
6319#define STAT_IDX(x) ((A_TP_MIB_TCP_##x) - A_TP_MIB_TCP_OUT_RST)
6320#define STAT(x)     val[STAT_IDX(x)]
6321#define STAT64(x)   (((u64)STAT(x##_HI) << 32) | STAT(x##_LO))
6322
6323	if (v4) {
6324		t4_tp_mib_read(adap, val, ARRAY_SIZE(val),
6325			       A_TP_MIB_TCP_OUT_RST, sleep_ok);
6326		v4->tcp_out_rsts = STAT(OUT_RST);
6327		v4->tcp_in_segs  = STAT64(IN_SEG);
6328		v4->tcp_out_segs = STAT64(OUT_SEG);
6329		v4->tcp_retrans_segs = STAT64(RXT_SEG);
6330	}
6331	if (v6) {
6332		t4_tp_mib_read(adap, val, ARRAY_SIZE(val),
6333			       A_TP_MIB_TCP_V6OUT_RST, sleep_ok);
6334		v6->tcp_out_rsts = STAT(OUT_RST);
6335		v6->tcp_in_segs  = STAT64(IN_SEG);
6336		v6->tcp_out_segs = STAT64(OUT_SEG);
6337		v6->tcp_retrans_segs = STAT64(RXT_SEG);
6338	}
6339#undef STAT64
6340#undef STAT
6341#undef STAT_IDX
6342}
6343
6344/**
6345 *	t4_tp_get_err_stats - read TP's error MIB counters
6346 *	@adap: the adapter
6347 *	@st: holds the counter values
6348 * 	@sleep_ok: if true we may sleep while awaiting command completion
6349 *
6350 *	Returns the values of TP's error counters.
6351 */
6352void t4_tp_get_err_stats(struct adapter *adap, struct tp_err_stats *st,
6353			 bool sleep_ok)
6354{
6355	int nchan = adap->params.arch.nchan;
6356
6357	t4_tp_mib_read(adap, st->mac_in_errs, nchan, A_TP_MIB_MAC_IN_ERR_0,
6358		       sleep_ok);
6359
6360	t4_tp_mib_read(adap, st->hdr_in_errs, nchan, A_TP_MIB_HDR_IN_ERR_0,
6361		       sleep_ok);
6362
6363	t4_tp_mib_read(adap, st->tcp_in_errs, nchan, A_TP_MIB_TCP_IN_ERR_0,
6364		       sleep_ok);
6365
6366	t4_tp_mib_read(adap, st->tnl_cong_drops, nchan,
6367		       A_TP_MIB_TNL_CNG_DROP_0, sleep_ok);
6368
6369	t4_tp_mib_read(adap, st->ofld_chan_drops, nchan,
6370		       A_TP_MIB_OFD_CHN_DROP_0, sleep_ok);
6371
6372	t4_tp_mib_read(adap, st->tnl_tx_drops, nchan, A_TP_MIB_TNL_DROP_0,
6373		       sleep_ok);
6374
6375	t4_tp_mib_read(adap, st->ofld_vlan_drops, nchan,
6376		       A_TP_MIB_OFD_VLN_DROP_0, sleep_ok);
6377
6378	t4_tp_mib_read(adap, st->tcp6_in_errs, nchan,
6379		       A_TP_MIB_TCP_V6IN_ERR_0, sleep_ok);
6380
6381	t4_tp_mib_read(adap, &st->ofld_no_neigh, 2, A_TP_MIB_OFD_ARP_DROP,
6382		       sleep_ok);
6383}
6384
6385/**
6386 *	t4_tp_get_cpl_stats - read TP's CPL MIB counters
6387 *	@adap: the adapter
6388 *	@st: holds the counter values
6389 * 	@sleep_ok: if true we may sleep while awaiting command completion
6390 *
6391 *	Returns the values of TP's CPL counters.
6392 */
6393void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st,
6394			 bool sleep_ok)
6395{
6396	int nchan = adap->params.arch.nchan;
6397
6398	t4_tp_mib_read(adap, st->req, nchan, A_TP_MIB_CPL_IN_REQ_0, sleep_ok);
6399
6400	t4_tp_mib_read(adap, st->rsp, nchan, A_TP_MIB_CPL_OUT_RSP_0, sleep_ok);
6401}
6402
6403/**
6404 *	t4_tp_get_rdma_stats - read TP's RDMA MIB counters
6405 *	@adap: the adapter
6406 *	@st: holds the counter values
6407 *
6408 *	Returns the values of TP's RDMA counters.
6409 */
6410void t4_tp_get_rdma_stats(struct adapter *adap, struct tp_rdma_stats *st,
6411			  bool sleep_ok)
6412{
6413	t4_tp_mib_read(adap, &st->rqe_dfr_pkt, 2, A_TP_MIB_RQE_DFR_PKT,
6414		       sleep_ok);
6415}
6416
6417/**
6418 *	t4_get_fcoe_stats - read TP's FCoE MIB counters for a port
6419 *	@adap: the adapter
6420 *	@idx: the port index
6421 *	@st: holds the counter values
6422 * 	@sleep_ok: if true we may sleep while awaiting command completion
6423 *
6424 *	Returns the values of TP's FCoE counters for the selected port.
6425 */
6426void t4_get_fcoe_stats(struct adapter *adap, unsigned int idx,
6427		       struct tp_fcoe_stats *st, bool sleep_ok)
6428{
6429	u32 val[2];
6430
6431	t4_tp_mib_read(adap, &st->frames_ddp, 1, A_TP_MIB_FCOE_DDP_0 + idx,
6432		       sleep_ok);
6433
6434	t4_tp_mib_read(adap, &st->frames_drop, 1,
6435		       A_TP_MIB_FCOE_DROP_0 + idx, sleep_ok);
6436
6437	t4_tp_mib_read(adap, val, 2, A_TP_MIB_FCOE_BYTE_0_HI + 2 * idx,
6438		       sleep_ok);
6439
6440	st->octets_ddp = ((u64)val[0] << 32) | val[1];
6441}
6442
6443/**
6444 *	t4_get_usm_stats - read TP's non-TCP DDP MIB counters
6445 *	@adap: the adapter
6446 *	@st: holds the counter values
6447 * 	@sleep_ok: if true we may sleep while awaiting command completion
6448 *
6449 *	Returns the values of TP's counters for non-TCP directly-placed packets.
6450 */
6451void t4_get_usm_stats(struct adapter *adap, struct tp_usm_stats *st,
6452		      bool sleep_ok)
6453{
6454	u32 val[4];
6455
6456	t4_tp_mib_read(adap, val, 4, A_TP_MIB_USM_PKTS, sleep_ok);
6457
6458	st->frames = val[0];
6459	st->drops = val[1];
6460	st->octets = ((u64)val[2] << 32) | val[3];
6461}
6462
6463/**
6464 *	t4_read_mtu_tbl - returns the values in the HW path MTU table
6465 *	@adap: the adapter
6466 *	@mtus: where to store the MTU values
6467 *	@mtu_log: where to store the MTU base-2 log (may be %NULL)
6468 *
6469 *	Reads the HW path MTU table.
6470 */
6471void t4_read_mtu_tbl(struct adapter *adap, u16 *mtus, u8 *mtu_log)
6472{
6473	u32 v;
6474	int i;
6475
6476	for (i = 0; i < NMTUS; ++i) {
6477		t4_write_reg(adap, A_TP_MTU_TABLE,
6478			     V_MTUINDEX(0xffU) | V_MTUVALUE(i));
6479		v = t4_read_reg(adap, A_TP_MTU_TABLE);
6480		mtus[i] = G_MTUVALUE(v);
6481		if (mtu_log)
6482			mtu_log[i] = G_MTUWIDTH(v);
6483	}
6484}
6485
6486/**
6487 *	t4_read_cong_tbl - reads the congestion control table
6488 *	@adap: the adapter
6489 *	@incr: where to store the alpha values
6490 *
6491 *	Reads the additive increments programmed into the HW congestion
6492 *	control table.
6493 */
6494void t4_read_cong_tbl(struct adapter *adap, u16 incr[NMTUS][NCCTRL_WIN])
6495{
6496	unsigned int mtu, w;
6497
6498	for (mtu = 0; mtu < NMTUS; ++mtu)
6499		for (w = 0; w < NCCTRL_WIN; ++w) {
6500			t4_write_reg(adap, A_TP_CCTRL_TABLE,
6501				     V_ROWINDEX(0xffffU) | (mtu << 5) | w);
6502			incr[mtu][w] = (u16)t4_read_reg(adap,
6503						A_TP_CCTRL_TABLE) & 0x1fff;
6504		}
6505}
6506
6507/**
6508 *	t4_tp_wr_bits_indirect - set/clear bits in an indirect TP register
6509 *	@adap: the adapter
6510 *	@addr: the indirect TP register address
6511 *	@mask: specifies the field within the register to modify
6512 *	@val: new value for the field
6513 *
6514 *	Sets a field of an indirect TP register to the given value.
6515 */
6516void t4_tp_wr_bits_indirect(struct adapter *adap, unsigned int addr,
6517			    unsigned int mask, unsigned int val)
6518{
6519	t4_write_reg(adap, A_TP_PIO_ADDR, addr);
6520	val |= t4_read_reg(adap, A_TP_PIO_DATA) & ~mask;
6521	t4_write_reg(adap, A_TP_PIO_DATA, val);
6522}
6523
6524/**
6525 *	init_cong_ctrl - initialize congestion control parameters
6526 *	@a: the alpha values for congestion control
6527 *	@b: the beta values for congestion control
6528 *
6529 *	Initialize the congestion control parameters.
6530 */
6531static void init_cong_ctrl(unsigned short *a, unsigned short *b)
6532{
6533	a[0] = a[1] = a[2] = a[3] = a[4] = a[5] = a[6] = a[7] = a[8] = 1;
6534	a[9] = 2;
6535	a[10] = 3;
6536	a[11] = 4;
6537	a[12] = 5;
6538	a[13] = 6;
6539	a[14] = 7;
6540	a[15] = 8;
6541	a[16] = 9;
6542	a[17] = 10;
6543	a[18] = 14;
6544	a[19] = 17;
6545	a[20] = 21;
6546	a[21] = 25;
6547	a[22] = 30;
6548	a[23] = 35;
6549	a[24] = 45;
6550	a[25] = 60;
6551	a[26] = 80;
6552	a[27] = 100;
6553	a[28] = 200;
6554	a[29] = 300;
6555	a[30] = 400;
6556	a[31] = 500;
6557
6558	b[0] = b[1] = b[2] = b[3] = b[4] = b[5] = b[6] = b[7] = b[8] = 0;
6559	b[9] = b[10] = 1;
6560	b[11] = b[12] = 2;
6561	b[13] = b[14] = b[15] = b[16] = 3;
6562	b[17] = b[18] = b[19] = b[20] = b[21] = 4;
6563	b[22] = b[23] = b[24] = b[25] = b[26] = b[27] = 5;
6564	b[28] = b[29] = 6;
6565	b[30] = b[31] = 7;
6566}
6567
6568/* The minimum additive increment value for the congestion control table */
6569#define CC_MIN_INCR 2U
6570
6571/**
6572 *	t4_load_mtus - write the MTU and congestion control HW tables
6573 *	@adap: the adapter
6574 *	@mtus: the values for the MTU table
6575 *	@alpha: the values for the congestion control alpha parameter
6576 *	@beta: the values for the congestion control beta parameter
6577 *
6578 *	Write the HW MTU table with the supplied MTUs and the high-speed
6579 *	congestion control table with the supplied alpha, beta, and MTUs.
6580 *	We write the two tables together because the additive increments
6581 *	depend on the MTUs.
6582 */
6583void t4_load_mtus(struct adapter *adap, const unsigned short *mtus,
6584		  const unsigned short *alpha, const unsigned short *beta)
6585{
6586	static const unsigned int avg_pkts[NCCTRL_WIN] = {
6587		2, 6, 10, 14, 20, 28, 40, 56, 80, 112, 160, 224, 320, 448, 640,
6588		896, 1281, 1792, 2560, 3584, 5120, 7168, 10240, 14336, 20480,
6589		28672, 40960, 57344, 81920, 114688, 163840, 229376
6590	};
6591
6592	unsigned int i, w;
6593
6594	for (i = 0; i < NMTUS; ++i) {
6595		unsigned int mtu = mtus[i];
6596		unsigned int log2 = fls(mtu);
6597
6598		if (!(mtu & ((1 << log2) >> 2)))     /* round */
6599			log2--;
6600		t4_write_reg(adap, A_TP_MTU_TABLE, V_MTUINDEX(i) |
6601			     V_MTUWIDTH(log2) | V_MTUVALUE(mtu));
6602
6603		for (w = 0; w < NCCTRL_WIN; ++w) {
6604			unsigned int inc;
6605
6606			inc = max(((mtu - 40) * alpha[w]) / avg_pkts[w],
6607				  CC_MIN_INCR);
6608
6609			t4_write_reg(adap, A_TP_CCTRL_TABLE, (i << 21) |
6610				     (w << 16) | (beta[w] << 13) | inc);
6611		}
6612	}
6613}
6614
6615/*
6616 * Calculates a rate in bytes/s given the number of 256-byte units per 4K core
6617 * clocks.  The formula is
6618 *
6619 * bytes/s = bytes256 * 256 * ClkFreq / 4096
6620 *
6621 * which is equivalent to
6622 *
6623 * bytes/s = 62.5 * bytes256 * ClkFreq_ms
6624 */
6625static u64 chan_rate(struct adapter *adap, unsigned int bytes256)
6626{
6627	u64 v = bytes256 * adap->params.vpd.cclk;
6628
6629	return v * 62 + v / 2;
6630}
6631
6632/**
6633 *	t4_get_chan_txrate - get the current per channel Tx rates
6634 *	@adap: the adapter
6635 *	@nic_rate: rates for NIC traffic
6636 *	@ofld_rate: rates for offloaded traffic
6637 *
6638 *	Return the current Tx rates in bytes/s for NIC and offloaded traffic
6639 *	for each channel.
6640 */
6641void t4_get_chan_txrate(struct adapter *adap, u64 *nic_rate, u64 *ofld_rate)
6642{
6643	u32 v;
6644
6645	v = t4_read_reg(adap, A_TP_TX_TRATE);
6646	nic_rate[0] = chan_rate(adap, G_TNLRATE0(v));
6647	nic_rate[1] = chan_rate(adap, G_TNLRATE1(v));
6648	if (adap->params.arch.nchan == NCHAN) {
6649		nic_rate[2] = chan_rate(adap, G_TNLRATE2(v));
6650		nic_rate[3] = chan_rate(adap, G_TNLRATE3(v));
6651	}
6652
6653	v = t4_read_reg(adap, A_TP_TX_ORATE);
6654	ofld_rate[0] = chan_rate(adap, G_OFDRATE0(v));
6655	ofld_rate[1] = chan_rate(adap, G_OFDRATE1(v));
6656	if (adap->params.arch.nchan == NCHAN) {
6657		ofld_rate[2] = chan_rate(adap, G_OFDRATE2(v));
6658		ofld_rate[3] = chan_rate(adap, G_OFDRATE3(v));
6659	}
6660}
6661
6662/**
6663 *	t4_set_trace_filter - configure one of the tracing filters
6664 *	@adap: the adapter
6665 *	@tp: the desired trace filter parameters
6666 *	@idx: which filter to configure
6667 *	@enable: whether to enable or disable the filter
6668 *
6669 *	Configures one of the tracing filters available in HW.  If @enable is
6670 *	%0 @tp is not examined and may be %NULL. The user is responsible to
6671 *	set the single/multiple trace mode by writing to A_MPS_TRC_CFG register
6672 *	by using "cxgbtool iface reg reg_addr=val" command. See t4_sniffer/
6673 *	docs/readme.txt for a complete description of how to setup traceing on
6674 *	T4.
6675 */
6676int t4_set_trace_filter(struct adapter *adap, const struct trace_params *tp, int idx,
6677			int enable)
6678{
6679	int i, ofst = idx * 4;
6680	u32 data_reg, mask_reg, cfg;
6681
6682	if (!enable) {
6683		t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, 0);
6684		return 0;
6685	}
6686
6687	/*
6688	 * TODO - After T4 data book is updated, specify the exact
6689	 * section below.
6690	 *
6691	 * See T4 data book - MPS section for a complete description
6692	 * of the below if..else handling of A_MPS_TRC_CFG register
6693	 * value.
6694	 */
6695	cfg = t4_read_reg(adap, A_MPS_TRC_CFG);
6696	if (cfg & F_TRCMULTIFILTER) {
6697		/*
6698		 * If multiple tracers are enabled, then maximum
6699		 * capture size is 2.5KB (FIFO size of a single channel)
6700		 * minus 2 flits for CPL_TRACE_PKT header.
6701		 */
6702		if (tp->snap_len > ((10 * 1024 / 4) - (2 * 8)))
6703			return -EINVAL;
6704	}
6705	else {
6706		/*
6707		 * If multiple tracers are disabled, to avoid deadlocks
6708		 * maximum packet capture size of 9600 bytes is recommended.
6709		 * Also in this mode, only trace0 can be enabled and running.
6710		 */
6711		if (tp->snap_len > 9600 || idx)
6712			return -EINVAL;
6713	}
6714
6715	if (tp->port > (is_t4(adap->params.chip) ? 11 : 19) || tp->invert > 1 ||
6716	    tp->skip_len > M_TFLENGTH || tp->skip_ofst > M_TFOFFSET ||
6717	    tp->min_len > M_TFMINPKTSIZE)
6718		return -EINVAL;
6719
6720	/* stop the tracer we'll be changing */
6721	t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst, 0);
6722
6723	idx *= (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH);
6724	data_reg = A_MPS_TRC_FILTER0_MATCH + idx;
6725	mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + idx;
6726
6727	for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
6728		t4_write_reg(adap, data_reg, tp->data[i]);
6729		t4_write_reg(adap, mask_reg, ~tp->mask[i]);
6730	}
6731	t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst,
6732		     V_TFCAPTUREMAX(tp->snap_len) |
6733		     V_TFMINPKTSIZE(tp->min_len));
6734	t4_write_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst,
6735		     V_TFOFFSET(tp->skip_ofst) | V_TFLENGTH(tp->skip_len) |
6736		     (is_t4(adap->params.chip) ?
6737		     V_TFPORT(tp->port) | F_TFEN | V_TFINVERTMATCH(tp->invert) :
6738		     V_T5_TFPORT(tp->port) | F_T5_TFEN |
6739		     V_T5_TFINVERTMATCH(tp->invert)));
6740
6741	return 0;
6742}
6743
6744/**
6745 *	t4_get_trace_filter - query one of the tracing filters
6746 *	@adap: the adapter
6747 *	@tp: the current trace filter parameters
6748 *	@idx: which trace filter to query
6749 *	@enabled: non-zero if the filter is enabled
6750 *
6751 *	Returns the current settings of one of the HW tracing filters.
6752 */
6753void t4_get_trace_filter(struct adapter *adap, struct trace_params *tp, int idx,
6754			 int *enabled)
6755{
6756	u32 ctla, ctlb;
6757	int i, ofst = idx * 4;
6758	u32 data_reg, mask_reg;
6759
6760	ctla = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_A + ofst);
6761	ctlb = t4_read_reg(adap, A_MPS_TRC_FILTER_MATCH_CTL_B + ofst);
6762
6763	if (is_t4(adap->params.chip)) {
6764		*enabled = !!(ctla & F_TFEN);
6765		tp->port =  G_TFPORT(ctla);
6766		tp->invert = !!(ctla & F_TFINVERTMATCH);
6767	} else {
6768		*enabled = !!(ctla & F_T5_TFEN);
6769		tp->port = G_T5_TFPORT(ctla);
6770		tp->invert = !!(ctla & F_T5_TFINVERTMATCH);
6771	}
6772	tp->snap_len = G_TFCAPTUREMAX(ctlb);
6773	tp->min_len = G_TFMINPKTSIZE(ctlb);
6774	tp->skip_ofst = G_TFOFFSET(ctla);
6775	tp->skip_len = G_TFLENGTH(ctla);
6776
6777	ofst = (A_MPS_TRC_FILTER1_MATCH - A_MPS_TRC_FILTER0_MATCH) * idx;
6778	data_reg = A_MPS_TRC_FILTER0_MATCH + ofst;
6779	mask_reg = A_MPS_TRC_FILTER0_DONT_CARE + ofst;
6780
6781	for (i = 0; i < TRACE_LEN / 4; i++, data_reg += 4, mask_reg += 4) {
6782		tp->mask[i] = ~t4_read_reg(adap, mask_reg);
6783		tp->data[i] = t4_read_reg(adap, data_reg) & tp->mask[i];
6784	}
6785}
6786
6787/**
6788 *	t4_read_tcb - read a hardware TCP Control Block structure
6789 *	@adap: the adapter
6790 *	@win: PCI-E Memory Window to use
6791 *	@tid: the TCB ID
6792 *	@tcb: the buffer to return the TCB in
6793 *
6794 *	Reads the indicated hardware TCP Control Block and returns it in
6795 *	the supplied buffer.  Returns 0 on success.
6796 */
6797int t4_read_tcb(struct adapter *adap, int win, int tid, u32 tcb[TCB_SIZE/4])
6798{
6799	u32 tcb_base = t4_read_reg(adap, A_TP_CMM_TCB_BASE);
6800	u32 tcb_addr = tcb_base + tid * TCB_SIZE;
6801	__be32 raw_tcb[TCB_SIZE/4];
6802	int ret, word;
6803
6804	ret = t4_memory_rw_addr(adap, win,
6805				tcb_addr, sizeof raw_tcb, raw_tcb,
6806				T4_MEMORY_READ);
6807	if (ret)
6808		return ret;
6809
6810	for (word = 0; word < 32; word++)
6811		tcb[word] = be32_to_cpu(raw_tcb[word]);
6812	return 0;
6813}
6814
6815/**
6816 *	t4_pmtx_get_stats - returns the HW stats from PMTX
6817 *	@adap: the adapter
6818 *	@cnt: where to store the count statistics
6819 *	@cycles: where to store the cycle statistics
6820 *
6821 *	Returns performance statistics from PMTX.
6822 */
6823void t4_pmtx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
6824{
6825	int i;
6826	u32 data[2];
6827
6828	for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) {
6829		t4_write_reg(adap, A_PM_TX_STAT_CONFIG, i + 1);
6830		cnt[i] = t4_read_reg(adap, A_PM_TX_STAT_COUNT);
6831		if (is_t4(adap->params.chip)) {
6832			cycles[i] = t4_read_reg64(adap, A_PM_TX_STAT_LSB);
6833		} else {
6834			t4_read_indirect(adap, A_PM_TX_DBG_CTRL,
6835					 A_PM_TX_DBG_DATA, data, 2,
6836					 A_PM_TX_DBG_STAT_MSB);
6837			cycles[i] = (((u64)data[0] << 32) | data[1]);
6838		}
6839	}
6840}
6841
6842/**
6843 *	t4_pmrx_get_stats - returns the HW stats from PMRX
6844 *	@adap: the adapter
6845 *	@cnt: where to store the count statistics
6846 *	@cycles: where to store the cycle statistics
6847 *
6848 *	Returns performance statistics from PMRX.
6849 */
6850void t4_pmrx_get_stats(struct adapter *adap, u32 cnt[], u64 cycles[])
6851{
6852	int i;
6853	u32 data[2];
6854
6855	for (i = 0; i < adap->params.arch.pm_stats_cnt; i++) {
6856		t4_write_reg(adap, A_PM_RX_STAT_CONFIG, i + 1);
6857		cnt[i] = t4_read_reg(adap, A_PM_RX_STAT_COUNT);
6858		if (is_t4(adap->params.chip)) {
6859			cycles[i] = t4_read_reg64(adap, A_PM_RX_STAT_LSB);
6860		} else {
6861			t4_read_indirect(adap, A_PM_RX_DBG_CTRL,
6862					 A_PM_RX_DBG_DATA, data, 2,
6863					 A_PM_RX_DBG_STAT_MSB);
6864			cycles[i] = (((u64)data[0] << 32) | data[1]);
6865		}
6866	}
6867}
6868
6869/**
6870 *	compute_mps_bg_map - compute the MPS Buffer Group Map for a Port
6871 *	@adapter: the adapter
6872 *	@pidx: the port index
6873 *
6874 *	Compuytes and returns a bitmap indicating which MPS buffer groups are
6875 *	associated with the given Port.  Bit i is set if buffer group i is
6876 *	used by the Port.
6877 */
6878static inline unsigned int compute_mps_bg_map(struct adapter *adapter,
6879					      int pidx)
6880{
6881	unsigned int chip_version, nports;
6882
6883	chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);
6884	nports = 1 << G_NUMPORTS(t4_read_reg(adapter, A_MPS_CMN_CTL));
6885
6886	switch (chip_version) {
6887	case CHELSIO_T4:
6888	case CHELSIO_T5:
6889		switch (nports) {
6890		case 1: return 0xf;
6891		case 2: return 3 << (2 * pidx);
6892		case 4: return 1 << pidx;
6893		}
6894		break;
6895
6896	case CHELSIO_T6:
6897		switch (nports) {
6898		case 2: return 1 << (2 * pidx);
6899		}
6900		break;
6901	}
6902
6903	CH_ERR(adapter, "Need MPS Buffer Group Map for Chip %0x, Nports %d\n",
6904	       chip_version, nports);
6905
6906	return 0;
6907}
6908
6909/**
6910 *	t4_get_mps_bg_map - return the buffer groups associated with a port
6911 *	@adapter: the adapter
6912 *	@pidx: the port index
6913 *
6914 *	Returns a bitmap indicating which MPS buffer groups are associated
6915 *	with the given Port.  Bit i is set if buffer group i is used by the
6916 *	Port.
6917 */
6918unsigned int t4_get_mps_bg_map(struct adapter *adapter, int pidx)
6919{
6920	u8 *mps_bg_map;
6921	unsigned int nports;
6922
6923	nports = 1 << G_NUMPORTS(t4_read_reg(adapter, A_MPS_CMN_CTL));
6924	if (pidx >= nports) {
6925		CH_WARN(adapter, "MPS Port Index %d >= Nports %d\n", pidx, nports);
6926		return 0;
6927	}
6928
6929	/* If we've already retrieved/computed this, just return the result.
6930	 */
6931	mps_bg_map = adapter->params.mps_bg_map;
6932	if (mps_bg_map[pidx])
6933		return mps_bg_map[pidx];
6934
6935	/* Newer Firmware can tell us what the MPS Buffer Group Map is.
6936	 * If we're talking to such Firmware, let it tell us.  If the new
6937	 * API isn't supported, revert back to old hardcoded way.  The value
6938	 * obtained from Firmware is encoded in below format:
6939	 *
6940	 * val = (( MPSBGMAP[Port 3] << 24 ) |
6941	 *        ( MPSBGMAP[Port 2] << 16 ) |
6942	 *        ( MPSBGMAP[Port 1] <<  8 ) |
6943	 *        ( MPSBGMAP[Port 0] <<  0 ))
6944	 */
6945	if (adapter->flags & FW_OK) {
6946		u32 param, val;
6947		int ret;
6948
6949		param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
6950			 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_MPSBGMAP));
6951		ret = t4_query_params_ns(adapter, adapter->mbox, adapter->