xref: /illumos-gate/usr/src/uts/common/io/cxgbe/common/t4_hw.c

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source. A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * This file is part of the Chelsio T4/T5/T6 Ethernet driver.
 *
 * Copyright (C) 2003-2019 Chelsio Communications.  All rights reserved.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
 * release for licensing terms and conditions.
 */

/*
 * Copyright 2020 RackTop Systems, Inc.
 */

#include "common.h"
#include "t4_regs.h"
#include "t4_regs_values.h"
#include "t4fw_interface.h"

/**
 *	t4_wait_op_done_val - wait until an operation is completed
 *	@adapter: the adapter performing the operation
 *	@reg: the register to check for completion
 *	@mask: a single-bit field within @reg that indicates completion
 *	@polarity: the value of the field when the operation is completed
 *	@attempts: number of check iterations
 *	@delay: delay in usecs between iterations
 *	@valp: where to store the value of the register at completion time
 *
 *	Wait until an operation is completed by checking a bit in a register
 *	up to @attempts times.  If @valp is not NULL the value of the register
 *	at the time it indicated completion is stored there.  Returns 0 if the
 *	operation completes and	-EAGAIN	otherwise.
 */
static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask,
			       int polarity, int attempts, int delay, u32 *valp)
{
	while (1) {
		u32 val = t4_read_reg(adapter, reg);

		if (!!(val & mask) == polarity) {
			if (valp)
				*valp = val;
			return 0;
		}
		if (--attempts == 0)
			return -EAGAIN;
		if (delay)
			udelay(delay);
	}
}

static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
				  int polarity, int attempts, int delay)
{
	return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts,
				   delay, NULL);
}

/**
 *	t4_set_reg_field - set a register field to a value
 *	@adapter: the adapter to program
 *	@addr: the register address
 *	@mask: specifies the portion of the register to modify
 *	@val: the new value for the register field
 *
 *	Sets a register field specified by the supplied mask to the
 *	given value.
 */
void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask,
		      u32 val)
{
	u32 v = t4_read_reg(adapter, addr) & ~mask;

	t4_write_reg(adapter, addr, v | val);
	(void) t4_read_reg(adapter, addr);      /* flush */
}

/**
 *	t4_read_indirect - read indirectly addressed registers
 *	@adap: the adapter
 *	@addr_reg: register holding the indirect address
 *	@data_reg: register holding the value of the indirect register
 *	@vals: where the read register values are stored
 *	@nregs: how many indirect registers to read
 *	@start_idx: index of first indirect register to read
 *
 *	Reads registers that are accessed indirectly through an address/data
 *	register pair.
 */
void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
			     unsigned int data_reg, u32 *vals,
			     unsigned int nregs, unsigned int start_idx)
{
	while (nregs--) {
		t4_write_reg(adap, addr_reg, start_idx);
		*vals++ = t4_read_reg(adap, data_reg);
		start_idx++;
	}
}

/**
 *	t4_write_indirect - write indirectly addressed registers
 *	@adap: the adapter
 *	@addr_reg: register holding the indirect addresses
 *	@data_reg: register holding the value for the indirect registers
 *	@vals: values to write
 *	@nregs: how many indirect registers to write
 *	@start_idx: address of first indirect register to write
 *
 *	Writes a sequential block of registers that are accessed indirectly
 *	through an address/data register pair.
 */
void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
		       unsigned int data_reg, const u32 *vals,
		       unsigned int nregs, unsigned int start_idx)
{
	while (nregs--) {
		t4_write_reg(adap, addr_reg, start_idx++);
		t4_write_reg(adap, data_reg, *vals++);
	}
}

/*
 * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor
 * mechanism.  This guarantees that we get the real value even if we're
 * operating within a Virtual Machine and the Hypervisor is trapping our
 * Configuration Space accesses.
 *
 * N.B. This routine should only be used as a last resort: the firmware uses
 *      the backdoor registers on a regular basis and we can end up
 *      conflicting with it's uses!
 */
void t4_hw_pci_read_cfg4(struct adapter *adap, int reg, u32 *val)
{
	u32 req = V_FUNCTION(adap->pf) | V_REGISTER(reg);

	if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
		req |= F_ENABLE;
	else
		req |= F_T6_ENABLE;

	if (is_t4(adap->params.chip))
		req |= F_LOCALCFG;

	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, req);
	*val = t4_read_reg(adap, A_PCIE_CFG_SPACE_DATA);

	/* Reset F_ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a
	 * Configuration Space read.  (None of the other fields matter when
	 * F_ENABLE is 0 so a simple register write is easier than a
	 * read-modify-write via t4_set_reg_field().)
	 */
	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, 0);
}

/*
 * t4_report_fw_error - report firmware error
 * @adap: the adapter
 *
 * The adapter firmware can indicate error conditions to the host.
 * If the firmware has indicated an error, print out the reason for
 * the firmware error.
 */
static void t4_report_fw_error(struct adapter *adap)
{
	static const char *const reason[] = {
		"Crash",			/* PCIE_FW_EVAL_CRASH */
		"During Device Preparation",	/* PCIE_FW_EVAL_PREP */
		"During Device Configuration",	/* PCIE_FW_EVAL_CONF */
		"During Device Initialization",	/* PCIE_FW_EVAL_INIT */
		"Unexpected Event",		/* PCIE_FW_EVAL_UNEXPECTEDEVENT */
		"Insufficient Airflow",		/* PCIE_FW_EVAL_OVERHEAT */
		"Device Shutdown",		/* PCIE_FW_EVAL_DEVICESHUTDOWN */
		"Reserved",			/* reserved */
	};
	u32 pcie_fw;

	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
	if (pcie_fw & F_PCIE_FW_ERR) {
		CH_ERR(adap, "Firmware reports adapter error: %s\n",
			reason[G_PCIE_FW_EVAL(pcie_fw)]);
		adap->flags &= ~FW_OK;
	}
}

/*
 * Get the reply to a mailbox command and store it in @rpl in big-endian order.
 */
static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit,
			 u32 mbox_addr)
{
	for ( ; nflit; nflit--, mbox_addr += 8)
		*rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr));
}

/*
 * Handle a FW assertion reported in a mailbox.
 */
static void fw_asrt(struct adapter *adap, struct fw_debug_cmd *asrt)
{
	CH_ALERT(adap,
		  "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n",
		  asrt->u.assert.filename_0_7,
		  be32_to_cpu(asrt->u.assert.line),
		  be32_to_cpu(asrt->u.assert.x),
		  be32_to_cpu(asrt->u.assert.y));
}

#define X_CIM_PF_NOACCESS 0xeeeeeeee

/*
 * If the Host OS Driver needs locking arround accesses to the mailbox, this
 * can be turned on via the T4_OS_NEEDS_MBOX_LOCKING CPP define ...
 */
/* makes single-statement usage a bit cleaner ... */
#ifdef T4_OS_NEEDS_MBOX_LOCKING
#define T4_OS_MBOX_LOCKING(x) x
#else
#define T4_OS_MBOX_LOCKING(x) do {} while (0)
#endif

/*
 * If the OS Driver wants busy waits to keep a watchdog happy, tap it during
 * busy loops which don't sleep.
 */
#ifdef T4_OS_NEEDS_TOUCH_NMI_WATCHDOG
#define T4_OS_TOUCH_NMI_WATCHDOG()	t4_os_touch_nmi_watchdog()
#else
#define T4_OS_TOUCH_NMI_WATCHDOG()
#endif

#ifdef T4_OS_LOG_MBOX_CMDS
/**
 *	t4_record_mbox - record a Firmware Mailbox Command/Reply in the log
 *	@adapter: the adapter
 *	@cmd: the Firmware Mailbox Command or Reply
 *	@size: command length in bytes
 *	@access: the time (ms) needed to access the Firmware Mailbox
 *	@execute: the time (ms) the command spent being executed
 */
static void t4_record_mbox(struct adapter *adapter,
			   const __be64 *cmd, unsigned int size,
			   int access, int execute)
{
	struct mbox_cmd_log *log = adapter->mbox_log;
	struct mbox_cmd *entry;
	int i;

	entry = mbox_cmd_log_entry(log, log->cursor++);
	if (log->cursor == log->size)
		log->cursor = 0;

	for (i = 0; i < size/8; i++)
		entry->cmd[i] = be64_to_cpu(cmd[i]);
	while (i < MBOX_LEN/8)
		entry->cmd[i++] = 0;
	entry->timestamp = t4_os_timestamp();
	entry->seqno = log->seqno++;
	entry->access = access;
	entry->execute = execute;
}

#define T4_RECORD_MBOX(__adapter, __cmd, __size, __access, __execute) \
	t4_record_mbox(__adapter, __cmd, __size, __access, __execute)

#else /* !T4_OS_LOG_MBOX_CMDS */

#define T4_RECORD_MBOX(__adapter, __cmd, __size, __access, __execute) \
	/* nothing */

#endif /* !T4_OS_LOG_MBOX_CMDS */

/**
 *	t4_record_mbox_marker - record a marker in the mailbox log
 *	@adapter: the adapter
 *	@marker: byte array marker
 *	@size: marker size in bytes
 *
 *	We inject a "fake mailbox command" into the Firmware Mailbox Log
 *	using a known command token and then the bytes of the specified
 *	marker.  This lets debugging code inject markers into the log to
 *	help identify which commands are in response to higher level code.
 */
void t4_record_mbox_marker(struct adapter *adapter,
			   const void *marker, unsigned int size)
{
#ifdef T4_OS_LOG_MBOX_CMDS
	__be64 marker_cmd[MBOX_LEN/8];
	const unsigned int max_marker = sizeof marker_cmd - sizeof (__be64);
	unsigned int marker_cmd_size;

	if (size > max_marker)
		size = max_marker;

	marker_cmd[0] = cpu_to_be64(~0LLU);
	memcpy(&marker_cmd[1], marker, size);
	memset((unsigned char *)&marker_cmd[1] + size, 0, max_marker - size);
	marker_cmd_size = sizeof (__be64) + roundup(size, sizeof (__be64));

	t4_record_mbox(adapter, marker_cmd, marker_cmd_size, 0, 0);
#endif /* T4_OS_LOG_MBOX_CMDS */
}

/*
 * Delay time in microseconds to wait for mailbox access/fw reply
 * to mailbox command
 */
#define MIN_MBOX_CMD_DELAY 900
#define MBOX_CMD_DELAY 1000

/**
 *	t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox
 *	@adap: the adapter
 *	@mbox: index of the mailbox to use
 *	@cmd: the command to write
 *	@size: command length in bytes
 *	@rpl: where to optionally store the reply
 *	@sleep_ok: if true we may sleep while awaiting command completion
 *	@timeout: time to wait for command to finish before timing out
 *		(negative implies @sleep_ok=false)
 *
 *	Sends the given command to FW through the selected mailbox and waits
 *	for the FW to execute the command.  If @rpl is not %NULL it is used to
 *	store the FW's reply to the command.  The command and its optional
 *	reply are of the same length.  Some FW commands like RESET and
 *	INITIALIZE can take a considerable amount of time to execute.
 *	@sleep_ok determines whether we may sleep while awaiting the response.
 *	If sleeping is allowed we use progressive backoff otherwise we spin.
 *	Note that passing in a negative @timeout is an alternate mechanism
 *	for specifying @sleep_ok=false.  This is useful when a higher level
 *	interface allows for specification of @timeout but not @sleep_ok ...
 *
 *	The return value is 0 on success or a negative errno on failure.  A
 *	failure can happen either because we are not able to execute the
 *	command or FW executes it but signals an error.  In the latter case
 *	the return value is the error code indicated by FW (negated).
 */
int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
			    int size, void *rpl, bool sleep_ok, int timeout)
{
#ifdef T4_OS_NEEDS_MBOX_LOCKING
	u16 access = 0;
#endif
	u32 v;
	u64 res;
	int i, ret;
	const __be64 *p = cmd;
	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
	u32 ctl;
	__be64 cmd_rpl[MBOX_LEN/8];
	T4_OS_MBOX_LOCKING(t4_os_list_t entry);
	u32 pcie_fw;

	if ((size & 15) || size > MBOX_LEN)
		return -EINVAL;

	/*
	 * If we have a negative timeout, that implies that we can't sleep.
	 */
	if (timeout < 0) {
		sleep_ok = false;
		timeout = -timeout;
	}

#ifdef T4_OS_NEEDS_MBOX_LOCKING
	/*
	 * Queue ourselves onto the mailbox access list.  When our entry is at
	 * the front of the list, we have rights to access the mailbox.  So we
	 * wait [for a while] till we're at the front [or bail out with an
	 * EBUSY] ...
	 */
	t4_os_atomic_add_tail(&entry, &adap->mbox_list, &adap->mbox_lock);

	for (i = 0; ; i++) {
		/*
		 * If we've waited too long, return a busy indication.  This
		 * really ought to be based on our initial position in the
		 * mailbox access list but this is a start.  We very rarely
		 * contend on access to the mailbox ...  Also check for a
		 * firmware error which we'll report as a device error.
		 */
		pcie_fw = t4_read_reg(adap, A_PCIE_FW);
		if (i > 4*timeout || (pcie_fw & F_PCIE_FW_ERR)) {
			t4_os_atomic_list_del(&entry, &adap->mbox_lock);
			t4_report_fw_error(adap);
			ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -EBUSY;
			T4_RECORD_MBOX(adap, cmd, size, ret, 0);
			return ret;
		}

		/*
		 * If we're at the head, break out and start the mailbox
		 * protocol.
		 */
		if (t4_os_list_first_entry(&adap->mbox_list) == &entry)
			break;

		/*
		 * Delay for a bit before checking again ...
		 */
		if (sleep_ok) {
			usleep_range(MIN_MBOX_CMD_DELAY, MBOX_CMD_DELAY);
		} else {
			T4_OS_TOUCH_NMI_WATCHDOG();
			udelay(MBOX_CMD_DELAY);
		}
	}
	access = i;
#endif /* T4_OS_NEEDS_MBOX_LOCKING */

	/*
	 * Attempt to gain access to the mailbox.
	 */
	for (i = 0; i < 4; i++) {
		ctl = t4_read_reg(adap, ctl_reg);
		v = G_MBOWNER(ctl);
		if (v != X_MBOWNER_NONE)
			break;
	}

	/*
	 * If we were unable to gain access, dequeue ourselves from the
	 * mailbox atomic access list and report the error to our caller.
	 */
	if (v != X_MBOWNER_PL) {
		T4_OS_MBOX_LOCKING(t4_os_atomic_list_del(&entry,
							 &adap->mbox_lock));
		t4_report_fw_error(adap);
		ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT;
		T4_RECORD_MBOX(adap, cmd, size, access, ret);
		return ret;
	}

	/*
	 * If we gain ownership of the mailbox and there's a "valid" message
	 * in it, this is likely an asynchronous error message from the
	 * firmware.  So we'll report that and then proceed on with attempting
	 * to issue our own command ... which may well fail if the error
	 * presaged the firmware crashing ...
	 */
	if (ctl & F_MBMSGVALID) {
		CH_ERR(adap, "found VALID command in mbox %u: "
		       "%llx %llx %llx %llx %llx %llx %llx %llx\n", mbox,
		       (unsigned long long)t4_read_reg64(adap, data_reg),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 8),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 16),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 24),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 32),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 40),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 48),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 56));
	}

	/*
	 * Copy in the new mailbox command and send it on its way ...
	 */
	T4_RECORD_MBOX(adap, cmd, size, access, 0);
	for (i = 0; i < size; i += 8, p++)
		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));

	/*
	 * XXX It's not clear that we need this anymore now
	 * XXX that we have mailbox logging ...
	 */
	CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);

	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
	(void) t4_read_reg(adap, ctl_reg);	/* flush write */

	/*
	 * Loop waiting for the reply; bail out if we time out or the firmware
	 * reports an error.
	 */
	for (i = 0;
	     !((pcie_fw = t4_read_reg(adap, A_PCIE_FW)) & F_PCIE_FW_ERR) &&
	     i < timeout;
	     i++) {
		if (sleep_ok) {
			usleep_range(MIN_MBOX_CMD_DELAY, MBOX_CMD_DELAY);
		} else {
			T4_OS_TOUCH_NMI_WATCHDOG();
			udelay(MBOX_CMD_DELAY);
		}

		v = t4_read_reg(adap, ctl_reg);
		if (v == X_CIM_PF_NOACCESS)
			continue;
		if (G_MBOWNER(v) == X_MBOWNER_PL) {
			if (!(v & F_MBMSGVALID)) {
				t4_write_reg(adap, ctl_reg,
					     V_MBOWNER(X_MBOWNER_NONE));
				continue;
			}

			/*
			 * Retrieve the command reply and release the mailbox.
			 */
			get_mbox_rpl(adap, cmd_rpl, size/8, data_reg);
			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));
			T4_OS_MBOX_LOCKING(t4_os_atomic_list_del(&entry,
								 &adap->mbox_lock));

			T4_RECORD_MBOX(adap, cmd_rpl, size, access, i + 1);

			/*
			 * XXX It's not clear that we need this anymore now
			 * XXX that we have mailbox logging ...
			 */
			CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
			CH_MSG(adap, INFO, HW,
			       "command completed in %d ms (%ssleeping)\n",
			       i + 1, sleep_ok ? "" : "non-");

			res = be64_to_cpu(cmd_rpl[0]);
			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
				fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl);
				res = V_FW_CMD_RETVAL(EIO);
			} else if (rpl)
				memcpy(rpl, cmd_rpl, size);
			return -G_FW_CMD_RETVAL((int)res);
		}
	}

	/*
	 * We timed out waiting for a reply to our mailbox command.  Report
	 * the error and also check to see if the firmware reported any
	 * errors ...
	 */
	T4_OS_MBOX_LOCKING(t4_os_atomic_list_del(&entry, &adap->mbox_lock));

	ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -ETIMEDOUT;
	T4_RECORD_MBOX(adap, cmd, size, access, ret);
	CH_ERR(adap, "command %#x in mailbox %d timed out\n",
	       *(const u8 *)cmd, mbox);

	t4_report_fw_error(adap);
	t4_fatal_err(adap);
	return ret;
}

#ifdef CONFIG_CUDBG
/*
 * The maximum number of times to iterate for FW reply before
 * issuing a mailbox timeout
 */
#define FW_REPLY_WAIT_LOOP 6000000

/**
 *	t4_wr_mbox_meat_timeout_panic - send a command to FW through the given
 *	mailbox. This function is a minimal version of t4_wr_mbox_meat_timeout()
 *	and is only invoked during a kernel crash. Since this function is
 *	called through a atomic notifier chain ,we cannot sleep awaiting a
 *	response from FW, hence repeatedly loop until we get a reply.
 *
 *	@adap: the adapter
 *	@mbox: index of the mailbox to use
 *	@cmd: the command to write
 *	@size: command length in bytes
 *	@rpl: where to optionally store the reply
 */

static int t4_wr_mbox_meat_timeout_panic(struct adapter *adap, int mbox,
			    const void *cmd, int size, void *rpl)
{
	u32 v;
	u64 res;
	int i, ret;
	u64 cnt;
	const __be64 *p = cmd;
	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
	u32 ctl;
	__be64 cmd_rpl[MBOX_LEN/8];
	u32 pcie_fw;

	if ((size & 15) || size > MBOX_LEN)
		return -EINVAL;

	/*
	 * Check for a firmware error which we'll report as a
	 * device error.
	 */
	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
	if (pcie_fw & F_PCIE_FW_ERR) {
		t4_report_fw_error(adap);
		ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -EBUSY;
		return ret;
	}

	/*
	 * Attempt to gain access to the mailbox.
	 */
	for (i = 0; i < 4; i++) {
		ctl = t4_read_reg(adap, ctl_reg);
		v = G_MBOWNER(ctl);
		if (v != X_MBOWNER_NONE)
			break;
	}

	/*
	 * If we were unable to gain access, report the error to our caller.
	 */
	if (v != X_MBOWNER_PL) {
		t4_report_fw_error(adap);
		ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT;
		return ret;
	}

	/*
	 * If we gain ownership of the mailbox and there's a "valid" message
	 * in it, this is likely an asynchronous error message from the
	 * firmware.  So we'll report that and then proceed on with attempting
	 * to issue our own command ... which may well fail if the error
	 * presaged the firmware crashing ...
	 */
	if (ctl & F_MBMSGVALID) {
		CH_ERR(adap, "found VALID command in mbox %u: "
		       "%llx %llx %llx %llx %llx %llx %llx %llx\n", mbox,
		       (unsigned long long)t4_read_reg64(adap, data_reg),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 8),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 16),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 24),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 32),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 40),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 48),
		       (unsigned long long)t4_read_reg64(adap, data_reg + 56));
	}

	/*
	 * Copy in the new mailbox command and send it on its way ...
	 */
	for (i = 0; i < size; i += 8, p++)
		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));

	CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);

	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
	t4_read_reg(adap, ctl_reg);	/* flush write */

	/*
	 * Loop waiting for the reply; bail out if we time out or the firmware
	 * reports an error.
	 */
	for (cnt = 0;
	    !((pcie_fw = t4_read_reg(adap, A_PCIE_FW)) & F_PCIE_FW_ERR) &&
	    cnt < FW_REPLY_WAIT_LOOP;
	    cnt++) {
		v = t4_read_reg(adap, ctl_reg);
		if (v == X_CIM_PF_NOACCESS)
			continue;
		if (G_MBOWNER(v) == X_MBOWNER_PL) {
			if (!(v & F_MBMSGVALID)) {
				t4_write_reg(adap, ctl_reg,
					     V_MBOWNER(X_MBOWNER_NONE));
				continue;
			}

			/*
			 * Retrieve the command reply and release the mailbox.
			 */
			get_mbox_rpl(adap, cmd_rpl, size/8, data_reg);
			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));

			CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);

			res = be64_to_cpu(cmd_rpl[0]);
			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
				fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl);
				res = V_FW_CMD_RETVAL(EIO);
			} else if (rpl)
				memcpy(rpl, cmd_rpl, size);
			return -G_FW_CMD_RETVAL((int)res);
		}
	}

	/*
	 * We timed out waiting for a reply to our mailbox command.  Report
	 * the error and also check to see if the firmware reported any
	 * errors ...
	 */
	ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -ETIMEDOUT;
	CH_ERR(adap, "command %#x in mailbox %d timed out\n",
	       *(const u8 *)cmd, mbox);

	t4_report_fw_error(adap);
	t4_fatal_err(adap);
	return ret;
}
#endif

int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
		    void *rpl, bool sleep_ok)
{
#ifdef CONFIG_CUDBG
	if (adap->flags & K_CRASH)
		return t4_wr_mbox_meat_timeout_panic(adap, mbox, cmd, size,
						     rpl);
	else
#endif
		return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl,
					       sleep_ok, FW_CMD_MAX_TIMEOUT);

}

static int t4_edc_err_read(struct adapter *adap, int idx)
{
	u32 edc_ecc_err_addr_reg;
	u32 edc_bist_status_rdata_reg;

	if (is_t4(adap->params.chip)) {
		CH_WARN(adap, "%s: T4 NOT supported.\n", __func__);
		return 0;
	}
	if (idx != MEM_EDC0 && idx != MEM_EDC1) {
		CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx);
		return 0;
	}

	edc_ecc_err_addr_reg = EDC_T5_REG(A_EDC_H_ECC_ERR_ADDR, idx);
	edc_bist_status_rdata_reg = EDC_T5_REG(A_EDC_H_BIST_STATUS_RDATA, idx);

	CH_WARN(adap,
		"edc%d err addr 0x%x: 0x%x.\n",
		idx, edc_ecc_err_addr_reg,
		t4_read_reg(adap, edc_ecc_err_addr_reg));
	CH_WARN(adap,
	 	"bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
		edc_bist_status_rdata_reg,
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 8),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 16),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 24),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 32),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 40),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 48),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 56),
		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 64));

	return 0;
}

/**
 *	t4_memory_rw_addr - read/write adapter memory via PCIE memory window
 *	@adap: the adapter
 *	@win: PCI-E Memory Window to use
 *	@addr: address within adapter memory
 *	@len: amount of memory to transfer
 *	@hbuf: host memory buffer
 *	@dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
 *
 *	Reads/writes an [almost] arbitrary memory region in the firmware: the
 *	firmware memory address and host buffer must be aligned on 32-bit
 *	boudaries; the length may be arbitrary.
 *
 *	NOTES:
 *	 1. The memory is transferred as a raw byte sequence from/to the
 *	    firmware's memory.  If this memory contains data structures which
 *	    contain multi-byte integers, it's the caller's responsibility to
 *	    perform appropriate byte order conversions.
 *
 *	 2. It is the Caller's responsibility to ensure that no other code
 *	    uses the specified PCI-E Memory Window while this routine is
 *	    using it.  This is typically done via the use of OS-specific
 *	    locks, etc.
 */
int t4_memory_rw_addr(struct adapter *adap, int win, u32 addr,
		      u32 len, void *hbuf, int dir)
{
	u32 pos, offset, resid;
	u32 win_pf, mem_reg, mem_aperture, mem_base;
	u32 *buf;

	/* Argument sanity checks ...
	 */
	if (addr & 0x3 || (uintptr_t)hbuf & 0x3)
		return -EINVAL;
	buf = (u32 *)hbuf;

	/* It's convenient to be able to handle lengths which aren't a
	 * multiple of 32-bits because we often end up transferring files to
	 * the firmware.  So we'll handle that by normalizing the length here
	 * and then handling any residual transfer at the end.
	 */
	resid = len & 0x3;
	len -= resid;

	/* Each PCI-E Memory Window is programmed with a window size -- or
	 * "aperture" -- which controls the granularity of its mapping onto
	 * adapter memory.  We need to grab that aperture in order to know
	 * how to use the specified window.  The window is also programmed
	 * with the base address of the Memory Window in BAR0's address
	 * space.  For T4 this is an absolute PCI-E Bus Address.  For T5
	 * the address is relative to BAR0.
	 */
	mem_reg = t4_read_reg(adap,
			      PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN,
						  win));

	/* a dead adapter will return 0xffffffff for PIO reads */
	if (mem_reg == 0xffffffff) {
		CH_WARN(adap, "Unable to read PCI-E Memory Window Base[%d]\n",
			win);
		return -ENXIO;
	}

	mem_aperture = 1 << (G_WINDOW(mem_reg) + X_WINDOW_SHIFT);
	mem_base = G_PCIEOFST(mem_reg) << X_PCIEOFST_SHIFT;
	if (is_t4(adap->params.chip))
		mem_base -= adap->t4_bar0;
	win_pf = is_t4(adap->params.chip) ? 0 : V_PFNUM(adap->pf);

	/* Calculate our initial PCI-E Memory Window Position and Offset into
	 * that Window.
	 */
	pos = addr & ~(mem_aperture-1);
	offset = addr - pos;

	/* Set up initial PCI-E Memory Window to cover the start of our
	 * transfer.  (Read it back to ensure that changes propagate before we
	 * attempt to use the new value.)
	 */
	t4_write_reg(adap,
		     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win),
		     pos | win_pf);
	t4_read_reg(adap,
		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win));

	/* Transfer data to/from the adapter as long as there's an integral
	 * number of 32-bit transfers to complete.
	 *
	 * A note on Endianness issues:
	 *
	 * The "register" reads and writes below from/to the PCI-E Memory
	 * Window invoke the standard adapter Big-Endian to PCI-E Link
	 * Little-Endian "swizzel."  As a result, if we have the following
	 * data in adapter memory:
	 *
	 *     Memory:  ... | b0 | b1 | b2 | b3 | ...
	 *     Address:      i+0  i+1  i+2  i+3
	 *
	 * Then a read of the adapter memory via the PCI-E Memory Window
	 * will yield:
	 *
	 *     x = readl(i)
	 *	   31                  0
	 *         [ b3 | b2 | b1 | b0 ]
	 *
	 * If this value is stored into local memory on a Little-Endian system
	 * it will show up correctly in local memory as:
	 *
	 *     ( ..., b0, b1, b2, b3, ... )
	 *
	 * But on a Big-Endian system, the store will show up in memory
	 * incorrectly swizzled as:
	 *
	 *     ( ..., b3, b2, b1, b0, ... )
	 *
	 * So we need to account for this in the reads and writes to the
	 * PCI-E Memory Window below by undoing the register read/write
	 * swizzels.
	 */
	while (len > 0) {
		if (dir == T4_MEMORY_READ)
			*buf++ = le32_to_cpu((__force __le32)t4_read_reg(adap,
						mem_base + offset));
		else
			t4_write_reg(adap, mem_base + offset,
				     (__force u32)cpu_to_le32(*buf++));
		offset += sizeof(__be32);
		len -= sizeof(__be32);

		/* If we've reached the end of our current window aperture,
		 * move the PCI-E Memory Window on to the next.  Note that
		 * doing this here after "len" may be 0 allows us to set up
		 * the PCI-E Memory Window for a possible final residual
		 * transfer below ...
		 */
		if (offset == mem_aperture) {
			pos += mem_aperture;
			offset = 0;
			t4_write_reg(adap,
				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
						    win), pos | win_pf);
			t4_read_reg(adap,
				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
						    win));
		}
	}

	/* If the original transfer had a length which wasn't a multiple of
	 * 32-bits, now's where we need to finish off the transfer of the
	 * residual amount.  The PCI-E Memory Window has already been moved
	 * above (if necessary) to cover this final transfer.
	 */
	if (resid) {
		union {
			u32 word;
			char byte[4];
		} last;
		unsigned char *bp;
		int i;

		if (dir == T4_MEMORY_READ) {
			last.word = le32_to_cpu(
					(__force __le32)t4_read_reg(adap,
						mem_base + offset));
			for (bp = (unsigned char *)buf, i = resid; i < 4; i++)
				bp[i] = last.byte[i];
		} else {
			last.word = *buf;
			for (i = resid; i < 4; i++)
				last.byte[i] = 0;
			t4_write_reg(adap, mem_base + offset,
				     (__force u32)cpu_to_le32(last.word));
		}
	}

	return 0;
}

/**
 *	t4_memory_rw_mtype - read/write EDC 0, EDC 1 or MC via PCIE memory window
 *	@adap: the adapter
 *	@win: PCI-E Memory Window to use
 *	@mtype: memory type: MEM_EDC0, MEM_EDC1, MEM_HMA or MEM_MC
 *	@maddr: address within indicated memory type
 *	@len: amount of memory to transfer
 *	@hbuf: host memory buffer
 *	@dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
 *
 *	Reads/writes adapter memory using t4_memory_rw_addr().  This routine
 *	provides an (memory type, address withing memory type) interface.
 */
int t4_memory_rw_mtype(struct adapter *adap, int win, int mtype, u32 maddr,
		       u32 len, void *hbuf, int dir)
{
	u32 mtype_offset;
	u32 edc_size, mc_size;

	/* Offset into the region of memory which is being accessed
	 * MEM_EDC0 = 0
	 * MEM_EDC1 = 1
	 * MEM_MC   = 2 -- MEM_MC for chips with only 1 memory controller
	 * MEM_MC1  = 3 -- for chips with 2 memory controllers (e.g. T5)
	 * MEM_HMA  = 4
	 */
	edc_size  = G_EDRAM0_SIZE(t4_read_reg(adap, A_MA_EDRAM0_BAR));
	if (mtype == MEM_HMA) {
		mtype_offset = 2 * (edc_size * 1024 * 1024);
	} else if (mtype != MEM_MC1)
		mtype_offset = (mtype * (edc_size * 1024 * 1024));
	else {
		mc_size = G_EXT_MEM0_SIZE(t4_read_reg(adap,
						      A_MA_EXT_MEMORY0_BAR));
		mtype_offset = (MEM_MC0 * edc_size + mc_size) * 1024 * 1024;
	}

	return t4_memory_rw_addr(adap, win,
				 mtype_offset + maddr, len,
				 hbuf, dir);
}

/*
 * Return the specified PCI-E Configuration Space register from our Physical
 * Function.  We try first via a Firmware LDST Command (if fw_attach != 0)
 * since we prefer to let the firmware own all of these registers, but if that
 * fails we go for it directly ourselves.
 */
u32 t4_read_pcie_cfg4(struct adapter *adap, int reg, int drv_fw_attach)
{
	u32 val;

	/*
	 * If fw_attach != 0, construct and send the Firmware LDST Command to
	 * retrieve the specified PCI-E Configuration Space register.
	 */
	if (drv_fw_attach != 0) {
		struct fw_ldst_cmd ldst_cmd;
		int ret;

		memset(&ldst_cmd, 0, sizeof(ldst_cmd));
		ldst_cmd.op_to_addrspace =
			cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
				    F_FW_CMD_REQUEST |
				    F_FW_CMD_READ |
				    V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_FUNC_PCIE));
		ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd));
		ldst_cmd.u.pcie.select_naccess = V_FW_LDST_CMD_NACCESS(1);
		ldst_cmd.u.pcie.ctrl_to_fn =
			(F_FW_LDST_CMD_LC | V_FW_LDST_CMD_FN(adap->pf));
		ldst_cmd.u.pcie.r = reg;

		/*
		 * If the LDST Command succeeds, return the result, otherwise
		 * fall through to reading it directly ourselves ...
		 */
		ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
				 &ldst_cmd);
		if (ret == 0)
			return be32_to_cpu(ldst_cmd.u.pcie.data[0]);

		CH_WARN(adap, "Firmware failed to return "
			"Configuration Space register %d, err = %d\n",
			reg, -ret);
	}

	/*
	 * Read the desired Configuration Space register via the PCI-E
	 * Backdoor mechanism.
	 */
	t4_hw_pci_read_cfg4(adap, reg, &val);
	return val;
}

/*
 * Get the window based on base passed to it.
 * Window aperture is currently unhandled, but there is no use case for it
 * right now
 */
static int t4_get_window(struct adapter *adap, u64 pci_base, u64 pci_mask, u64 memwin_base, int drv_fw_attach)
{
	if (is_t4(adap->params.chip)) {
		u32 bar0;

		/*
		 * Truncation intentional: we only read the bottom 32-bits of
		 * the 64-bit BAR0/BAR1 ...  We use the hardware backdoor
		 * mechanism to read BAR0 instead of using
		 * pci_resource_start() because we could be operating from
		 * within a Virtual Machine which is trapping our accesses to
		 * our Configuration Space and we need to set up the PCI-E
		 * Memory Window decoders with the actual addresses which will
		 * be coming across the PCI-E link.
		 */
		bar0 = t4_read_pcie_cfg4(adap, pci_base, drv_fw_attach);
		bar0 &= pci_mask;
		adap->t4_bar0 = bar0;

		return bar0 + memwin_base;
	} else {
		/* For T5, only relative offset inside the PCIe BAR is passed */
		return memwin_base;
	}
}

/* Get the default utility window (win0) used by everyone */
int t4_get_util_window(struct adapter *adap, int drv_fw_attach)
{
	return t4_get_window(adap, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_MEM_MASK, MEMWIN0_BASE, drv_fw_attach);
}

/*
 * Set up memory window for accessing adapter memory ranges.  (Read
 * back MA register to ensure that changes propagate before we attempt
 * to use the new values.)
 */
void t4_setup_memwin(struct adapter *adap, u32 memwin_base, u32 window)
{
	t4_write_reg(adap, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, window),
		     memwin_base | V_BIR(0) |
		     V_WINDOW(ilog2(MEMWIN0_APERTURE) - X_WINDOW_SHIFT));
	t4_read_reg(adap, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, window));
}

/**
 *	t4_get_regs_len - return the size of the chips register set
 *	@adapter: the adapter
 *
 *	Returns the size of the chip's BAR0 register space.
 */
unsigned int t4_get_regs_len(struct adapter *adapter)
{
	unsigned int chip_version = CHELSIO_CHIP_VERSION(adapter->params.chip);

	switch (chip_version) {
	case CHELSIO_T4:
		return T4_REGMAP_SIZE;

	case CHELSIO_T5:
	case CHELSIO_T6:
		return T5_REGMAP_SIZE;
	}

	CH_ERR(adapter,
		"Unsupported chip version %d\n", chip_version);
	return 0;
}

/**
 *	t4_get_regs - read chip registers into provided buffer
 *	@adap: the adapter
 *	@buf: register buffer
 *	@buf_size: size (in bytes) of register buffer
 *
 *	If the provided register buffer isn't large enough for the chip's
 *	full register range, the register dump will be truncated to the
 *	register buffer's size.
 */
void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
{
	static const unsigned int t4_reg_ranges[] = {
		0x1008, 0x1108,
		0x1180, 0x1184,
		0x1190, 0x1194,
		0x11a0, 0x11a4,
		0x11b0, 0x11b4,
		0x11fc, 0x123c,
		0x1300, 0x173c,
		0x1800, 0x18fc,
		0x3000, 0x30d8,
		0x30e0, 0x30e4,
		0x30ec, 0x5910,
		0x5920, 0x5924,
		0x5960, 0x5960,
		0x5968, 0x5968,
		0x5970, 0x5970,
		0x5978, 0x5978,
		0x5980, 0x5980,
		0x5988, 0x5988,
		0x5990, 0x5990,
		0x5998, 0x5998,
		0x59a0, 0x59d4,
		0x5a00, 0x5ae0,
		0x5ae8, 0x5ae8,
		0x5af0, 0x5af0,
		0x5af8, 0x5af8,
		0x6000, 0x6098,
		0x6100, 0x6150,
		0x6200, 0x6208,
		0x6240, 0x6248,
		0x6280, 0x62b0,
		0x62c0, 0x6338,
		0x6370, 0x638c,
		0x6400, 0x643c,
		0x6500, 0x6524,
		0x6a00, 0x6a04,
		0x6a14, 0x6a38,
		0x6a60, 0x6a70,
		0x6a78, 0x6a78,
		0x6b00, 0x6b0c,
		0x6b1c, 0x6b84,
		0x6bf0, 0x6bf8,
		0x6c00, 0x6c0c,