156b2bdd1SGireesh Nagabhushana /*
256b2bdd1SGireesh Nagabhushana  * This file and its contents are supplied under the terms of the
356b2bdd1SGireesh Nagabhushana  * Common Development and Distribution License ("CDDL"), version 1.0.
456b2bdd1SGireesh Nagabhushana  * You may only use this file in accordance with the terms of version
556b2bdd1SGireesh Nagabhushana  * 1.0 of the CDDL.
656b2bdd1SGireesh Nagabhushana  *
756b2bdd1SGireesh Nagabhushana  * A full copy of the text of the CDDL should have accompanied this
856b2bdd1SGireesh Nagabhushana  * source. A copy of the CDDL is also available via the Internet at
956b2bdd1SGireesh Nagabhushana  * http://www.illumos.org/license/CDDL.
1056b2bdd1SGireesh Nagabhushana  */
1156b2bdd1SGireesh Nagabhushana 
1256b2bdd1SGireesh Nagabhushana /*
133dde7c95SVishal Kulkarni  * This file is part of the Chelsio T4/T5/T6 Ethernet driver.
1456b2bdd1SGireesh Nagabhushana  *
157e6ad469SVishal Kulkarni  * Copyright (C) 2003-2019 Chelsio Communications.  All rights reserved.
1656b2bdd1SGireesh Nagabhushana  *
1756b2bdd1SGireesh Nagabhushana  * This program is distributed in the hope that it will be useful, but WITHOUT
1856b2bdd1SGireesh Nagabhushana  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1956b2bdd1SGireesh Nagabhushana  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
2056b2bdd1SGireesh Nagabhushana  * release for licensing terms and conditions.
2156b2bdd1SGireesh Nagabhushana  */
223dde7c95SVishal Kulkarni 
23d77e6e0fSPaul Winder /*
24d77e6e0fSPaul Winder  * Copyright 2020 RackTop Systems, Inc.
25d77e6e0fSPaul Winder  */
26d77e6e0fSPaul Winder 
2756b2bdd1SGireesh Nagabhushana #include "common.h"
2856b2bdd1SGireesh Nagabhushana #include "t4_regs.h"
2956b2bdd1SGireesh Nagabhushana #include "t4_regs_values.h"
3056b2bdd1SGireesh Nagabhushana #include "t4fw_interface.h"
3156b2bdd1SGireesh Nagabhushana 
323dde7c95SVishal Kulkarni /**
3356b2bdd1SGireesh Nagabhushana  *	t4_wait_op_done_val - wait until an operation is completed
3456b2bdd1SGireesh Nagabhushana  *	@adapter: the adapter performing the operation
3556b2bdd1SGireesh Nagabhushana  *	@reg: the register to check for completion
3656b2bdd1SGireesh Nagabhushana  *	@mask: a single-bit field within @reg that indicates completion
3756b2bdd1SGireesh Nagabhushana  *	@polarity: the value of the field when the operation is completed
3856b2bdd1SGireesh Nagabhushana  *	@attempts: number of check iterations
3956b2bdd1SGireesh Nagabhushana  *	@delay: delay in usecs between iterations
4056b2bdd1SGireesh Nagabhushana  *	@valp: where to store the value of the register at completion time
4156b2bdd1SGireesh Nagabhushana  *
4256b2bdd1SGireesh Nagabhushana  *	Wait until an operation is completed by checking a bit in a register
4356b2bdd1SGireesh Nagabhushana  *	up to @attempts times.  If @valp is not NULL the value of the register
4456b2bdd1SGireesh Nagabhushana  *	at the time it indicated completion is stored there.  Returns 0 if the
4556b2bdd1SGireesh Nagabhushana  *	operation completes and	-EAGAIN	otherwise.
4656b2bdd1SGireesh Nagabhushana  */
t4_wait_op_done_val(struct adapter * adapter,int reg,u32 mask,int polarity,int attempts,int delay,u32 * valp)473dde7c95SVishal Kulkarni static int t4_wait_op_done_val(struct adapter *adapter, int reg, u32 mask,
483dde7c95SVishal Kulkarni 			       int polarity, int attempts, int delay, u32 *valp)
4956b2bdd1SGireesh Nagabhushana {
5056b2bdd1SGireesh Nagabhushana 	while (1) {
5156b2bdd1SGireesh Nagabhushana 		u32 val = t4_read_reg(adapter, reg);
5256b2bdd1SGireesh Nagabhushana 
5356b2bdd1SGireesh Nagabhushana 		if (!!(val & mask) == polarity) {
543dde7c95SVishal Kulkarni 			if (valp)
5556b2bdd1SGireesh Nagabhushana 				*valp = val;
563dde7c95SVishal Kulkarni 			return 0;
5756b2bdd1SGireesh Nagabhushana 		}
583dde7c95SVishal Kulkarni 		if (--attempts == 0)
593dde7c95SVishal Kulkarni 			return -EAGAIN;
603dde7c95SVishal Kulkarni 		if (delay)
61de483253SVishal Kulkarni 			udelay(delay);
6256b2bdd1SGireesh Nagabhushana 	}
633dde7c95SVishal Kulkarni }
6456b2bdd1SGireesh Nagabhushana 
t4_wait_op_done(struct adapter * adapter,int reg,u32 mask,int polarity,int attempts,int delay)653dde7c95SVishal Kulkarni static inline int t4_wait_op_done(struct adapter *adapter, int reg, u32 mask,
663dde7c95SVishal Kulkarni 				  int polarity, int attempts, int delay)
673dde7c95SVishal Kulkarni {
683dde7c95SVishal Kulkarni 	return t4_wait_op_done_val(adapter, reg, mask, polarity, attempts,
693dde7c95SVishal Kulkarni 				   delay, NULL);
7056b2bdd1SGireesh Nagabhushana }
7156b2bdd1SGireesh Nagabhushana 
723dde7c95SVishal Kulkarni /**
7356b2bdd1SGireesh Nagabhushana  *	t4_set_reg_field - set a register field to a value
7456b2bdd1SGireesh Nagabhushana  *	@adapter: the adapter to program
7556b2bdd1SGireesh Nagabhushana  *	@addr: the register address
7656b2bdd1SGireesh Nagabhushana  *	@mask: specifies the portion of the register to modify
7756b2bdd1SGireesh Nagabhushana  *	@val: the new value for the register field
7856b2bdd1SGireesh Nagabhushana  *
7956b2bdd1SGireesh Nagabhushana  *	Sets a register field specified by the supplied mask to the
8056b2bdd1SGireesh Nagabhushana  *	given value.
8156b2bdd1SGireesh Nagabhushana  */
t4_set_reg_field(struct adapter * adapter,unsigned int addr,u32 mask,u32 val)823dde7c95SVishal Kulkarni void t4_set_reg_field(struct adapter *adapter, unsigned int addr, u32 mask,
833dde7c95SVishal Kulkarni 		      u32 val)
8456b2bdd1SGireesh Nagabhushana {
8556b2bdd1SGireesh Nagabhushana 	u32 v = t4_read_reg(adapter, addr) & ~mask;
8656b2bdd1SGireesh Nagabhushana 
8756b2bdd1SGireesh Nagabhushana 	t4_write_reg(adapter, addr, v | val);
883dde7c95SVishal Kulkarni 	(void) t4_read_reg(adapter, addr);      /* flush */
8956b2bdd1SGireesh Nagabhushana }
9056b2bdd1SGireesh Nagabhushana 
913dde7c95SVishal Kulkarni /**
9256b2bdd1SGireesh Nagabhushana  *	t4_read_indirect - read indirectly addressed registers
9356b2bdd1SGireesh Nagabhushana  *	@adap: the adapter
9456b2bdd1SGireesh Nagabhushana  *	@addr_reg: register holding the indirect address
9556b2bdd1SGireesh Nagabhushana  *	@data_reg: register holding the value of the indirect register
9656b2bdd1SGireesh Nagabhushana  *	@vals: where the read register values are stored
9756b2bdd1SGireesh Nagabhushana  *	@nregs: how many indirect registers to read
9856b2bdd1SGireesh Nagabhushana  *	@start_idx: index of first indirect register to read
9956b2bdd1SGireesh Nagabhushana  *
10056b2bdd1SGireesh Nagabhushana  *	Reads registers that are accessed indirectly through an address/data
10156b2bdd1SGireesh Nagabhushana  *	register pair.
10256b2bdd1SGireesh Nagabhushana  */
t4_read_indirect(struct adapter * adap,unsigned int addr_reg,unsigned int data_reg,u32 * vals,unsigned int nregs,unsigned int start_idx)1033dde7c95SVishal Kulkarni void t4_read_indirect(struct adapter *adap, unsigned int addr_reg,
1043dde7c95SVishal Kulkarni 			     unsigned int data_reg, u32 *vals,
1053dde7c95SVishal Kulkarni 			     unsigned int nregs, unsigned int start_idx)
10656b2bdd1SGireesh Nagabhushana {
10756b2bdd1SGireesh Nagabhushana 	while (nregs--) {
10856b2bdd1SGireesh Nagabhushana 		t4_write_reg(adap, addr_reg, start_idx);
10956b2bdd1SGireesh Nagabhushana 		*vals++ = t4_read_reg(adap, data_reg);
11056b2bdd1SGireesh Nagabhushana 		start_idx++;
11156b2bdd1SGireesh Nagabhushana 	}
11256b2bdd1SGireesh Nagabhushana }
11356b2bdd1SGireesh Nagabhushana 
1143dde7c95SVishal Kulkarni /**
11556b2bdd1SGireesh Nagabhushana  *	t4_write_indirect - write indirectly addressed registers
11656b2bdd1SGireesh Nagabhushana  *	@adap: the adapter
11756b2bdd1SGireesh Nagabhushana  *	@addr_reg: register holding the indirect addresses
11856b2bdd1SGireesh Nagabhushana  *	@data_reg: register holding the value for the indirect registers
11956b2bdd1SGireesh Nagabhushana  *	@vals: values to write
12056b2bdd1SGireesh Nagabhushana  *	@nregs: how many indirect registers to write
12156b2bdd1SGireesh Nagabhushana  *	@start_idx: address of first indirect register to write
12256b2bdd1SGireesh Nagabhushana  *
12356b2bdd1SGireesh Nagabhushana  *	Writes a sequential block of registers that are accessed indirectly
12456b2bdd1SGireesh Nagabhushana  *	through an address/data register pair.
12556b2bdd1SGireesh Nagabhushana  */
t4_write_indirect(struct adapter * adap,unsigned int addr_reg,unsigned int data_reg,const u32 * vals,unsigned int nregs,unsigned int start_idx)1263dde7c95SVishal Kulkarni void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
1273dde7c95SVishal Kulkarni 		       unsigned int data_reg, const u32 *vals,
1283dde7c95SVishal Kulkarni 		       unsigned int nregs, unsigned int start_idx)
12956b2bdd1SGireesh Nagabhushana {
13056b2bdd1SGireesh Nagabhushana 	while (nregs--) {
13156b2bdd1SGireesh Nagabhushana 		t4_write_reg(adap, addr_reg, start_idx++);
13256b2bdd1SGireesh Nagabhushana 		t4_write_reg(adap, data_reg, *vals++);
13356b2bdd1SGireesh Nagabhushana 	}
13456b2bdd1SGireesh Nagabhushana }
13556b2bdd1SGireesh Nagabhushana 
136de483253SVishal Kulkarni /*
1373dde7c95SVishal Kulkarni  * Read a 32-bit PCI Configuration Space register via the PCI-E backdoor
1383dde7c95SVishal Kulkarni  * mechanism.  This guarantees that we get the real value even if we're
1393dde7c95SVishal Kulkarni  * operating within a Virtual Machine and the Hypervisor is trapping our
1403dde7c95SVishal Kulkarni  * Configuration Space accesses.
1413dde7c95SVishal Kulkarni  *
1423dde7c95SVishal Kulkarni  * N.B. This routine should only be used as a last resort: the firmware uses
1433dde7c95SVishal Kulkarni  *      the backdoor registers on a regular basis and we can end up
1443dde7c95SVishal Kulkarni  *      conflicting with it's uses!
1453dde7c95SVishal Kulkarni  */
t4_hw_pci_read_cfg4(struct adapter * adap,int reg,u32 * val)1463dde7c95SVishal Kulkarni void t4_hw_pci_read_cfg4(struct adapter *adap, int reg, u32 *val)
1473dde7c95SVishal Kulkarni {
1483dde7c95SVishal Kulkarni 	u32 req = V_FUNCTION(adap->pf) | V_REGISTER(reg);
1493dde7c95SVishal Kulkarni 
1503dde7c95SVishal Kulkarni 	if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
1513dde7c95SVishal Kulkarni 		req |= F_ENABLE;
1523dde7c95SVishal Kulkarni 	else
1533dde7c95SVishal Kulkarni 		req |= F_T6_ENABLE;
1543dde7c95SVishal Kulkarni 
1553dde7c95SVishal Kulkarni 	if (is_t4(adap->params.chip))
1563dde7c95SVishal Kulkarni 		req |= F_LOCALCFG;
1573dde7c95SVishal Kulkarni 
1583dde7c95SVishal Kulkarni 	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, req);
1593dde7c95SVishal Kulkarni 	*val = t4_read_reg(adap, A_PCIE_CFG_SPACE_DATA);
1603dde7c95SVishal Kulkarni 
1613dde7c95SVishal Kulkarni 	/* Reset F_ENABLE to 0 so reads of PCIE_CFG_SPACE_DATA won't cause a
1623dde7c95SVishal Kulkarni 	 * Configuration Space read.  (None of the other fields matter when
1633dde7c95SVishal Kulkarni 	 * F_ENABLE is 0 so a simple register write is easier than a
1643dde7c95SVishal Kulkarni 	 * read-modify-write via t4_set_reg_field().)
1653dde7c95SVishal Kulkarni 	 */
1663dde7c95SVishal Kulkarni 	t4_write_reg(adap, A_PCIE_CFG_SPACE_REQ, 0);
1673dde7c95SVishal Kulkarni }
1683dde7c95SVishal Kulkarni 
1693dde7c95SVishal Kulkarni /*
1703dde7c95SVishal Kulkarni  * t4_report_fw_error - report firmware error
1713dde7c95SVishal Kulkarni  * @adap: the adapter
172de483253SVishal Kulkarni  *
1733dde7c95SVishal Kulkarni  * The adapter firmware can indicate error conditions to the host.
1743dde7c95SVishal Kulkarni  * If the firmware has indicated an error, print out the reason for
1753dde7c95SVishal Kulkarni  * the firmware error.
176de483253SVishal Kulkarni  */
t4_report_fw_error(struct adapter * adap)177de483253SVishal Kulkarni static void t4_report_fw_error(struct adapter *adap)
178de483253SVishal Kulkarni {
1793dde7c95SVishal Kulkarni 	static const char *const reason[] = {
180de483253SVishal Kulkarni 		"Crash",			/* PCIE_FW_EVAL_CRASH */
181de483253SVishal Kulkarni 		"During Device Preparation",	/* PCIE_FW_EVAL_PREP */
182de483253SVishal Kulkarni 		"During Device Configuration",	/* PCIE_FW_EVAL_CONF */
183de483253SVishal Kulkarni 		"During Device Initialization",	/* PCIE_FW_EVAL_INIT */
184de483253SVishal Kulkarni 		"Unexpected Event",		/* PCIE_FW_EVAL_UNEXPECTEDEVENT */
185de483253SVishal Kulkarni 		"Insufficient Airflow",		/* PCIE_FW_EVAL_OVERHEAT */
186de483253SVishal Kulkarni 		"Device Shutdown",		/* PCIE_FW_EVAL_DEVICESHUTDOWN */
187de483253SVishal Kulkarni 		"Reserved",			/* reserved */
188de483253SVishal Kulkarni 	};
189de483253SVishal Kulkarni 	u32 pcie_fw;
190de483253SVishal Kulkarni 
191de483253SVishal Kulkarni 	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
1927e6ad469SVishal Kulkarni 	if (pcie_fw & F_PCIE_FW_ERR) {
193de483253SVishal Kulkarni 		CH_ERR(adap, "Firmware reports adapter error: %s\n",
1943dde7c95SVishal Kulkarni 			reason[G_PCIE_FW_EVAL(pcie_fw)]);
1957e6ad469SVishal Kulkarni 		adap->flags &= ~FW_OK;
1967e6ad469SVishal Kulkarni 	}
197de483253SVishal Kulkarni }
198de483253SVishal Kulkarni 
19956b2bdd1SGireesh Nagabhushana /*
20056b2bdd1SGireesh Nagabhushana  * Get the reply to a mailbox command and store it in @rpl in big-endian order.
20156b2bdd1SGireesh Nagabhushana  */
get_mbox_rpl(struct adapter * adap,__be64 * rpl,int nflit,u32 mbox_addr)2023dde7c95SVishal Kulkarni static void get_mbox_rpl(struct adapter *adap, __be64 *rpl, int nflit,
2033dde7c95SVishal Kulkarni 			 u32 mbox_addr)
20456b2bdd1SGireesh Nagabhushana {
2053dde7c95SVishal Kulkarni 	for ( ; nflit; nflit--, mbox_addr += 8)
20656b2bdd1SGireesh Nagabhushana 		*rpl++ = cpu_to_be64(t4_read_reg64(adap, mbox_addr));
20756b2bdd1SGireesh Nagabhushana }
20856b2bdd1SGireesh Nagabhushana 
20956b2bdd1SGireesh Nagabhushana /*
21056b2bdd1SGireesh Nagabhushana  * Handle a FW assertion reported in a mailbox.
21156b2bdd1SGireesh Nagabhushana  */
fw_asrt(struct adapter * adap,struct fw_debug_cmd * asrt)2123dde7c95SVishal Kulkarni static void fw_asrt(struct adapter *adap, struct fw_debug_cmd *asrt)
2133dde7c95SVishal Kulkarni {
2143dde7c95SVishal Kulkarni 	CH_ALERT(adap,
2153dde7c95SVishal Kulkarni 		  "FW assertion at %.16s:%u, val0 %#x, val1 %#x\n",
2163dde7c95SVishal Kulkarni 		  asrt->u.assert.filename_0_7,
2173dde7c95SVishal Kulkarni 		  be32_to_cpu(asrt->u.assert.line),
2183dde7c95SVishal Kulkarni 		  be32_to_cpu(asrt->u.assert.x),
2193dde7c95SVishal Kulkarni 		  be32_to_cpu(asrt->u.assert.y));
2203dde7c95SVishal Kulkarni }
2213dde7c95SVishal Kulkarni 
2223dde7c95SVishal Kulkarni #define X_CIM_PF_NOACCESS 0xeeeeeeee
2233dde7c95SVishal Kulkarni 
2243dde7c95SVishal Kulkarni /*
2253dde7c95SVishal Kulkarni  * If the OS Driver wants busy waits to keep a watchdog happy, tap it during
2263dde7c95SVishal Kulkarni  * busy loops which don't sleep.
2273dde7c95SVishal Kulkarni  */
2283dde7c95SVishal Kulkarni #ifdef T4_OS_NEEDS_TOUCH_NMI_WATCHDOG
2293dde7c95SVishal Kulkarni #define T4_OS_TOUCH_NMI_WATCHDOG()	t4_os_touch_nmi_watchdog()
2303dde7c95SVishal Kulkarni #else
2313dde7c95SVishal Kulkarni #define T4_OS_TOUCH_NMI_WATCHDOG()
2323dde7c95SVishal Kulkarni #endif
2333dde7c95SVishal Kulkarni 
2343dde7c95SVishal Kulkarni #ifdef T4_OS_LOG_MBOX_CMDS
2353dde7c95SVishal Kulkarni /**
2363dde7c95SVishal Kulkarni  *	t4_record_mbox - record a Firmware Mailbox Command/Reply in the log
2373dde7c95SVishal Kulkarni  *	@adapter: the adapter
2383dde7c95SVishal Kulkarni  *	@cmd: the Firmware Mailbox Command or Reply
2393dde7c95SVishal Kulkarni  *	@size: command length in bytes
2403dde7c95SVishal Kulkarni  *	@access: the time (ms) needed to access the Firmware Mailbox
2413dde7c95SVishal Kulkarni  *	@execute: the time (ms) the command spent being executed
2423dde7c95SVishal Kulkarni  */
t4_record_mbox(struct adapter * adapter,const __be64 * cmd,unsigned int size,int access,int execute)2433dde7c95SVishal Kulkarni static void t4_record_mbox(struct adapter *adapter,
2443dde7c95SVishal Kulkarni 			   const __be64 *cmd, unsigned int size,
2453dde7c95SVishal Kulkarni 			   int access, int execute)
2463dde7c95SVishal Kulkarni {
2473dde7c95SVishal Kulkarni 	struct mbox_cmd_log *log = adapter->mbox_log;
2483dde7c95SVishal Kulkarni 	struct mbox_cmd *entry;
2493dde7c95SVishal Kulkarni 	int i;
2503dde7c95SVishal Kulkarni 
2513dde7c95SVishal Kulkarni 	entry = mbox_cmd_log_entry(log, log->cursor++);
2523dde7c95SVishal Kulkarni 	if (log->cursor == log->size)
2533dde7c95SVishal Kulkarni 		log->cursor = 0;
2543dde7c95SVishal Kulkarni 
2553dde7c95SVishal Kulkarni 	for (i = 0; i < size/8; i++)
2563dde7c95SVishal Kulkarni 		entry->cmd[i] = be64_to_cpu(cmd[i]);
2573dde7c95SVishal Kulkarni 	while (i < MBOX_LEN/8)
2583dde7c95SVishal Kulkarni 		entry->cmd[i++] = 0;
2593dde7c95SVishal Kulkarni 	entry->timestamp = t4_os_timestamp();
2603dde7c95SVishal Kulkarni 	entry->seqno = log->seqno++;
2613dde7c95SVishal Kulkarni 	entry->access = access;
2623dde7c95SVishal Kulkarni 	entry->execute = execute;
2633dde7c95SVishal Kulkarni }
2643dde7c95SVishal Kulkarni 
2653dde7c95SVishal Kulkarni #define T4_RECORD_MBOX(__adapter, __cmd, __size, __access, __execute) \
2663dde7c95SVishal Kulkarni 	t4_record_mbox(__adapter, __cmd, __size, __access, __execute)
2673dde7c95SVishal Kulkarni 
2683dde7c95SVishal Kulkarni #else /* !T4_OS_LOG_MBOX_CMDS */
2693dde7c95SVishal Kulkarni 
2703dde7c95SVishal Kulkarni #define T4_RECORD_MBOX(__adapter, __cmd, __size, __access, __execute) \
2713dde7c95SVishal Kulkarni 	/* nothing */
2723dde7c95SVishal Kulkarni 
2733dde7c95SVishal Kulkarni #endif /* !T4_OS_LOG_MBOX_CMDS */
2743dde7c95SVishal Kulkarni 
2753dde7c95SVishal Kulkarni /**
2763dde7c95SVishal Kulkarni  *	t4_record_mbox_marker - record a marker in the mailbox log
2773dde7c95SVishal Kulkarni  *	@adapter: the adapter
2783dde7c95SVishal Kulkarni  *	@marker: byte array marker
2793dde7c95SVishal Kulkarni  *	@size: marker size in bytes
2803dde7c95SVishal Kulkarni  *
2813dde7c95SVishal Kulkarni  *	We inject a "fake mailbox command" into the Firmware Mailbox Log
2823dde7c95SVishal Kulkarni  *	using a known command token and then the bytes of the specified
2833dde7c95SVishal Kulkarni  *	marker.  This lets debugging code inject markers into the log to
2843dde7c95SVishal Kulkarni  *	help identify which commands are in response to higher level code.
2853dde7c95SVishal Kulkarni  */
t4_record_mbox_marker(struct adapter * adapter,const void * marker,unsigned int size)2863dde7c95SVishal Kulkarni void t4_record_mbox_marker(struct adapter *adapter,
2873dde7c95SVishal Kulkarni 			   const void *marker, unsigned int size)
28856b2bdd1SGireesh Nagabhushana {
2893dde7c95SVishal Kulkarni #ifdef T4_OS_LOG_MBOX_CMDS
2903dde7c95SVishal Kulkarni 	__be64 marker_cmd[MBOX_LEN/8];
2913dde7c95SVishal Kulkarni 	const unsigned int max_marker = sizeof marker_cmd - sizeof (__be64);
2923dde7c95SVishal Kulkarni 	unsigned int marker_cmd_size;
2933dde7c95SVishal Kulkarni 
2943dde7c95SVishal Kulkarni 	if (size > max_marker)
2953dde7c95SVishal Kulkarni 		size = max_marker;
29656b2bdd1SGireesh Nagabhushana 
2973dde7c95SVishal Kulkarni 	marker_cmd[0] = cpu_to_be64(~0LLU);
2983dde7c95SVishal Kulkarni 	memcpy(&marker_cmd[1], marker, size);
2993dde7c95SVishal Kulkarni 	memset((unsigned char *)&marker_cmd[1] + size, 0, max_marker - size);
3003dde7c95SVishal Kulkarni 	marker_cmd_size = sizeof (__be64) + roundup(size, sizeof (__be64));
3013dde7c95SVishal Kulkarni 
3023dde7c95SVishal Kulkarni 	t4_record_mbox(adapter, marker_cmd, marker_cmd_size, 0, 0);
3033dde7c95SVishal Kulkarni #endif /* T4_OS_LOG_MBOX_CMDS */
30456b2bdd1SGireesh Nagabhushana }
30556b2bdd1SGireesh Nagabhushana 
30656b2bdd1SGireesh Nagabhushana /*
3073dde7c95SVishal Kulkarni  * Delay time in microseconds to wait for mailbox access/fw reply
3083dde7c95SVishal Kulkarni  * to mailbox command
3093dde7c95SVishal Kulkarni  */
3103dde7c95SVishal Kulkarni #define MIN_MBOX_CMD_DELAY 900
3113dde7c95SVishal Kulkarni #define MBOX_CMD_DELAY 1000
3123dde7c95SVishal Kulkarni 
3133dde7c95SVishal Kulkarni /**
3143dde7c95SVishal Kulkarni  *	t4_wr_mbox_meat_timeout - send a command to FW through the given mailbox
31556b2bdd1SGireesh Nagabhushana  *	@adap: the adapter
31656b2bdd1SGireesh Nagabhushana  *	@mbox: index of the mailbox to use
31756b2bdd1SGireesh Nagabhushana  *	@cmd: the command to write
31856b2bdd1SGireesh Nagabhushana  *	@size: command length in bytes
31956b2bdd1SGireesh Nagabhushana  *	@rpl: where to optionally store the reply
32056b2bdd1SGireesh Nagabhushana  *	@sleep_ok: if true we may sleep while awaiting command completion
3213dde7c95SVishal Kulkarni  *	@timeout: time to wait for command to finish before timing out
3223dde7c95SVishal Kulkarni  *		(negative implies @sleep_ok=false)
32356b2bdd1SGireesh Nagabhushana  *
32456b2bdd1SGireesh Nagabhushana  *	Sends the given command to FW through the selected mailbox and waits
32556b2bdd1SGireesh Nagabhushana  *	for the FW to execute the command.  If @rpl is not %NULL it is used to
32656b2bdd1SGireesh Nagabhushana  *	store the FW's reply to the command.  The command and its optional
32756b2bdd1SGireesh Nagabhushana  *	reply are of the same length.  Some FW commands like RESET and
32856b2bdd1SGireesh Nagabhushana  *	INITIALIZE can take a considerable amount of time to execute.
32956b2bdd1SGireesh Nagabhushana  *	@sleep_ok determines whether we may sleep while awaiting the response.
33056b2bdd1SGireesh Nagabhushana  *	If sleeping is allowed we use progressive backoff otherwise we spin.
3313dde7c95SVishal Kulkarni  *	Note that passing in a negative @timeout is an alternate mechanism
3323dde7c95SVishal Kulkarni  *	for specifying @sleep_ok=false.  This is useful when a higher level
3333dde7c95SVishal Kulkarni  *	interface allows for specification of @timeout but not @sleep_ok ...
33456b2bdd1SGireesh Nagabhushana  *
33556b2bdd1SGireesh Nagabhushana  *	The return value is 0 on success or a negative errno on failure.  A
33656b2bdd1SGireesh Nagabhushana  *	failure can happen either because we are not able to execute the
33756b2bdd1SGireesh Nagabhushana  *	command or FW executes it but signals an error.  In the latter case
33856b2bdd1SGireesh Nagabhushana  *	the return value is the error code indicated by FW (negated).
33956b2bdd1SGireesh Nagabhushana  */
t4_wr_mbox_meat_timeout(struct adapter * adap,int mbox,const void * cmd,int size,void * rpl,bool sleep_ok,int timeout)3403dde7c95SVishal Kulkarni int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
3413dde7c95SVishal Kulkarni 			    int size, void *rpl, bool sleep_ok, int timeout)
34256b2bdd1SGireesh Nagabhushana {
343*6feac2e3SRahul Lakkireddy #ifdef T4_OS_LOG_MBOX_CMDS
3443dde7c95SVishal Kulkarni 	u16 access = 0;
345*6feac2e3SRahul Lakkireddy #endif /* T4_OS_LOG_MBOX_CMDS */
34656b2bdd1SGireesh Nagabhushana 	u32 v;
34756b2bdd1SGireesh Nagabhushana 	u64 res;
3483dde7c95SVishal Kulkarni 	int i, ret;
34956b2bdd1SGireesh Nagabhushana 	const __be64 *p = cmd;
35056b2bdd1SGireesh Nagabhushana 	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
35156b2bdd1SGireesh Nagabhushana 	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
3523dde7c95SVishal Kulkarni 	u32 ctl;
3533dde7c95SVishal Kulkarni 	__be64 cmd_rpl[MBOX_LEN/8];
354*6feac2e3SRahul Lakkireddy 	struct t4_mbox_list entry;
3553dde7c95SVishal Kulkarni 	u32 pcie_fw;
35656b2bdd1SGireesh Nagabhushana 
35756b2bdd1SGireesh Nagabhushana 	if ((size & 15) || size > MBOX_LEN)
3583dde7c95SVishal Kulkarni 		return -EINVAL;
3593dde7c95SVishal Kulkarni 
3603dde7c95SVishal Kulkarni 	/*
3613dde7c95SVishal Kulkarni 	 * If we have a negative timeout, that implies that we can't sleep.
3623dde7c95SVishal Kulkarni 	 */
3633dde7c95SVishal Kulkarni 	if (timeout < 0) {
3643dde7c95SVishal Kulkarni 		sleep_ok = false;
3653dde7c95SVishal Kulkarni 		timeout = -timeout;
3663dde7c95SVishal Kulkarni 	}
3673dde7c95SVishal Kulkarni 
3683dde7c95SVishal Kulkarni 	/*
3693dde7c95SVishal Kulkarni 	 * Queue ourselves onto the mailbox access list.  When our entry is at
3703dde7c95SVishal Kulkarni 	 * the front of the list, we have rights to access the mailbox.  So we
3713dde7c95SVishal Kulkarni 	 * wait [for a while] till we're at the front [or bail out with an
3723dde7c95SVishal Kulkarni 	 * EBUSY] ...
3733dde7c95SVishal Kulkarni 	 */
374*6feac2e3SRahul Lakkireddy 	t4_mbox_list_add(adap, &entry);
3753dde7c95SVishal Kulkarni 
3763dde7c95SVishal Kulkarni 	for (i = 0; ; i++) {
3773dde7c95SVishal Kulkarni 		/*
3783dde7c95SVishal Kulkarni 		 * If we've waited too long, return a busy indication.  This
3793dde7c95SVishal Kulkarni 		 * really ought to be based on our initial position in the
3803dde7c95SVishal Kulkarni 		 * mailbox access list but this is a start.  We very rarely
3813dde7c95SVishal Kulkarni 		 * contend on access to the mailbox ...  Also check for a
3823dde7c95SVishal Kulkarni 		 * firmware error which we'll report as a device error.
3833dde7c95SVishal Kulkarni 		 */
3843dde7c95SVishal Kulkarni 		pcie_fw = t4_read_reg(adap, A_PCIE_FW);
3853dde7c95SVishal Kulkarni 		if (i > 4*timeout || (pcie_fw & F_PCIE_FW_ERR)) {
386*6feac2e3SRahul Lakkireddy 			t4_mbox_list_del(adap, &entry);
3873dde7c95SVishal Kulkarni 			t4_report_fw_error(adap);
3883dde7c95SVishal Kulkarni 			ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -EBUSY;
3893dde7c95SVishal Kulkarni 			T4_RECORD_MBOX(adap, cmd, size, ret, 0);
3903dde7c95SVishal Kulkarni 			return ret;
3913dde7c95SVishal Kulkarni 		}
3923dde7c95SVishal Kulkarni 
3933dde7c95SVishal Kulkarni 		/*
3943dde7c95SVishal Kulkarni 		 * If we're at the head, break out and start the mailbox
3953dde7c95SVishal Kulkarni 		 * protocol.
3963dde7c95SVishal Kulkarni 		 */
397*6feac2e3SRahul Lakkireddy 		if (t4_mbox_list_first_entry(adap) == &entry)
3983dde7c95SVishal Kulkarni 			break;
3993dde7c95SVishal Kulkarni 
4003dde7c95SVishal Kulkarni 		/*
4013dde7c95SVishal Kulkarni 		 * Delay for a bit before checking again ...
4023dde7c95SVishal Kulkarni 		 */
4033dde7c95SVishal Kulkarni 		if (sleep_ok) {
4043dde7c95SVishal Kulkarni 			usleep_range(MIN_MBOX_CMD_DELAY, MBOX_CMD_DELAY);
4053dde7c95SVishal Kulkarni 		} else {
4063dde7c95SVishal Kulkarni 			T4_OS_TOUCH_NMI_WATCHDOG();
4073dde7c95SVishal Kulkarni 			udelay(MBOX_CMD_DELAY);
4083dde7c95SVishal Kulkarni 		}
4093dde7c95SVishal Kulkarni 	}
410*6feac2e3SRahul Lakkireddy #ifdef T4_OS_LOG_MBOX_CMDS
4113dde7c95SVishal Kulkarni 	access = i;
412*6feac2e3SRahul Lakkireddy #endif /* T4_OS_LOG_MBOX_CMDS */
4133dde7c95SVishal Kulkarni 
4143dde7c95SVishal Kulkarni 	/*
4153dde7c95SVishal Kulkarni 	 * Attempt to gain access to the mailbox.
4163dde7c95SVishal Kulkarni 	 */
4173dde7c95SVishal Kulkarni 	for (i = 0; i < 4; i++) {
4183dde7c95SVishal Kulkarni 		ctl = t4_read_reg(adap, ctl_reg);
4193dde7c95SVishal Kulkarni 		v = G_MBOWNER(ctl);
4203dde7c95SVishal Kulkarni 		if (v != X_MBOWNER_NONE)
4213dde7c95SVishal Kulkarni 			break;
4223dde7c95SVishal Kulkarni 	}
42356b2bdd1SGireesh Nagabhushana 
4243dde7c95SVishal Kulkarni 	/*
4253dde7c95SVishal Kulkarni 	 * If we were unable to gain access, dequeue ourselves from the
4263dde7c95SVishal Kulkarni 	 * mailbox atomic access list and report the error to our caller.
4273dde7c95SVishal Kulkarni 	 */
4283dde7c95SVishal Kulkarni 	if (v != X_MBOWNER_PL) {
429*6feac2e3SRahul Lakkireddy 		t4_mbox_list_del(adap, &entry);
4303dde7c95SVishal Kulkarni 		t4_report_fw_error(adap);
4313dde7c95SVishal Kulkarni 		ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT;
4323dde7c95SVishal Kulkarni 		T4_RECORD_MBOX(adap, cmd, size, access, ret);
4333dde7c95SVishal Kulkarni 		return ret;
4343dde7c95SVishal Kulkarni 	}
43556b2bdd1SGireesh Nagabhushana 
4363dde7c95SVishal Kulkarni 	/*
4373dde7c95SVishal Kulkarni 	 * If we gain ownership of the mailbox and there's a "valid" message
4383dde7c95SVishal Kulkarni 	 * in it, this is likely an asynchronous error message from the
4393dde7c95SVishal Kulkarni 	 * firmware.  So we'll report that and then proceed on with attempting
4403dde7c95SVishal Kulkarni 	 * to issue our own command ... which may well fail if the error
4413dde7c95SVishal Kulkarni 	 * presaged the firmware crashing ...
4423dde7c95SVishal Kulkarni 	 */
4433dde7c95SVishal Kulkarni 	if (ctl & F_MBMSGVALID) {
4443dde7c95SVishal Kulkarni 		CH_ERR(adap, "found VALID command in mbox %u: "
4453dde7c95SVishal Kulkarni 		       "%llx %llx %llx %llx %llx %llx %llx %llx\n", mbox,
4463dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg),
4473dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 8),
4483dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 16),
4493dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 24),
4503dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 32),
4513dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 40),
4523dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 48),
4533dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 56));
4543dde7c95SVishal Kulkarni 	}
45556b2bdd1SGireesh Nagabhushana 
4563dde7c95SVishal Kulkarni 	/*
4573dde7c95SVishal Kulkarni 	 * Copy in the new mailbox command and send it on its way ...
4583dde7c95SVishal Kulkarni 	 */
4593dde7c95SVishal Kulkarni 	T4_RECORD_MBOX(adap, cmd, size, access, 0);
46056b2bdd1SGireesh Nagabhushana 	for (i = 0; i < size; i += 8, p++)
46156b2bdd1SGireesh Nagabhushana 		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));
46256b2bdd1SGireesh Nagabhushana 
4633dde7c95SVishal Kulkarni 	/*
4643dde7c95SVishal Kulkarni 	 * XXX It's not clear that we need this anymore now
4653dde7c95SVishal Kulkarni 	 * XXX that we have mailbox logging ...
4663dde7c95SVishal Kulkarni 	 */
4673dde7c95SVishal Kulkarni 	CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
4683dde7c95SVishal Kulkarni 
46956b2bdd1SGireesh Nagabhushana 	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
47056b2bdd1SGireesh Nagabhushana 	(void) t4_read_reg(adap, ctl_reg);	/* flush write */
47156b2bdd1SGireesh Nagabhushana 
4723dde7c95SVishal Kulkarni 	/*
4733dde7c95SVishal Kulkarni 	 * Loop waiting for the reply; bail out if we time out or the firmware
4743dde7c95SVishal Kulkarni 	 * reports an error.
4753dde7c95SVishal Kulkarni 	 */
4763dde7c95SVishal Kulkarni 	for (i = 0;
4773dde7c95SVishal Kulkarni 	     !((pcie_fw = t4_read_reg(adap, A_PCIE_FW)) & F_PCIE_FW_ERR) &&
4783dde7c95SVishal Kulkarni 	     i < timeout;
4793dde7c95SVishal Kulkarni 	     i++) {
4803dde7c95SVishal Kulkarni 		if (sleep_ok) {
4813dde7c95SVishal Kulkarni 			usleep_range(MIN_MBOX_CMD_DELAY, MBOX_CMD_DELAY);
4823dde7c95SVishal Kulkarni 		} else {
4833dde7c95SVishal Kulkarni 			T4_OS_TOUCH_NMI_WATCHDOG();
4843dde7c95SVishal Kulkarni 			udelay(MBOX_CMD_DELAY);
4853dde7c95SVishal Kulkarni 		}
48656b2bdd1SGireesh Nagabhushana 
48756b2bdd1SGireesh Nagabhushana 		v = t4_read_reg(adap, ctl_reg);
48856b2bdd1SGireesh Nagabhushana 		if (v == X_CIM_PF_NOACCESS)
48956b2bdd1SGireesh Nagabhushana 			continue;
49056b2bdd1SGireesh Nagabhushana 		if (G_MBOWNER(v) == X_MBOWNER_PL) {
49156b2bdd1SGireesh Nagabhushana 			if (!(v & F_MBMSGVALID)) {
49256b2bdd1SGireesh Nagabhushana 				t4_write_reg(adap, ctl_reg,
4933dde7c95SVishal Kulkarni 					     V_MBOWNER(X_MBOWNER_NONE));
49456b2bdd1SGireesh Nagabhushana 				continue;
49556b2bdd1SGireesh Nagabhushana 			}
49656b2bdd1SGireesh Nagabhushana 
4973dde7c95SVishal Kulkarni 			/*
4983dde7c95SVishal Kulkarni 			 * Retrieve the command reply and release the mailbox.
4993dde7c95SVishal Kulkarni 			 */
5003dde7c95SVishal Kulkarni 			get_mbox_rpl(adap, cmd_rpl, size/8, data_reg);
5013dde7c95SVishal Kulkarni 			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));
502*6feac2e3SRahul Lakkireddy 			t4_mbox_list_del(adap, &entry);
5033dde7c95SVishal Kulkarni 
504baa1db26SToomas Soome 			T4_RECORD_MBOX(adap, cmd_rpl, size, access, i + 1);
5053dde7c95SVishal Kulkarni 
5063dde7c95SVishal Kulkarni 			/*
5073dde7c95SVishal Kulkarni 			 * XXX It's not clear that we need this anymore now
5083dde7c95SVishal Kulkarni 			 * XXX that we have mailbox logging ...
5093dde7c95SVishal Kulkarni 			 */
5103dde7c95SVishal Kulkarni 			CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
5113dde7c95SVishal Kulkarni 			CH_MSG(adap, INFO, HW,
5123dde7c95SVishal Kulkarni 			       "command completed in %d ms (%ssleeping)\n",
513baa1db26SToomas Soome 			       i + 1, sleep_ok ? "" : "non-");
5143dde7c95SVishal Kulkarni 
5153dde7c95SVishal Kulkarni 			res = be64_to_cpu(cmd_rpl[0]);
51656b2bdd1SGireesh Nagabhushana 			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
5173dde7c95SVishal Kulkarni 				fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl);
51856b2bdd1SGireesh Nagabhushana 				res = V_FW_CMD_RETVAL(EIO);
5193dde7c95SVishal Kulkarni 			} else if (rpl)
5203dde7c95SVishal Kulkarni 				memcpy(rpl, cmd_rpl, size);
5213dde7c95SVishal Kulkarni 			return -G_FW_CMD_RETVAL((int)res);
52256b2bdd1SGireesh Nagabhushana 		}
52356b2bdd1SGireesh Nagabhushana 	}
52456b2bdd1SGireesh Nagabhushana 
525de483253SVishal Kulkarni 	/*
5263dde7c95SVishal Kulkarni 	 * We timed out waiting for a reply to our mailbox command.  Report
527de483253SVishal Kulkarni 	 * the error and also check to see if the firmware reported any
528de483253SVishal Kulkarni 	 * errors ...
529de483253SVishal Kulkarni 	 */
530*6feac2e3SRahul Lakkireddy 	t4_mbox_list_del(adap, &entry);
5313dde7c95SVishal Kulkarni 
5323dde7c95SVishal Kulkarni 	ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -ETIMEDOUT;
5333dde7c95SVishal Kulkarni 	T4_RECORD_MBOX(adap, cmd, size, access, ret);
534*6feac2e3SRahul Lakkireddy 	CH_ERR(adap, "command 0x%x in mailbox %d timed out\n",
5353dde7c95SVishal Kulkarni 	       *(const u8 *)cmd, mbox);
5363dde7c95SVishal Kulkarni 
5373dde7c95SVishal Kulkarni 	t4_report_fw_error(adap);
5383dde7c95SVishal Kulkarni 	t4_fatal_err(adap);
5393dde7c95SVishal Kulkarni 	return ret;
54056b2bdd1SGireesh Nagabhushana }
54156b2bdd1SGireesh Nagabhushana 
5423dde7c95SVishal Kulkarni #ifdef CONFIG_CUDBG
54356b2bdd1SGireesh Nagabhushana /*
5443dde7c95SVishal Kulkarni  * The maximum number of times to iterate for FW reply before
5453dde7c95SVishal Kulkarni  * issuing a mailbox timeout
54656b2bdd1SGireesh Nagabhushana  */
5473dde7c95SVishal Kulkarni #define FW_REPLY_WAIT_LOOP 6000000
54856b2bdd1SGireesh Nagabhushana 
5493dde7c95SVishal Kulkarni /**
5503dde7c95SVishal Kulkarni  *	t4_wr_mbox_meat_timeout_panic - send a command to FW through the given
5513dde7c95SVishal Kulkarni  *	mailbox. This function is a minimal version of t4_wr_mbox_meat_timeout()
5523dde7c95SVishal Kulkarni  *	and is only invoked during a kernel crash. Since this function is
5533dde7c95SVishal Kulkarni  *	called through a atomic notifier chain ,we cannot sleep awaiting a
5543dde7c95SVishal Kulkarni  *	response from FW, hence repeatedly loop until we get a reply.
55556b2bdd1SGireesh Nagabhushana  *
5563dde7c95SVishal Kulkarni  *	@adap: the adapter
5573dde7c95SVishal Kulkarni  *	@mbox: index of the mailbox to use
5583dde7c95SVishal Kulkarni  *	@cmd: the command to write
5593dde7c95SVishal Kulkarni  *	@size: command length in bytes
5603dde7c95SVishal Kulkarni  *	@rpl: where to optionally store the reply
56156b2bdd1SGireesh Nagabhushana  */
5623dde7c95SVishal Kulkarni 
t4_wr_mbox_meat_timeout_panic(struct adapter * adap,int mbox,const void * cmd,int size,void * rpl)5633dde7c95SVishal Kulkarni static int t4_wr_mbox_meat_timeout_panic(struct adapter *adap, int mbox,
5643dde7c95SVishal Kulkarni 			    const void *cmd, int size, void *rpl)
56556b2bdd1SGireesh Nagabhushana {
5663dde7c95SVishal Kulkarni 	u32 v;
5673dde7c95SVishal Kulkarni 	u64 res;
5683dde7c95SVishal Kulkarni 	int i, ret;
5693dde7c95SVishal Kulkarni 	u64 cnt;
5703dde7c95SVishal Kulkarni 	const __be64 *p = cmd;
5713dde7c95SVishal Kulkarni 	u32 data_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_DATA);
5723dde7c95SVishal Kulkarni 	u32 ctl_reg = PF_REG(mbox, A_CIM_PF_MAILBOX_CTRL);
5733dde7c95SVishal Kulkarni 	u32 ctl;
5743dde7c95SVishal Kulkarni 	__be64 cmd_rpl[MBOX_LEN/8];
5753dde7c95SVishal Kulkarni 	u32 pcie_fw;
57656b2bdd1SGireesh Nagabhushana 
5773dde7c95SVishal Kulkarni 	if ((size & 15) || size > MBOX_LEN)
5783dde7c95SVishal Kulkarni 		return -EINVAL;
579de483253SVishal Kulkarni 
5803dde7c95SVishal Kulkarni 	/*
5813dde7c95SVishal Kulkarni 	 * Check for a firmware error which we'll report as a
5823dde7c95SVishal Kulkarni 	 * device error.
5833dde7c95SVishal Kulkarni 	 */
5843dde7c95SVishal Kulkarni 	pcie_fw = t4_read_reg(adap, A_PCIE_FW);
5853dde7c95SVishal Kulkarni 	if (pcie_fw & F_PCIE_FW_ERR) {
5863dde7c95SVishal Kulkarni 		t4_report_fw_error(adap);
5873dde7c95SVishal Kulkarni 		ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -EBUSY;
5883dde7c95SVishal Kulkarni 		return ret;
5893dde7c95SVishal Kulkarni 	}
59056b2bdd1SGireesh Nagabhushana 
5913dde7c95SVishal Kulkarni 	/*
5923dde7c95SVishal Kulkarni 	 * Attempt to gain access to the mailbox.
5933dde7c95SVishal Kulkarni 	 */
5943dde7c95SVishal Kulkarni 	for (i = 0; i < 4; i++) {
5953dde7c95SVishal Kulkarni 		ctl = t4_read_reg(adap, ctl_reg);
5963dde7c95SVishal Kulkarni 		v = G_MBOWNER(ctl);
5973dde7c95SVishal Kulkarni 		if (v != X_MBOWNER_NONE)
5983dde7c95SVishal Kulkarni 			break;
5993dde7c95SVishal Kulkarni 	}
60056b2bdd1SGireesh Nagabhushana 
60156b2bdd1SGireesh Nagabhushana 	/*
6023dde7c95SVishal Kulkarni 	 * If we were unable to gain access, report the error to our caller.
60356b2bdd1SGireesh Nagabhushana 	 */
6043dde7c95SVishal Kulkarni 	if (v != X_MBOWNER_PL) {
6053dde7c95SVishal Kulkarni 		t4_report_fw_error(adap);
6063dde7c95SVishal Kulkarni 		ret = (v == X_MBOWNER_FW) ? -EBUSY : -ETIMEDOUT;
6073dde7c95SVishal Kulkarni 		return ret;
6083dde7c95SVishal Kulkarni 	}
60956b2bdd1SGireesh Nagabhushana 
61056b2bdd1SGireesh Nagabhushana 	/*
6113dde7c95SVishal Kulkarni 	 * If we gain ownership of the mailbox and there's a "valid" message
6123dde7c95SVishal Kulkarni 	 * in it, this is likely an asynchronous error message from the
6133dde7c95SVishal Kulkarni 	 * firmware.  So we'll report that and then proceed on with attempting
6143dde7c95SVishal Kulkarni 	 * to issue our own command ... which may well fail if the error
6153dde7c95SVishal Kulkarni 	 * presaged the firmware crashing ...
61656b2bdd1SGireesh Nagabhushana 	 */
6173dde7c95SVishal Kulkarni 	if (ctl & F_MBMSGVALID) {
6183dde7c95SVishal Kulkarni 		CH_ERR(adap, "found VALID command in mbox %u: "
6193dde7c95SVishal Kulkarni 		       "%llx %llx %llx %llx %llx %llx %llx %llx\n", mbox,
6203dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg),
6213dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 8),
6223dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 16),
6233dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 24),
6243dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 32),
6253dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 40),
6263dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 48),
6273dde7c95SVishal Kulkarni 		       (unsigned long long)t4_read_reg64(adap, data_reg + 56));
6283dde7c95SVishal Kulkarni 	}
62956b2bdd1SGireesh Nagabhushana 
6303dde7c95SVishal Kulkarni 	/*
6313dde7c95SVishal Kulkarni 	 * Copy in the new mailbox command and send it on its way ...
6323dde7c95SVishal Kulkarni 	 */
6333dde7c95SVishal Kulkarni 	for (i = 0; i < size; i += 8, p++)
6343dde7c95SVishal Kulkarni 		t4_write_reg64(adap, data_reg + i, be64_to_cpu(*p));
63556b2bdd1SGireesh Nagabhushana 
6363dde7c95SVishal Kulkarni 	CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
63756b2bdd1SGireesh Nagabhushana 
6383dde7c95SVishal Kulkarni 	t4_write_reg(adap, ctl_reg, F_MBMSGVALID | V_MBOWNER(X_MBOWNER_FW));
6393dde7c95SVishal Kulkarni 	t4_read_reg(adap, ctl_reg);	/* flush write */
64056b2bdd1SGireesh Nagabhushana 
6413dde7c95SVishal Kulkarni 	/*
6423dde7c95SVishal Kulkarni 	 * Loop waiting for the reply; bail out if we time out or the firmware
6433dde7c95SVishal Kulkarni 	 * reports an error.
6443dde7c95SVishal Kulkarni 	 */
6453dde7c95SVishal Kulkarni 	for (cnt = 0;
6463dde7c95SVishal Kulkarni 	    !((pcie_fw = t4_read_reg(adap, A_PCIE_FW)) & F_PCIE_FW_ERR) &&
6473dde7c95SVishal Kulkarni 	    cnt < FW_REPLY_WAIT_LOOP;
6483dde7c95SVishal Kulkarni 	    cnt++) {
6493dde7c95SVishal Kulkarni 		v = t4_read_reg(adap, ctl_reg);
6503dde7c95SVishal Kulkarni 		if (v == X_CIM_PF_NOACCESS)
6513dde7c95SVishal Kulkarni 			continue;
6523dde7c95SVishal Kulkarni 		if (G_MBOWNER(v) == X_MBOWNER_PL) {
6533dde7c95SVishal Kulkarni 			if (!(v & F_MBMSGVALID)) {
6543dde7c95SVishal Kulkarni 				t4_write_reg(adap, ctl_reg,
6553dde7c95SVishal Kulkarni 					     V_MBOWNER(X_MBOWNER_NONE));
6563dde7c95SVishal Kulkarni 				continue;
6573dde7c95SVishal Kulkarni 			}
65856b2bdd1SGireesh Nagabhushana 
6593dde7c95SVishal Kulkarni 			/*
6603dde7c95SVishal Kulkarni 			 * Retrieve the command reply and release the mailbox.
6613dde7c95SVishal Kulkarni 			 */
6623dde7c95SVishal Kulkarni 			get_mbox_rpl(adap, cmd_rpl, size/8, data_reg);
6633dde7c95SVishal Kulkarni 			t4_write_reg(adap, ctl_reg, V_MBOWNER(X_MBOWNER_NONE));
6643dde7c95SVishal Kulkarni 
6653dde7c95SVishal Kulkarni 			CH_DUMP_MBOX(adap, mbox, data_reg, size / 8);
6663dde7c95SVishal Kulkarni 
6673dde7c95SVishal Kulkarni 			res = be64_to_cpu(cmd_rpl[0]);
6683dde7c95SVishal Kulkarni 			if (G_FW_CMD_OP(res >> 32) == FW_DEBUG_CMD) {
6693dde7c95SVishal Kulkarni 				fw_asrt(adap, (struct fw_debug_cmd *)cmd_rpl);
6703dde7c95SVishal Kulkarni 				res = V_FW_CMD_RETVAL(EIO);
6713dde7c95SVishal Kulkarni 			} else if (rpl)
6723dde7c95SVishal Kulkarni 				memcpy(rpl, cmd_rpl, size);
6733dde7c95SVishal Kulkarni 			return -G_FW_CMD_RETVAL((int)res);
6743dde7c95SVishal Kulkarni 		}
6753dde7c95SVishal Kulkarni 	}
67656b2bdd1SGireesh Nagabhushana 
67756b2bdd1SGireesh Nagabhushana 	/*
6783dde7c95SVishal Kulkarni 	 * We timed out waiting for a reply to our mailbox command.  Report
6793dde7c95SVishal Kulkarni 	 * the error and also check to see if the firmware reported any
6803dde7c95SVishal Kulkarni 	 * errors ...
68156b2bdd1SGireesh Nagabhushana 	 */
6823dde7c95SVishal Kulkarni 	ret = (pcie_fw & F_PCIE_FW_ERR) ? -ENXIO : -ETIMEDOUT;
6833dde7c95SVishal Kulkarni 	CH_ERR(adap, "command %#x in mailbox %d timed out\n",
6843dde7c95SVishal Kulkarni 	       *(const u8 *)cmd, mbox);
68556b2bdd1SGireesh Nagabhushana 
6863dde7c95SVishal Kulkarni 	t4_report_fw_error(adap);
6873dde7c95SVishal Kulkarni 	t4_fatal_err(adap);
6883dde7c95SVishal Kulkarni 	return ret;
68956b2bdd1SGireesh Nagabhushana }
6903dde7c95SVishal Kulkarni #endif
69156b2bdd1SGireesh Nagabhushana 
t4_wr_mbox_meat(struct adapter * adap,int mbox,const void * cmd,int size,void * rpl,bool sleep_ok)6923dde7c95SVishal Kulkarni int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
6933dde7c95SVishal Kulkarni 		    void *rpl, bool sleep_ok)
69456b2bdd1SGireesh Nagabhushana {
6953dde7c95SVishal Kulkarni #ifdef CONFIG_CUDBG
6963dde7c95SVishal Kulkarni 	if (adap->flags & K_CRASH)
6973dde7c95SVishal Kulkarni 		return t4_wr_mbox_meat_timeout_panic(adap, mbox, cmd, size,
6983dde7c95SVishal Kulkarni 						     rpl);
6993dde7c95SVishal Kulkarni 	else
7003dde7c95SVishal Kulkarni #endif
7013dde7c95SVishal Kulkarni 		return t4_wr_mbox_meat_timeout(adap, mbox, cmd, size, rpl,
7023dde7c95SVishal Kulkarni 					       sleep_ok, FW_CMD_MAX_TIMEOUT);
7033dde7c95SVishal Kulkarni 
70456b2bdd1SGireesh Nagabhushana }
70556b2bdd1SGireesh Nagabhushana 
t4_edc_err_read(struct adapter * adap,int idx)7063dde7c95SVishal Kulkarni static int t4_edc_err_read(struct adapter *adap, int idx)
7073dde7c95SVishal Kulkarni {
7083dde7c95SVishal Kulkarni 	u32 edc_ecc_err_addr_reg;
7093dde7c95SVishal Kulkarni 	u32 edc_bist_status_rdata_reg;
71056b2bdd1SGireesh Nagabhushana 
7113dde7c95SVishal Kulkarni 	if (is_t4(adap->params.chip)) {
7123dde7c95SVishal Kulkarni 		CH_WARN(adap, "%s: T4 NOT supported.\n", __func__);
7133dde7c95SVishal Kulkarni 		return 0;
7143dde7c95SVishal Kulkarni 	}
7153dde7c95SVishal Kulkarni 	if (idx != MEM_EDC0 && idx != MEM_EDC1) {
7163dde7c95SVishal Kulkarni 		CH_WARN(adap, "%s: idx %d NOT supported.\n", __func__, idx);
7173dde7c95SVishal Kulkarni 		return 0;
7183dde7c95SVishal Kulkarni 	}
71956b2bdd1SGireesh Nagabhushana 
7203dde7c95SVishal Kulkarni 	edc_ecc_err_addr_reg = EDC_T5_REG(A_EDC_H_ECC_ERR_ADDR, idx);
7213dde7c95SVishal Kulkarni 	edc_bist_status_rdata_reg = EDC_T5_REG(A_EDC_H_BIST_STATUS_RDATA, idx);
7223dde7c95SVishal Kulkarni 
7233dde7c95SVishal Kulkarni 	CH_WARN(adap,
7243dde7c95SVishal Kulkarni 		"edc%d err addr 0x%x: 0x%x.\n",
7253dde7c95SVishal Kulkarni 		idx, edc_ecc_err_addr_reg,
7263dde7c95SVishal Kulkarni 		t4_read_reg(adap, edc_ecc_err_addr_reg));
7273dde7c95SVishal Kulkarni 	CH_WARN(adap,
7283dde7c95SVishal Kulkarni 	 	"bist: 0x%x, status %llx %llx %llx %llx %llx %llx %llx %llx %llx.\n",
7293dde7c95SVishal Kulkarni 		edc_bist_status_rdata_reg,
7303dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg),
7313dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 8),
7323dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 16),
7333dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 24),
7343dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 32),
7353dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 40),
7363dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 48),
7373dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 56),
7383dde7c95SVishal Kulkarni 		(unsigned long long)t4_read_reg64(adap, edc_bist_status_rdata_reg + 64));
73956b2bdd1SGireesh Nagabhushana 
7403dde7c95SVishal Kulkarni 	return 0;
7413dde7c95SVishal Kulkarni }
74256b2bdd1SGireesh Nagabhushana 
7433dde7c95SVishal Kulkarni /**
7443dde7c95SVishal Kulkarni  *	t4_memory_rw_addr - read/write adapter memory via PCIE memory window
7453dde7c95SVishal Kulkarni  *	@adap: the adapter
7463dde7c95SVishal Kulkarni  *	@win: PCI-E Memory Window to use
7473dde7c95SVishal Kulkarni  *	@addr: address within adapter memory
7483dde7c95SVishal Kulkarni  *	@len: amount of memory to transfer
7493dde7c95SVishal Kulkarni  *	@hbuf: host memory buffer
7503dde7c95SVishal Kulkarni  *	@dir: direction of transfer T4_MEMORY_READ (1) or T4_MEMORY_WRITE (0)
7513dde7c95SVishal Kulkarni  *
7523dde7c95SVishal Kulkarni  *	Reads/writes an [almost] arbitrary memory region in the firmware: the
7533dde7c95SVishal Kulkarni  *	firmware memory address and host buffer must be aligned on 32-bit
7543dde7c95SVishal Kulkarni  *	boudaries; the length may be arbitrary.
7553dde7c95SVishal Kulkarni  *
7563dde7c95SVishal Kulkarni  *	NOTES:
7573dde7c95SVishal Kulkarni  *	 1. The memory is transferred as a raw byte sequence from/to the
7583dde7c95SVishal Kulkarni  *	    firmware's memory.  If this memory contains data structures which
7593dde7c95SVishal Kulkarni  *	    contain multi-byte integers, it's the caller's responsibility to
7603dde7c95SVishal Kulkarni  *	    perform appropriate byte order conversions.
7613dde7c95SVishal Kulkarni  *
7623dde7c95SVishal Kulkarni  *	 2. It is the Caller's responsibility to ensure that no other code
7633dde7c95SVishal Kulkarni  *	    uses the specified PCI-E Memory Window while this routine is
7643dde7c95SVishal Kulkarni  *	    using it.  This is typically done via the use of OS-specific
7653dde7c95SVishal Kulkarni  *	    locks, etc.
7663dde7c95SVishal Kulkarni  */
t4_memory_rw_addr(struct adapter * adap,int win,u32 addr,u32 len,void * hbuf,int dir)7673dde7c95SVishal Kulkarni int t4_memory_rw_addr(struct adapter *adap, int win, u32 addr,
7683dde7c95SVishal Kulkarni 		      u32 len, void *hbuf, int dir)
7693dde7c95SVishal Kulkarni {
7703dde7c95SVishal Kulkarni 	u32 pos, offset, resid;
7713dde7c95SVishal Kulkarni 	u32 win_pf, mem_reg, mem_aperture, mem_base;
7723dde7c95SVishal Kulkarni 	u32 *buf;
7733dde7c95SVishal Kulkarni 
7743dde7c95SVishal Kulkarni 	/* Argument sanity checks ...
7753dde7c95SVishal Kulkarni 	 */
7763dde7c95SVishal Kulkarni 	if (addr & 0x3 || (uintptr_t)hbuf & 0x3)
7773dde7c95SVishal Kulkarni 		return -EINVAL;
7783dde7c95SVishal Kulkarni 	buf = (u32 *)hbuf;
77956b2bdd1SGireesh Nagabhushana 
7803dde7c95SVishal Kulkarni 	/* It's convenient to be able to handle lengths which aren't a
7813dde7c95SVishal Kulkarni 	 * multiple of 32-bits because we often end up transferring files to
7823dde7c95SVishal Kulkarni 	 * the firmware.  So we'll handle that by normalizing the length here
7833dde7c95SVishal Kulkarni 	 * and then handling any residual transfer at the end.
7843dde7c95SVishal Kulkarni 	 */
7853dde7c95SVishal Kulkarni 	resid = len & 0x3;
7863dde7c95SVishal Kulkarni 	len -= resid;
7873dde7c95SVishal Kulkarni 
7883dde7c95SVishal Kulkarni 	/* Each PCI-E Memory Window is programmed with a window size -- or
7893dde7c95SVishal Kulkarni 	 * "aperture" -- which controls the granularity of its mapping onto
7903dde7c95SVishal Kulkarni 	 * adapter memory.  We need to grab that aperture in order to know
7913dde7c95SVishal Kulkarni 	 * how to use the specified window.  The window is also programmed
7923dde7c95SVishal Kulkarni 	 * with the base address of the Memory Window in BAR0's address
7933dde7c95SVishal Kulkarni 	 * space.  For T4 this is an absolute PCI-E Bus Address.  For T5
7943dde7c95SVishal Kulkarni 	 * the address is relative to BAR0.
7953dde7c95SVishal Kulkarni 	 */
7963dde7c95SVishal Kulkarni 	mem_reg = t4_read_reg(adap,
7973dde7c95SVishal Kulkarni 			      PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN,
7983dde7c95SVishal Kulkarni 						  win));
7993dde7c95SVishal Kulkarni 
8003dde7c95SVishal Kulkarni 	/* a dead adapter will return 0xffffffff for PIO reads */
8013dde7c95SVishal Kulkarni 	if (mem_reg == 0xffffffff) {
8023dde7c95SVishal Kulkarni 		CH_WARN(adap, "Unable to read PCI-E Memory Window Base[%d]\n",
8033dde7c95SVishal Kulkarni 			win);
8043dde7c95SVishal Kulkarni 		return -ENXIO;
80556b2bdd1SGireesh Nagabhushana 	}
80656b2bdd1SGireesh Nagabhushana 
8073dde7c95SVishal Kulkarni 	mem_aperture = 1 << (G_WINDOW(mem_reg) + X_WINDOW_SHIFT);
8083dde7c95SVishal Kulkarni 	mem_base = G_PCIEOFST(mem_reg) << X_PCIEOFST_SHIFT;
8093dde7c95SVishal Kulkarni 	if (is_t4(adap->params.chip))
8103dde7c95SVishal Kulkarni 		mem_base -= adap->t4_bar0;
8113dde7c95SVishal Kulkarni 	win_pf = is_t4(adap->params.chip) ? 0 : V_PFNUM(adap->pf);
8123dde7c95SVishal Kulkarni 
8133dde7c95SVishal Kulkarni 	/* Calculate our initial PCI-E Memory Window Position and Offset into
8143dde7c95SVishal Kulkarni 	 * that Window.
8153dde7c95SVishal Kulkarni 	 */
8163dde7c95SVishal Kulkarni 	pos = addr & ~(mem_aperture-1);
8173dde7c95SVishal Kulkarni 	offset = addr - pos;
8183dde7c95SVishal Kulkarni 
8193dde7c95SVishal Kulkarni 	/* Set up initial PCI-E Memory Window to cover the start of our
8203dde7c95SVishal Kulkarni 	 * transfer.  (Read it back to ensure that changes propagate before we
8213dde7c95SVishal Kulkarni 	 * attempt to use the new value.)
8223dde7c95SVishal Kulkarni 	 */
8233dde7c95SVishal Kulkarni 	t4_write_reg(adap,
8243dde7c95SVishal Kulkarni 		     PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win),
8253dde7c95SVishal Kulkarni 		     pos | win_pf);
8263dde7c95SVishal Kulkarni 	t4_read_reg(adap,
8273dde7c95SVishal Kulkarni 		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, win));
8283dde7c95SVishal Kulkarni 
8293dde7c95SVishal Kulkarni 	/* Transfer data to/from the adapter as long as there's an integral
8303dde7c95SVishal Kulkarni 	 * number of 32-bit transfers to complete.
8313dde7c95SVishal Kulkarni 	 *
8323dde7c95SVishal Kulkarni 	 * A note on Endianness issues:
8333dde7c95SVishal Kulkarni 	 *
8343dde7c95SVishal Kulkarni 	 * The "register" reads and writes below from/to the PCI-E Memory
8353dde7c95SVishal Kulkarni 	 * Window invoke the standard adapter Big-Endian to PCI-E Link
8363dde7c95SVishal Kulkarni 	 * Little-Endian "swizzel."  As a result, if we have the following
8373dde7c95SVishal Kulkarni 	 * data in adapter memory:
8383dde7c95SVishal Kulkarni 	 *
8393dde7c95SVishal Kulkarni 	 *     Memory:  ... | b0 | b1 | b2 | b3 | ...
8403dde7c95SVishal Kulkarni 	 *     Address:      i+0  i+1  i+2  i+3
8413dde7c95SVishal Kulkarni 	 *
8423dde7c95SVishal Kulkarni 	 * Then a read of the adapter memory via the PCI-E Memory Window
8433dde7c95SVishal Kulkarni 	 * will yield:
8443dde7c95SVishal Kulkarni 	 *
8453dde7c95SVishal Kulkarni 	 *     x = readl(i)
8463dde7c95SVishal Kulkarni 	 *	   31                  0
8473dde7c95SVishal Kulkarni 	 *         [ b3 | b2 | b1 | b0 ]
8483dde7c95SVishal Kulkarni 	 *
8493dde7c95SVishal Kulkarni 	 * If this value is stored into local memory on a Little-Endian system
8503dde7c95SVishal Kulkarni 	 * it will show up correctly in local memory as:
8513dde7c95SVishal Kulkarni 	 *
8523dde7c95SVishal Kulkarni 	 *     ( ..., b0, b1, b2, b3, ... )
8533dde7c95SVishal Kulkarni 	 *
8543dde7c95SVishal Kulkarni 	 * But on a Big-Endian system, the store will show up in memory
8553dde7c95SVishal Kulkarni 	 * incorrectly swizzled as:
8563dde7c95SVishal Kulkarni 	 *
8573dde7c95SVishal Kulkarni 	 *     ( ..., b3, b2, b1, b0, ... )
8583dde7c95SVishal Kulkarni 	 *
8593dde7c95SVishal Kulkarni 	 * So we need to account for this in the reads and writes to the
8603dde7c95SVishal Kulkarni 	 * PCI-E Memory Window below by undoing the register read/write
8613dde7c95SVishal Kulkarni 	 * swizzels.
8623dde7c95SVishal Kulkarni 	 */
8633dde7c95SVishal Kulkarni 	while (len > 0) {
8643dde7c95SVishal Kulkarni 		if (dir == T4_MEMORY_READ)
8653dde7c95SVishal Kulkarni 			*buf++ = le32_to_cpu((__force __le32)t4_read_reg(adap,
8663dde7c95SVishal Kulkarni 						mem_base + offset));
8673dde7c95SVishal Kulkarni 		else
8683dde7c95SVishal Kulkarni 			t4_write_reg(adap, mem_base + offset,
8693dde7c95SVishal Kulkarni 				     (__force u32)cpu_to_le32(*buf++));
8703dde7c95SVishal Kulkarni 		offset += sizeof(__be32);
8713dde7c95SVishal Kulkarni 		len -= sizeof(__be32);
8723dde7c95SVishal Kulkarni 
8733dde7c95SVishal Kulkarni 		/* If we've reached the end of our current window aperture,
8743dde7c95SVishal Kulkarni 		 * move the PCI-E Memory Window on to the next.  Note that
8753dde7c95SVishal Kulkarni 		 * doing this here after "len" may be 0 allows us to set up
8763dde7c95SVishal Kulkarni 		 * the PCI-E Memory Window for a possible final residual
8773dde7c95SVishal Kulkarni 		 * transfer below ...
8783dde7c95SVishal Kulkarni 		 */
8793dde7c95SVishal Kulkarni 		if (offset == mem_aperture) {
8803dde7c95SVishal Kulkarni 			pos += mem_aperture;
8813dde7c95SVishal Kulkarni 			offset = 0;
8823dde7c95SVishal Kulkarni 			t4_write_reg(adap,
8833dde7c95SVishal Kulkarni 				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
8843dde7c95SVishal Kulkarni 						    win), pos | win_pf);
8853dde7c95SVishal Kulkarni 			t4_read_reg(adap,
8863dde7c95SVishal Kulkarni 				PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET,
8873dde7c95SVishal Kulkarni 						    win));
8883dde7c95SVishal Kulkarni 		}
8893dde7c95SVishal Kulkarni 	}
8903dde7c95SVishal Kulkarni 
8913dde7c95SVishal Kulkarni 	/* If the original transfer had a length which wasn't a multiple of
8923dde7c95SVishal Kulkarni 	 * 32-bits, now's where we need to finish off the transfer of the
8933dde7c95SVishal Kulkarni 	 * residual amount.  The PCI-E Memory Window has already been moved
8943dde7c95SVishal Kulkarni 	 * above (if necessary) to cover this final transfer.
8953dde7c95SVishal Kulkarni 	 */
8963dde7c95SVishal Kulkarni 	if (resid) {
8973dde7c95SVishal Kulkarni 		union {
8983dde7c95SVishal Kulkarni 			u32 word;
8993dde7c95SVishal Kulkarni 			char byte[4];
9003dde7c95SVishal Kulkarni 		} last;
9013dde7c95SVishal Kulkarni 		unsigned char *bp;
9023dde7c95SVishal Kulkarni 		int i;
9033dde7c95SVishal Kulkarni 
9043dde7c95SVishal Kulkarni 		if (dir == T4_MEMORY_READ) {
9053dde7c95SVishal Kulkarni 			last.