15c8e2d7hselasky/*-
2a7cccbdhselasky * Copyright (c) 2013-2019, Mellanox Technologies, Ltd.  All rights reserved.
35c8e2d7hselasky *
45c8e2d7hselasky * Redistribution and use in source and binary forms, with or without
55c8e2d7hselasky * modification, are permitted provided that the following conditions
65c8e2d7hselasky * are met:
75c8e2d7hselasky * 1. Redistributions of source code must retain the above copyright
85c8e2d7hselasky *    notice, this list of conditions and the following disclaimer.
95c8e2d7hselasky * 2. Redistributions in binary form must reproduce the above copyright
105c8e2d7hselasky *    notice, this list of conditions and the following disclaimer in the
115c8e2d7hselasky *    documentation and/or other materials provided with the distribution.
125c8e2d7hselasky *
135c8e2d7hselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
145c8e2d7hselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
155c8e2d7hselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
165c8e2d7hselasky * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
175c8e2d7hselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
185c8e2d7hselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
195c8e2d7hselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
205c8e2d7hselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
215c8e2d7hselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
225c8e2d7hselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
235c8e2d7hselasky * SUCH DAMAGE.
245c8e2d7hselasky *
255c8e2d7hselasky * $FreeBSD$
265c8e2d7hselasky */
275c8e2d7hselasky
285c8e2d7hselasky#ifndef MLX5_DRIVER_H
295c8e2d7hselasky#define MLX5_DRIVER_H
305c8e2d7hselasky
31a09337chselasky#include "opt_ratelimit.h"
32a09337chselasky
335c8e2d7hselasky#include <linux/kernel.h>
345c8e2d7hselasky#include <linux/completion.h>
355c8e2d7hselasky#include <linux/pci.h>
365c8e2d7hselasky#include <linux/cache.h>
375c8e2d7hselasky#include <linux/rbtree.h>
3852fa249hselasky#include <linux/if_ether.h>
395c8e2d7hselasky#include <linux/semaphore.h>
405c8e2d7hselasky#include <linux/slab.h>
415c8e2d7hselasky#include <linux/vmalloc.h>
425c8e2d7hselasky#include <linux/radix-tree.h>
4316b9405slavash#include <linux/idr.h>
445c8e2d7hselasky
455c8e2d7hselasky#include <dev/mlx5/device.h>
465c8e2d7hselasky#include <dev/mlx5/doorbell.h>
47b363020hselasky#include <dev/mlx5/srq.h>
485c8e2d7hselasky
496ef4747hselasky#define MLX5_QCOUNTER_SETS_NETDEV 64
50120993bhselasky#define MLX5_MAX_NUMBER_OF_VFS 128
516ef4747hselasky
525c8e2d7hselaskyenum {
535c8e2d7hselasky	MLX5_BOARD_ID_LEN = 64,
545c8e2d7hselasky	MLX5_MAX_NAME_LEN = 16,
555c8e2d7hselasky};
565c8e2d7hselasky
575c8e2d7hselaskyenum {
58ad1e2f9hselasky	MLX5_CMD_TIMEOUT_MSEC	= 60 * 1000,
595c8e2d7hselasky};
605c8e2d7hselasky
615c8e2d7hselaskyenum {
625c8e2d7hselasky	CMD_OWNER_SW		= 0x0,
635c8e2d7hselasky	CMD_OWNER_HW		= 0x1,
645c8e2d7hselasky	CMD_STATUS_SUCCESS	= 0,
655c8e2d7hselasky};
665c8e2d7hselasky
675c8e2d7hselaskyenum mlx5_sqp_t {
685c8e2d7hselasky	MLX5_SQP_SMI		= 0,
695c8e2d7hselasky	MLX5_SQP_GSI		= 1,
705c8e2d7hselasky	MLX5_SQP_IEEE_1588	= 2,
715c8e2d7hselasky	MLX5_SQP_SNIFFER	= 3,
725c8e2d7hselasky	MLX5_SQP_SYNC_UMR	= 4,
735c8e2d7hselasky};
745c8e2d7hselasky
755c8e2d7hselaskyenum {
765c8e2d7hselasky	MLX5_MAX_PORTS	= 2,
775c8e2d7hselasky};
785c8e2d7hselasky
795c8e2d7hselaskyenum {
805c8e2d7hselasky	MLX5_EQ_VEC_PAGES	 = 0,
815c8e2d7hselasky	MLX5_EQ_VEC_CMD		 = 1,
825c8e2d7hselasky	MLX5_EQ_VEC_ASYNC	 = 2,
835c8e2d7hselasky	MLX5_EQ_VEC_COMP_BASE,
845c8e2d7hselasky};
855c8e2d7hselasky
865c8e2d7hselaskyenum {
876ef4747hselasky	MLX5_ATOMIC_MODE_OFF		= 16,
886ef4747hselasky	MLX5_ATOMIC_MODE_NONE		= 0 << MLX5_ATOMIC_MODE_OFF,
896ef4747hselasky	MLX5_ATOMIC_MODE_IB_COMP	= 1 << MLX5_ATOMIC_MODE_OFF,
906ef4747hselasky	MLX5_ATOMIC_MODE_CX		= 2 << MLX5_ATOMIC_MODE_OFF,
916ef4747hselasky	MLX5_ATOMIC_MODE_8B		= 3 << MLX5_ATOMIC_MODE_OFF,
926ef4747hselasky	MLX5_ATOMIC_MODE_16B		= 4 << MLX5_ATOMIC_MODE_OFF,
936ef4747hselasky	MLX5_ATOMIC_MODE_32B		= 5 << MLX5_ATOMIC_MODE_OFF,
946ef4747hselasky	MLX5_ATOMIC_MODE_64B		= 6 << MLX5_ATOMIC_MODE_OFF,
956ef4747hselasky	MLX5_ATOMIC_MODE_128B		= 7 << MLX5_ATOMIC_MODE_OFF,
966ef4747hselasky	MLX5_ATOMIC_MODE_256B		= 8 << MLX5_ATOMIC_MODE_OFF,
976ef4747hselasky};
986ef4747hselasky
996ef4747hselaskyenum {
1006ef4747hselasky	MLX5_ATOMIC_MODE_DCT_OFF	= 20,
1016ef4747hselasky	MLX5_ATOMIC_MODE_DCT_NONE	= 0 << MLX5_ATOMIC_MODE_DCT_OFF,
1026ef4747hselasky	MLX5_ATOMIC_MODE_DCT_IB_COMP	= 1 << MLX5_ATOMIC_MODE_DCT_OFF,
1036ef4747hselasky	MLX5_ATOMIC_MODE_DCT_CX		= 2 << MLX5_ATOMIC_MODE_DCT_OFF,
1046ef4747hselasky	MLX5_ATOMIC_MODE_DCT_8B		= 3 << MLX5_ATOMIC_MODE_DCT_OFF,
1056ef4747hselasky	MLX5_ATOMIC_MODE_DCT_16B	= 4 << MLX5_ATOMIC_MODE_DCT_OFF,
1066ef4747hselasky	MLX5_ATOMIC_MODE_DCT_32B	= 5 << MLX5_ATOMIC_MODE_DCT_OFF,
1076ef4747hselasky	MLX5_ATOMIC_MODE_DCT_64B	= 6 << MLX5_ATOMIC_MODE_DCT_OFF,
1086ef4747hselasky	MLX5_ATOMIC_MODE_DCT_128B	= 7 << MLX5_ATOMIC_MODE_DCT_OFF,
1096ef4747hselasky	MLX5_ATOMIC_MODE_DCT_256B	= 8 << MLX5_ATOMIC_MODE_DCT_OFF,
1106ef4747hselasky};
1116ef4747hselasky
1126ef4747hselaskyenum {
1136ef4747hselasky	MLX5_ATOMIC_OPS_CMP_SWAP		= 1 << 0,
1146ef4747hselasky	MLX5_ATOMIC_OPS_FETCH_ADD		= 1 << 1,
1156ef4747hselasky	MLX5_ATOMIC_OPS_MASKED_CMP_SWAP		= 1 << 2,
1166ef4747hselasky	MLX5_ATOMIC_OPS_MASKED_FETCH_ADD	= 1 << 3,
1175c8e2d7hselasky};
1185c8e2d7hselasky
1195c8e2d7hselaskyenum {
120c90970bhselasky	MLX5_REG_QPTS		 = 0x4002,
1215c8e2d7hselasky	MLX5_REG_QETCR		 = 0x4005,
1225c8e2d7hselasky	MLX5_REG_QPDP		 = 0x4007,
1235c8e2d7hselasky	MLX5_REG_QTCT		 = 0x400A,
124c90970bhselasky	MLX5_REG_QPDPM		 = 0x4013,
1254f9623fhselasky	MLX5_REG_QHLL		 = 0x4016,
126c90970bhselasky	MLX5_REG_QCAM		 = 0x4019,
1276ef4747hselasky	MLX5_REG_DCBX_PARAM	 = 0x4020,
1286ef4747hselasky	MLX5_REG_DCBX_APP	 = 0x4021,
12916b9405slavash	MLX5_REG_FPGA_CAP	 = 0x4022,
13016b9405slavash	MLX5_REG_FPGA_CTRL	 = 0x4023,
13116b9405slavash	MLX5_REG_FPGA_ACCESS_REG = 0x4024,
13216b9405slavash	MLX5_REG_FPGA_SHELL_CNTR = 0x4025,
1336fb43c1hselasky	MLX5_REG_PCAP		 = 0x5001,
1346fb43c1hselasky	MLX5_REG_PMLP		 = 0x5002,
1355c8e2d7hselasky	MLX5_REG_PMTU		 = 0x5003,
1365c8e2d7hselasky	MLX5_REG_PTYS		 = 0x5004,
1375c8e2d7hselasky	MLX5_REG_PAOS		 = 0x5006,
1385c8e2d7hselasky	MLX5_REG_PFCC		 = 0x5007,
1395c8e2d7hselasky	MLX5_REG_PPCNT		 = 0x5008,
1405c8e2d7hselasky	MLX5_REG_PUDE		 = 0x5009,
1415c8e2d7hselasky	MLX5_REG_PPTB		 = 0x500B,
1425c8e2d7hselasky	MLX5_REG_PBMC		 = 0x500C,
1436fb43c1hselasky	MLX5_REG_PELC		 = 0x500E,
1446fb43c1hselasky	MLX5_REG_PVLC		 = 0x500F,
1455c8e2d7hselasky	MLX5_REG_PMPE		 = 0x5010,
1466fb43c1hselasky	MLX5_REG_PMAOS		 = 0x5012,
1477a5500ahselasky	MLX5_REG_PPLM		 = 0x5023,
14856e46b5hselasky	MLX5_REG_PBSR		 = 0x5038,
14923804e3hselasky	MLX5_REG_PCAM		 = 0x507f,
1505c8e2d7hselasky	MLX5_REG_NODE_DESC	 = 0x6001,
1515c8e2d7hselasky	MLX5_REG_HOST_ENDIANNESS = 0x7004,
1529ba28d9slavash	MLX5_REG_MTMP		 = 0x900a,
1535c8e2d7hselasky	MLX5_REG_MCIA		 = 0x9014,
1545289a2bhselasky	MLX5_REG_MFRL		 = 0x9028,
1556ef4747hselasky	MLX5_REG_MPCNT		 = 0x9051,
156e726ac7hselasky	MLX5_REG_MCQI		 = 0x9061,
157e726ac7hselasky	MLX5_REG_MCC		 = 0x9062,
158e726ac7hselasky	MLX5_REG_MCDA		 = 0x9063,
15923804e3hselasky	MLX5_REG_MCAM		 = 0x907f,
1605c8e2d7hselasky};
1615c8e2d7hselasky
1625c8e2d7hselaskyenum dbg_rsc_type {
1635c8e2d7hselasky	MLX5_DBG_RSC_QP,
1645c8e2d7hselasky	MLX5_DBG_RSC_EQ,
1655c8e2d7hselasky	MLX5_DBG_RSC_CQ,
1665c8e2d7hselasky};
1675c8e2d7hselasky
1686ef4747hselaskyenum {
1696ef4747hselasky	MLX5_INTERFACE_PROTOCOL_IB  = 0,
1706ef4747hselasky	MLX5_INTERFACE_PROTOCOL_ETH = 1,
1716ef4747hselasky	MLX5_INTERFACE_NUMBER       = 2,
1726ef4747hselasky};
1736ef4747hselasky
1745c8e2d7hselaskystruct mlx5_field_desc {
1755c8e2d7hselasky	struct dentry	       *dent;
1765c8e2d7hselasky	int			i;
1775c8e2d7hselasky};
1785c8e2d7hselasky
1795c8e2d7hselaskystruct mlx5_rsc_debug {
1805c8e2d7hselasky	struct mlx5_core_dev   *dev;
1815c8e2d7hselasky	void		       *object;
1825c8e2d7hselasky	enum dbg_rsc_type	type;
1835c8e2d7hselasky	struct dentry	       *root;
1845c8e2d7hselasky	struct mlx5_field_desc	fields[0];
1855c8e2d7hselasky};
1865c8e2d7hselasky
1875c8e2d7hselaskyenum mlx5_dev_event {
1885c8e2d7hselasky	MLX5_DEV_EVENT_SYS_ERROR,
1895c8e2d7hselasky	MLX5_DEV_EVENT_PORT_UP,
1905c8e2d7hselasky	MLX5_DEV_EVENT_PORT_DOWN,
1915c8e2d7hselasky	MLX5_DEV_EVENT_PORT_INITIALIZED,
1925c8e2d7hselasky	MLX5_DEV_EVENT_LID_CHANGE,
1935c8e2d7hselasky	MLX5_DEV_EVENT_PKEY_CHANGE,
1945c8e2d7hselasky	MLX5_DEV_EVENT_GUID_CHANGE,
1955c8e2d7hselasky	MLX5_DEV_EVENT_CLIENT_REREG,
1965c8e2d7hselasky	MLX5_DEV_EVENT_VPORT_CHANGE,
1976ef4747hselasky	MLX5_DEV_EVENT_ERROR_STATE_DCBX,
1986ef4747hselasky	MLX5_DEV_EVENT_REMOTE_CONFIG_CHANGE,
1996ef4747hselasky	MLX5_DEV_EVENT_LOCAL_OPER_CHANGE,
2006ef4747hselasky	MLX5_DEV_EVENT_REMOTE_CONFIG_APPLICATION_PRIORITY_CHANGE,
2015c8e2d7hselasky};
2025c8e2d7hselasky
2035c8e2d7hselaskyenum mlx5_port_status {
2045c8e2d7hselasky	MLX5_PORT_UP        = 1 << 0,
2055c8e2d7hselasky	MLX5_PORT_DOWN      = 1 << 1,
2065c8e2d7hselasky};
2075c8e2d7hselasky
2082514deehselaskyenum {
2092514deehselasky	MLX5_VSC_SPACE_SUPPORTED = 0x1,
2102514deehselasky	MLX5_VSC_SPACE_OFFSET	 = 0x4,
2112514deehselasky	MLX5_VSC_COUNTER_OFFSET	 = 0x8,
2122514deehselasky	MLX5_VSC_SEMA_OFFSET	 = 0xC,
2132514deehselasky	MLX5_VSC_ADDR_OFFSET	 = 0x10,
2142514deehselasky	MLX5_VSC_DATA_OFFSET	 = 0x14,
2152514deehselasky	MLX5_VSC_MAX_RETRIES	 = 0x1000,
2162514deehselasky};
2172514deehselasky
2185c8e2d7hselasky#define MLX5_PROT_MASK(link_mode) (1 << link_mode)
2195c8e2d7hselasky
2205c8e2d7hselaskystruct mlx5_uuar_info {
2215c8e2d7hselasky	struct mlx5_uar	       *uars;
2225c8e2d7hselasky	int			num_uars;
2235c8e2d7hselasky	int			num_low_latency_uuars;
2245c8e2d7hselasky	unsigned long	       *bitmap;
2255c8e2d7hselasky	unsigned int	       *count;
2265c8e2d7hselasky	struct mlx5_bf	       *bfs;
2275c8e2d7hselasky
2285c8e2d7hselasky	/*
2295c8e2d7hselasky	 * protect uuar allocation data structs
2305c8e2d7hselasky	 */
2315c8e2d7hselasky	struct mutex		lock;
2325c8e2d7hselasky	u32			ver;
2335c8e2d7hselasky};
2345c8e2d7hselasky
2355c8e2d7hselaskystruct mlx5_bf {
2365c8e2d7hselasky	void __iomem	       *reg;
2375c8e2d7hselasky	void __iomem	       *regreg;
2385c8e2d7hselasky	int			buf_size;
2395c8e2d7hselasky	struct mlx5_uar	       *uar;
2405c8e2d7hselasky	unsigned long		offset;
2415c8e2d7hselasky	int			need_lock;
2425c8e2d7hselasky	/* protect blue flame buffer selection when needed
2435c8e2d7hselasky	 */
2445c8e2d7hselasky	spinlock_t		lock;
2455c8e2d7hselasky
2465c8e2d7hselasky	/* serialize 64 bit writes when done as two 32 bit accesses
2475c8e2d7hselasky	 */
2485c8e2d7hselasky	spinlock_t		lock32;
2495c8e2d7hselasky	int			uuarn;
2505c8e2d7hselasky};
2515c8e2d7hselasky
2525c8e2d7hselaskystruct mlx5_cmd_first {
2535c8e2d7hselasky	__be32		data[4];
2545c8e2d7hselasky};
2555c8e2d7hselasky
25685824cdhselaskystruct cache_ent;
25785824cdhselaskystruct mlx5_fw_page {
25885824cdhselasky	union {
25985824cdhselasky		struct rb_node rb_node;
26085824cdhselasky		struct list_head list;
26185824cdhselasky	};
26285824cdhselasky	struct mlx5_cmd_first first;
26385824cdhselasky	struct mlx5_core_dev *dev;
26485824cdhselasky	bus_dmamap_t dma_map;
26585824cdhselasky	bus_addr_t dma_addr;
26685824cdhselasky	void *virt_addr;
26785824cdhselasky	struct cache_ent *cache;
26885824cdhselasky	u32 numpages;
26985824cdhselasky	u16 load_done;
27085824cdhselasky#define	MLX5_LOAD_ST_NONE 0
27185824cdhselasky#define	MLX5_LOAD_ST_SUCCESS 1
27285824cdhselasky#define	MLX5_LOAD_ST_FAILURE 2
27385824cdhselasky	u16 func_id;
27485824cdhselasky};
27585824cdhselasky#define	mlx5_cmd_msg mlx5_fw_page
2765c8e2d7hselasky
2775c8e2d7hselaskystruct mlx5_cmd_debug {
2785c8e2d7hselasky	struct dentry	       *dbg_root;
2795c8e2d7hselasky	struct dentry	       *dbg_in;
2805c8e2d7hselasky	struct dentry	       *dbg_out;
2815c8e2d7hselasky	struct dentry	       *dbg_outlen;
2825c8e2d7hselasky	struct dentry	       *dbg_status;
2835c8e2d7hselasky	struct dentry	       *dbg_run;
2845c8e2d7hselasky	void		       *in_msg;
2855c8e2d7hselasky	void		       *out_msg;
2865c8e2d7hselasky	u8			status;
2875c8e2d7hselasky	u16			inlen;
2885c8e2d7hselasky	u16			outlen;
2895c8e2d7hselasky};
2905c8e2d7hselasky
2915c8e2d7hselaskystruct cache_ent {
2925c8e2d7hselasky	/* protect block chain allocations
2935c8e2d7hselasky	 */
2945c8e2d7hselasky	spinlock_t		lock;
2955c8e2d7hselasky	struct list_head	head;
2965c8e2d7hselasky};
2975c8e2d7hselasky
2985c8e2d7hselaskystruct cmd_msg_cache {
2995c8e2d7hselasky	struct cache_ent	large;
3005c8e2d7hselasky	struct cache_ent	med;
3015c8e2d7hselasky
3025c8e2d7hselasky};
3035c8e2d7hselasky
30475528d7hselaskystruct mlx5_traffic_counter {
30575528d7hselasky	u64         packets;
30675528d7hselasky	u64         octets;
30775528d7hselasky};
30875528d7hselasky
309d26314fslavashenum mlx5_cmd_mode {
310d26314fslavash	MLX5_CMD_MODE_POLLING,
311d26314fslavash	MLX5_CMD_MODE_EVENTS
312d26314fslavash};
313d26314fslavash
3145c8e2d7hselaskystruct mlx5_cmd_stats {
3155c8e2d7hselasky	u64		sum;
3165c8e2d7hselasky	u64		n;
3175c8e2d7hselasky	struct dentry  *root;
3185c8e2d7hselasky	struct dentry  *avg;
3195c8e2d7hselasky	struct dentry  *count;
3205c8e2d7hselasky	/* protect command average calculations */
3215c8e2d7hselasky	spinlock_t	lock;
3225c8e2d7hselasky};
3235c8e2d7hselasky
3245c8e2d7hselaskystruct mlx5_cmd {
32585824cdhselasky	struct mlx5_fw_page *cmd_page;
32685824cdhselasky	bus_dma_tag_t dma_tag;
32785824cdhselasky	struct sx dma_sx;
32885824cdhselasky	struct mtx dma_mtx;
32985824cdhselasky#define	MLX5_DMA_OWNED(dev) mtx_owned(&(dev)->cmd.dma_mtx)
33085824cdhselasky#define	MLX5_DMA_LOCK(dev) mtx_lock(&(dev)->cmd.dma_mtx)
33185824cdhselasky#define	MLX5_DMA_UNLOCK(dev) mtx_unlock(&(dev)->cmd.dma_mtx)
33285824cdhselasky	struct cv dma_cv;
33385824cdhselasky#define	MLX5_DMA_DONE(dev) cv_broadcast(&(dev)->cmd.dma_cv)
33485824cdhselasky#define	MLX5_DMA_WAIT(dev) cv_wait(&(dev)->cmd.dma_cv, &(dev)->cmd.dma_mtx)
3355c8e2d7hselasky	void	       *cmd_buf;
3365c8e2d7hselasky	dma_addr_t	dma;
3375c8e2d7hselasky	u16		cmdif_rev;
3385c8e2d7hselasky	u8		log_sz;
3395c8e2d7hselasky	u8		log_stride;
3405c8e2d7hselasky	int		max_reg_cmds;
3415c8e2d7hselasky	int		events;
3425c8e2d7hselasky	u32 __iomem    *vector;
3435c8e2d7hselasky
3445c8e2d7hselasky	/* protect command queue allocations
3455c8e2d7hselasky	 */
3465c8e2d7hselasky	spinlock_t	alloc_lock;
3475c8e2d7hselasky
3485c8e2d7hselasky	/* protect token allocations
3495c8e2d7hselasky	 */
3505c8e2d7hselasky	spinlock_t	token_lock;
3515c8e2d7hselasky	u8		token;
3525c8e2d7hselasky	unsigned long	bitmask;
3535c8e2d7hselasky	struct semaphore sem;
3545c8e2d7hselasky	struct semaphore pages_sem;
355d26314fslavash	enum mlx5_cmd_mode mode;
356d26314fslavash	struct mlx5_cmd_work_ent * volatile ent_arr[MLX5_MAX_COMMANDS];
357d26314fslavash	volatile enum mlx5_cmd_mode ent_mode[MLX5_MAX_COMMANDS];
3585c8e2d7hselasky	struct mlx5_cmd_debug dbg;
3595c8e2d7hselasky	struct cmd_msg_cache cache;
3605c8e2d7hselasky	int checksum_disabled;
3615c8e2d7hselasky	struct mlx5_cmd_stats stats[MLX5_CMD_OP_MAX];
3625c8e2d7hselasky};
3635c8e2d7hselasky
3645c8e2d7hselaskystruct mlx5_port_caps {
3655c8e2d7hselasky	int	gid_table_len;
3665c8e2d7hselasky	int	pkey_table_len;
3675c8e2d7hselasky	u8	ext_port_cap;
3685c8e2d7hselasky};
3695c8e2d7hselasky
3705c8e2d7hselaskystruct mlx5_buf {
37185824cdhselasky	bus_dma_tag_t		dma_tag;
37285824cdhselasky	bus_dmamap_t		dma_map;
37385824cdhselasky	struct mlx5_core_dev   *dev;
37485824cdhselasky	struct {
37585824cdhselasky		void	       *buf;
37685824cdhselasky	} direct;
37785824cdhselasky	u64		       *page_list;
3785c8e2d7hselasky	int			npages;
3795c8e2d7hselasky	int			size;
3805c8e2d7hselasky	u8			page_shift;
38185824cdhselasky	u8			load_done;
3825c8e2d7hselasky};
3835c8e2d7hselasky
38416b9405slavashstruct mlx5_frag_buf {
38516b9405slavash	struct mlx5_buf_list	*frags;
38616b9405slavash	int			npages;
38716b9405slavash	int			size;
38816b9405slavash	u8			page_shift;
38916b9405slavash};
39016b9405slavash
3915c8e2d7hselaskystruct mlx5_eq {
3925c8e2d7hselasky	struct mlx5_core_dev   *dev;
3935c8e2d7hselasky	__be32 __iomem	       *doorbell;
3945c8e2d7hselasky	u32			cons_index;
3955c8e2d7hselasky	struct mlx5_buf		buf;
3965c8e2d7hselasky	int			size;
3975c8e2d7hselasky	u8			irqn;
3985c8e2d7hselasky	u8			eqn;
3995c8e2d7hselasky	int			nent;
4005c8e2d7hselasky	u64			mask;
4015c8e2d7hselasky	struct list_head	list;
4025c8e2d7hselasky	int			index;
4035c8e2d7hselasky	struct mlx5_rsc_debug	*dbg;
4045c8e2d7hselasky};
4055c8e2d7hselasky
4065c8e2d7hselaskystruct mlx5_core_psv {
4075c8e2d7hselasky	u32	psv_idx;
4085c8e2d7hselasky	struct psv_layout {
4095c8e2d7hselasky		u32	pd;
4105c8e2d7hselasky		u16	syndrome;
4115c8e2d7hselasky		u16	reserved;
4125c8e2d7hselasky		u16	bg;
4135c8e2d7hselasky		u16	app_tag;
4145c8e2d7hselasky		u32	ref_tag;
4155c8e2d7hselasky	} psv;
4165c8e2d7hselasky};
4175c8e2d7hselasky
4185c8e2d7hselaskystruct mlx5_core_sig_ctx {
4195c8e2d7hselasky	struct mlx5_core_psv	psv_memory;
4205c8e2d7hselasky	struct mlx5_core_psv	psv_wire;
4215c8e2d7hselasky#if (__FreeBSD_version >= 1100000)
4225c8e2d7hselasky	struct ib_sig_err       err_item;
4235c8e2d7hselasky#endif
4245c8e2d7hselasky	bool			sig_status_checked;
4255c8e2d7hselasky	bool			sig_err_exists;
4265c8e2d7hselasky	u32			sigerr_count;
4275c8e2d7hselasky};
4285c8e2d7hselasky
42916b9405slavashenum {
43016b9405slavash	MLX5_MKEY_MR = 1,
43116b9405slavash	MLX5_MKEY_MW,
43216b9405slavash	MLX5_MKEY_MR_USER,
43316b9405slavash};
43416b9405slavash
43516b9405slavashstruct mlx5_core_mkey {
43616b9405slavash	u64			iova;
43716b9405slavash	u64			size;
43816b9405slavash	u32			key;
43916b9405slavash	u32			pd;
44016b9405slavash	u32			type;
44116b9405slavash};
44216b9405slavash
4435c8e2d7hselaskystruct mlx5_core_mr {
4445c8e2d7hselasky	u64			iova;
4455c8e2d7hselasky	u64			size;
4465c8e2d7hselasky	u32			key;
4475c8e2d7hselasky	u32			pd;
4485c8e2d7hselasky};
4495c8e2d7hselasky
4505c8e2d7hselaskyenum mlx5_res_type {
4516ef4747hselasky	MLX5_RES_QP	= MLX5_EVENT_QUEUE_TYPE_QP,
4526ef4747hselasky	MLX5_RES_RQ	= MLX5_EVENT_QUEUE_TYPE_RQ,
4536ef4747hselasky	MLX5_RES_SQ	= MLX5_EVENT_QUEUE_TYPE_SQ,
4546ef4747hselasky	MLX5_RES_SRQ	= 3,
4556ef4747hselasky	MLX5_RES_XSRQ	= 4,
4566ef4747hselasky	MLX5_RES_DCT	= 5,
4575c8e2d7hselasky};
4585c8e2d7hselasky
4595c8e2d7hselaskystruct mlx5_core_rsc_common {
4605c8e2d7hselasky	enum mlx5_res_type	res;
4615c8e2d7hselasky	atomic_t		refcount;
4625c8e2d7hselasky	struct completion	free;
4635c8e2d7hselasky};
4645c8e2d7hselasky
4655c8e2d7hselaskystruct mlx5_core_srq {
4665c8e2d7hselasky	struct mlx5_core_rsc_common	common; /* must be first */
4675c8e2d7hselasky	u32				srqn;
4685c8e2d7hselasky	int				max;
469b6bd33aslavash	size_t				max_gs;
470b6bd33aslavash	size_t				max_avail_gather;
4715c8e2d7hselasky	int				wqe_shift;
4725c8e2d7hselasky	void				(*event)(struct mlx5_core_srq *, int);
4735c8e2d7hselasky	atomic_t			refcount;
4745c8e2d7hselasky	struct completion		free;
4755c8e2d7hselasky};
4765c8e2d7hselasky
4775c8e2d7hselaskystruct mlx5_eq_table {
4785c8e2d7hselasky	void __iomem	       *update_ci;
4795c8e2d7hselasky	void __iomem	       *update_arm_ci;
4805c8e2d7hselasky	struct list_head	comp_eqs_list;
4815c8e2d7hselasky	struct mlx5_eq		pages_eq;
4825c8e2d7hselasky	struct mlx5_eq		async_eq;
4835c8e2d7hselasky	struct mlx5_eq		cmd_eq;
4845c8e2d7hselasky	int			num_comp_vectors;
4855c8e2d7hselasky	/* protect EQs list
4865c8e2d7hselasky	 */
4875c8e2d7hselasky	spinlock_t		lock;
4885c8e2d7hselasky};
4895c8e2d7hselasky
4905c8e2d7hselaskystruct mlx5_uar {
4915c8e2d7hselasky	u32			index;
4925c8e2d7hselasky	void __iomem	       *bf_map;
4935c8e2d7hselasky	void __iomem	       *map;
4945c8e2d7hselasky};
4955c8e2d7hselasky
4965c8e2d7hselasky
4975c8e2d7hselaskystruct mlx5_core_health {
4985c8e2d7hselasky	struct mlx5_health_buffer __iomem	*health;
4995c8e2d7hselasky	__be32 __iomem		       *health_counter;
5005c8e2d7hselasky	struct timer_list		timer;
5015c8e2d7hselasky	u32				prev;
5025c8e2d7hselasky	int				miss_counter;
5034c421e8hselasky	u32				fatal_error;
504a7cccbdhselasky	struct workqueue_struct	       *wq_watchdog;
5056e21fa7hselasky	struct work_struct		work_watchdog;
5063b71a31hselasky	/* wq spinlock to synchronize draining */
5073b71a31hselasky	spinlock_t			wq_lock;
508f5d1640hselasky	struct workqueue_struct	       *wq;
5093b71a31hselasky	unsigned long			flags;
510f5d1640hselasky	struct work_struct		work;
511947b9c0hselasky	struct delayed_work		recover_work;
5120ca4d48hselasky	unsigned int			last_reset_req;
513da8c8b9hselasky	struct work_struct		work_cmd_completion;
514450ce30hselasky	struct workqueue_struct	       *wq_cmd;
5155c8e2d7hselasky};
5165c8e2d7hselasky
5175c8e2d7hselasky#define	MLX5_CQ_LINEAR_ARRAY_SIZE	1024
5185c8e2d7hselasky
5195c8e2d7hselaskystruct mlx5_cq_linear_array_entry {
5205c8e2d7hselasky	struct mlx5_core_cq * volatile cq;
5215c8e2d7hselasky};
5225c8e2d7hselasky
5235c8e2d7hselaskystruct mlx5_cq_table {
5245c8e2d7hselasky	/* protect radix tree
5255c8e2d7hselasky	 */
5260a18c75hselasky	spinlock_t		writerlock;
5270a18c75hselasky	atomic_t		writercount;
5285c8e2d7hselasky	struct radix_tree_root	tree;
5295c8e2d7hselasky	struct mlx5_cq_linear_array_entry linear_array[MLX5_CQ_LINEAR_ARRAY_SIZE];
5305c8e2d7hselasky};
5315c8e2d7hselasky
5325c8e2d7hselaskystruct mlx5_qp_table {
5335c8e2d7hselasky	/* protect radix tree
5345c8e2d7hselasky	 */
5355c8e2d7hselasky	spinlock_t		lock;
5365c8e2d7hselasky	struct radix_tree_root	tree;
5375c8e2d7hselasky};
5385c8e2d7hselasky
5395c8e2d7hselaskystruct mlx5_srq_table {
5405c8e2d7hselasky	/* protect radix tree
5415c8e2d7hselasky	 */
5425c8e2d7hselasky	spinlock_t		lock;
5435c8e2d7hselasky	struct radix_tree_root	tree;
5445c8e2d7hselasky};
5455c8e2d7hselasky
5465c8e2d7hselaskystruct mlx5_mr_table {
5475c8e2d7hselasky	/* protect radix tree
5485c8e2d7hselasky	 */
5496ef4747hselasky	spinlock_t		lock;
5505c8e2d7hselasky	struct radix_tree_root	tree;
5515c8e2d7hselasky};
5525c8e2d7hselasky
553a09337chselasky#ifdef RATELIMIT
554a09337chselaskystruct mlx5_rl_entry {
555a09337chselasky	u32			rate;
556a09337chselasky	u16			burst;
557a09337chselasky	u16			index;
558a09337chselasky	u32			refcount;
559a09337chselasky};
560a09337chselasky
561a09337chselaskystruct mlx5_rl_table {
562a09337chselasky	struct mutex		rl_lock;
563a09337chselasky	u16			max_size;
564a09337chselasky	u32			max_rate;
565a09337chselasky	u32			min_rate;
566a09337chselasky	struct mlx5_rl_entry   *rl_entry;
567a09337chselasky};
568a09337chselasky#endif
569a09337chselasky
570d6e923chselaskystruct mlx5_pme_stats {
571d6e923chselasky	u64			status_counters[MLX5_MODULE_STATUS_NUM];
572d6e923chselasky	u64			error_counters[MLX5_MODULE_EVENT_ERROR_NUM];
573d6e923chselasky};
574d6e923chselasky
5755c8e2d7hselaskystruct mlx5_priv {
5765c8e2d7hselasky	char			name[MLX5_MAX_NAME_LEN];
5775c8e2d7hselasky	struct mlx5_eq_table	eq_table;
5785c8e2d7hselasky	struct msix_entry	*msix_arr;
5795c8e2d7hselasky	struct mlx5_uuar_info	uuari;
5805c8e2d7hselasky	MLX5_DECLARE_DOORBELL_LOCK(cq_uar_lock);
5814d6dbe0hselasky	int			disable_irqs;
5825c8e2d7hselasky
5835c8e2d7hselasky	struct io_mapping	*bf_mapping;
5845c8e2d7hselasky
5855c8e2d7hselasky	/* pages stuff */
5865c8e2d7hselasky	struct workqueue_struct *pg_wq;
5875c8e2d7hselasky	struct rb_root		page_root;
588cf42c74hselasky	s64			fw_pages;
5896ef4747hselasky	atomic_t		reg_pages;
590120993bhselasky	s64			pages_per_func[MLX5_MAX_NUMBER_OF_VFS];
5915c8e2d7hselasky	struct mlx5_core_health health;
5925c8e2d7hselasky
5935c8e2d7hselasky	struct mlx5_srq_table	srq_table;
5945c8e2d7hselasky
5955c8e2d7hselasky	/* start: qp staff */
5965c8e2d7hselasky	struct mlx5_qp_table	qp_table;
5975c8e2d7hselasky	struct dentry	       *qp_debugfs;
5985c8e2d7hselasky	struct dentry	       *eq_debugfs;
5995c8e2d7hselasky	struct dentry	       *cq_debugfs;
6005c8e2d7hselasky	struct dentry	       *cmdif_debugfs;
6015c8e2d7hselasky	/* end: qp staff */
6025c8e2d7hselasky
6035c8e2d7hselasky	/* start: cq staff */
6045c8e2d7hselasky	struct mlx5_cq_table	cq_table;
6055c8e2d7hselasky	/* end: cq staff */
6065c8e2d7hselasky
6075c8e2d7hselasky	/* start: mr staff */
6085c8e2d7hselasky	struct mlx5_mr_table	mr_table;
6095c8e2d7hselasky	/* end: mr staff */
6105c8e2d7hselasky
6115c8e2d7hselasky	/* start: alloc staff */
6125c8e2d7hselasky	int			numa_node;
6135c8e2d7hselasky
6145c8e2d7hselasky	struct mutex   pgdir_mutex;
6155c8e2d7hselasky	struct list_head        pgdir_list;
6165c8e2d7hselasky	/* end: alloc staff */
6175c8e2d7hselasky	struct dentry	       *dbg_root;
6185c8e2d7hselasky
6195c8e2d7hselasky	/* protect mkey key part */
6205c8e2d7hselasky	spinlock_t		mkey_lock;
6215c8e2d7hselasky	u8			mkey_key;
6225c8e2d7hselasky
6235c8e2d7hselasky	struct list_head        dev_list;
6245c8e2d7hselasky	struct list_head        ctx_list;
6255c8e2d7hselasky	spinlock_t              ctx_lock;
6266ef4747hselasky	unsigned long		pci_dev_data;
627a09337chselasky#ifdef RATELIMIT
628a09337chselasky	struct mlx5_rl_table	rl_table;
629a09337chselasky#endif
630d6e923chselasky	struct mlx5_pme_stats pme_stats;
631761449ekib
632761449ekib	struct mlx5_eswitch	*eswitch;
6336ef4747hselasky};
6346ef4747hselasky
6356ef4747hselaskyenum mlx5_device_state {
6366ef4747hselasky	MLX5_DEVICE_STATE_UP,
6376ef4747hselasky	MLX5_DEVICE_STATE_INTERNAL_ERROR,
6385c8e2d7hselasky};
6395c8e2d7hselasky
640f5d1640hselaskyenum mlx5_interface_state {
64172af986kib	MLX5_INTERFACE_STATE_UP = 0x1,
64272af986kib	MLX5_INTERFACE_STATE_TEARDOWN = 0x2,
643f5d1640hselasky};
644f5d1640hselasky
645f5d1640hselaskyenum mlx5_pci_status {
646f5d1640hselasky	MLX5_PCI_STATUS_DISABLED,
647f5d1640hselasky	MLX5_PCI_STATUS_ENABLED,
648f5d1640hselasky};
649f5d1640hselasky
65016b9405slavash#define	MLX5_MAX_RESERVED_GIDS	8
65116b9405slavash
65216b9405slavashstruct mlx5_rsvd_gids {
65316b9405slavash	unsigned int start;
65416b9405slavash	unsigned int count;
65516b9405slavash	struct ida ida;
65616b9405slavash};
65716b9405slavash
6585c8e2d7hselaskystruct mlx5_special_contexts {
6595c8e2d7hselasky	int resd_lkey;
6605c8e2d7hselasky};
6615c8e2d7hselasky
662