1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source. A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * This file is part of the Chelsio T4 support code.
14  *
15  * Copyright (C) 2011-2013 Chelsio Communications.  All rights reserved.
16  *
17  * This program is distributed in the hope that it will be useful, but WITHOUT
18  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
20  * release for licensing terms and conditions.
21  */
22 
23 #ifndef __CXGBE_ADAPTER_H
24 #define	__CXGBE_ADAPTER_H
25 
26 #include <sys/ddi.h>
27 #include <sys/mac_provider.h>
28 #include <sys/ethernet.h>
29 #include <sys/queue.h>
30 #include <sys/containerof.h>
31 #include <sys/ddi_ufm.h>
32 
33 #include "offload.h"
34 #include "firmware/t4fw_interface.h"
35 #include "shared.h"
36 
37 struct adapter;
38 typedef struct adapter adapter_t;
39 
40 enum {
41 	FW_IQ_QSIZE = 256,
42 	FW_IQ_ESIZE = 64,	/* At least 64 mandated by the firmware spec */
43 
44 	RX_IQ_QSIZE = 1024,
45 	RX_IQ_ESIZE = 64,	/* At least 64 so CPL_RX_PKT will fit */
46 
47 	EQ_ESIZE = 64,		/* All egres queues use this entry size */
48 
49 	RX_FL_ESIZE = 64,	/* 8 64bit addresses */
50 
51 	FL_BUF_SIZES = 4,
52 
53 	CTRL_EQ_QSIZE = 128,
54 
55 	TX_EQ_QSIZE = 1024,
56 	TX_SGL_SEGS = 36,
57 	TX_WR_FLITS = SGE_MAX_WR_LEN / 8
58 };
59 
60 enum {
61 	/* adapter flags */
62 	FULL_INIT_DONE	= (1 << 0),
63 	FW_OK		= (1 << 1),
64 	INTR_FWD	= (1 << 2),
65 	INTR_ALLOCATED	= (1 << 3),
66 	MASTER_PF	= (1 << 4),
67 
68 	CXGBE_BUSY	= (1 << 9),
69 
70 	/* port flags */
71 	DOOMED		= (1 << 0),
72 	PORT_INIT_DONE	= (1 << 1),
73 };
74 
75 enum {
76 	/* Features */
77 	CXGBE_HW_LSO	= (1 << 0),
78 	CXGBE_HW_CSUM	= (1 << 1),
79 };
80 
81 enum {
82 	UDBS_SEG_SHIFT	= 7,	/* log2(UDBS_SEG_SIZE) */
83 	UDBS_DB_OFFSET	= 8,	/* offset of the 4B doorbell in a segment */
84 	UDBS_WR_OFFSET	= 64,	/* offset of the work request in a segment */
85 };
86 
87 #define	IS_DOOMED(pi)	(pi->flags & DOOMED)
88 #define	SET_DOOMED(pi)	do { pi->flags |= DOOMED; } while (0)
89 #define	IS_BUSY(sc)	(sc->flags & CXGBE_BUSY)
90 #define	SET_BUSY(sc)	do { sc->flags |= CXGBE_BUSY; } while (0)
91 #define	CLR_BUSY(sc)	do { sc->flags &= ~CXGBE_BUSY; } while (0)
92 
93 struct port_info {
94 	PORT_INFO_HDR;
95 
96 	kmutex_t lock;
97 	struct adapter *adapter;
98 
99 #ifdef TCP_OFFLOAD_ENABLE
100 	void *tdev;
101 #endif
102 
103 	unsigned int flags;
104 
105 	uint16_t viid;
106 	int16_t  xact_addr_filt; /* index of exact MAC address filter */
107 	uint16_t rss_size;	/* size of VI's RSS table slice */
108 	uint16_t ntxq;		/* # of tx queues */
109 	uint16_t first_txq;	/* index of first tx queue */
110 	uint16_t nrxq;		/* # of rx queues */
111 	uint16_t first_rxq;	/* index of first rx queue */
112 #ifdef TCP_OFFLOAD_ENABLE
113 	uint16_t nofldtxq;		/* # of offload tx queues */
114 	uint16_t first_ofld_txq;	/* index of first offload tx queue */
115 	uint16_t nofldrxq;		/* # of offload rx queues */
116 	uint16_t first_ofld_rxq;	/* index of first offload rx queue */
117 #endif
118 	uint8_t  lport;		/* associated offload logical port */
119 	int8_t   mdio_addr;
120 	uint8_t  port_type;
121 	uint8_t  mod_type;
122 	uint8_t  port_id;
123 	uint8_t  tx_chan;
124 	uint8_t  rx_chan;
125 	uint8_t  rx_cchan;
126 	uint8_t instance; /* Associated adapter instance */
127 	uint8_t child_inst; /* Associated child instance */
128 	uint8_t	tmr_idx;
129 	int8_t	pktc_idx;
130 	struct link_config link_cfg;
131 	struct port_stats stats;
132 	uint32_t features;
133 	uint8_t macaddr_cnt;
134 	u8 rss_mode;
135 	u16 viid_mirror;
136 	kstat_t *ksp_config;
137 	kstat_t *ksp_info;
138 	kstat_t *ksp_fec;
139 
140 	u8 vivld;
141 	u8 vin;
142 	u8 smt_idx;
143 
144 	u8 vivld_mirror;
145 	u8 vin_mirror;
146 	u8 smt_idx_mirror;
147 };
148 
149 struct fl_sdesc {
150 	struct rxbuf *rxb;
151 };
152 
153 struct tx_desc {
154 	__be64 flit[8];
155 };
156 
157 /* DMA maps used for tx */
158 struct tx_maps {
159 	ddi_dma_handle_t *map;
160 	uint32_t map_total;	/* # of DMA maps */
161 	uint32_t map_pidx;	/* next map to be used */
162 	uint32_t map_cidx;	/* reclaimed up to this index */
163 	uint32_t map_avail;	/* # of available maps */
164 };
165 
166 struct tx_sdesc {
167 	mblk_t *m;
168 	uint32_t txb_used;	/* # of bytes of tx copy buffer used */
169 	uint16_t hdls_used;	/* # of dma handles used */
170 	uint16_t desc_used;	/* # of hardware descriptors used */
171 };
172 
173 enum {
174 	/* iq flags */
175 	IQ_ALLOCATED	= (1 << 0),	/* firmware resources allocated */
176 	IQ_INTR		= (1 << 1),	/* iq takes direct interrupt */
177 	IQ_HAS_FL	= (1 << 2),	/* iq has fl */
178 
179 	/* iq state */
180 	IQS_DISABLED	= 0,
181 	IQS_BUSY	= 1,
182 	IQS_IDLE	= 2,
183 };
184 
185 /*
186  * Ingress Queue: T4 is producer, driver is consumer.
187  */
188 struct sge_iq {
189 	unsigned int flags;
190 	ddi_dma_handle_t dhdl;
191 	ddi_acc_handle_t ahdl;
192 
193 	volatile uint_t state;
194 	__be64 *desc;		/* KVA of descriptor ring */
195 	uint64_t ba;		/* bus address of descriptor ring */
196 	const __be64 *cdesc;	/* current descriptor */
197 	struct adapter *adapter; /* associated  adapter */
198 	uint8_t  gen;		/* generation bit */
199 	uint8_t  intr_params;	/* interrupt holdoff parameters */
200 	int8_t   intr_pktc_idx;	/* packet count threshold index */
201 	uint8_t  intr_next;	/* holdoff for next interrupt */
202 	uint8_t  esize;		/* size (bytes) of each entry in the queue */
203 	uint16_t qsize;		/* size (# of entries) of the queue */
204 	uint16_t cidx;		/* consumer index */
205 	uint16_t pending;	/* # of descs processed since last doorbell */
206 	uint16_t cntxt_id;	/* SGE context id  for the iq */
207 	uint16_t abs_id;	/* absolute SGE id for the iq */
208 	kmutex_t lock;		/* Rx access lock */
209 	uint8_t polling;
210 
211 	STAILQ_ENTRY(sge_iq) link;
212 };
213 
214 enum {
215 	EQ_CTRL		= 1,
216 	EQ_ETH		= 2,
217 #ifdef TCP_OFFLOAD_ENABLE
218 	EQ_OFLD		= 3,
219 #endif
220 
221 	/* eq flags */
222 	EQ_TYPEMASK	= 7,		/* 3 lsbits hold the type */
223 	EQ_ALLOCATED	= (1 << 3),	/* firmware resources allocated */
224 	EQ_DOOMED	= (1 << 4),	/* about to be destroyed */
225 	EQ_CRFLUSHED	= (1 << 5),	/* expecting an update from SGE */
226 	EQ_STALLED	= (1 << 6),	/* out of hw descriptors or dmamaps */
227 	EQ_MTX		= (1 << 7),	/* mutex has been initialized */
228 	EQ_STARTED	= (1 << 8),	/* started */
229 };
230 
231 /* Listed in order of preference.  Update t4_sysctls too if you change these */
232 enum {DOORBELL_UDB=0x1 , DOORBELL_WCWR=0x2, DOORBELL_UDBWC=0x4, DOORBELL_KDB=0x8};
233 
234 /*
235  * Egress Queue: driver is producer, T4 is consumer.
236  *
237  * Note: A free list is an egress queue (driver produces the buffers and T4
238  * consumes them) but it's special enough to have its own struct (see sge_fl).
239  */
240 struct sge_eq {
241 	ddi_dma_handle_t desc_dhdl;
242 	ddi_acc_handle_t desc_ahdl;
243 	unsigned int flags;
244 	kmutex_t lock;
245 
246 	struct tx_desc *desc;	/* KVA of descriptor ring */
247 	uint64_t ba;		/* bus address of descriptor ring */
248 	struct sge_qstat *spg;	/* status page, for convenience */
249 	int doorbells;
250 	volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */
251 	u_int udb_qid;		/* relative qid within the doorbell page */
252 	uint16_t cap;		/* max # of desc, for convenience */
253 	uint16_t avail;		/* available descriptors, for convenience */
254 	uint16_t qsize;		/* size (# of entries) of the queue */
255 	uint16_t cidx;		/* consumer idx (desc idx) */
256 	uint16_t pidx;		/* producer idx (desc idx) */
257 	uint16_t pending;	/* # of descriptors used since last doorbell */
258 	uint16_t iqid;		/* iq that gets egr_update for the eq */
259 	uint8_t tx_chan;	/* tx channel used by the eq */
260 	uint32_t cntxt_id;	/* SGE context id for the eq */
261 };
262 
263 enum {
264 	/* fl flags */
265 	FL_MTX		= (1 << 0),	/* mutex has been initialized */
266 	FL_STARVING	= (1 << 1),	/* on the list of starving fl's */
267 	FL_DOOMED	= (1 << 2),	/* about to be destroyed */
268 };
269 
270 #define	FL_RUNNING_LOW(fl)	(fl->cap - fl->needed <= fl->lowat)
271 #define	FL_NOT_RUNNING_LOW(fl)	(fl->cap - fl->needed >= 2 * fl->lowat)
272 
273 struct sge_fl {
274 	unsigned int flags;
275 	kmutex_t lock;
276 	ddi_dma_handle_t dhdl;
277 	ddi_acc_handle_t ahdl;
278 
279 	__be64 *desc;		/* KVA of descriptor ring, ptr to addresses */
280 	uint64_t ba;		/* bus address of descriptor ring */
281 	struct fl_sdesc *sdesc;	/* KVA of software descriptor ring */
282 	uint32_t cap;		/* max # of buffers, for convenience */
283 	uint16_t qsize;		/* size (# of entries) of the queue */
284 	uint16_t cntxt_id;	/* SGE context id for the freelist */
285 	uint32_t cidx;		/* consumer idx (buffer idx, NOT hw desc idx) */
286 	uint32_t pidx;		/* producer idx (buffer idx, NOT hw desc idx) */
287 	uint32_t needed;	/* # of buffers needed to fill up fl. */
288 	uint32_t lowat;		/* # of buffers <= this means fl needs help */
289 	uint32_t pending;	/* # of bufs allocated since last doorbell */
290 	uint32_t offset;	/* current packet within the larger buffer */
291 	uint16_t copy_threshold; /* anything this size or less is copied up */
292 
293 	uint64_t copied_up;	/* # of frames copied into mblk and handed up */
294 	uint64_t passed_up;	/* # of frames wrapped in mblk and handed up */
295 	uint64_t allocb_fail;	/* # of mblk allocation failures */
296 
297 	TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
298 };
299 
300 /* txq: SGE egress queue + miscellaneous items */
301 struct sge_txq {
302 	struct sge_eq eq;	/* MUST be first */
303 
304 	struct port_info *port;	/* the port this txq belongs to */
305 	struct tx_sdesc *sdesc;	/* KVA of software descriptor ring */
306 	mac_ring_handle_t ring_handle;
307 
308 	/* DMA handles used for tx */
309 	ddi_dma_handle_t *tx_dhdl;
310 	uint32_t tx_dhdl_total;	/* Total # of handles */
311 	uint32_t tx_dhdl_pidx;	/* next handle to be used */
312 	uint32_t tx_dhdl_cidx;	/* reclaimed up to this index */
313 	uint32_t tx_dhdl_avail;	/* # of available handles */
314 
315 	/* Copy buffers for tx */
316 	ddi_dma_handle_t txb_dhdl;
317 	ddi_acc_handle_t txb_ahdl;
318 	caddr_t txb_va;		/* KVA of copy buffers area */
319 	uint64_t txb_ba;	/* bus address of copy buffers area */
320 	uint32_t txb_size;	/* total size */
321 	uint32_t txb_next;	/* offset of next useable area in the buffer */
322 	uint32_t txb_avail;	/* # of bytes available */
323 	uint16_t copy_threshold; /* anything this size or less is copied up */
324 
325 	uint64_t txpkts;	/* # of ethernet packets */
326 	uint64_t txbytes;	/* # of ethernet bytes */
327 	kstat_t *ksp;
328 
329 	/* stats for common events first */
330 
331 	uint64_t txcsum;	/* # of times hardware assisted with checksum */
332 	uint64_t tso_wrs;	/* # of IPv4 TSO work requests */
333 	uint64_t imm_wrs;	/* # of work requests with immediate data */
334 	uint64_t sgl_wrs;	/* # of work requests with direct SGL */
335 	uint64_t txpkt_wrs;	/* # of txpkt work requests (not coalesced) */
336 	uint64_t txpkts_wrs;	/* # of coalesced tx work requests */
337 	uint64_t txpkts_pkts;	/* # of frames in coalesced tx work requests */
338 	uint64_t txb_used;	/* # of tx copy buffers used (64 byte each) */
339 	uint64_t hdl_used;	/* # of DMA handles used */
340 
341 	/* stats for not-that-common events */
342 
343 	uint32_t txb_full;	/* txb ran out of space */
344 	uint32_t dma_hdl_failed; /* couldn't obtain DMA handle */
345 	uint32_t dma_map_failed; /* couldn't obtain DMA mapping */
346 	uint32_t qfull;		/* out of hardware descriptors */
347 	uint32_t qflush;	/* # of SGE_EGR_UPDATE notifications for txq */
348 	uint32_t pullup_early;	/* # of pullups before starting frame's SGL */
349 	uint32_t pullup_late;	/* # of pullups while building frame's SGL */
350 	uint32_t pullup_failed;	/* # of failed pullups */
351 };
352 
353 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
354 struct sge_rxq {
355 	struct sge_iq iq;	/* MUST be first */
356 	struct sge_fl fl;
357 
358 	struct port_info *port;	/* the port this rxq belongs to */
359 	kstat_t *ksp;
360 
361 	mac_ring_handle_t ring_handle;
362 	uint64_t ring_gen_num;
363 
364 	/* stats for common events first */
365 
366 	uint64_t rxcsum;	/* # of times hardware assisted with checksum */
367 	uint64_t rxpkts;	/* # of ethernet packets */
368 	uint64_t rxbytes;	/* # of ethernet bytes */
369 
370 	/* stats for not-that-common events */
371 
372 	uint32_t nomem;		/* mblk allocation during rx failed */
373 };
374 
375 #ifdef TCP_OFFLOAD_ENABLE
376 /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
377 struct sge_ofld_rxq {
378 	struct sge_iq iq;	/* MUST be first */
379 	struct sge_fl fl;
380 };
381 
382 /*
383  * wrq: SGE egress queue that is given prebuilt work requests.  Both the control
384  * and offload tx queues are of this type.
385  */
386 struct sge_wrq {
387 	struct sge_eq eq;	/* MUST be first */
388 
389 	struct adapter *adapter;
390 
391 	/* List of WRs held up due to lack of tx descriptors */
392 	struct mblk_pair wr_list;
393 
394 	/* stats for common events first */
395 
396 	uint64_t tx_wrs;	/* # of tx work requests */
397 
398 	/* stats for not-that-common events */
399 
400 	uint32_t no_desc;	/* out of hardware descriptors */
401 };
402 #endif
403 
404 struct sge {
405 	int fl_starve_threshold;
406 	int s_qpp;
407 
408 	int nrxq;	/* total rx queues (all ports and the rest) */
409 	int ntxq;	/* total tx queues (all ports and the rest) */
410 #ifdef TCP_OFFLOAD_ENABLE
411 	int nofldrxq;	/* total # of TOE rx queues */
412 	int nofldtxq;	/* total # of TOE tx queues */
413 #endif
414 	int niq;	/* total ingress queues */
415 	int neq;	/* total egress queues */
416 	int stat_len;	/* length of status page at ring end */
417 	int pktshift;	/* padding between CPL & packet data */
418 	int fl_align;	/* response queue message alignment */
419 
420 	struct sge_iq fwq;	/* Firmware event queue */
421 #ifdef TCP_OFFLOAD_ENABLE
422 	struct sge_wrq mgmtq;	/* Management queue (Control queue) */
423 #endif
424 	struct sge_txq *txq;	/* NIC tx queues */
425 	struct sge_rxq *rxq;	/* NIC rx queues */
426 #ifdef TCP_OFFLOAD_ENABLE
427 	struct sge_wrq *ctrlq;	/* Control queues */
428 	struct sge_wrq *ofld_txq;	/* TOE tx queues */
429 	struct sge_ofld_rxq *ofld_rxq;	/* TOE rx queues */
430 #endif
431 
432 	int iq_start; /* iq context id map start index */
433 	int eq_start; /* eq context id map start index */
434 	int iqmap_sz; /* size of iq context id map */
435 	int eqmap_sz; /* size of eq context id map */
436 	struct sge_iq **iqmap;	/* iq->cntxt_id to iq mapping */
437 	struct sge_eq **eqmap;	/* eq->cntxt_id to eq mapping */
438 
439 	/* Device access and DMA attributes for all the descriptor rings */
440 	ddi_device_acc_attr_t acc_attr_desc;
441 	ddi_dma_attr_t	dma_attr_desc;
442 
443 	/* Device access and DMA attributes for tx buffers */
444 	ddi_device_acc_attr_t acc_attr_tx;
445 	ddi_dma_attr_t	dma_attr_tx;
446 
447 	/* Device access and DMA attributes for rx buffers are in rxb_params */
448 	kmem_cache_t *rxbuf_cache;
449 	struct rxbuf_cache_params rxb_params;
450 };
451 
452 struct driver_properties {
453 	/* There is a driver.conf variable for each of these */
454 	int max_ntxq_10g;
455 	int max_nrxq_10g;
456 	int max_ntxq_1g;
457 	int max_nrxq_1g;
458 #ifdef TCP_OFFLOAD_ENABLE
459 	int max_nofldtxq_10g;
460 	int max_nofldrxq_10g;
461 	int max_nofldtxq_1g;
462 	int max_nofldrxq_1g;
463 #endif
464 	int intr_types;
465 	int tmr_idx_10g;
466 	int pktc_idx_10g;
467 	int tmr_idx_1g;
468 	int pktc_idx_1g;
469 	int qsize_txq;
470 	int qsize_rxq;
471 
472 	int timer_val[SGE_NTIMERS];
473 	int counter_val[SGE_NCOUNTERS];
474 
475 	int wc;
476 
477 	int multi_rings;
478 	int t4_fw_install;
479 };
480 
481 struct rss_header;
482 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
483     mblk_t *);
484 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
485 
486 struct t4_mbox_list {
487 	STAILQ_ENTRY(t4_mbox_list) link;
488 };
489 
490 struct adapter {
491 	SLIST_ENTRY(adapter) link;
492 	dev_info_t *dip;
493 	dev_t dev;
494 
495 	unsigned int pf;
496 	unsigned int mbox;
497 
498 	unsigned int vpd_busy;
499 	unsigned int vpd_flag;
500 
501 	u32 t4_bar0;
502 
503 	uint_t open;	/* character device is open */
504 
505 	/* PCI config space access handle */
506 	ddi_acc_handle_t pci_regh;
507 
508 	/* MMIO register access handle */
509 	ddi_acc_handle_t regh;
510 	caddr_t regp;
511 	/* BAR1 register access handle */
512 	ddi_acc_handle_t reg1h;
513 	caddr_t reg1p;
514 
515 	/* Interrupt information */
516 	int intr_type;
517 	int intr_count;
518 	int intr_cap;
519 	uint_t intr_pri;
520 	ddi_intr_handle_t *intr_handle;
521 
522 	struct driver_properties props;
523 	kstat_t *ksp;
524 	kstat_t *ksp_stat;
525 
526 	struct sge sge;
527 
528 	struct port_info *port[MAX_NPORTS];
529 	ddi_taskq_t *tq[NCHAN];
530 	uint8_t chan_map[NCHAN];
531 	uint32_t filter_mode;
532 
533 	struct l2t_data *l2t;	/* L2 table */
534 	struct tid_info tids;
535 
536 	int doorbells;
537 	int registered_device_map;
538 	int open_device_map;
539 	int flags;
540 
541 	unsigned int cfcsum;
542 	struct adapter_params params;
543 	struct t4_virt_res vres;
544 
545 #ifdef TCP_OFFLOAD_ENABLE
546 	struct uld_softc tom;
547 	struct tom_tunables tt;
548 #endif
549 
550 #ifdef TCP_OFFLOAD_ENABLE
551 	int offload_map;
552 #endif
553 	uint16_t linkcaps;
554 	uint16_t niccaps;
555 	uint16_t toecaps;
556 	uint16_t rdmacaps;
557 	uint16_t iscsicaps;
558 	uint16_t fcoecaps;
559 
560 	fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */
561 	cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */
562 
563 	kmutex_t lock;
564 	kcondvar_t cv;
565 
566 	/* Starving free lists */
567 	kmutex_t sfl_lock;	/* same cache-line as sc_lock? but that's ok */
568 	TAILQ_HEAD(, sge_fl) sfl;
569 	timeout_id_t sfl_timer;
570 
571 	/* Sensors */
572 	id_t temp_sensor;
573 	id_t volt_sensor;
574 
575 	ddi_ufm_handle_t *ufm_hdl;
576 
577 	/* support for single-threading access to adapter mailbox registers */
578 	kmutex_t mbox_lock;
579 	STAILQ_HEAD(, t4_mbox_list) mbox_list;
580 };
581 
582 enum {
583 	NIC_H = 0,
584 	TOM_H,
585 	IW_H,
586 	ISCSI_H
587 };
588 
589 struct memwin {
590 	uint32_t base;
591 	uint32_t aperture;
592 };
593 
594 #define	ADAPTER_LOCK(sc)		mutex_enter(&(sc)->lock)
595 #define	ADAPTER_UNLOCK(sc)		mutex_exit(&(sc)->lock)
596 #define	ADAPTER_LOCK_ASSERT_OWNED(sc)	ASSERT(mutex_owned(&(sc)->lock))
597 #define	ADAPTER_LOCK_ASSERT_NOTOWNED(sc) ASSERT(!mutex_owned(&(sc)->lock))
598 
599 #define	PORT_LOCK(pi)			mutex_enter(&(pi)->lock)
600 #define	PORT_UNLOCK(pi)			mutex_exit(&(pi)->lock)
601 #define	PORT_LOCK_ASSERT_OWNED(pi)	ASSERT(mutex_owned(&(pi)->lock))
602 #define	PORT_LOCK_ASSERT_NOTOWNED(pi)	ASSERT(!mutex_owned(&(pi)->lock))
603 
604 #define	IQ_LOCK(iq)			mutex_enter(&(iq)->lock)
605 #define	IQ_UNLOCK(iq)			mutex_exit(&(iq)->lock)
606 #define	IQ_LOCK_ASSERT_OWNED(iq)	ASSERT(mutex_owned(&(iq)->lock))
607 #define	IQ_LOCK_ASSERT_NOTOWNED(iq)	ASSERT(!mutex_owned(&(iq)->lock))
608 
609 #define	FL_LOCK(fl)			mutex_enter(&(fl)->lock)
610 #define	FL_UNLOCK(fl)			mutex_exit(&(fl)->lock)
611 #define	FL_LOCK_ASSERT_OWNED(fl)	ASSERT(mutex_owned(&(fl)->lock))
612 #define	FL_LOCK_ASSERT_NOTOWNED(fl)	ASSERT(!mutex_owned(&(fl)->lock))
613 
614 #define	RXQ_LOCK(rxq)			IQ_LOCK(&(rxq)->iq)
615 #define	RXQ_UNLOCK(rxq)			IQ_UNLOCK(&(rxq)->iq)
616 #define	RXQ_LOCK_ASSERT_OWNED(rxq)	IQ_LOCK_ASSERT_OWNED(&(rxq)->iq)
617 #define	RXQ_LOCK_ASSERT_NOTOWNED(rxq)	IQ_LOCK_ASSERT_NOTOWNED(&(rxq)->iq)
618 
619 #define	RXQ_FL_LOCK(rxq)		FL_LOCK(&(rxq)->fl)
620 #define	RXQ_FL_UNLOCK(rxq)		FL_UNLOCK(&(rxq)->fl)
621 #define	RXQ_FL_LOCK_ASSERT_OWNED(rxq)	FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
622 #define	RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
623 
624 #define	EQ_LOCK(eq)			mutex_enter(&(eq)->lock)
625 #define	EQ_UNLOCK(eq)			mutex_exit(&(eq)->lock)
626 #define	EQ_LOCK_ASSERT_OWNED(eq)	ASSERT(mutex_owned(&(eq)->lock))
627 #define	EQ_LOCK_ASSERT_NOTOWNED(eq)	ASSERT(!mutex_owned(&(eq)->lock))
628 
629 #define	TXQ_LOCK(txq)			EQ_LOCK(&(txq)->eq)
630 #define	TXQ_UNLOCK(txq)			EQ_UNLOCK(&(txq)->eq)
631 #define	TXQ_LOCK_ASSERT_OWNED(txq)	EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
632 #define	TXQ_LOCK_ASSERT_NOTOWNED(txq)	EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
633 
634 #define	for_each_txq(pi, iter, txq) \
635 	txq = &pi->adapter->sge.txq[pi->first_txq]; \
636 	for (iter = 0; iter < pi->ntxq; ++iter, ++txq)
637 #define	for_each_rxq(pi, iter, rxq) \
638 	rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \
639 	for (iter = 0; iter < pi->nrxq; ++iter, ++rxq)
640 #define	for_each_ofld_txq(pi, iter, ofld_txq) \
641 	ofld_txq = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq]; \
642 	for (iter = 0; iter < pi->nofldtxq; ++iter, ++ofld_txq)
643 #define	for_each_ofld_rxq(pi, iter, ofld_rxq) \
644 	ofld_rxq = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq]; \
645 	for (iter = 0; iter < pi->nofldrxq; ++iter, ++ofld_rxq)
646 
647 #define	NFIQ(sc) ((sc)->intr_count > 1 ? (sc)->intr_count - 1 : 1)
648 
649 /* One for errors, one for firmware events */
650 #define	T4_EXTRA_INTR 2
651 
652 typedef kmutex_t t4_os_lock_t;
653 
t4_os_lock(t4_os_lock_t * lock)654 static inline void t4_os_lock(t4_os_lock_t *lock)
655 {
656 	mutex_enter(lock);
657 }
658 
t4_os_unlock(t4_os_lock_t * lock)659 static inline void t4_os_unlock(t4_os_lock_t *lock)
660 {
661 	mutex_exit(lock);
662 }
663 
t4_mbox_list_add(struct adapter * adap,struct t4_mbox_list * entry)664 static inline void t4_mbox_list_add(struct adapter *adap,
665 				    struct t4_mbox_list *entry)
666 {
667 	t4_os_lock(&adap->mbox_lock);
668 	STAILQ_INSERT_TAIL(&adap->mbox_list, entry, link);
669 	t4_os_unlock(&adap->mbox_lock);
670 }
671 
t4_mbox_list_del(struct adapter * adap,struct t4_mbox_list * entry)672 static inline void t4_mbox_list_del(struct adapter *adap,
673 				    struct t4_mbox_list *entry)
674 {
675 	t4_os_lock(&adap->mbox_lock);
676 	STAILQ_REMOVE(&adap->mbox_list, entry, t4_mbox_list, link);
677 	t4_os_unlock(&adap->mbox_lock);
678 }
679 
680 static inline struct t4_mbox_list *
t4_mbox_list_first_entry(struct adapter * adap)681 t4_mbox_list_first_entry(struct adapter *adap)
682 {
683 	return STAILQ_FIRST(&adap->mbox_list);
684 }
685 
686 static inline uint32_t
t4_read_reg(struct adapter * sc,uint32_t reg)687 t4_read_reg(struct adapter *sc, uint32_t reg)
688 {
689 	/* LINTED: E_BAD_PTR_CAST_ALIGN */
690 	return (ddi_get32(sc->regh, (uint32_t *)(sc->regp + reg)));
691 }
692 
693 static inline void
t4_write_reg(struct adapter * sc,uint32_t reg,uint32_t val)694 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
695 {
696 	/* LINTED: E_BAD_PTR_CAST_ALIGN */
697 	ddi_put32(sc->regh, (uint32_t *)(sc->regp + reg), val);
698 }
699 
700 static inline void
t4_os_pci_read_cfg1(struct adapter * sc,int reg,uint8_t * val)701 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
702 {
703 	*val = pci_config_get8(sc->pci_regh, reg);
704 }
705 
706 static inline void
t4_os_pci_write_cfg1(struct adapter * sc,int reg,uint8_t val)707 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
708 {
709 	pci_config_put8(sc->pci_regh, reg, val);
710 }
711 
712 static inline void
t4_os_pci_read_cfg2(struct adapter * sc,int reg,uint16_t * val)713 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
714 {
715 	*val = pci_config_get16(sc->pci_regh, reg);
716 }
717 
718 static inline void
t4_os_pci_write_cfg2(struct adapter * sc,int reg,uint16_t val)719 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
720 {
721 	pci_config_put16(sc->pci_regh, reg, val);
722 }
723 
724 static inline void
t4_os_pci_read_cfg4(struct adapter * sc,int reg,uint32_t * val)725 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
726 {
727 	*val = pci_config_get32(sc->pci_regh, reg);
728 }
729 
730 static inline void
t4_os_pci_write_cfg4(struct adapter * sc,int reg,uint32_t val)731 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
732 {
733 	pci_config_put32(sc->pci_regh, reg, val);
734 }
735 
736 static inline uint32_t
t4_read_reg32(struct adapter * sc,uint32_t reg)737 t4_read_reg32(struct adapter *sc, uint32_t reg)
738 {
739 	return (ddi_get32(sc->regh, (uint32_t *)(sc->regp + reg)));
740 }
741 
742 static inline uint64_t
t4_read_reg64(struct adapter * sc,uint32_t reg)743 t4_read_reg64(struct adapter *sc, uint32_t reg)
744 {
745 	return (ddi_get64(sc->regh, (uint64_t *)(sc->regp + reg)));
746 }
747 
748 static inline void
t4_write_reg64(struct adapter * sc,uint32_t reg,uint64_t val)749 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
750 {
751 	ddi_put64(sc->regh, (uint64_t *)(sc->regp + reg), val);
752 }
753 
754 static inline struct port_info *
adap2pinfo(struct adapter * sc,int idx)755 adap2pinfo(struct adapter *sc, int idx)
756 {
757 	return (sc->port[idx]);
758 }
759 
760 static inline void
t4_os_set_hw_addr(struct adapter * sc,int idx,uint8_t hw_addr[])761 t4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[])
762 {
763 	bcopy(hw_addr, sc->port[idx]->hw_addr, ETHERADDRL);
764 }
765 
766 static inline bool
is_10G_port(const struct port_info * pi)767 is_10G_port(const struct port_info *pi)
768 {
769 	return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) != 0);
770 }
771 
772 static inline struct sge_rxq *
iq_to_rxq(struct sge_iq * iq)773 iq_to_rxq(struct sge_iq *iq)
774 {
775 	return (__containerof(iq, struct sge_rxq, iq));
776 }
777 
778 static inline bool
is_25G_port(const struct port_info * pi)779 is_25G_port(const struct port_info *pi)
780 {
781 	return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) != 0);
782 }
783 
784 static inline bool
is_40G_port(const struct port_info * pi)785 is_40G_port(const struct port_info *pi)
786 {
787 	return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) != 0);
788 }
789 
790 static inline bool
is_50G_port(const struct port_info * pi)791 is_50G_port(const struct port_info *pi)
792 {
793 	return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) != 0);
794 }
795 
796 static inline bool
is_100G_port(const struct port_info * pi)797 is_100G_port(const struct port_info *pi)
798 {
799 	return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) != 0);
800 }
801 
802 static inline bool
is_10XG_port(const struct port_info * pi)803 is_10XG_port(const struct port_info *pi)
804 {
805 	return (is_10G_port(pi) || is_40G_port(pi) ||
806 		is_25G_port(pi) || is_50G_port(pi) ||
807 		is_100G_port(pi));
808 }
809 
810 #ifdef TCP_OFFLOAD_ENABLE
811 int t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0);
812 
813 static inline int
t4_wrq_tx(struct adapter * sc,struct sge_wrq * wrq,mblk_t * m)814 t4_wrq_tx(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m)
815 {
816 	int rc;
817 
818 	TXQ_LOCK(wrq);
819 	rc = t4_wrq_tx_locked(sc, wrq, m);
820 	TXQ_UNLOCK(wrq);
821 	return (rc);
822 }
823 #endif
824 
825 /**
826  * t4_os_pci_read_seeprom - read four bytes of SEEPROM/VPD contents
827  * @adapter: the adapter
828  * @addr: SEEPROM/VPD Address to read
829  * @valp: where to store the value read
830  *
831  * Read a 32-bit value from the given address in the SEEPROM/VPD.  The address
832  * must be four-byte aligned.  Returns 0 on success, a negative erro number
833  * on failure.
834  */
t4_os_pci_read_seeprom(adapter_t * adapter,int addr,u32 * valp)835 static inline int t4_os_pci_read_seeprom(adapter_t *adapter,
836 					 int addr, u32 *valp)
837 {
838 	int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data);
839 	int ret;
840 
841 	ret = t4_seeprom_read(adapter, addr, valp);
842 
843 	return ret >= 0 ? 0 : ret;
844 }
845 
846 /**
847  * t4_os_pci_write_seeprom - write four bytes of SEEPROM/VPD contents
848  * @adapter: the adapter
849  * @addr: SEEPROM/VPD Address to write
850  * @val: the value write
851  *
852  * Write a 32-bit value to the given address in the SEEPROM/VPD.  The address
853  * must be four-byte aligned.  Returns 0 on success, a negative erro number
854  * on failure.
855  */
t4_os_pci_write_seeprom(adapter_t * adapter,int addr,u32 val)856 static inline int t4_os_pci_write_seeprom(adapter_t *adapter,
857 					  int addr, u32 val)
858 {
859 	int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data);
860 	int ret;
861 
862 	ret = t4_seeprom_write(adapter, addr, val);
863 
864 	return ret >= 0 ? 0 : ret;
865 }
866 
t4_os_pci_set_vpd_size(struct adapter * adapter,size_t len)867 static inline int t4_os_pci_set_vpd_size(struct adapter *adapter, size_t len)
868 {
869 	return 0;
870 }
871 
t4_use_ldst(struct adapter * adap)872 static inline unsigned int t4_use_ldst(struct adapter *adap)
873 {
874 	return (adap->flags & FW_OK);
875 }
876 #define t4_os_alloc(_size)	kmem_alloc(_size, KM_SLEEP)
877 
t4_db_full(struct adapter * adap)878 static inline void t4_db_full(struct adapter *adap) {}
t4_db_dropped(struct adapter * adap)879 static inline void t4_db_dropped(struct adapter *adap) {}
880 
881 /* t4_nexus.c */
882 int t4_os_find_pci_capability(struct adapter *sc, int cap);
883 void t4_os_portmod_changed(struct adapter *sc, int idx);
884 int adapter_full_init(struct adapter *sc);
885 int adapter_full_uninit(struct adapter *sc);
886 int port_full_init(struct port_info *pi);
887 int port_full_uninit(struct port_info *pi);
888 void enable_port_queues(struct port_info *pi);
889 void disable_port_queues(struct port_info *pi);
890 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
891 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
892 void t4_iterate(void (*func)(int, void *), void *arg);
893 
894 /* t4_sge.c */
895 void t4_sge_init(struct adapter *sc);
896 int t4_setup_adapter_queues(struct adapter *sc);
897 int t4_teardown_adapter_queues(struct adapter *sc);
898 int t4_setup_port_queues(struct port_info *pi);
899 int t4_teardown_port_queues(struct port_info *pi);
900 uint_t t4_intr_all(caddr_t arg1, caddr_t arg2);
901 uint_t t4_intr(caddr_t arg1, caddr_t arg2);
902 uint_t t4_intr_err(caddr_t arg1, caddr_t arg2);
903 int t4_mgmt_tx(struct adapter *sc, mblk_t *m);
904 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
905 uint32_t position_memwin(struct adapter *, int, uint32_t);
906 
907 mblk_t *t4_eth_tx(void *, mblk_t *);
908 mblk_t *t4_mc_tx(void *arg, mblk_t *m);
909 mblk_t *t4_ring_rx(struct sge_rxq *rxq, int poll_bytes);
910 int t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps,  int count,
911     int flags);
912 
913 /* t4_mac.c */
914 void t4_mc_init(struct port_info *pi);
915 void t4_mc_cb_init(struct port_info *);
916 void t4_os_link_changed(struct adapter *sc, int idx, int link_stat);
917 void t4_mac_rx(struct port_info *pi, struct sge_rxq *rxq, mblk_t *m);
918 void t4_mac_tx_update(struct port_info *pi, struct sge_txq *txq);
919 int t4_addmac(void *arg, const uint8_t *ucaddr);
920 
921 /* t4_ioctl.c */
922 int t4_ioctl(struct adapter *sc, int cmd, void *data, int mode);
923 
924 struct l2t_data *t4_init_l2t(struct adapter *sc);
925 int begin_synchronized_op(struct port_info *pi, int hold, int waitok);
926 void end_synchronized_op(struct port_info *pi, int held);
927 #endif /* __CXGBE_ADAPTER_H */
928