xref: /illumos-gate/usr/src/uts/common/io/cxgbe/t4nex/t4_sge.c (revision 4e0c5eff9af325c80994e9527b7cb8b3a1ffd1d4)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source. A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * This file is part of the Chelsio T4 support code.
14  *
15  * Copyright (C) 2010-2013 Chelsio Communications.  All rights reserved.
16  *
17  * This program is distributed in the hope that it will be useful, but WITHOUT
18  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
20  * release for licensing terms and conditions.
21  */
22 
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/atomic.h>
27 #include <sys/dlpi.h>
28 #include <sys/pattr.h>
29 #include <sys/strsubr.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <inet/ip.h>
33 #include <inet/tcp.h>
34 
35 #include "version.h"
36 #include "common/common.h"
37 #include "common/t4_msg.h"
38 #include "common/t4_regs.h"
39 #include "common/t4_regs_values.h"
40 
41 /* TODO: Tune. */
42 int rx_buf_size = 8192;
43 int tx_copy_threshold = 256;
44 uint16_t rx_copy_threshold = 256;
45 
46 /* Used to track coalesced tx work request */
47 struct txpkts {
48 	mblk_t *tail;		/* head is in the software descriptor */
49 	uint64_t *flitp;	/* ptr to flit where next pkt should start */
50 	uint8_t npkt;		/* # of packets in this work request */
51 	uint8_t nflits;		/* # of flits used by this work request */
52 	uint16_t plen;		/* total payload (sum of all packets) */
53 };
54 
55 /* All information needed to tx a frame */
56 struct txinfo {
57 	uint32_t len;		/* Total length of frame */
58 	uint32_t flags;		/* Checksum and LSO flags */
59 	uint32_t mss;		/* MSS for LSO */
60 	uint8_t nsegs;		/* # of segments in the SGL, 0 means imm. tx */
61 	uint8_t nflits;		/* # of flits needed for the SGL */
62 	uint8_t hdls_used;	/* # of DMA handles used */
63 	uint32_t txb_used;	/* txb_space used */
64 	struct ulptx_sgl sgl __attribute__((aligned(8)));
65 	struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
66 };
67 
68 static int service_iq(struct sge_iq *iq, int budget);
69 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
70     int8_t pktc_idx, int qsize, uint8_t esize);
71 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
72 static inline void init_eq(struct adapter *sc, struct sge_eq *eq,
73     uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid);
74 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
75     struct sge_fl *fl, int intr_idx, int cong);
76 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
77     struct sge_fl *fl);
78 static int alloc_fwq(struct adapter *sc);
79 static int free_fwq(struct adapter *sc);
80 #ifdef TCP_OFFLOAD_ENABLE
81 static int alloc_mgmtq(struct adapter *sc);
82 #endif
83 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
84     int i);
85 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
86 #ifdef TCP_OFFLOAD_ENABLE
87 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
88 	int intr_idx);
89 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq);
90 #endif
91 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq);
92 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
93     struct sge_eq *eq);
94 #ifdef TCP_OFFLOAD_ENABLE
95 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi,
96     struct sge_eq *eq);
97 #endif
98 static int alloc_eq(struct adapter *sc, struct port_info *pi,
99     struct sge_eq *eq);
100 static int free_eq(struct adapter *sc, struct sge_eq *eq);
101 #ifdef TCP_OFFLOAD_ENABLE
102 static int alloc_wrq(struct adapter *sc, struct port_info *pi,
103     struct sge_wrq *wrq, int idx);
104 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq);
105 #endif
106 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
107 static int free_txq(struct port_info *pi, struct sge_txq *txq);
108 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
109     ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
110     ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
111     caddr_t *pva);
112 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
113 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
114     ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
115     caddr_t *pva);
116 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
117 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
118     ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
119     caddr_t *pva);
120 static inline bool is_new_response(const struct sge_iq *iq,
121     struct rsp_ctrl **ctrl);
122 static inline void iq_next(struct sge_iq *iq);
123 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
124 static void refill_sfl(void *arg);
125 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
126 static void free_fl_bufs(struct sge_fl *fl);
127 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl,
128     uint32_t len_newbuf, int *fl_bufs_used);
129 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
130     struct txinfo *txinfo, int sgl_only);
131 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
132 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
133     struct txinfo *txinfo);
134 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
135 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
136     mblk_t *m, int len);
137 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
138 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
139     struct txinfo *txinfo);
140 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
141 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
142     struct txinfo *txinfo);
143 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
144     struct txpkts *txpkts, struct txinfo *txinfo);
145 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
146     int len);
147 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
148 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
149 static void write_txqflush_wr(struct sge_txq *txq);
150 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
151     mblk_t *m);
152 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
153 static kstat_t *setup_port_config_kstats(struct port_info *pi);
154 static kstat_t *setup_port_info_kstats(struct port_info *pi);
155 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
156     int idx);
157 static int update_rxq_kstats(kstat_t *ksp, int rw);
158 static int update_port_info_kstats(kstat_t *ksp, int rw);
159 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
160     int idx);
161 static int update_txq_kstats(kstat_t *ksp, int rw);
162 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
163     mblk_t *);
164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss,
165     mblk_t *m);
166 
167 static inline int
168 reclaimable(struct sge_eq *eq)
169 {
170 	unsigned int cidx;
171 
172 	cidx = eq->spg->cidx;   /* stable snapshot */
173 	cidx = be16_to_cpu(cidx);
174 
175 	if (cidx >= eq->cidx)
176 		return (cidx - eq->cidx);
177 	else
178 		return (cidx + eq->cap - eq->cidx);
179 }
180 
181 void
182 t4_sge_init(struct adapter *sc)
183 {
184 	struct driver_properties *p = &sc->props;
185 	ddi_dma_attr_t *dma_attr;
186 	ddi_device_acc_attr_t *acc_attr;
187 	uint32_t sge_control, sge_conm_ctrl;
188 	int egress_threshold;
189 
190 	/*
191 	 * Device access and DMA attributes for descriptor rings
192 	 */
193 	acc_attr = &sc->sge.acc_attr_desc;
194 	acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
195 	acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
196 	acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
197 
198 	dma_attr = &sc->sge.dma_attr_desc;
199 	dma_attr->dma_attr_version = DMA_ATTR_V0;
200 	dma_attr->dma_attr_addr_lo = 0;
201 	dma_attr->dma_attr_addr_hi = UINT64_MAX;
202 	dma_attr->dma_attr_count_max = UINT64_MAX;
203 	dma_attr->dma_attr_align = 512;
204 	dma_attr->dma_attr_burstsizes = 0xfff;
205 	dma_attr->dma_attr_minxfer = 1;
206 	dma_attr->dma_attr_maxxfer = UINT64_MAX;
207 	dma_attr->dma_attr_seg = UINT64_MAX;
208 	dma_attr->dma_attr_sgllen = 1;
209 	dma_attr->dma_attr_granular = 1;
210 	dma_attr->dma_attr_flags = 0;
211 
212 	/*
213 	 * Device access and DMA attributes for tx buffers
214 	 */
215 	acc_attr = &sc->sge.acc_attr_tx;
216 	acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
217 	acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
218 
219 	dma_attr = &sc->sge.dma_attr_tx;
220 	dma_attr->dma_attr_version = DMA_ATTR_V0;
221 	dma_attr->dma_attr_addr_lo = 0;
222 	dma_attr->dma_attr_addr_hi = UINT64_MAX;
223 	dma_attr->dma_attr_count_max = UINT64_MAX;
224 	dma_attr->dma_attr_align = 1;
225 	dma_attr->dma_attr_burstsizes = 0xfff;
226 	dma_attr->dma_attr_minxfer = 1;
227 	dma_attr->dma_attr_maxxfer = UINT64_MAX;
228 	dma_attr->dma_attr_seg = UINT64_MAX;
229 	dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
230 	dma_attr->dma_attr_granular = 1;
231 	dma_attr->dma_attr_flags = 0;
232 
233 	/*
234 	 * Ingress Padding Boundary and Egress Status Page Size are set up by
235 	 * t4_fixup_host_params().
236 	 */
237 	sge_control = t4_read_reg(sc, A_SGE_CONTROL);
238 	sc->sge.pktshift = G_PKTSHIFT(sge_control);
239 	sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
240 
241 	/* t4_nex uses FLM packed mode */
242 	sc->sge.fl_align = t4_fl_pkt_align(sc, true);
243 
244 	/*
245 	 * Device access and DMA attributes for rx buffers
246 	 */
247 	sc->sge.rxb_params.dip = sc->dip;
248 	sc->sge.rxb_params.buf_size = rx_buf_size;
249 
250 	acc_attr = &sc->sge.rxb_params.acc_attr_rx;
251 	acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
252 	acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
253 
254 	dma_attr = &sc->sge.rxb_params.dma_attr_rx;
255 	dma_attr->dma_attr_version = DMA_ATTR_V0;
256 	dma_attr->dma_attr_addr_lo = 0;
257 	dma_attr->dma_attr_addr_hi = UINT64_MAX;
258 	dma_attr->dma_attr_count_max = UINT64_MAX;
259 	/*
260 	 * Low 4 bits of an rx buffer address have a special meaning to the SGE
261 	 * and an rx buf cannot have an address with any of these bits set.
262 	 * FL_ALIGN is >= 32 so we're sure things are ok.
263 	 */
264 	dma_attr->dma_attr_align = sc->sge.fl_align;
265 	dma_attr->dma_attr_burstsizes = 0xfff;
266 	dma_attr->dma_attr_minxfer = 1;
267 	dma_attr->dma_attr_maxxfer = UINT64_MAX;
268 	dma_attr->dma_attr_seg = UINT64_MAX;
269 	dma_attr->dma_attr_sgllen = 1;
270 	dma_attr->dma_attr_granular = 1;
271 	dma_attr->dma_attr_flags = 0;
272 
273 	sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
274 
275 	/*
276 	 * A FL with <= fl_starve_thres buffers is starving and a periodic
277 	 * timer will attempt to refill it.  This needs to be larger than the
278 	 * SGE's Egress Congestion Threshold.  If it isn't, then we can get
279 	 * stuck waiting for new packets while the SGE is waiting for us to
280 	 * give it more Free List entries.  (Note that the SGE's Egress
281 	 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
282 	 * there was only a single field to control this.  For T5 there's the
283 	 * original field which now only applies to Unpacked Mode Free List
284 	 * buffers and a new field which only applies to Packed Mode Free List
285 	 * buffers.
286 	 */
287 
288 	sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL);
289 	switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
290 	case CHELSIO_T4:
291 		egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
292 		break;
293 	case CHELSIO_T5:
294 		egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
295 		break;
296 	case CHELSIO_T6:
297 	default:
298 		egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl);
299 	}
300 	sc->sge.fl_starve_threshold = 2*egress_threshold + 1;
301 
302 	t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
303 
304 	t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
305 	    V_THRESHOLD_0(p->counter_val[0]) |
306 	    V_THRESHOLD_1(p->counter_val[1]) |
307 	    V_THRESHOLD_2(p->counter_val[2]) |
308 	    V_THRESHOLD_3(p->counter_val[3]));
309 
310 	t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
311 	    V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
312 	    V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
313 	t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
314 	    V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
315 	    V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
316 	t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
317 	    V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
318 	    V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
319 
320 	(void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
321 	(void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
322 	(void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
323 	(void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
324 	(void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL,
325 		    t4_handle_fw_rpl);
326 }
327 
328 /*
329  * Allocate and initialize the firmware event queue and the forwarded interrupt
330  * queues, if any.  The adapter owns all these queues as they are not associated
331  * with any particular port.
332  *
333  * Returns errno on failure.  Resources allocated up to that point may still be
334  * allocated.  Caller is responsible for cleanup in case this function fails.
335  */
336 int
337 t4_setup_adapter_queues(struct adapter *sc)
338 {
339 	int rc;
340 
341 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
342 
343 	/*
344 	 * Firmware event queue
345 	 */
346 	rc = alloc_fwq(sc);
347 	if (rc != 0)
348 		return (rc);
349 
350 #ifdef TCP_OFFLOAD_ENABLE
351 	/*
352 	 * Management queue.  This is just a control queue that uses the fwq as
353 	 * its associated iq.
354 	 */
355 	rc = alloc_mgmtq(sc);
356 #endif
357 
358 	return (rc);
359 }
360 
361 /*
362  * Idempotent
363  */
364 int
365 t4_teardown_adapter_queues(struct adapter *sc)
366 {
367 
368 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
369 
370 	(void) free_fwq(sc);
371 
372 	return (0);
373 }
374 
375 static inline int
376 first_vector(struct port_info *pi)
377 {
378 	struct adapter *sc = pi->adapter;
379 	int rc = T4_EXTRA_INTR, i;
380 
381 	if (sc->intr_count == 1)
382 		return (0);
383 
384 	for_each_port(sc, i) {
385 		struct port_info *p = sc->port[i];
386 
387 		if (i == pi->port_id)
388 			break;
389 
390 #ifdef TCP_OFFLOAD_ENABLE
391 		if (!(sc->flags & INTR_FWD))
392 			rc += p->nrxq + p->nofldrxq;
393 		else
394 			rc += max(p->nrxq, p->nofldrxq);
395 #else
396 		/*
397 		 * Not compiled with offload support and intr_count > 1.  Only
398 		 * NIC queues exist and they'd better be taking direct
399 		 * interrupts.
400 		 */
401 		ASSERT(!(sc->flags & INTR_FWD));
402 		rc += p->nrxq;
403 #endif
404 	}
405 	return (rc);
406 }
407 
408 /*
409  * Given an arbitrary "index," come up with an iq that can be used by other
410  * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
411  * The iq returned is guaranteed to be something that takes direct interrupts.
412  */
413 static struct sge_iq *
414 port_intr_iq(struct port_info *pi, int idx)
415 {
416 	struct adapter *sc = pi->adapter;
417 	struct sge *s = &sc->sge;
418 	struct sge_iq *iq = NULL;
419 
420 	if (sc->intr_count == 1)
421 		return (&sc->sge.fwq);
422 
423 #ifdef TCP_OFFLOAD_ENABLE
424 	if (!(sc->flags & INTR_FWD)) {
425 		idx %= pi->nrxq + pi->nofldrxq;
426 
427 		if (idx >= pi->nrxq) {
428 			idx -= pi->nrxq;
429 			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
430 		} else
431 			iq = &s->rxq[pi->first_rxq + idx].iq;
432 
433 	} else {
434 		idx %= max(pi->nrxq, pi->nofldrxq);
435 
436 		if (pi->nrxq >= pi->nofldrxq)
437 			iq = &s->rxq[pi->first_rxq + idx].iq;
438 		else
439 			iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
440 	}
441 #else
442 	/*
443 	 * Not compiled with offload support and intr_count > 1.  Only NIC
444 	 * queues exist and they'd better be taking direct interrupts.
445 	 */
446 	ASSERT(!(sc->flags & INTR_FWD));
447 
448 	idx %= pi->nrxq;
449 	iq = &s->rxq[pi->first_rxq + idx].iq;
450 #endif
451 
452 	return (iq);
453 }
454 
455 int
456 t4_setup_port_queues(struct port_info *pi)
457 {
458 	int rc = 0, i, intr_idx, j;
459 	struct sge_rxq *rxq;
460 	struct sge_txq *txq;
461 #ifdef TCP_OFFLOAD_ENABLE
462 	int iqid;
463 	struct sge_wrq *ctrlq;
464 	struct sge_ofld_rxq *ofld_rxq;
465 	struct sge_wrq *ofld_txq;
466 #endif
467 	struct adapter *sc = pi->adapter;
468 	struct driver_properties *p = &sc->props;
469 
470 	pi->ksp_config = setup_port_config_kstats(pi);
471 	pi->ksp_info   = setup_port_info_kstats(pi);
472 
473 	/* Interrupt vector to start from (when using multiple vectors) */
474 	intr_idx = first_vector(pi);
475 
476 	/*
477 	 * First pass over all rx queues (NIC and TOE):
478 	 * a) initialize iq and fl
479 	 * b) allocate queue iff it will take direct interrupts.
480 	 */
481 
482 	for_each_rxq(pi, i, rxq) {
483 
484 		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
485 		    RX_IQ_ESIZE);
486 
487 		init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
488 
489 		if ((!(sc->flags & INTR_FWD))
490 #ifdef TCP_OFFLOAD_ENABLE
491 		    || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
492 #else
493 		    || (sc->intr_count > 1 && pi->nrxq)
494 #endif
495 		   ) {
496 			rxq->iq.flags |= IQ_INTR;
497 			rc = alloc_rxq(pi, rxq, intr_idx, i);
498 			if (rc != 0)
499 				goto done;
500 			intr_idx++;
501 		}
502 
503 	}
504 
505 #ifdef TCP_OFFLOAD_ENABLE
506 	for_each_ofld_rxq(pi, i, ofld_rxq) {
507 
508 		init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
509 		    p->qsize_rxq, RX_IQ_ESIZE);
510 
511 		init_fl(&ofld_rxq->fl, p->qsize_rxq / 8);
512 
513 		if (!(sc->flags & INTR_FWD) ||
514 		    (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
515 			ofld_rxq->iq.flags = IQ_INTR;
516 			rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
517 			if (rc != 0)
518 				goto done;
519 
520 			intr_idx++;
521 		}
522 	}
523 #endif
524 
525 	/*
526 	 * Second pass over all rx queues (NIC and TOE).  The queues forwarding
527 	 * their interrupts are allocated now.
528 	 */
529 	j = 0;
530 	for_each_rxq(pi, i, rxq) {
531 		if (rxq->iq.flags & IQ_INTR)
532 			continue;
533 
534 		intr_idx = port_intr_iq(pi, j)->abs_id;
535 
536 		rc = alloc_rxq(pi, rxq, intr_idx, i);
537 		if (rc != 0)
538 			goto done;
539 		j++;
540 	}
541 
542 #ifdef TCP_OFFLOAD_ENABLE
543 	for_each_ofld_rxq(pi, i, ofld_rxq) {
544 		if (ofld_rxq->iq.flags & IQ_INTR)
545 			continue;
546 
547 		intr_idx = port_intr_iq(pi, j)->abs_id;
548 		rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
549 		if (rc != 0)
550 			goto done;
551 		j++;
552 	}
553 #endif
554 	/*
555 	 * Now the tx queues.  Only one pass needed.
556 	 */
557 	j = 0;
558 	for_each_txq(pi, i, txq) {
559 		uint16_t iqid;
560 
561 		iqid = port_intr_iq(pi, j)->cntxt_id;
562 		init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid);
563 		rc = alloc_txq(pi, txq, i);
564 		if (rc != 0)
565 			goto done;
566 	}
567 
568 #ifdef TCP_OFFLOAD_ENABLE
569 	for_each_ofld_txq(pi, i, ofld_txq) {
570 		uint16_t iqid;
571 
572 		iqid = port_intr_iq(pi, j)->cntxt_id;
573 		init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan,
574 		    iqid);
575 		rc = alloc_wrq(sc, pi, ofld_txq, i);
576 		if (rc != 0)
577 			goto done;
578 	}
579 
580 	/*
581 	 * Finally, the control queue.
582 	 */
583 	ctrlq = &sc->sge.ctrlq[pi->port_id];
584 	iqid = port_intr_iq(pi, 0)->cntxt_id;
585 	init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid);
586 	rc = alloc_wrq(sc, pi, ctrlq, 0);
587 #endif
588 
589 done:
590 	if (rc != 0)
591 		(void) t4_teardown_port_queues(pi);
592 
593 	return (rc);
594 }
595 
596 /*
597  * Idempotent
598  */
599 int
600 t4_teardown_port_queues(struct port_info *pi)
601 {
602 	int i;
603 	struct sge_rxq *rxq;
604 	struct sge_txq *txq;
605 #ifdef TCP_OFFLOAD_ENABLE
606 	struct adapter *sc = pi->adapter;
607 	struct sge_ofld_rxq *ofld_rxq;
608 	struct sge_wrq *ofld_txq;
609 #endif
610 
611 	if (pi->ksp_config != NULL) {
612 		kstat_delete(pi->ksp_config);
613 		pi->ksp_config = NULL;
614 	}
615 	if (pi->ksp_info != NULL) {
616 		kstat_delete(pi->ksp_info);
617 		pi->ksp_info = NULL;
618 	}
619 
620 #ifdef TCP_OFFLOAD_ENABLE
621 	(void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
622 #endif
623 
624 	for_each_txq(pi, i, txq) {
625 		(void) free_txq(pi, txq);
626 	}
627 
628 #ifdef TCP_OFFLOAD_ENABLE
629 	for_each_ofld_txq(pi, i, ofld_txq) {
630 		(void) free_wrq(sc, ofld_txq);
631 	}
632 
633 	for_each_ofld_rxq(pi, i, ofld_rxq) {
634 		if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
635 			(void) free_ofld_rxq(pi, ofld_rxq);
636 	}
637 #endif
638 
639 	for_each_rxq(pi, i, rxq) {
640 		if ((rxq->iq.flags & IQ_INTR) == 0)
641 			(void) free_rxq(pi, rxq);
642 	}
643 
644 	/*
645 	 * Then take down the rx queues that take direct interrupts.
646 	 */
647 
648 	for_each_rxq(pi, i, rxq) {
649 		if (rxq->iq.flags & IQ_INTR)
650 			(void) free_rxq(pi, rxq);
651 	}
652 
653 #ifdef TCP_OFFLOAD_ENABLE
654 	for_each_ofld_rxq(pi, i, ofld_rxq) {
655 		if (ofld_rxq->iq.flags & IQ_INTR)
656 			(void) free_ofld_rxq(pi, ofld_rxq);
657 	}
658 #endif
659 
660 	return (0);
661 }
662 
663 /* Deals with errors and forwarded interrupts */
664 uint_t
665 t4_intr_all(caddr_t arg1, caddr_t arg2)
666 {
667 
668 	(void) t4_intr_err(arg1, arg2);
669 	(void) t4_intr(arg1, arg2);
670 
671 	return (DDI_INTR_CLAIMED);
672 }
673 
674 static void
675 t4_intr_rx_work(struct sge_iq *iq)
676 {
677 	mblk_t *mp = NULL;
678 	struct sge_rxq *rxq = iq_to_rxq(iq);	/* Use iff iq is part of rxq */
679 	RXQ_LOCK(rxq);
680 	if (!iq->polling) {
681 		mp = t4_ring_rx(rxq, iq->qsize/8);
682 		t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
683 		     V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
684 	}
685 	RXQ_UNLOCK(rxq);
686 	if (mp != NULL)
687 		mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
688 			    rxq->ring_gen_num);
689 }
690 
691 /* Deals with interrupts on the given ingress queue */
692 /* ARGSUSED */
693 uint_t
694 t4_intr(caddr_t arg1, caddr_t arg2)
695 {
696 	struct sge_iq *iq = (struct sge_iq *)arg2;
697 	int state;
698 
699 	/* Right now receive polling is only enabled for MSI-X and
700 	 * when we have enough msi-x vectors i.e no interrupt forwarding.
701 	 */
702 	if (iq->adapter->props.multi_rings) {
703 		t4_intr_rx_work(iq);
704 	} else {
705 		state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
706 		if (state == IQS_IDLE) {
707 			(void) service_iq(iq, 0);
708 			(void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
709 		}
710 	}
711 	return (DDI_INTR_CLAIMED);
712 }
713 
714 /* Deals with error interrupts */
715 /* ARGSUSED */
716 uint_t
717 t4_intr_err(caddr_t arg1, caddr_t arg2)
718 {
719 	/* LINTED: E_BAD_PTR_CAST_ALIGN */
720 	struct adapter *sc = (struct adapter *)arg1;
721 
722 	t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
723 	(void) t4_slow_intr_handler(sc);
724 
725 	return (DDI_INTR_CLAIMED);
726 }
727 
728 /*
729  * t4_ring_rx - Process responses from an SGE response queue.
730  *
731  * This function processes responses from an SGE response queue up to the supplied budget.
732  * Responses include received packets as well as control messages from FW
733  * or HW.
734  * It returns a chain of mblks containing the received data, to be
735  * passed up to mac_ring_rx().
736  */
737 mblk_t *
738 t4_ring_rx(struct sge_rxq *rxq, int budget)
739 {
740 	struct sge_iq *iq = &rxq->iq;
741 	struct sge_fl *fl = &rxq->fl;           /* Use iff IQ_HAS_FL */
742 	struct adapter *sc = iq->adapter;
743 	struct rsp_ctrl *ctrl;
744 	const struct rss_header *rss;
745 	int ndescs = 0, fl_bufs_used = 0;
746 	int rsp_type;
747 	uint32_t lq;
748 	mblk_t *mblk_head = NULL, **mblk_tail, *m;
749 	struct cpl_rx_pkt *cpl;
750 	uint32_t received_bytes = 0, pkt_len = 0;
751 	bool csum_ok;
752 	uint16_t err_vec;
753 
754 	mblk_tail = &mblk_head;
755 
756 	while (is_new_response(iq, &ctrl)) {
757 
758 		membar_consumer();
759 
760 		m = NULL;
761 		rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
762 		lq = be32_to_cpu(ctrl->pldbuflen_qid);
763 		rss = (const void *)iq->cdesc;
764 
765 		switch (rsp_type) {
766 		case X_RSPD_TYPE_FLBUF:
767 
768 			ASSERT(iq->flags & IQ_HAS_FL);
769 
770 			if (CPL_RX_PKT == rss->opcode) {
771 				cpl = (void *)(rss + 1);
772 				pkt_len = be16_to_cpu(cpl->len);
773 
774 				if (iq->polling && ((received_bytes + pkt_len) > budget))
775 					goto done;
776 
777 				m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
778 				if (m == NULL) {
779 					panic("%s: line %d.", __func__,
780 					    __LINE__);
781 				}
782 
783 				iq->intr_next = iq->intr_params;
784 				m->b_rptr += sc->sge.pktshift;
785 				if (sc->params.tp.rx_pkt_encap)
786 				/* It is enabled only in T6 config file */
787 					err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
788 				else
789 					err_vec = ntohs(cpl->err_vec);
790 
791 				csum_ok = cpl->csum_calc && !err_vec;
792 
793 				/* TODO: what about cpl->ip_frag? */
794 				if (csum_ok && !cpl->ip_frag) {
795 					mac_hcksum_set(m, 0, 0, 0, 0xffff,
796 					    HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
797 					    HCK_IPV4_HDRCKSUM_OK);
798 					rxq->rxcsum++;
799 				}
800 				rxq->rxpkts++;
801 				rxq->rxbytes += pkt_len;
802 				received_bytes += pkt_len;
803 
804 				*mblk_tail = m;
805 				mblk_tail = &m->b_next;
806 
807 				break;
808 			}
809 
810 			m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
811 			if (m == NULL) {
812 				panic("%s: line %d.", __func__,
813 				    __LINE__);
814 			}
815 			/* FALLTHROUGH */
816 
817 		case X_RSPD_TYPE_CPL:
818 			ASSERT(rss->opcode < NUM_CPL_CMDS);
819 			sc->cpl_handler[rss->opcode](iq, rss, m);
820 			break;
821 
822 		default:
823 			break;
824 		}
825 		iq_next(iq);
826 		++ndescs;
827 		if (!iq->polling && (ndescs == budget))
828 			break;
829 	}
830 
831 done:
832 
833 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
834 		     V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
835 		     V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
836 
837 	if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) {
838 		int starved;
839 		FL_LOCK(fl);
840 		fl->needed += fl_bufs_used;
841 		starved = refill_fl(sc, fl, fl->cap / 8);
842 		FL_UNLOCK(fl);
843 		if (starved)
844 			add_fl_to_sfl(sc, fl);
845 	}
846 	return (mblk_head);
847 }
848 
849 /*
850  * Deals with anything and everything on the given ingress queue.
851  */
852 static int
853 service_iq(struct sge_iq *iq, int budget)
854 {
855 	struct sge_iq *q;
856 	struct sge_rxq *rxq = iq_to_rxq(iq);	/* Use iff iq is part of rxq */
857 	struct sge_fl *fl = &rxq->fl;		/* Use iff IQ_HAS_FL */
858 	struct adapter *sc = iq->adapter;
859 	struct rsp_ctrl *ctrl;
860 	const struct rss_header *rss;
861 	int ndescs = 0, limit, fl_bufs_used = 0;
862 	int rsp_type;
863 	uint32_t lq;
864 	mblk_t *m;
865 	STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
866 
867 	limit = budget ? budget : iq->qsize / 8;
868 
869 	/*
870 	 * We always come back and check the descriptor ring for new indirect
871 	 * interrupts and other responses after running a single handler.
872 	 */
873 	for (;;) {
874 		while (is_new_response(iq, &ctrl)) {
875 
876 			membar_consumer();
877 
878 			m = NULL;
879 			rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
880 			lq = be32_to_cpu(ctrl->pldbuflen_qid);
881 			rss = (const void *)iq->cdesc;
882 
883 			switch (rsp_type) {
884 			case X_RSPD_TYPE_FLBUF:
885 
886 				ASSERT(iq->flags & IQ_HAS_FL);
887 
888 				m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
889 				if (m == NULL) {
890 					panic("%s: line %d.", __func__,
891 					    __LINE__);
892 				}
893 
894 			/* FALLTHRU */
895 			case X_RSPD_TYPE_CPL:
896 
897 				ASSERT(rss->opcode < NUM_CPL_CMDS);
898 				sc->cpl_handler[rss->opcode](iq, rss, m);
899 				break;
900 
901 			case X_RSPD_TYPE_INTR:
902 
903 				/*
904 				 * Interrupts should be forwarded only to queues
905 				 * that are not forwarding their interrupts.
906 				 * This means service_iq can recurse but only 1
907 				 * level deep.
908 				 */
909 				ASSERT(budget == 0);
910 
911 				q = sc->sge.iqmap[lq - sc->sge.iq_start];
912 				if (atomic_cas_uint(&q->state, IQS_IDLE,
913 				    IQS_BUSY) == IQS_IDLE) {
914 					if (service_iq(q, q->qsize / 8) == 0) {
915 						(void) atomic_cas_uint(
916 						    &q->state, IQS_BUSY,
917 						    IQS_IDLE);
918 					} else {
919 						STAILQ_INSERT_TAIL(&iql, q,
920 						    link);
921 					}
922 				}
923 				break;
924 
925 			default:
926 				break;
927 			}
928 
929 			iq_next(iq);
930 			if (++ndescs == limit) {
931 				t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
932 				    V_CIDXINC(ndescs) |
933 				    V_INGRESSQID(iq->cntxt_id) |
934 				    V_SEINTARM(V_QINTR_TIMER_IDX(
935 				    X_TIMERREG_UPDATE_CIDX)));
936 				ndescs = 0;
937 
938 				if (fl_bufs_used > 0) {
939 					ASSERT(iq->flags & IQ_HAS_FL);
940 					FL_LOCK(fl);
941 					fl->needed += fl_bufs_used;
942 					(void) refill_fl(sc, fl, fl->cap / 8);
943 					FL_UNLOCK(fl);
944 					fl_bufs_used = 0;
945 				}
946 
947 				if (budget != 0)
948 					return (EINPROGRESS);
949 			}
950 		}
951 
952 		if (STAILQ_EMPTY(&iql) != 0)
953 			break;
954 
955 		/*
956 		 * Process the head only, and send it to the back of the list if
957 		 * it's still not done.
958 		 */
959 		q = STAILQ_FIRST(&iql);
960 		STAILQ_REMOVE_HEAD(&iql, link);
961 		if (service_iq(q, q->qsize / 8) == 0)
962 			(void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
963 		else
964 			STAILQ_INSERT_TAIL(&iql, q, link);
965 	}
966 
967 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
968 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
969 
970 	if (iq->flags & IQ_HAS_FL) {
971 		int starved;
972 
973 		FL_LOCK(fl);
974 		fl->needed += fl_bufs_used;
975 		starved = refill_fl(sc, fl, fl->cap / 4);
976 		FL_UNLOCK(fl);
977 		if (starved != 0)
978 			add_fl_to_sfl(sc, fl);
979 	}
980 
981 	return (0);
982 }
983 
984 #ifdef TCP_OFFLOAD_ENABLE
985 int
986 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
987 {
988 	return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
989 }
990 
991 /*
992  * Doesn't fail.  Holds on to work requests it can't send right away.
993  */
994 int
995 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0)
996 {
997 	struct sge_eq *eq = &wrq->eq;
998 	struct mblk_pair *wr_list = &wrq->wr_list;
999 	int can_reclaim;
1000 	caddr_t dst;
1001 	mblk_t *wr, *next;
1002 
1003 	TXQ_LOCK_ASSERT_OWNED(wrq);
1004 #ifdef TCP_OFFLOAD_ENABLE
1005 	ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
1006 	    (eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1007 #else
1008 	ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1009 #endif
1010 
1011 	if (m0 != NULL) {
1012 		if (wr_list->head != NULL)
1013 			wr_list->tail->b_next = m0;
1014 		else
1015 			wr_list->head = m0;
1016 		while (m0->b_next)
1017 			m0 = m0->b_next;
1018 		wr_list->tail = m0;
1019 	}
1020 
1021 	can_reclaim = reclaimable(eq);
1022 	eq->cidx += can_reclaim;
1023 	eq->avail += can_reclaim;
1024 	if (eq->cidx >= eq->cap)
1025 		eq->cidx -= eq->cap;
1026 
1027 	for (wr = wr_list->head; wr; wr = next) {
1028 		int ndesc, len = 0;
1029 		mblk_t *m;
1030 
1031 		next = wr->b_next;
1032 		wr->b_next = NULL;
1033 
1034 		for (m = wr; m; m = m->b_cont)
1035 			len += MBLKL(m);
1036 
1037 		ASSERT(len > 0 && (len & 0x7) == 0);
1038 		ASSERT(len <= SGE_MAX_WR_LEN);
1039 
1040 		ndesc = howmany(len, EQ_ESIZE);
1041 		if (eq->avail < ndesc) {
1042 			wr->b_next = next;
1043 			wrq->no_desc++;
1044 			break;
1045 		}
1046 
1047 		dst = (void *)&eq->desc[eq->pidx];
1048 		for (m = wr; m; m = m->b_cont)
1049 			copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
1050 
1051 		eq->pidx += ndesc;
1052 		eq->avail -= ndesc;
1053 		if (eq->pidx >= eq->cap)
1054 			eq->pidx -= eq->cap;
1055 
1056 		eq->pending += ndesc;
1057 		if (eq->pending > 16)
1058 			ring_tx_db(sc, eq);
1059 
1060 		wrq->tx_wrs++;
1061 		freemsg(wr);
1062 
1063 		if (eq->avail < 8) {
1064 			can_reclaim = reclaimable(eq);
1065 			eq->cidx += can_reclaim;
1066 			eq->avail += can_reclaim;
1067 			if (eq->cidx >= eq->cap)
1068 				eq->cidx -= eq->cap;
1069 		}
1070 	}
1071 
1072 	if (eq->pending != 0)
1073 		ring_tx_db(sc, eq);
1074 
1075 	if (wr == NULL)
1076 		wr_list->head = wr_list->tail = NULL;
1077 	else {
1078 		wr_list->head = wr;
1079 
1080 		ASSERT(wr_list->tail->b_next == NULL);
1081 	}
1082 
1083 	return (0);
1084 }
1085 #endif
1086 
1087 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
1088 #define	TXPKTS_PKT_HDR ((\
1089 	sizeof (struct ulp_txpkt) + \
1090 	sizeof (struct ulptx_idata) + \
1091 	sizeof (struct cpl_tx_pkt_core)) / 8)
1092 
1093 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
1094 #define	TXPKTS_WR_HDR (\
1095 	sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
1096 	TXPKTS_PKT_HDR)
1097 
1098 /* Header of a tx WR, before SGL of first packet (in flits) */
1099 #define	TXPKT_WR_HDR ((\
1100 	sizeof (struct fw_eth_tx_pkt_wr) + \
1101 	sizeof (struct cpl_tx_pkt_core)) / 8)
1102 
1103 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
1104 #define	TXPKT_LSO_WR_HDR ((\
1105 	sizeof (struct fw_eth_tx_pkt_wr) + \
1106 	sizeof(struct cpl_tx_pkt_lso_core) + \
1107 	sizeof (struct cpl_tx_pkt_core)) / 8)
1108 
1109 mblk_t *
1110 t4_eth_tx(void *arg, mblk_t *frame)
1111 {
1112 	struct sge_txq *txq = (struct sge_txq *) arg;
1113 	struct port_info *pi = txq->port;
1114 	struct adapter *sc = pi->adapter;
1115 	struct sge_eq *eq = &txq->eq;
1116 	mblk_t *next_frame;
1117 	int rc, coalescing;
1118 	struct txpkts txpkts;
1119 	struct txinfo txinfo;
1120 
1121 	txpkts.npkt = 0; /* indicates there's nothing in txpkts */
1122 	coalescing = 0;
1123 
1124 	TXQ_LOCK(txq);
1125 	if (eq->avail < 8)
1126 		(void) reclaim_tx_descs(txq, 8);
1127 	for (; frame; frame = next_frame) {
1128 
1129 		if (eq->avail < 8)
1130 			break;
1131 
1132 		next_frame = frame->b_next;
1133 		frame->b_next = NULL;
1134 
1135 		if (next_frame != NULL)
1136 			coalescing = 1;
1137 
1138 		rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
1139 		if (rc != 0) {
1140 			if (rc == ENOMEM) {
1141 
1142 				/* Short of resources, suspend tx */
1143 
1144 				frame->b_next = next_frame;
1145 				break;
1146 			}
1147 
1148 			/*
1149 			 * Unrecoverable error for this frame, throw it
1150 			 * away and move on to the next.
1151 			 */
1152 
1153 			freemsg(frame);
1154 			continue;
1155 		}
1156 
1157 		if (coalescing != 0 &&
1158 		    add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
1159 
1160 			/* Successfully absorbed into txpkts */
1161 
1162 			write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
1163 			goto doorbell;
1164 		}
1165 
1166 		/*
1167 		 * We weren't coalescing to begin with, or current frame could
1168 		 * not be coalesced (add_to_txpkts flushes txpkts if a frame
1169 		 * given to it can't be coalesced).  Either way there should be
1170 		 * nothing in txpkts.
1171 		 */
1172 		ASSERT(txpkts.npkt == 0);
1173 
1174 		/* We're sending out individual frames now */
1175 		coalescing = 0;
1176 
1177 		if (eq->avail < 8)
1178 			(void) reclaim_tx_descs(txq, 8);
1179 		rc = write_txpkt_wr(pi, txq, frame, &txinfo);
1180 		if (rc != 0) {
1181 
1182 			/* Short of hardware descriptors, suspend tx */
1183 
1184 			/*
1185 			 * This is an unlikely but expensive failure.  We've
1186 			 * done all the hard work (DMA bindings etc.) and now we
1187 			 * can't send out the frame.  What's worse, we have to
1188 			 * spend even more time freeing up everything in txinfo.
1189 			 */
1190 			txq->qfull++;
1191 			free_txinfo_resources(txq, &txinfo);
1192 
1193 			frame->b_next = next_frame;
1194 			break;
1195 		}
1196 
1197 doorbell:
1198 		/* Fewer and fewer doorbells as the queue fills up */
1199 		if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) {
1200 			txq->txbytes += txinfo.len;
1201 			txq->txpkts++;
1202 			ring_tx_db(sc, eq);
1203 		}
1204 		(void) reclaim_tx_descs(txq, 32);
1205 	}
1206 
1207 	if (txpkts.npkt > 0)
1208 		write_txpkts_wr(txq, &txpkts);
1209 
1210 	/*
1211 	 * frame not NULL means there was an error but we haven't thrown it
1212 	 * away.  This can happen when we're short of tx descriptors (qfull) or
1213 	 * maybe even DMA handles (dma_hdl_failed).  Either way, a credit flush
1214 	 * and reclaim will get things going again.
1215 	 *
1216 	 * If eq->avail is already 0 we know a credit flush was requested in the
1217 	 * WR that reduced it to 0 so we don't need another flush (we don't have
1218 	 * any descriptor for a flush WR anyway, duh).
1219 	 */
1220 	if (frame && eq->avail > 0)
1221 		write_txqflush_wr(txq);
1222 
1223 	if (eq->pending != 0)
1224 		ring_tx_db(sc, eq);
1225 
1226 	(void) reclaim_tx_descs(txq, eq->qsize);
1227 	TXQ_UNLOCK(txq);
1228 
1229 	return (frame);
1230 }
1231 
1232 static inline void
1233 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx,
1234 	int qsize, uint8_t esize)
1235 {
1236 	ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1237 	ASSERT(pktc_idx < SGE_NCOUNTERS);	/* -ve is ok, means don't use */
1238 
1239 	iq->flags = 0;
1240 	iq->adapter = sc;
1241 	iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1242 	iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1243 	if (pktc_idx >= 0) {
1244 		iq->intr_params |= F_QINTR_CNT_EN;
1245 		iq->intr_pktc_idx = pktc_idx;
1246 	}
1247 	iq->qsize = roundup(qsize, 16);		/* See FW_IQ_CMD/iqsize */
1248 	iq->esize = max(esize, 16);		/* See FW_IQ_CMD/iqesize */
1249 }
1250 
1251 static inline void
1252 init_fl(struct sge_fl *fl, uint16_t qsize)
1253 {
1254 
1255 	fl->qsize = qsize;
1256 }
1257 
1258 static inline void
1259 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1260     uint8_t tx_chan, uint16_t iqid)
1261 {
1262 	struct sge *s = &sc->sge;
1263 	uint32_t r;
1264 
1265 	ASSERT(tx_chan < NCHAN);
1266 	ASSERT(eqtype <= EQ_TYPEMASK);
1267 
1268 	if (is_t5(sc->params.chip)) {
1269 		r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1270 		r >>= S_QUEUESPERPAGEPF0 +
1271 		    (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1272 		s->s_qpp = r & M_QUEUESPERPAGEPF0;
1273 	}
1274 
1275 	eq->flags = eqtype & EQ_TYPEMASK;
1276 	eq->tx_chan = tx_chan;
1277 	eq->iqid = iqid;
1278 	eq->qsize = qsize;
1279 }
1280 
1281 /*
1282  * Allocates the ring for an ingress queue and an optional freelist.  If the
1283  * freelist is specified it will be allocated and then associated with the
1284  * ingress queue.
1285  *
1286  * Returns errno on failure.  Resources allocated up to that point may still be
1287  * allocated.  Caller is responsible for cleanup in case this function fails.
1288  *
1289  * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1290  * the intr_idx specifies the vector, starting from 0.  Otherwise it specifies
1291  * the index of the queue to which its interrupts will be forwarded.
1292  */
1293 static int
1294 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1295     int intr_idx, int cong)
1296 {
1297 	int rc, i, cntxt_id;
1298 	size_t len;
1299 	struct fw_iq_cmd c;
1300 	struct adapter *sc = iq->adapter;
1301 	uint32_t v = 0;
1302 
1303 	len = iq->qsize * iq->esize;
1304 	rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1305 	    &iq->ba, (caddr_t *)&iq->desc);
1306 	if (rc != 0)
1307 		return (rc);
1308 
1309 	bzero(&c, sizeof (c));
1310 	c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1311 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1312 	    V_FW_IQ_CMD_VFN(0));
1313 
1314 	c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1315 	    FW_LEN16(c));
1316 
1317 	/* Special handling for firmware event queue */
1318 	if (iq == &sc->sge.fwq)
1319 		v |= F_FW_IQ_CMD_IQASYNCH;
1320 
1321 	if (iq->flags & IQ_INTR)
1322 		ASSERT(intr_idx < sc->intr_count);
1323 	else
1324 		v |= F_FW_IQ_CMD_IQANDST;
1325 	v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1326 
1327 	c.type_to_iqandstindex = cpu_to_be32(v |
1328 	    V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1329 	    V_FW_IQ_CMD_VIID(pi->viid) |
1330 	    V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1331 	c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1332 	    F_FW_IQ_CMD_IQGTSMODE |
1333 	    V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1334 	    V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1335 	c.iqsize = cpu_to_be16(iq->qsize);
1336 	c.iqaddr = cpu_to_be64(iq->ba);
1337 	if (cong >= 0)
1338 		c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN);
1339 
1340 	if (fl != NULL) {
1341 		unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip);
1342 
1343 		mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1344 		    DDI_INTR_PRI(sc->intr_pri));
1345 		fl->flags |= FL_MTX;
1346 
1347 		len = fl->qsize * RX_FL_ESIZE;
1348 		rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1349 		    &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1350 		if (rc != 0)
1351 			return (rc);
1352 
1353 		/* Allocate space for one software descriptor per buffer. */
1354 		fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8;
1355 		fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1356 		    KM_SLEEP);
1357 		fl->needed = fl->cap;
1358 		fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1359 
1360 		c.iqns_to_fl0congen |=
1361 		    cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1362 		    F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1363 		if (cong >= 0) {
1364 			c.iqns_to_fl0congen |=
1365 			    BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1366 			    F_FW_IQ_CMD_FL0CONGCIF |
1367 			    F_FW_IQ_CMD_FL0CONGEN);
1368 		}
1369 
1370 		/* In T6, for egress queue type FL there is internal overhead
1371 		 * of 16B for header going into FLM module.  Hence the maximum
1372 		 * allowed burst size is 448 bytes.  For T4/T5, the hardware
1373 		 * doesn't coalesce fetch requests if more than 64 bytes of
1374 		 * Free List pointers are provided, so we use a 128-byte Fetch
1375 		 * Burst Minimum there (T6 implements coalescing so we can use
1376 		 * the smaller 64-byte value there).
1377 		 */
1378 
1379 		c.fl0dcaen_to_fl0cidxfthresh =
1380 		    cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5
1381 						     ? X_FETCHBURSTMIN_128B
1382 						     : X_FETCHBURSTMIN_64B) |
1383 		    V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5
1384 					 ? X_FETCHBURSTMAX_512B
1385 					 : X_FETCHBURSTMAX_256B));
1386 		c.fl0size = cpu_to_be16(fl->qsize);
1387 		c.fl0addr = cpu_to_be64(fl->ba);
1388 	}
1389 
1390 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1391 	if (rc != 0) {
1392 		cxgb_printf(sc->dip, CE_WARN,
1393 		    "failed to create ingress queue: %d", rc);
1394 		return (rc);
1395 	}
1396 
1397 	iq->cdesc = iq->desc;
1398 	iq->cidx = 0;
1399 	iq->gen = 1;
1400 	iq->intr_next = iq->intr_params;
1401 	iq->adapter = sc;
1402 	iq->cntxt_id = be16_to_cpu(c.iqid);
1403 	iq->abs_id = be16_to_cpu(c.physiqid);
1404 	iq->flags |= IQ_ALLOCATED;
1405 	mutex_init(&iq->lock, NULL,
1406 		    MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri)));
1407 	iq->polling = 0;
1408 
1409 	cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1410 	if (cntxt_id >= sc->sge.niq) {
1411 		panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1412 		    cntxt_id, sc->sge.niq - 1);
1413 	}
1414 	sc->sge.iqmap[cntxt_id] = iq;
1415 
1416 	if (fl != NULL) {
1417 		fl->cntxt_id = be16_to_cpu(c.fl0id);
1418 		fl->pidx = fl->cidx = 0;
1419 		fl->copy_threshold = rx_copy_threshold;
1420 
1421 		cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1422 		if (cntxt_id >= sc->sge.neq) {
1423 			panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1424 			    __func__, cntxt_id, sc->sge.neq - 1);
1425 		}
1426 		sc->sge.eqmap[cntxt_id] = (void *)fl;
1427 
1428 		FL_LOCK(fl);
1429 		(void) refill_fl(sc, fl, fl->lowat);
1430 		FL_UNLOCK(fl);
1431 
1432 		iq->flags |= IQ_HAS_FL;
1433 	}
1434 
1435 	if (is_t5(sc->params.chip) && cong >= 0) {
1436 		uint32_t param, val;
1437 
1438 		param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1439 			V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
1440 			V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
1441 		if (cong == 0)
1442 			val = 1 << 19;
1443 		else {
1444 			val = 2 << 19;
1445 			for (i = 0; i < 4; i++) {
1446 				if (cong & (1 << i))
1447 					val |= 1 << (i << 2);
1448 			}
1449 		}
1450 
1451 		rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1452 		if (rc != 0) {
1453 			/* report error but carry on */
1454 			cxgb_printf(sc->dip, CE_WARN,
1455 			    "failed to set congestion manager context for "
1456 			    "ingress queue %d: %d", iq->cntxt_id, rc);
1457 		}
1458 	}
1459 
1460 	/* Enable IQ interrupts */
1461 	iq->state = IQS_IDLE;
1462 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1463 	    V_INGRESSQID(iq->cntxt_id));
1464 
1465 	return (0);
1466 }
1467 
1468 static int
1469 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1470 {
1471 	int rc;
1472 	struct adapter *sc = iq->adapter;
1473 	dev_info_t *dip;
1474 
1475 	dip = pi ? pi->dip : sc->dip;
1476 
1477 	if (iq != NULL) {
1478 		if (iq->flags & IQ_ALLOCATED) {
1479 			rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1480 			    FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1481 			    fl ? fl->cntxt_id : 0xffff, 0xffff);
1482 			if (rc != 0) {
1483 				cxgb_printf(dip, CE_WARN,
1484 				    "failed to free queue %p: %d", iq, rc);
1485 				return (rc);
1486 			}
1487 			mutex_destroy(&iq->lock);
1488 			iq->flags &= ~IQ_ALLOCATED;
1489 		}
1490 
1491 		if (iq->desc != NULL) {
1492 			(void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1493 			iq->desc = NULL;
1494 		}
1495 
1496 		bzero(iq, sizeof (*iq));
1497 	}
1498 
1499 	if (fl != NULL) {
1500 		if (fl->sdesc != NULL) {
1501 			FL_LOCK(fl);
1502 			free_fl_bufs(fl);
1503 			FL_UNLOCK(fl);
1504 
1505 			kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1506 			    fl->cap);
1507 			fl->sdesc = NULL;
1508 		}
1509 
1510 		if (fl->desc != NULL) {
1511 			(void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1512 			fl->desc = NULL;
1513 		}
1514 
1515 		if (fl->flags & FL_MTX) {
1516 			mutex_destroy(&fl->lock);
1517 			fl->flags &= ~FL_MTX;
1518 		}
1519 
1520 		bzero(fl, sizeof (struct sge_fl));
1521 	}
1522 
1523 	return (0);
1524 }
1525 
1526 static int
1527 alloc_fwq(struct adapter *sc)
1528 {
1529 	int rc, intr_idx;
1530 	struct sge_iq *fwq = &sc->sge.fwq;
1531 
1532 	init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1533 	fwq->flags |= IQ_INTR;	/* always */
1534 	intr_idx = sc->intr_count > 1 ? 1 : 0;
1535 	rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1536 	if (rc != 0) {
1537 		cxgb_printf(sc->dip, CE_WARN,
1538 		    "failed to create firmware event queue: %d.", rc);
1539 		return (rc);
1540 	}
1541 
1542 	return (0);
1543 }
1544 
1545 static int
1546 free_fwq(struct adapter *sc)
1547 {
1548 
1549 	return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1550 }
1551 
1552 #ifdef TCP_OFFLOAD_ENABLE
1553 static int
1554 alloc_mgmtq(struct adapter *sc)
1555 {
1556 	int rc;
1557 	struct sge_wrq *mgmtq = &sc->sge.mgmtq;
1558 
1559 	init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
1560 	    sc->sge.fwq.cntxt_id);
1561 	rc = alloc_wrq(sc, NULL, mgmtq, 0);
1562 	if (rc != 0) {
1563 		cxgb_printf(sc->dip, CE_WARN,
1564 		    "failed to create management queue: %d\n", rc);
1565 		return (rc);
1566 	}
1567 
1568 	return (0);
1569 }
1570 #endif
1571 
1572 static int
1573 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1574 {
1575 	int rc;
1576 
1577 	rxq->port = pi;
1578 	rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1 << pi->tx_chan);
1579 	if (rc != 0)
1580 		return (rc);
1581 
1582 	rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1583 
1584 	return (rc);
1585 }
1586 
1587 static int
1588 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1589 {
1590 	int rc;
1591 
1592 	if (rxq->ksp != NULL) {
1593 		kstat_delete(rxq->ksp);
1594 		rxq->ksp = NULL;
1595 	}
1596 
1597 	rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1598 	if (rc == 0)
1599 		bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1600 
1601 	return (rc);
1602 }
1603 
1604 #ifdef TCP_OFFLOAD_ENABLE
1605 static int
1606 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
1607 	int intr_idx)
1608 {
1609 	int rc;
1610 
1611 	rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
1612 	    1 << pi->tx_chan);
1613 	if (rc != 0)
1614 		return (rc);
1615 
1616 	return (rc);
1617 }
1618 
1619 static int
1620 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
1621 {
1622 	int rc;
1623 
1624 	rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
1625 	if (rc == 0)
1626 		bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) -
1627 		    offsetof(struct sge_ofld_rxq, fl));
1628 
1629 	return (rc);
1630 }
1631 #endif
1632 
1633 static int
1634 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
1635 {
1636 	int rc, cntxt_id;
1637 	struct fw_eq_ctrl_cmd c;
1638 
1639 	bzero(&c, sizeof (c));
1640 
1641 	c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
1642 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
1643 	    V_FW_EQ_CTRL_CMD_VFN(0));
1644 	c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC |
1645 	    F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
1646 	c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */
1647 	c.physeqid_pkd = BE_32(0);
1648 	c.fetchszm_to_iqid =
1649 	    BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1650 	    V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
1651 	    F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
1652 	c.dcaen_to_eqsize =
1653 	    BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1654 	    V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1655 	    V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1656 	    V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
1657 	c.eqaddr = BE_64(eq->ba);
1658 
1659 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1660 	if (rc != 0) {
1661 		cxgb_printf(sc->dip, CE_WARN,
1662 		    "failed to create control queue %d: %d", eq->tx_chan, rc);
1663 		return (rc);
1664 	}
1665 	eq->flags |= EQ_ALLOCATED;
1666 
1667 	eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid));
1668 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1669 	if (cntxt_id >= sc->sge.neq)
1670 		panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1671 		    cntxt_id, sc->sge.neq - 1);
1672 	sc->sge.eqmap[cntxt_id] = eq;
1673 
1674 	return (rc);
1675 }
1676 
1677 static int
1678 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1679 {
1680 	int rc, cntxt_id;
1681 	struct fw_eq_eth_cmd c;
1682 
1683 	bzero(&c, sizeof (c));
1684 
1685 	c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1686 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1687 	    V_FW_EQ_ETH_CMD_VFN(0));
1688 	c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1689 	    F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1690 	c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1691 	    F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1692 	c.fetchszm_to_iqid =
1693 	    BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1694 	    V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1695 	    V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1696 	c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1697 	    V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1698 	    V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1699 	    V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1700 	c.eqaddr = BE_64(eq->ba);
1701 
1702 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1703 	if (rc != 0) {
1704 		cxgb_printf(pi->dip, CE_WARN,
1705 		    "failed to create Ethernet egress queue: %d", rc);
1706 		return (rc);
1707 	}
1708 	eq->flags |= EQ_ALLOCATED;
1709 
1710 	eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1711 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1712 	if (cntxt_id >= sc->sge.neq)
1713 		panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1714 		    cntxt_id, sc->sge.neq - 1);
1715 	sc->sge.eqmap[cntxt_id] = eq;
1716 
1717 	return (rc);
1718 }
1719 
1720 #ifdef TCP_OFFLOAD_ENABLE
1721 static int
1722 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1723 {
1724 	int rc, cntxt_id;
1725 	struct fw_eq_ofld_cmd c;
1726 
1727 	bzero(&c, sizeof (c));
1728 
1729 	c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
1730 	    F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
1731 	    V_FW_EQ_OFLD_CMD_VFN(0));
1732 	c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
1733 	    F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
1734 	c.fetchszm_to_iqid =
1735 	    htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1736 	    V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
1737 	    F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
1738 	c.dcaen_to_eqsize =
1739 	    BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1740 	    V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1741 	    V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1742 	    V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
1743 	c.eqaddr = BE_64(eq->ba);
1744 
1745 	rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1746 	if (rc != 0) {
1747 		cxgb_printf(pi->dip, CE_WARN,
1748 		    "failed to create egress queue for TCP offload: %d", rc);
1749 		return (rc);
1750 	}
1751 	eq->flags |= EQ_ALLOCATED;
1752 
1753 	eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd));
1754 	cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1755 	if (cntxt_id >= sc->sge.neq)
1756 		panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1757 		    cntxt_id, sc->sge.neq - 1);
1758 	sc->sge.eqmap[cntxt_id] = eq;
1759 
1760 	return (rc);
1761 }
1762 #endif
1763 
1764 static int
1765 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1766 {
1767 	int rc;
1768 	size_t len;
1769 
1770 	mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1771 	eq->flags |= EQ_MTX;
1772 
1773 	len = eq->qsize * EQ_ESIZE;
1774 	rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1775 	    &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1776 	if (rc != 0)
1777 		return (rc);
1778 
1779 	eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE;
1780 	eq->spg = (void *)&eq->desc[eq->cap];
1781 	eq->avail = eq->cap - 1;	/* one less to avoid cidx = pidx */
1782 	eq->pidx = eq->cidx = 0;
1783 	eq->doorbells = sc->doorbells;
1784 
1785 	switch (eq->flags & EQ_TYPEMASK) {
1786 	case EQ_CTRL:
1787 		rc = ctrl_eq_alloc(sc, eq);
1788 		break;
1789 
1790 	case EQ_ETH:
1791 		rc = eth_eq_alloc(sc, pi, eq);
1792 		break;
1793 
1794 #ifdef TCP_OFFLOAD_ENABLE
1795 	case EQ_OFLD:
1796 		rc = ofld_eq_alloc(sc, pi, eq);
1797 		break;
1798 #endif
1799 
1800 	default:
1801 		panic("%s: invalid eq type %d.", __func__,
1802 		    eq->flags & EQ_TYPEMASK);
1803 	}
1804 
1805 	if (eq->doorbells &
1806 		(DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) {
1807 		uint32_t s_qpp = sc->sge.s_qpp;
1808 		uint32_t mask = (1 << s_qpp) - 1;
1809 		volatile uint8_t *udb;
1810 
1811 		udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET;
1812 		udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT;   /* pg offset */
1813 		eq->udb_qid = eq->cntxt_id & mask;              /* id in page */
1814 		if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
1815 			eq->doorbells &= ~DOORBELL_WCWR;
1816 		else {
1817 			udb += eq->udb_qid << UDBS_SEG_SHIFT;   /* seg offset */
1818 			eq->udb_qid = 0;
1819 		}
1820 		eq->udb = (volatile void *)udb;
1821 	}
1822 
1823 	if (rc != 0) {
1824 		cxgb_printf(sc->dip, CE_WARN,
1825 		    "failed to allocate egress queue(%d): %d",
1826 		    eq->flags & EQ_TYPEMASK, rc);
1827 	}
1828 
1829 	return (rc);
1830 }
1831 
1832 static int
1833 free_eq(struct adapter *sc, struct sge_eq *eq)
1834 {
1835 	int rc;
1836 
1837 	if (eq->flags & EQ_ALLOCATED) {
1838 		switch (eq->flags & EQ_TYPEMASK) {
1839 		case EQ_CTRL:
1840 			rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
1841 			    eq->cntxt_id);
1842 			break;
1843 
1844 		case EQ_ETH:
1845 			rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
1846 			    eq->cntxt_id);
1847 			break;
1848 
1849 #ifdef TCP_OFFLOAD_ENABLE
1850 		case EQ_OFLD:
1851 			rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
1852 			    eq->cntxt_id);
1853 			break;
1854 #endif
1855 
1856 		default:
1857 			panic("%s: invalid eq type %d.", __func__,
1858 			    eq->flags & EQ_TYPEMASK);
1859 		}
1860 		if (rc != 0) {
1861 			cxgb_printf(sc->dip, CE_WARN,
1862 			    "failed to free egress queue (%d): %d",
1863 			    eq->flags & EQ_TYPEMASK, rc);
1864 			return (rc);
1865 		}
1866 		eq->flags &= ~EQ_ALLOCATED;
1867 	}
1868 
1869 	if (eq->desc != NULL) {
1870 		(void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1871 		eq->desc = NULL;
1872 	}
1873 
1874 	if (eq->flags & EQ_MTX)
1875 		mutex_destroy(&eq->lock);
1876 
1877 	bzero(eq, sizeof (*eq));
1878 	return (0);
1879 }
1880 
1881 #ifdef TCP_OFFLOAD_ENABLE
1882 /* ARGSUSED */
1883 static int
1884 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
1885     int idx)
1886 {
1887 	int rc;
1888 
1889 	rc = alloc_eq(sc, pi, &wrq->eq);
1890 	if (rc != 0)
1891 		return (rc);
1892 
1893 	wrq->adapter = sc;
1894 	wrq->wr_list.head = NULL;
1895 	wrq->wr_list.tail = NULL;
1896 
1897 	/*
1898 	 * TODO: use idx to figure out what kind of wrq this is and install
1899 	 * useful kstats for it.
1900 	 */
1901 
1902 	return (rc);
1903 }
1904 
1905 static int
1906 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
1907 {
1908 	int rc;
1909 
1910 	rc = free_eq(sc, &wrq->eq);
1911 	if (rc != 0)
1912 		return (rc);
1913 
1914 	bzero(wrq, sizeof (*wrq));
1915 	return (0);
1916 }
1917 #endif
1918 
1919 static int
1920 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1921 {
1922 	int rc, i;
1923 	struct adapter *sc = pi->adapter;
1924 	struct sge_eq *eq = &txq->eq;
1925 
1926 	rc = alloc_eq(sc, pi, eq);
1927 	if (rc != 0)
1928 		return (rc);
1929 
1930 	txq->port = pi;
1931 	txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1932 	txq->txb_size = eq->qsize * tx_copy_threshold;
1933 	rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1934 	    &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1935 	if (rc == 0)
1936 		txq->txb_avail = txq->txb_size;
1937 	else
1938 		txq->txb_avail = txq->txb_size = 0;
1939 
1940 	/*
1941 	 * TODO: is this too low?  Worst case would need around 4 times qsize
1942 	 * (all tx descriptors filled to the brim with SGLs, with each entry in
1943 	 * the SGL coming from a distinct DMA handle).  Increase tx_dhdl_total
1944 	 * if you see too many dma_hdl_failed.
1945 	 */
1946 	txq->tx_dhdl_total = eq->qsize * 2;
1947 	txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1948 	    txq->tx_dhdl_total, KM_SLEEP);
1949 	for (i = 0; i < txq->tx_dhdl_total; i++) {
1950 		rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1951 		    DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1952 		if (rc != DDI_SUCCESS) {
1953 			cxgb_printf(sc->dip, CE_WARN,
1954 			    "%s: failed to allocate DMA handle (%d)",
1955 			    __func__, rc);
1956 			return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1957 		}
1958 		txq->tx_dhdl_avail++;
1959 	}
1960 
1961 	txq->ksp = setup_txq_kstats(pi, txq, idx);
1962 
1963 	return (rc);
1964 }
1965 
1966 static int
1967 free_txq(struct port_info *pi, struct sge_txq *txq)
1968 {
1969 	int i;
1970 	struct adapter *sc = pi->adapter;
1971 	struct sge_eq *eq = &txq->eq;
1972 
1973 	if (txq->ksp != NULL) {
1974 		kstat_delete(txq->ksp);
1975 		txq->ksp = NULL;
1976 	}
1977 
1978 	if (txq->txb_va != NULL) {
1979 		(void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1980 		txq->txb_va = NULL;
1981 	}
1982 
1983 	if (txq->sdesc != NULL) {
1984 		struct tx_sdesc *sd;
1985 		ddi_dma_handle_t hdl;
1986 
1987 		TXQ_LOCK(txq);
1988 		while (eq->cidx != eq->pidx) {
1989 			sd = &txq->sdesc[eq->cidx];
1990 
1991 			for (i = sd->hdls_used; i; i--) {
1992 				hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
1993 				(void) ddi_dma_unbind_handle(hdl);
1994 				if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
1995 					txq->tx_dhdl_cidx = 0;
1996 			}
1997 
1998 			ASSERT(sd->m);
1999 			freemsgchain(sd->m);
2000 
2001 			eq->cidx += sd->desc_used;
2002 			if (eq->cidx >= eq->cap)
2003 				eq->cidx -= eq->cap;
2004 
2005 			txq->txb_avail += txq->txb_used;
2006 		}
2007 		ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
2008 		ASSERT(txq->txb_avail == txq->txb_size);
2009 		TXQ_UNLOCK(txq);
2010 
2011 		kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
2012 		txq->sdesc = NULL;
2013 	}
2014 
2015 	if (txq->tx_dhdl != NULL) {
2016 		for (i = 0; i < txq->tx_dhdl_total; i++) {
2017 			if (txq->tx_dhdl[i] != NULL)
2018 				ddi_dma_free_handle(&txq->tx_dhdl[i]);
2019 		}
2020 	}
2021 
2022 	(void) free_eq(sc, &txq->eq);
2023 
2024 	bzero(txq, sizeof (*txq));
2025 	return (0);
2026 }
2027 
2028 /*
2029  * Allocates a block of contiguous memory for DMA.  Can be used to allocate
2030  * memory for descriptor rings or for tx/rx copy buffers.
2031  *
2032  * Caller does not have to clean up anything if this function fails, it cleans
2033  * up after itself.
2034  *
2035  * Caller provides the following:
2036  * len		length of the block of memory to allocate.
2037  * flags	DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
2038  * acc_attr	device access attributes for the allocation.
2039  * dma_attr	DMA attributes for the allocation
2040  *
2041  * If the function is successful it fills up this information:
2042  * dma_hdl	DMA handle for the allocated memory
2043  * acc_hdl	access handle for the allocated memory
2044  * ba		bus address of the allocated memory
2045  * va		KVA of the allocated memory.
2046  */
2047 static int
2048 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
2049     ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
2050     ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2051     uint64_t *pba, caddr_t *pva)
2052 {
2053 	int rc;
2054 	ddi_dma_handle_t dhdl;
2055 	ddi_acc_handle_t ahdl;
2056 	ddi_dma_cookie_t cookie;
2057 	uint_t ccount;
2058 	caddr_t va;
2059 	size_t real_len;
2060 
2061 	*pva = NULL;
2062 
2063 	/*
2064 	 * DMA handle.
2065 	 */
2066 	rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
2067 	if (rc != DDI_SUCCESS) {
2068 		cxgb_printf(sc->dip, CE_WARN,
2069 		    "failed to allocate DMA handle: %d", rc);
2070 
2071 		return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
2072 	}
2073 
2074 	/*
2075 	 * Memory suitable for DMA.
2076 	 */
2077 	rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
2078 	    flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
2079 	    DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
2080 	if (rc != DDI_SUCCESS) {
2081 		cxgb_printf(sc->dip, CE_WARN,
2082 		    "failed to allocate DMA memory: %d", rc);
2083 
2084 		ddi_dma_free_handle(&dhdl);
2085 		return (ENOMEM);
2086 	}
2087 
2088 	if (len != real_len) {
2089 		cxgb_printf(sc->dip, CE_WARN,
2090 		    "%s: len (%u) != real_len (%u)\n", len, real_len);
2091 	}
2092 
2093 	/*
2094 	 * DMA bindings.
2095 	 */
2096 	rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
2097 	    NULL, &cookie, &ccount);
2098 	if (rc != DDI_DMA_MAPPED) {
2099 		cxgb_printf(sc->dip, CE_WARN,
2100 		    "failed to map DMA memory: %d", rc);
2101 
2102 		ddi_dma_mem_free(&ahdl);
2103 		ddi_dma_free_handle(&dhdl);
2104 		return (ENOMEM);
2105 	}
2106 	if (ccount != 1) {
2107 		cxgb_printf(sc->dip, CE_WARN,
2108 		    "unusable DMA mapping (%d segments)", ccount);
2109 		(void) free_desc_ring(&dhdl, &ahdl);
2110 	}
2111 
2112 	bzero(va, real_len);
2113 	*dma_hdl = dhdl;
2114 	*acc_hdl = ahdl;
2115 	*pba = cookie.dmac_laddress;
2116 	*pva = va;
2117 
2118 	return (0);
2119 }
2120 
2121 static int
2122 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2123 {
2124 	(void) ddi_dma_unbind_handle(*dhdl);
2125 	ddi_dma_mem_free(ahdl);
2126 	ddi_dma_free_handle(dhdl);
2127 
2128 	return (0);
2129 }
2130 
2131 static int
2132 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
2133     ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2134     uint64_t *pba, caddr_t *pva)
2135 {
2136 	ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
2137 	ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
2138 
2139 	return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
2140 	    dma_attr, dma_hdl, acc_hdl, pba, pva));
2141 }
2142 
2143 static int
2144 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2145 {
2146 	return (free_dma_memory(dhdl, ahdl));
2147 }
2148 
2149 static int
2150 alloc_tx_copybuffer(struct adapter *sc, size_t len,
2151     ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2152     uint64_t *pba, caddr_t *pva)
2153 {
2154 	ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
2155 	ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
2156 
2157 	return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
2158 	    acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
2159 }
2160 
2161 static inline bool
2162 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
2163 {
2164 	(void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
2165 	    (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
2166 
2167 	*ctrl = (void *)((uintptr_t)iq->cdesc +
2168 	    (iq->esize - sizeof (struct rsp_ctrl)));
2169 
2170 	return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
2171 }
2172 
2173 static inline void
2174 iq_next(struct sge_iq *iq)
2175 {
2176 	iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
2177 	if (++iq->cidx == iq->qsize - 1) {
2178 		iq->cidx = 0;
2179 		iq->gen ^= 1;
2180 		iq->cdesc = iq->desc;
2181 	}
2182 }
2183 
2184 /*
2185  * Fill up the freelist by upto nbufs and maybe ring its doorbell.
2186  *
2187  * Returns non-zero to indicate that it should be added to the list of starving
2188  * freelists.
2189  */
2190 static int
2191 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
2192 {
2193 	uint64_t *d = &fl->desc[fl->pidx];
2194 	struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
2195 
2196 	FL_LOCK_ASSERT_OWNED(fl);
2197 	ASSERT(nbufs >= 0);
2198 
2199 	if (nbufs > fl->needed)
2200 		nbufs = fl->needed;
2201 
2202 	while (nbufs--) {
2203 		if (sd->rxb != NULL) {
2204 			if (sd->rxb->ref_cnt == 1) {
2205 				/*
2206 				 * Buffer is available for recycling.  Two ways
2207 				 * this can happen:
2208 				 *
2209 				 * a) All the packets DMA'd into it last time
2210 				 *    around were within the rx_copy_threshold
2211 				 *    and no part of the buffer was ever passed
2212 				 *    up (ref_cnt never went over 1).
2213 				 *
2214 				 * b) Packets DMA'd into the buffer were passed
2215 				 *    up but have all been freed by the upper
2216 				 *    layers by now (ref_cnt went over 1 but is
2217 				 *    now back to 1).
2218 				 *
2219 				 * Either way the bus address in the descriptor
2220 				 * ring is already valid.
2221 				 */
2222 				ASSERT(*d == cpu_to_be64(sd->rxb->ba));
2223 				d++;
2224 				goto recycled;
2225 			} else {
2226 				/*
2227 				 * Buffer still in use and we need a
2228 				 * replacement. But first release our reference
2229 				 * on the existing buffer.
2230 				 */
2231 				rxbuf_free(sd->rxb);
2232 			}
2233 		}
2234 
2235 		sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
2236 		if (sd->rxb == NULL)
2237 			break;
2238 		*d++ = cpu_to_be64(sd->rxb->ba);
2239 
2240 recycled:	fl->pending++;
2241 		sd++;
2242 		fl->needed--;
2243 		if (++fl->pidx == fl->cap) {
2244 			fl->pidx = 0;
2245 			sd = fl->sdesc;
2246 			d = fl->desc;
2247 		}
2248 	}
2249 
2250 	if (fl->pending >= 8)
2251 		ring_fl_db(sc, fl);
2252 
2253 	return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
2254 }
2255 
2256 #ifndef TAILQ_FOREACH_SAFE
2257 #define	TAILQ_FOREACH_SAFE(var, head, field, tvar)			\
2258 	for ((var) = TAILQ_FIRST((head));				\
2259 	    (var) && ((tvar) = TAILQ_NEXT((var), field), 1);		\
2260 	    (var) = (tvar))
2261 #endif
2262 
2263 /*
2264  * Attempt to refill all starving freelists.
2265  */
2266 static void
2267 refill_sfl(void *arg)
2268 {
2269 	struct adapter *sc = arg;
2270 	struct sge_fl *fl, *fl_temp;
2271 
2272 	mutex_enter(&sc->sfl_lock);
2273 	TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
2274 		FL_LOCK(fl);
2275 		(void) refill_fl(sc, fl, 64);
2276 		if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
2277 			TAILQ_REMOVE(&sc->sfl, fl, link);
2278 			fl->flags &= ~FL_STARVING;
2279 		}
2280 		FL_UNLOCK(fl);
2281 	}
2282 
2283 	if (!TAILQ_EMPTY(&sc->sfl) != 0)
2284 		sc->sfl_timer =  timeout(refill_sfl, sc, drv_usectohz(100000));
2285 	mutex_exit(&sc->sfl_lock);
2286 }
2287 
2288 static void
2289 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
2290 {
2291 	mutex_enter(&sc->sfl_lock);
2292 	FL_LOCK(fl);
2293 	if ((fl->flags & FL_DOOMED) == 0) {
2294 		if (TAILQ_EMPTY(&sc->sfl) != 0) {
2295 			sc->sfl_timer = timeout(refill_sfl, sc,
2296 			    drv_usectohz(100000));
2297 		}
2298 		fl->flags |= FL_STARVING;
2299 		TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
2300 	}
2301 	FL_UNLOCK(fl);
2302 	mutex_exit(&sc->sfl_lock);
2303 }
2304 
2305 static void
2306 free_fl_bufs(struct sge_fl *fl)
2307 {
2308 	struct fl_sdesc *sd;
2309 	unsigned int i;
2310 
2311 	FL_LOCK_ASSERT_OWNED(fl);
2312 
2313 	for (i = 0; i < fl->cap; i++) {
2314 		sd = &fl->sdesc[i];
2315 
2316 		if (sd->rxb != NULL) {
2317 			rxbuf_free(sd->rxb);
2318 			sd->rxb = NULL;
2319 		}
2320 	}
2321 }
2322 
2323 /*
2324  * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2325  */
2326 static mblk_t *
2327 get_fl_payload(struct adapter *sc, struct sge_fl *fl,
2328 	       uint32_t len_newbuf, int *fl_bufs_used)
2329 {
2330 	struct mblk_pair frame = {0};
2331 	struct rxbuf *rxb;
2332 	mblk_t *m = NULL;
2333 	uint_t nbuf = 0, len, copy, n;
2334 	uint32_t cidx, offset;
2335 
2336 	/*
2337 	 * The SGE won't pack a new frame into the current buffer if the entire
2338 	 * payload doesn't fit in the remaining space.  Move on to the next buf
2339 	 * in that case.
2340 	 */
2341 	if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2342 		fl->offset = 0;
2343 		if (++fl->cidx == fl->cap)
2344 			fl->cidx = 0;
2345 		nbuf++;
2346 	}
2347 	cidx = fl->cidx;
2348 	offset = fl->offset;
2349 
2350 	len = G_RSPD_LEN(len_newbuf);	/* pktshift + payload length */
2351 	copy = (len <= fl->copy_threshold);
2352 	if (copy != 0) {
2353 		frame.head = m = allocb(len, BPRI_HI);
2354 		if (m == NULL)
2355 			return (NULL);
2356 	}
2357 
2358 	while (len) {
2359 		rxb = fl->sdesc[cidx].rxb;
2360 		n = min(len, rxb->buf_size - offset);
2361 
2362 		(void) ddi_dma_sync(rxb->dhdl, offset, n,
2363 		    DDI_DMA_SYNC_FORKERNEL);
2364 
2365 		if (copy != 0)
2366 			bcopy(rxb->va + offset, m->b_wptr, n);
2367 		else {
2368 			m = desballoc((unsigned char *)rxb->va + offset, n,
2369 			    BPRI_HI, &rxb->freefunc);
2370 			if (m == NULL) {
2371 				freemsg(frame.head);
2372 				return (NULL);
2373 			}
2374 			atomic_inc_uint(&rxb->ref_cnt);
2375 			if (frame.head != NULL)
2376 				frame.tail->b_cont = m;
2377 			else
2378 				frame.head = m;
2379 			frame.tail = m;
2380 		}
2381 		m->b_wptr += n;
2382 		len -= n;
2383 		offset += roundup(n, sc->sge.fl_align);
2384 		ASSERT(offset <= rxb->buf_size);
2385 		if (offset == rxb->buf_size) {
2386 			offset = 0;
2387 			if (++cidx == fl->cap)
2388 				cidx = 0;
2389 			nbuf++;
2390 		}
2391 	}
2392 
2393 	fl->cidx = cidx;
2394 	fl->offset = offset;
2395 	(*fl_bufs_used) += nbuf;
2396 
2397 	ASSERT(frame.head != NULL);
2398 	return (frame.head);
2399 }
2400 
2401 /*
2402  * We'll do immediate data tx for non-LSO, but only when not coalescing.  We're
2403  * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2404  * of immediate data.
2405  */
2406 #define	IMM_LEN ( \
2407 	2 * EQ_ESIZE \
2408 	- sizeof (struct fw_eth_tx_pkt_wr) \
2409 	- sizeof (struct cpl_tx_pkt_core))
2410 
2411 /*
2412  * Returns non-zero on failure, no need to cleanup anything in that case.
2413  *
2414  * Note 1: We always try to pull up the mblk if required and return E2BIG only
2415  * if this fails.
2416  *
2417  * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
2418  * does not have the TCP header in it.
2419  */
2420 static int
2421 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2422     int sgl_only)
2423 {
2424 	uint32_t flags = 0, len, n;
2425 	mblk_t *m = *fp;
2426 	int rc;
2427 
2428 	TXQ_LOCK_ASSERT_OWNED(txq);	/* will manipulate txb and dma_hdls */
2429 
2430 	mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2431 	txinfo->flags = flags;
2432 
2433 	mac_lso_get(m, &txinfo->mss, &flags);
2434 	txinfo->flags |= flags;
2435 
2436 	if (flags & HW_LSO)
2437 		sgl_only = 1;	/* Do not allow immediate data with LSO */
2438 
2439 start:	txinfo->nsegs = 0;
2440 	txinfo->hdls_used = 0;
2441 	txinfo->txb_used = 0;
2442 	txinfo->len = 0;
2443 
2444 	/* total length and a rough estimate of # of segments */
2445 	n = 0;
2446 	for (; m; m = m->b_cont) {
2447 		len = MBLKL(m);
2448 		n += (len / PAGE_SIZE) + 1;
2449 		txinfo->len += len;
2450 	}
2451 	m = *fp;
2452 
2453 	if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2454 		txq->pullup_early++;
2455 		m = msgpullup(*fp, -1);
2456 		if (m == NULL) {
2457 			txq->pullup_failed++;
2458 			return (E2BIG);	/* (*fp) left as it was */
2459 		}
2460 		freemsg(*fp);
2461 		*fp = m;
2462 		mac_hcksum_set(m, NULL, NULL, NULL, NULL, txinfo->flags);
2463 	}
2464 
2465 	if (txinfo->len <= IMM_LEN && !sgl_only)
2466 		return (0);	/* nsegs = 0 tells caller to use imm. tx */
2467 
2468 	if (txinfo->len <= txq->copy_threshold &&
2469 	    copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2470 		goto done;
2471 
2472 	for (; m; m = m->b_cont) {
2473 
2474 		len = MBLKL(m);
2475 
2476 		/* Use tx copy buffer if this mblk is small enough */
2477 		if (len <= txq->copy_threshold &&
2478 		    copy_into_txb(txq, m, len, txinfo) == 0)
2479 			continue;
2480 
2481 		/* Add DMA bindings for this mblk to the SGL */
2482 		rc = add_mblk(txq, txinfo, m, len);
2483 
2484 		if (rc == E2BIG ||
2485 		    (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2486 
2487 			txq->pullup_late++;
2488 			m = msgpullup(*fp, -1);
2489 			if (m != NULL) {
2490 				free_txinfo_resources(txq, txinfo);
2491 				freemsg(*fp);
2492 				*fp = m;
2493 				mac_hcksum_set(m, NULL, NULL, NULL, NULL,
2494 				    txinfo->flags);
2495 				goto start;
2496 			}
2497 
2498 			txq->pullup_failed++;
2499 			rc = E2BIG;
2500 		}
2501 
2502 		if (rc != 0) {
2503 			free_txinfo_resources(txq, txinfo);
2504 			return (rc);
2505 		}
2506 	}
2507 
2508 	ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2509 
2510 done:
2511 
2512 	/*
2513 	 * Store the # of flits required to hold this frame's SGL in nflits.  An
2514 	 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2515 	 * multiple (len0 + len1, addr0, addr1) tuples.  If addr1 is not used
2516 	 * then len1 must be set to 0.
2517 	 */
2518 	n = txinfo->nsegs - 1;
2519 	txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2520 	if (n & 1)
2521 		txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2522 
2523 	txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2524 	    V_ULPTX_NSGE(txinfo->nsegs));
2525 
2526 	return (0);
2527 }
2528 
2529 static inline int
2530 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2531 {
2532 	if (txq->txb_avail < len)
2533 		return (0);
2534 
2535 	if (txq->txb_next + len <= txq->txb_size) {
2536 		*waste = 0;
2537 		return (1);
2538 	}
2539 
2540 	*waste = txq->txb_size - txq->txb_next;
2541 
2542 	return (txq->txb_avail - *waste < len ? 0 : 1);
2543 }
2544 
2545 #define	TXB_CHUNK	64
2546 
2547 /*
2548  * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2549  * and txq to indicate resources used.  Caller has to make sure that those many
2550  * bytes are available in the mblk chain (b_cont linked).
2551  */
2552 static inline int
2553 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2554 {
2555 	int waste, n;
2556 
2557 	TXQ_LOCK_ASSERT_OWNED(txq);	/* will manipulate txb */
2558 
2559 	if (!fits_in_txb(txq, len, &waste)) {
2560 		txq->txb_full++;
2561 		return (ENOMEM);
2562 	}
2563 
2564 	if (waste != 0) {
2565 		ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2566 		txinfo->txb_used += waste;
2567 		txq->txb_avail -= waste;
2568 		txq->txb_next = 0;
2569 	}
2570 
2571 	for (n = 0; n < len; m = m->b_cont) {
2572 		bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2573 		n += MBLKL(m);
2574 	}
2575 
2576 	add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2577 
2578 	n = roundup(len, TXB_CHUNK);
2579 	txinfo->txb_used += n;
2580 	txq->txb_avail -= n;
2581 	txq->txb_next += n;
2582 	ASSERT(txq->txb_next <= txq->txb_size);
2583 	if (txq->txb_next == txq->txb_size)
2584 		txq->txb_next = 0;
2585 
2586 	return (0);
2587 }
2588 
2589 static inline void
2590 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2591 {
2592 	ASSERT(txinfo->nsegs < TX_SGL_SEGS);	/* must have room */
2593 
2594 	if (txinfo->nsegs != 0) {
2595 		int idx = txinfo->nsegs - 1;
2596 		txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2597 		txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2598 	} else {
2599 		txinfo->sgl.len0 = cpu_to_be32(len);
2600 		txinfo->sgl.addr0 = cpu_to_be64(ba);
2601 	}
2602 	txinfo->nsegs++;
2603 }
2604 
2605 /*
2606  * This function cleans up any partially allocated resources when it fails so
2607  * there's nothing for the caller to clean up in that case.
2608  *
2609  * EIO indicates permanent failure.  Caller should drop the frame containing
2610  * this mblk and continue.
2611  *
2612  * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2613  * limit.  Caller should pull up the frame before trying to send it out.
2614  * (This error means our pullup_early heuristic did not work for this frame)
2615  *
2616  * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2617  * resources, etc.).  Caller should suspend the tx queue and wait for reclaim to
2618  * free up resources.
2619  */
2620 static inline int
2621 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2622 {
2623 	ddi_dma_handle_t dhdl;
2624 	ddi_dma_cookie_t cookie;
2625 	uint_t ccount = 0;
2626 	int rc;
2627 
2628 	TXQ_LOCK_ASSERT_OWNED(txq);	/* will manipulate dhdls */
2629 
2630 	if (txq->tx_dhdl_avail == 0) {
2631 		txq->dma_hdl_failed++;
2632 		return (ENOMEM);
2633 	}
2634 
2635 	dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2636 	rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2637 	    DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2638 	    &ccount);
2639 	if (rc != DDI_DMA_MAPPED) {
2640 		txq->dma_map_failed++;
2641 
2642 		ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2643 
2644 		return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2645 	}
2646 
2647 	if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2648 		(void) ddi_dma_unbind_handle(dhdl);
2649 		return (E2BIG);
2650 	}
2651 
2652 	add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2653 	while (--ccount) {
2654 		ddi_dma_nextcookie(dhdl, &cookie);
2655 		add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2656 	}
2657 
2658 	if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2659 		txq->tx_dhdl_pidx = 0;
2660 	txq->tx_dhdl_avail--;
2661 	txinfo->hdls_used++;
2662 
2663 	return (0);
2664 }
2665 
2666 /*
2667  * Releases all the txq resources used up in the specified txinfo.
2668  */
2669 static void
2670 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2671 {
2672 	int n;
2673 
2674 	TXQ_LOCK_ASSERT_OWNED(txq);	/* dhdls, txb */
2675 
2676 	n = txinfo->txb_used;
2677 	if (n > 0) {
2678 		txq->txb_avail += n;
2679 		if (n <= txq->txb_next)
2680 			txq->txb_next -= n;
2681 		else {
2682 			n -= txq->txb_next;
2683 			txq->txb_next = txq->txb_size - n;
2684 		}
2685 	}
2686 
2687 	for (n = txinfo->hdls_used; n > 0; n--) {
2688 		if (txq->tx_dhdl_pidx > 0)
2689 			txq->tx_dhdl_pidx--;
2690 		else
2691 			txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2692 		txq->tx_dhdl_avail++;
2693 		(void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2694 	}
2695 }
2696 
2697 /*
2698  * Returns 0 to indicate that m has been accepted into a coalesced tx work
2699  * request.  It has either been folded into txpkts or txpkts was flushed and m
2700  * has started a new coalesced work request (as the first frame in a fresh
2701  * txpkts).
2702  *
2703  * Returns non-zero to indicate a failure - caller is responsible for
2704  * transmitting m, if there was anything in txpkts it has been flushed.
2705  */
2706 static int
2707 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2708     struct txinfo *txinfo)
2709 {
2710 	struct sge_eq *eq = &txq->eq;
2711 	int can_coalesce;
2712 	struct tx_sdesc *txsd;
2713 	uint8_t flits;
2714 
2715 	TXQ_LOCK_ASSERT_OWNED(txq);
2716 
2717 	if (txpkts->npkt > 0) {
2718 		flits = TXPKTS_PKT_HDR + txinfo->nflits;
2719 		can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2720 		    txpkts->nflits + flits <= TX_WR_FLITS &&
2721 		    txpkts->nflits + flits <= eq->avail * 8 &&
2722 		    txpkts->plen + txinfo->len < 65536;
2723 
2724 		if (can_coalesce != 0) {
2725 			txpkts->tail->b_next = m;
2726 			txpkts->tail = m;
2727 			txpkts->npkt++;
2728 			txpkts->nflits += flits;
2729 			txpkts->plen += txinfo->len;
2730 
2731 			txsd = &txq->sdesc[eq->pidx];
2732 			txsd->txb_used += txinfo->txb_used;
2733 			txsd->hdls_used += txinfo->hdls_used;
2734 
2735 			return (0);
2736 		}
2737 
2738 		/*
2739 		 * Couldn't coalesce m into txpkts.  The first order of business
2740 		 * is to send txpkts on its way.  Then we'll revisit m.
2741 		 */
2742 		write_txpkts_wr(txq, txpkts);
2743 	}
2744 
2745 	/*
2746 	 * Check if we can start a new coalesced tx work request with m as
2747 	 * the first packet in it.
2748 	 */
2749 
2750 	ASSERT(txpkts->npkt == 0);
2751 	ASSERT(txinfo->len < 65536);
2752 
2753 	flits = TXPKTS_WR_HDR + txinfo->nflits;
2754 	can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2755 	    flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2756 
2757 	if (can_coalesce == 0)
2758 		return (EINVAL);
2759 
2760 	/*
2761 	 * Start a fresh coalesced tx WR with m as the first frame in it.
2762 	 */
2763 	txpkts->tail = m;
2764 	txpkts->npkt = 1;
2765 	txpkts->nflits = flits;
2766 	txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2767 	txpkts->plen = txinfo->len;
2768 
2769 	txsd = &txq->sdesc[eq->pidx];
2770 	txsd->m = m;
2771 	txsd->txb_used = txinfo->txb_used;
2772 	txsd->hdls_used = txinfo->hdls_used;
2773 
2774 	return (0);
2775 }
2776 
2777 /*
2778  * Note that write_txpkts_wr can never run out of hardware descriptors (but
2779  * write_txpkt_wr can).  add_to_txpkts ensures that a frame is accepted for
2780  * coalescing only if sufficient hardware descriptors are available.
2781  */
2782 static void
2783 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2784 {
2785 	struct sge_eq *eq = &txq->eq;
2786 	struct fw_eth_tx_pkts_wr *wr;
2787 	struct tx_sdesc *txsd;
2788 	uint32_t ctrl;
2789 	uint16_t ndesc;
2790 
2791 	TXQ_LOCK_ASSERT_OWNED(txq);	/* pidx, avail */
2792 
2793 	ndesc = howmany(txpkts->nflits, 8);
2794 
2795 	wr = (void *)&eq->desc[eq->pidx];
2796 	wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2797 	    V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2798 	ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2799 	if (eq->avail == ndesc)
2800 		ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2801 	wr->equiq_to_len16 = cpu_to_be32(ctrl);
2802 	wr->plen = cpu_to_be16(txpkts->plen);
2803 	wr->npkt = txpkts->npkt;
2804 	wr->r3 = wr->type = 0;
2805 
2806 	/* Everything else already written */
2807 
2808 	txsd = &txq->sdesc[eq->pidx];
2809 	txsd->desc_used = ndesc;
2810 
2811 	txq->txb_used += txsd->txb_used / TXB_CHUNK;
2812 	txq->hdl_used += txsd->hdls_used;
2813 
2814 	ASSERT(eq->avail >= ndesc);
2815 
2816 	eq->pending += ndesc;
2817 	eq->avail -= ndesc;
2818 	eq->pidx += ndesc;
2819 	if (eq->pidx >= eq->cap)
2820 		eq->pidx -= eq->cap;
2821 
2822 	txq->txpkts_pkts += txpkts->npkt;
2823 	txq->txpkts_wrs++;
2824 	txpkts->npkt = 0;	/* emptied */
2825 }
2826 
2827 static int
2828 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2829     struct txinfo *txinfo)
2830 {
2831 	struct sge_eq *eq = &txq->eq;
2832 	struct fw_eth_tx_pkt_wr *wr;
2833 	struct cpl_tx_pkt_core *cpl;
2834 	uint32_t ctrl;	/* used in many unrelated places */
2835 	uint64_t ctrl1;
2836 	int nflits, ndesc;
2837 	struct tx_sdesc *txsd;
2838 	caddr_t dst;
2839 
2840 	TXQ_LOCK_ASSERT_OWNED(txq);	/* pidx, avail */
2841 
2842 	/*
2843 	 * Do we have enough flits to send this frame out?
2844 	 */
2845 	ctrl = sizeof (struct cpl_tx_pkt_core);
2846 	if (txinfo->flags & HW_LSO) {
2847 		nflits = TXPKT_LSO_WR_HDR;
2848 		ctrl += sizeof(struct cpl_tx_pkt_lso_core);
2849 	} else
2850 		nflits = TXPKT_WR_HDR;
2851 	if (txinfo->nsegs > 0)
2852 		nflits += txinfo->nflits;
2853 	else {
2854 		nflits += howmany(txinfo->len, 8);
2855 		ctrl += txinfo->len;
2856 	}
2857 	ndesc = howmany(nflits, 8);
2858 	if (ndesc > eq->avail)
2859 		return (ENOMEM);
2860 
2861 	/* Firmware work request header */
2862 	wr = (void *)&eq->desc[eq->pidx];
2863 	wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2864 	    V_FW_WR_IMMDLEN(ctrl));
2865 	ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2866 	if (eq->avail == ndesc)
2867 		ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2868 	wr->equiq_to_len16 = cpu_to_be32(ctrl);
2869 	wr->r3 = 0;
2870 
2871 	if (txinfo->flags & HW_LSO) {
2872 		struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
2873 		char *p = (void *)m->b_rptr;
2874 		ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2875 		    F_LSO_LAST_SLICE;
2876 
2877 		/* LINTED: E_BAD_PTR_CAST_ALIGN */
2878 		if (((struct ether_header *)p)->ether_type ==
2879 		    htons(ETHERTYPE_VLAN)) {
2880 			ctrl |= V_LSO_ETHHDR_LEN(1);
2881 			p += sizeof (struct ether_vlan_header);
2882 		} else
2883 			p += sizeof (struct ether_header);
2884 
2885 		/* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2886 		ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4);
2887 		/* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2888 		p += IPH_HDR_LENGTH(p);
2889 		ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4);
2890 
2891 		lso->lso_ctrl = cpu_to_be32(ctrl);
2892 		lso->ipid_ofst = cpu_to_be16(0);
2893 		lso->mss = cpu_to_be16(txinfo->mss);
2894 		lso->seqno_offset = cpu_to_be32(0);
2895 		if (is_t4(pi->adapter->params.chip))
2896 			lso->len = cpu_to_be32(txinfo->len);
2897 		else
2898 			lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len));
2899 
2900 		cpl = (void *)(lso + 1);
2901 
2902 		txq->tso_wrs++;
2903 	} else
2904 		cpl = (void *)(wr + 1);
2905 
2906 	/* Checksum offload */
2907 	ctrl1 = 0;
2908 	if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2909 		ctrl1 |= F_TXPKT_IPCSUM_DIS;
2910 	if (!(txinfo->flags & HCK_FULLCKSUM))
2911 		ctrl1 |= F_TXPKT_L4CSUM_DIS;
2912 	if (ctrl1 == 0)
2913 		txq->txcsum++;	/* some hardware assistance provided */
2914 
2915 	/* CPL header */
2916 	cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2917 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2918 	cpl->pack = 0;
2919 	cpl->len = cpu_to_be16(txinfo->len);
2920 	cpl->ctrl1 = cpu_to_be64(ctrl1);
2921 
2922 	/* Software descriptor */
2923 	txsd = &txq->sdesc[eq->pidx];
2924 	txsd->m = m;
2925 	txsd->txb_used = txinfo->txb_used;
2926 	txsd->hdls_used = txinfo->hdls_used;
2927 	/* LINTED: E_ASSIGN_NARROW_CONV */
2928 	txsd->desc_used = ndesc;
2929 
2930 	txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2931 	txq->hdl_used += txinfo->hdls_used;
2932 
2933 	eq->pending += ndesc;
2934 	eq->avail -= ndesc;
2935 	eq->pidx += ndesc;
2936 	if (eq->pidx >= eq->cap)
2937 		eq->pidx -= eq->cap;
2938 
2939 	/* SGL */
2940 	dst = (void *)(cpl + 1);
2941 	if (txinfo->nsegs > 0) {
2942 		txq->sgl_wrs++;
2943 		copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2944 
2945 		/* Need to zero-pad to a 16 byte boundary if not on one */
2946 		if ((uintptr_t)dst & 0xf)
2947 			/* LINTED: E_BAD_PTR_CAST_ALIGN */
2948 			*(uint64_t *)dst = 0;
2949 
2950 	} else {
2951 		txq->imm_wrs++;
2952 #ifdef DEBUG
2953 		ctrl = txinfo->len;
2954 #endif
2955 		for (; m; m = m->b_cont) {
2956 			copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
2957 #ifdef DEBUG
2958 			ctrl -= MBLKL(m);
2959 #endif
2960 		}
2961 		ASSERT(ctrl == 0);
2962 	}
2963 
2964 	txq->txpkt_wrs++;
2965 	return (0);
2966 }
2967 
2968 static inline void
2969 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
2970     struct txpkts *txpkts, struct txinfo *txinfo)
2971 {
2972 	struct ulp_txpkt *ulpmc;
2973 	struct ulptx_idata *ulpsc;
2974 	struct cpl_tx_pkt_core *cpl;
2975 	uintptr_t flitp, start, end;
2976 	uint64_t ctrl;
2977 	caddr_t dst;
2978 
2979 	ASSERT(txpkts->npkt > 0);
2980 
2981 	start = (uintptr_t)txq->eq.desc;
2982 	end = (uintptr_t)txq->eq.spg;
2983 
2984 	/* Checksum offload */
2985 	ctrl = 0;
2986 	if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2987 		ctrl |= F_TXPKT_IPCSUM_DIS;
2988 	if (!(txinfo->flags & HCK_FULLCKSUM))
2989 		ctrl |= F_TXPKT_L4CSUM_DIS;
2990 	if (ctrl == 0)
2991 		txq->txcsum++;	/* some hardware assistance provided */
2992 
2993 	/*
2994 	 * The previous packet's SGL must have ended at a 16 byte boundary (this
2995 	 * is required by the firmware/hardware).  It follows that flitp cannot
2996 	 * wrap around between the ULPTX master command and ULPTX subcommand (8
2997 	 * bytes each), and that it can not wrap around in the middle of the
2998 	 * cpl_tx_pkt_core either.
2999 	 */
3000 	flitp = (uintptr_t)txpkts->flitp;
3001 	ASSERT((flitp & 0xf) == 0);
3002 
3003 	/* ULP master command */
3004 	ulpmc = (void *)flitp;
3005 	ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
3006 	ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
3007 	    sizeof (*cpl) + 8 * txinfo->nflits, 16));
3008 
3009 	/* ULP subcommand */
3010 	ulpsc = (void *)(ulpmc + 1);
3011 	ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
3012 	    F_ULP_TX_SC_MORE);
3013 	ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
3014 
3015 	flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
3016 	if (flitp == end)
3017 		flitp = start;
3018 
3019 	/* CPL_TX_PKT */
3020 	cpl = (void *)flitp;
3021 	cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3022 	    V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
3023 	cpl->pack = 0;
3024 	cpl->len = cpu_to_be16(txinfo->len);
3025 	cpl->ctrl1 = cpu_to_be64(ctrl);
3026 
3027 	flitp += sizeof (*cpl);
3028 	if (flitp == end)
3029 		flitp = start;
3030 
3031 	/* SGL for this frame */
3032 	dst = (caddr_t)flitp;
3033 	copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
3034 	flitp = (uintptr_t)dst;
3035 
3036 	/* Zero pad and advance to a 16 byte boundary if not already at one. */
3037 	if (flitp & 0xf) {
3038 
3039 		/* no matter what, flitp should be on an 8 byte boundary */
3040 		ASSERT((flitp & 0x7) == 0);
3041 
3042 		*(uint64_t *)flitp = 0;
3043 		flitp += sizeof (uint64_t);
3044 		txpkts->nflits++;
3045 	}
3046 
3047 	if (flitp == end)
3048 		flitp = start;
3049 
3050 	txpkts->flitp = (void *)flitp;
3051 }
3052 
3053 static inline void
3054 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
3055 {
3056 	if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
3057 		bcopy(from, *to, len);
3058 		(*to) += len;
3059 	} else {
3060 		int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
3061 
3062 		bcopy(from, *to, portion);
3063 		from += portion;
3064 		portion = len - portion;	/* remaining */
3065 		bcopy(from, (void *)eq->desc, portion);
3066 		(*to) = (caddr_t)eq->desc + portion;
3067 	}
3068 }
3069 
3070 static inline void
3071 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
3072 {
3073 	int val, db_mode;
3074 	u_int db = eq->doorbells;
3075 
3076 	if (eq->pending > 1)
3077 		db &= ~DOORBELL_WCWR;
3078 
3079 	if (eq->pending > eq->pidx) {
3080 		int offset = eq->cap - (eq->pending - eq->pidx);
3081 
3082 		/* pidx has wrapped around since last doorbell */
3083 
3084 		(void) ddi_dma_sync(eq->desc_dhdl,
3085 		    offset * sizeof (struct tx_desc), 0,
3086 		    DDI_DMA_SYNC_FORDEV);
3087 		(void) ddi_dma_sync(eq->desc_dhdl,
3088 		    0, eq->pidx * sizeof (struct tx_desc),
3089 		    DDI_DMA_SYNC_FORDEV);
3090 	} else if (eq->pending > 0) {
3091 		(void) ddi_dma_sync(eq->desc_dhdl,
3092 		    (eq->pidx - eq->pending) * sizeof (struct tx_desc),
3093 		    eq->pending * sizeof (struct tx_desc),
3094 		    DDI_DMA_SYNC_FORDEV);
3095 	}
3096 
3097 	membar_producer();
3098 
3099 	if (is_t4(sc->params.chip))
3100 		val = V_PIDX(eq->pending);
3101 	else
3102 		val = V_PIDX_T5(eq->pending);
3103 
3104 	db_mode = (1 << (ffs(db) - 1));
3105 	switch (db_mode) {
3106 		case DOORBELL_UDB:
3107 			*eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3108 			break;
3109 
3110 		case DOORBELL_WCWR:
3111 			{
3112 				volatile uint64_t *dst, *src;
3113 				int i;
3114 				/*
3115 				 * Queues whose 128B doorbell segment fits in
3116 				 * the page do not use relative qid
3117 				 * (udb_qid is always 0).  Only queues with
3118 				 * doorbell segments can do WCWR.
3119 				 */
3120 				ASSERT(eq->udb_qid == 0 && eq->pending == 1);
3121 
3122 				dst = (volatile void *)((uintptr_t)eq->udb +
3123 				    UDBS_WR_OFFSET - UDBS_DB_OFFSET);
3124 				i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
3125 				src = (void *)&eq->desc[i];
3126 				while (src != (void *)&eq->desc[i + 1])
3127 				        *dst++ = *src++;
3128 				membar_producer();
3129 				break;
3130 			}
3131 
3132 		case DOORBELL_UDBWC:
3133 			*eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3134 			membar_producer();
3135 			break;
3136 
3137 		case DOORBELL_KDB:
3138 			t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
3139 			    V_QID(eq->cntxt_id) | val);
3140 			break;
3141 	}
3142 
3143 	eq->pending = 0;
3144 }
3145 
3146 static int
3147 reclaim_tx_descs(struct sge_txq *txq, int howmany)
3148 {
3149 	struct tx_sdesc *txsd;
3150 	uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
3151 	struct sge_eq *eq = &txq->eq;
3152 
3153 	EQ_LOCK_ASSERT_OWNED(eq);
3154 
3155 	cidx = eq->spg->cidx;	/* stable snapshot */
3156 	cidx = be16_to_cpu(cidx);
3157 
3158 	if (cidx >= eq->cidx)
3159 		can_reclaim = cidx - eq->cidx;
3160 	else
3161 		can_reclaim = cidx + eq->cap - eq->cidx;
3162 
3163 	if (can_reclaim == 0)
3164 		return (0);
3165 
3166 	txb_freed = hdls_freed = reclaimed = 0;
3167 	do {
3168 		int ndesc;
3169 
3170 		txsd = &txq->sdesc[eq->cidx];
3171 		ndesc = txsd->desc_used;
3172 
3173 		/* Firmware doesn't return "partial" credits. */
3174 		ASSERT(can_reclaim >= ndesc);
3175 
3176 		/*
3177 		 * We always keep mblk around, even for immediate data.  If mblk
3178 		 * is NULL, this has to be the software descriptor for a credit
3179 		 * flush work request.
3180 		 */
3181 		if (txsd->m != NULL)
3182 			freemsgchain(txsd->m);
3183 #ifdef DEBUG
3184 		else {
3185 			ASSERT(txsd->txb_used == 0);
3186 			ASSERT(txsd->hdls_used == 0);
3187 			ASSERT(ndesc == 1);
3188 		}
3189 #endif
3190 
3191 		txb_freed += txsd->txb_used;
3192 		hdls_freed += txsd->hdls_used;
3193 		reclaimed += ndesc;
3194 
3195 		eq->cidx += ndesc;
3196 		if (eq->cidx >= eq->cap)
3197 			eq->cidx -= eq->cap;
3198 
3199 		can_reclaim -= ndesc;
3200 
3201 	} while (can_reclaim && reclaimed < howmany);
3202 
3203 	eq->avail += reclaimed;
3204 	ASSERT(eq->avail < eq->cap);	/* avail tops out at (cap - 1) */
3205 
3206 	txq->txb_avail += txb_freed;
3207 
3208 	txq->tx_dhdl_avail += hdls_freed;
3209 	ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
3210 	for (; hdls_freed; hdls_freed--) {
3211 		(void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
3212 		if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
3213 			txq->tx_dhdl_cidx = 0;
3214 	}
3215 
3216 	return (reclaimed);
3217 }
3218 
3219 static void
3220 write_txqflush_wr(struct sge_txq *txq)
3221 {
3222 	struct sge_eq *eq = &txq->eq;
3223 	struct fw_eq_flush_wr *wr;
3224 	struct tx_sdesc *txsd;
3225 
3226 	EQ_LOCK_ASSERT_OWNED(eq);
3227 	ASSERT(eq->avail > 0);
3228 
3229 	wr = (void *)&eq->desc[eq->pidx];
3230 	bzero(wr, sizeof (*wr));
3231 	wr->opcode = FW_EQ_FLUSH_WR;
3232 	wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
3233 	    F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3234 
3235 	txsd = &txq->sdesc[eq->pidx];
3236 	txsd->m = NULL;
3237 	txsd->txb_used = 0;
3238 	txsd->hdls_used = 0;
3239 	txsd->desc_used = 1;
3240 
3241 	eq->pending++;
3242 	eq->avail--;
3243 	if (++eq->pidx == eq->cap)
3244 		eq->pidx = 0;
3245 }
3246 
3247 static int
3248 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3249 {
3250 	bool csum_ok;
3251 	uint16_t err_vec;
3252 	struct sge_rxq *rxq = (void *)iq;
3253 	struct mblk_pair chain = {0};
3254 	struct adapter *sc = iq->adapter;
3255 	const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
3256 
3257 	iq->intr_next = iq->intr_params;
3258 
3259 	m->b_rptr += sc->sge.pktshift;
3260 
3261 	/* Compressed error vector is enabled for T6 only */
3262 	if (sc->params.tp.rx_pkt_encap)
3263 		/* It is enabled only in T6 config file */
3264 		err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
3265 	else
3266 		err_vec = ntohs(cpl->err_vec);
3267 
3268 	csum_ok = cpl->csum_calc && !err_vec;
3269 	/* TODO: what about cpl->ip_frag? */
3270 	if (csum_ok && !cpl->ip_frag) {
3271 		mac_hcksum_set(m, 0, 0, 0, 0xffff,
3272 		    HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
3273 		    HCK_IPV4_HDRCKSUM_OK);
3274 		rxq->rxcsum++;
3275 	}
3276 
3277 	/* Add to the chain that we'll send up */
3278 	if (chain.head != NULL)
3279 		chain.tail->b_next = m;
3280 	else
3281 		chain.head = m;
3282 	chain.tail = m;
3283 
3284 	t4_mac_rx(rxq->port, rxq, chain.head);
3285 
3286 	rxq->rxpkts++;
3287 	rxq->rxbytes  += be16_to_cpu(cpl->len);
3288 	return (0);
3289 }
3290 
3291 #define	FL_HW_IDX(idx)	((idx) >> 3)
3292 
3293 static inline void
3294 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3295 {
3296 	int desc_start, desc_last, ndesc;
3297 	uint32_t v = sc->params.arch.sge_fl_db ;
3298 
3299 	ndesc = FL_HW_IDX(fl->pending);
3300 
3301 	/* Hold back one credit if pidx = cidx */
3302 	if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3303 		ndesc--;
3304 
3305 	/*
3306 	 * There are chances of ndesc modified above (to avoid pidx = cidx).
3307 	 * If there is nothing to post, return.
3308 	 */
3309 	if (ndesc <= 0)
3310 		return;
3311 
3312 	desc_last = FL_HW_IDX(fl->pidx);
3313 
3314 	if (fl->pidx < fl->pending) {
3315 		/* There was a wrap */
3316 		desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
3317 
3318 		/* From desc_start to the end of list */
3319 		(void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
3320 		    DDI_DMA_SYNC_FORDEV);
3321 
3322 		/* From start of list to the desc_last */
3323 		if (desc_last != 0)
3324 			(void) ddi_dma_sync(fl->dhdl, 0, desc_last *
3325 			    RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3326 	} else {
3327 		/* There was no wrap, sync from start_desc to last_desc */
3328 		desc_start = FL_HW_IDX(fl->pidx - fl->pending);
3329 		(void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
3330 		    ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3331 	}
3332 
3333 	if (is_t4(sc->params.chip))
3334 		v |= V_PIDX(ndesc);
3335 	else
3336 		v |= V_PIDX_T5(ndesc);
3337 	v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3338 
3339 	membar_producer();
3340 
3341 	t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3342 
3343 	/*
3344 	 * Update pending count:
3345 	 * Deduct the number of descriptors posted
3346 	 */
3347 	fl->pending -= ndesc * 8;
3348 }
3349 
3350 static void
3351 tx_reclaim_task(void *arg)
3352 {
3353 	struct sge_txq *txq = arg;
3354 
3355 	TXQ_LOCK(txq);
3356 	reclaim_tx_descs(txq, txq->eq.qsize);
3357 	TXQ_UNLOCK(txq);
3358 }
3359 
3360 /* ARGSUSED */
3361 static int
3362 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3363                 mblk_t *m)
3364 {
3365 	const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3366 	unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3367 	struct adapter *sc = iq->adapter;
3368 	struct sge *s = &sc->sge;
3369 	struct sge_eq *eq;
3370 	struct sge_txq *txq;
3371 
3372 	txq = (void *)s->eqmap[qid - s->eq_start];
3373 	eq = &txq->eq;
3374 	txq->qflush++;
3375 	t4_mac_tx_update(txq->port, txq);
3376 
3377 	ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3378 		(void *)txq, DDI_NOSLEEP);
3379 
3380 	return (0);
3381 }
3382 
3383 static int
3384 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3385 {
3386 	struct adapter *sc = iq->adapter;
3387 	const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3388 
3389 	ASSERT(m == NULL);
3390 
3391 	if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3392 		const struct rss_header *rss2;
3393 
3394 		rss2 = (const struct rss_header *)&cpl->data[0];
3395 		return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3396 	}
3397 	return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3398 }
3399 
3400 int
3401 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3402     int flags)
3403 {
3404 	int i, rc;
3405 
3406 	txmaps->map_total =  count;
3407 	txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3408 
3409 	txmaps->map =  kmem_zalloc(sizeof (ddi_dma_handle_t) *
3410 	    txmaps->map_total, flags);
3411 
3412 	for (i = 0; i < count; i++) {
3413 		rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3414 		    DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3415 		if (rc != DDI_SUCCESS) {
3416 			cxgb_printf(sc->dip, CE_WARN,
3417 			    "%s: failed to allocate DMA handle (%d)",
3418 			    __func__, rc);
3419 			return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3420 		}
3421 		txmaps->map_avail++;
3422 	}
3423 
3424 	return (0);
3425 }
3426 
3427 #define	KS_UINIT(x)	kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3428 #define	KS_CINIT(x)	kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3429 #define	KS_U_SET(x, y)	kstatp->x.value.ul = (y)
3430 #define	KS_U_FROM(x, y)	kstatp->x.value.ul = (y)->x
3431 #define	KS_C_SET(x, ...)	\
3432 			(void) snprintf(kstatp->x.value.c, 16,  __VA_ARGS__)
3433 
3434 /*
3435  * cxgbe:X:config
3436  */
3437 struct cxgbe_port_config_kstats {
3438 	kstat_named_t idx;
3439 	kstat_named_t nrxq;
3440 	kstat_named_t ntxq;
3441 	kstat_named_t first_rxq;
3442 	kstat_named_t first_txq;
3443 	kstat_named_t controller;
3444 	kstat_named_t factory_mac_address;
3445 };
3446 
3447 /*
3448  * cxgbe:X:info
3449  */
3450 struct cxgbe_port_info_kstats {
3451 	kstat_named_t transceiver;
3452 	kstat_named_t rx_ovflow0;
3453 	kstat_named_t rx_ovflow1;
3454 	kstat_named_t rx_ovflow2;
3455 	kstat_named_t rx_ovflow3;
3456 	kstat_named_t rx_trunc0;
3457 	kstat_named_t rx_trunc1;
3458 	kstat_named_t rx_trunc2;
3459 	kstat_named_t rx_trunc3;
3460 	kstat_named_t tx_pause;
3461 	kstat_named_t rx_pause;
3462 };
3463 
3464 static kstat_t *
3465 setup_port_config_kstats(struct port_info *pi)
3466 {
3467 	kstat_t *ksp;
3468 	struct cxgbe_port_config_kstats *kstatp;
3469 	int ndata;
3470 	dev_info_t *pdip = ddi_get_parent(pi->dip);
3471 	uint8_t *ma = &pi->hw_addr[0];
3472 
3473 	ndata = sizeof (struct cxgbe_port_config_kstats) /
3474 	    sizeof (kstat_named_t);
3475 
3476 	ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3477 	    "net", KSTAT_TYPE_NAMED, ndata, 0);
3478 	if (ksp == NULL) {
3479 		cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3480 		return (NULL);
3481 	}
3482 
3483 	kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3484 
3485 	KS_UINIT(idx);
3486 	KS_UINIT(nrxq);
3487 	KS_UINIT(ntxq);
3488 	KS_UINIT(first_rxq);
3489 	KS_UINIT(first_txq);
3490 	KS_CINIT(controller);
3491 	KS_CINIT(factory_mac_address);
3492 
3493 	KS_U_SET(idx, pi->port_id);
3494 	KS_U_SET(nrxq, pi->nrxq);
3495 	KS_U_SET(ntxq, pi->ntxq);
3496 	KS_U_SET(first_rxq, pi->first_rxq);
3497 	KS_U_SET(first_txq, pi->first_txq);
3498 	KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3499 	    ddi_get_instance(pdip));
3500 	KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3501 	    ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3502 
3503 	/* Do NOT set ksp->ks_update.  These kstats do not change. */
3504 
3505 	/* Install the kstat */
3506 	ksp->ks_private = (void *)pi;
3507 	kstat_install(ksp);
3508 
3509 	return (ksp);
3510 }
3511 
3512 static kstat_t *
3513 setup_port_info_kstats(struct port_info *pi)
3514 {
3515 	kstat_t *ksp;
3516 	struct cxgbe_port_info_kstats *kstatp;
3517 	int ndata;
3518 
3519 	ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3520 
3521 	ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3522 	    "net", KSTAT_TYPE_NAMED, ndata, 0);
3523 	if (ksp == NULL) {
3524 		cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3525 		return (NULL);
3526 	}
3527 
3528 	kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3529 
3530 	KS_CINIT(transceiver);
3531 	KS_UINIT(rx_ovflow0);
3532 	KS_UINIT(rx_ovflow1);
3533 	KS_UINIT(rx_ovflow2);
3534 	KS_UINIT(rx_ovflow3);
3535 	KS_UINIT(rx_trunc0);
3536 	KS_UINIT(rx_trunc1);
3537 	KS_UINIT(rx_trunc2);
3538 	KS_UINIT(rx_trunc3);
3539 	KS_UINIT(tx_pause);
3540 	KS_UINIT(rx_pause);
3541 
3542 	/* Install the kstat */
3543 	ksp->ks_update = update_port_info_kstats;
3544 	ksp->ks_private = (void *)pi;
3545 	kstat_install(ksp);
3546 
3547 	return (ksp);
3548 }
3549 
3550 static int
3551 update_port_info_kstats(kstat_t *ksp, int rw)
3552 {
3553 	struct cxgbe_port_info_kstats *kstatp =
3554 	    (struct cxgbe_port_info_kstats *)ksp->ks_data;
3555 	struct port_info *pi = ksp->ks_private;
3556 	static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3557 	    "active TWINAX", "LRM" };
3558 	uint32_t bgmap;
3559 
3560 	if (rw == KSTAT_WRITE)
3561 		return (0);
3562 
3563 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3564 		KS_C_SET(transceiver, "unplugged");
3565 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3566 		KS_C_SET(transceiver, "unknown");
3567 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3568 		KS_C_SET(transceiver, "unsupported");
3569 	else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3570 		KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3571 	else
3572 		KS_C_SET(transceiver, "type %d", pi->mod_type);
3573 
3574 #define	GET_STAT(name) t4_read_reg64(pi->adapter, \
3575 	    PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3576 #define	GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3577 	    A_MPS_STAT_##name##_L)
3578 
3579 	bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3580 	if (bgmap == 0)
3581 		bgmap = (pi->port_id == 0) ? 0xf : 0;
3582 	else if (bgmap == 1)
3583 		bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3584 	else
3585 		bgmap = 1;
3586 
3587 	KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3588 	    GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3589 	KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3590 	    GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3591 	KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3592 	    GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3593 	KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3594 	    GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3595 	KS_U_SET(rx_trunc0,  (bgmap & 1) ?
3596 	    GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3597 	KS_U_SET(rx_trunc1,  (bgmap & 2) ?
3598 	    GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3599 	KS_U_SET(rx_trunc2,  (bgmap & 4) ?
3600 	    GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3601 	KS_U_SET(rx_trunc3,  (bgmap & 8) ?
3602 	    GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3603 
3604 	KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3605 	KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3606 
3607 	return (0);
3608 
3609 }
3610 
3611 /*
3612  * cxgbe:X:rxqY
3613  */
3614 struct rxq_kstats {
3615 	kstat_named_t rxcsum;
3616 	kstat_named_t rxpkts;
3617 	kstat_named_t rxbytes;
3618 	kstat_named_t nomem;
3619 };
3620 
3621 static kstat_t *
3622 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3623 {
3624 	struct kstat *ksp;
3625 	struct rxq_kstats *kstatp;
3626 	int ndata;
3627 	char str[16];
3628 
3629 	ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3630 	(void) snprintf(str, sizeof (str), "rxq%u", idx);
3631 
3632 	ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3633 	    KSTAT_TYPE_NAMED, ndata, 0);
3634 	if (ksp == NULL) {
3635 		cxgb_printf(pi->dip, CE_WARN,
3636 		    "%s: failed to initialize rxq kstats for queue %d.",
3637 		    __func__, idx);
3638 		return (NULL);
3639 	}
3640 
3641 	kstatp = (struct rxq_kstats *)ksp->ks_data;
3642 
3643 	KS_UINIT(rxcsum);
3644 	KS_UINIT(rxpkts);
3645 	KS_UINIT(rxbytes);
3646 	KS_UINIT(nomem);
3647 
3648 	ksp->ks_update = update_rxq_kstats;
3649 	ksp->ks_private = (void *)rxq;
3650 	kstat_install(ksp);
3651 
3652 	return (ksp);
3653 }
3654 
3655 static int
3656 update_rxq_kstats(kstat_t *ksp, int rw)
3657 {
3658 	struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3659 	struct sge_rxq *rxq = ksp->ks_private;
3660 
3661 	if (rw == KSTAT_WRITE)
3662 		return (0);
3663 
3664 	KS_U_FROM(rxcsum, rxq);
3665 	KS_U_FROM(rxpkts, rxq);
3666 	KS_U_FROM(rxbytes, rxq);
3667 	KS_U_FROM(nomem, rxq);
3668 
3669 	return (0);
3670 }
3671 
3672 /*
3673  * cxgbe:X:txqY
3674  */
3675 struct txq_kstats {
3676 	kstat_named_t txcsum;
3677 	kstat_named_t tso_wrs;
3678 	kstat_named_t imm_wrs;
3679 	kstat_named_t sgl_wrs;
3680 	kstat_named_t txpkt_wrs;
3681 	kstat_named_t txpkts_wrs;
3682 	kstat_named_t txpkts_pkts;
3683 	kstat_named_t txb_used;
3684 	kstat_named_t hdl_used;
3685 	kstat_named_t txb_full;
3686 	kstat_named_t dma_hdl_failed;
3687 	kstat_named_t dma_map_failed;
3688 	kstat_named_t qfull;
3689 	kstat_named_t qflush;
3690 	kstat_named_t pullup_early;
3691 	kstat_named_t pullup_late;
3692 	kstat_named_t pullup_failed;
3693 };
3694 
3695 static kstat_t *
3696 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3697 {
3698 	struct kstat *ksp;
3699 	struct txq_kstats *kstatp;
3700 	int ndata;
3701 	char str[16];
3702 
3703 	ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3704 	(void) snprintf(str, sizeof (str), "txq%u", idx);
3705 
3706 	ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3707 	    KSTAT_TYPE_NAMED, ndata, 0);
3708 	if (ksp == NULL) {
3709 		cxgb_printf(pi->dip, CE_WARN,
3710 		    "%s: failed to initialize txq kstats for queue %d.",
3711 		    __func__, idx);
3712 		return (NULL);
3713 	}
3714 
3715 	kstatp = (struct txq_kstats *)ksp->ks_data;
3716 
3717 	KS_UINIT(txcsum);
3718 	KS_UINIT(tso_wrs);
3719 	KS_UINIT(imm_wrs);
3720 	KS_UINIT(sgl_wrs);
3721 	KS_UINIT(txpkt_wrs);
3722 	KS_UINIT(txpkts_wrs);
3723 	KS_UINIT(txpkts_pkts);
3724 	KS_UINIT(txb_used);
3725 	KS_UINIT(hdl_used);
3726 	KS_UINIT(txb_full);
3727 	KS_UINIT(dma_hdl_failed);
3728 	KS_UINIT(dma_map_failed);
3729 	KS_UINIT(qfull);
3730 	KS_UINIT(qflush);
3731 	KS_UINIT(pullup_early);
3732 	KS_UINIT(pullup_late);
3733 	KS_UINIT(pullup_failed);
3734 
3735 	ksp->ks_update = update_txq_kstats;
3736 	ksp->ks_private = (void *)txq;
3737 	kstat_install(ksp);
3738 
3739 	return (ksp);
3740 }
3741 
3742 static int
3743 update_txq_kstats(kstat_t *ksp, int rw)
3744 {
3745 	struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3746 	struct sge_txq *txq = ksp->ks_private;
3747 
3748 	if (rw == KSTAT_WRITE)
3749 		return (0);
3750 
3751 	KS_U_FROM(txcsum, txq);
3752 	KS_U_FROM(tso_wrs, txq);
3753 	KS_U_FROM(imm_wrs, txq);
3754 	KS_U_FROM(sgl_wrs, txq);
3755 	KS_U_FROM(txpkt_wrs, txq);
3756 	KS_U_FROM(txpkts_wrs, txq);
3757 	KS_U_FROM(txpkts_pkts, txq);
3758 	KS_U_FROM(txb_used, txq);
3759 	KS_U_FROM(hdl_used, txq);
3760 	KS_U_FROM(txb_full, txq);
3761 	KS_U_FROM(dma_hdl_failed, txq);
3762 	KS_U_FROM(dma_map_failed, txq);
3763 	KS_U_FROM(qfull, txq);
3764 	KS_U_FROM(qflush, txq);
3765 	KS_U_FROM(pullup_early, txq);
3766 	KS_U_FROM(pullup_late, txq);
3767 	KS_U_FROM(pullup_failed, txq);
3768 
3769 	return (0);
3770 }
3771