1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <hxge_impl.h>
27 #include <hxge_txdma.h>
28 #include <sys/llc1.h>
29 
30 uint32_t hxge_reclaim_pending = TXDMA_RECLAIM_PENDING_DEFAULT;
31 uint32_t hxge_tx_minfree = 64;
32 uint32_t hxge_tx_intr_thres = 0;
33 uint32_t hxge_tx_max_gathers = TX_MAX_GATHER_POINTERS;
34 uint32_t hxge_tx_tiny_pack = 1;
35 uint32_t hxge_tx_use_bcopy = 1;
36 
37 extern uint32_t hxge_tx_ring_size;
38 extern uint32_t hxge_bcopy_thresh;
39 extern uint32_t hxge_dvma_thresh;
40 extern uint32_t hxge_dma_stream_thresh;
41 extern dma_method_t hxge_force_dma;
42 
43 /* Device register access attributes for PIO.  */
44 extern ddi_device_acc_attr_t hxge_dev_reg_acc_attr;
45 
46 /* Device descriptor access attributes for DMA.  */
47 extern ddi_device_acc_attr_t hxge_dev_desc_dma_acc_attr;
48 
49 /* Device buffer access attributes for DMA.  */
50 extern ddi_device_acc_attr_t hxge_dev_buf_dma_acc_attr;
51 extern ddi_dma_attr_t hxge_desc_dma_attr;
52 extern ddi_dma_attr_t hxge_tx_dma_attr;
53 
54 static hxge_status_t hxge_map_txdma(p_hxge_t hxgep);
55 static void hxge_unmap_txdma(p_hxge_t hxgep);
56 static hxge_status_t hxge_txdma_hw_start(p_hxge_t hxgep);
57 static void hxge_txdma_hw_stop(p_hxge_t hxgep);
58 
59 static hxge_status_t hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
60     p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
61     uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
62     p_tx_mbox_t *tx_mbox_p);
63 static void hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
64     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
65 static hxge_status_t hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t,
66     p_hxge_dma_common_t *, p_tx_ring_t *, uint32_t);
67 static void hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep,
68     p_tx_ring_t tx_ring_p);
69 static void hxge_map_txdma_channel_cfg_ring(p_hxge_t, uint16_t,
70     p_hxge_dma_common_t *, p_tx_ring_t, p_tx_mbox_t *);
71 static void hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
72     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
73 static hxge_status_t hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
74     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
75 static hxge_status_t hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
76     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
77 static p_tx_ring_t hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel);
78 static hxge_status_t hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index,
79     p_hxge_ldv_t ldvp, tdc_stat_t cs);
80 static p_tx_mbox_t hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel);
81 static hxge_status_t hxge_txdma_fatal_err_recover(p_hxge_t hxgep,
82     uint16_t channel, p_tx_ring_t tx_ring_p);
83 static hxge_status_t hxge_tx_port_fatal_err_recover(p_hxge_t hxgep);
84 
85 hxge_status_t
hxge_init_txdma_channels(p_hxge_t hxgep)86 hxge_init_txdma_channels(p_hxge_t hxgep)
87 {
88 	hxge_status_t	status = HXGE_OK;
89 	block_reset_t	reset_reg;
90 
91 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_init_txdma_channels"));
92 
93 	/*
94 	 * Reset TDC block from PEU to cleanup any unknown configuration.
95 	 * This may be resulted from previous reboot.
96 	 */
97 	reset_reg.value = 0;
98 	reset_reg.bits.tdc_rst = 1;
99 	HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
100 
101 	HXGE_DELAY(1000);
102 
103 	status = hxge_map_txdma(hxgep);
104 	if (status != HXGE_OK) {
105 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
106 		    "<== hxge_init_txdma_channels: status 0x%x", status));
107 		return (status);
108 	}
109 
110 	status = hxge_txdma_hw_start(hxgep);
111 	if (status != HXGE_OK) {
112 		hxge_unmap_txdma(hxgep);
113 		return (status);
114 	}
115 
116 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
117 	    "<== hxge_init_txdma_channels: status 0x%x", status));
118 
119 	return (HXGE_OK);
120 }
121 
122 void
hxge_uninit_txdma_channels(p_hxge_t hxgep)123 hxge_uninit_txdma_channels(p_hxge_t hxgep)
124 {
125 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_uninit_txdma_channels"));
126 
127 	hxge_txdma_hw_stop(hxgep);
128 	hxge_unmap_txdma(hxgep);
129 
130 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_uinit_txdma_channels"));
131 }
132 
133 void
hxge_setup_dma_common(p_hxge_dma_common_t dest_p,p_hxge_dma_common_t src_p,uint32_t entries,uint32_t size)134 hxge_setup_dma_common(p_hxge_dma_common_t dest_p, p_hxge_dma_common_t src_p,
135     uint32_t entries, uint32_t size)
136 {
137 	size_t tsize;
138 	*dest_p = *src_p;
139 	tsize = size * entries;
140 	dest_p->alength = tsize;
141 	dest_p->nblocks = entries;
142 	dest_p->block_size = size;
143 	dest_p->offset += tsize;
144 
145 	src_p->kaddrp = (caddr_t)dest_p->kaddrp + tsize;
146 	src_p->alength -= tsize;
147 	src_p->dma_cookie.dmac_laddress += tsize;
148 	src_p->dma_cookie.dmac_size -= tsize;
149 }
150 
151 hxge_status_t
hxge_reset_txdma_channel(p_hxge_t hxgep,uint16_t channel,uint64_t reg_data)152 hxge_reset_txdma_channel(p_hxge_t hxgep, uint16_t channel, uint64_t reg_data)
153 {
154 	hpi_status_t	rs = HPI_SUCCESS;
155 	hxge_status_t	status = HXGE_OK;
156 	hpi_handle_t	handle;
157 
158 	HXGE_DEBUG_MSG((hxgep, TX_CTL, " ==> hxge_reset_txdma_channel"));
159 
160 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
161 	if ((reg_data & TDC_TDR_RST_MASK) == TDC_TDR_RST_MASK) {
162 		rs = hpi_txdma_channel_reset(handle, channel);
163 	} else {
164 		rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel);
165 	}
166 
167 	if (rs != HPI_SUCCESS) {
168 		status = HXGE_ERROR | rs;
169 	}
170 
171 	/*
172 	 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
173 	 * overflow fatal error if tail is not set to 0 after reset!
174 	 */
175 	TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
176 
177 	HXGE_DEBUG_MSG((hxgep, TX_CTL, " <== hxge_reset_txdma_channel"));
178 
179 	return (status);
180 }
181 
182 hxge_status_t
hxge_init_txdma_channel_event_mask(p_hxge_t hxgep,uint16_t channel,tdc_int_mask_t * mask_p)183 hxge_init_txdma_channel_event_mask(p_hxge_t hxgep, uint16_t channel,
184     tdc_int_mask_t *mask_p)
185 {
186 	hpi_handle_t	handle;
187 	hpi_status_t	rs = HPI_SUCCESS;
188 	hxge_status_t	status = HXGE_OK;
189 
190 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
191 	    "<== hxge_init_txdma_channel_event_mask"));
192 
193 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
194 
195 	/*
196 	 * Mask off tx_rng_oflow since it is a false alarm. The driver
197 	 * ensures not over flowing the hardware and check the hardware
198 	 * status.
199 	 */
200 	mask_p->bits.tx_rng_oflow = 1;
201 	rs = hpi_txdma_event_mask(handle, OP_SET, channel, mask_p);
202 	if (rs != HPI_SUCCESS) {
203 		status = HXGE_ERROR | rs;
204 	}
205 
206 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
207 	    "==> hxge_init_txdma_channel_event_mask"));
208 	return (status);
209 }
210 
211 hxge_status_t
hxge_enable_txdma_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_desc_p,p_tx_mbox_t mbox_p)212 hxge_enable_txdma_channel(p_hxge_t hxgep,
213     uint16_t channel, p_tx_ring_t tx_desc_p, p_tx_mbox_t mbox_p)
214 {
215 	hpi_handle_t	handle;
216 	hpi_status_t	rs = HPI_SUCCESS;
217 	hxge_status_t	status = HXGE_OK;
218 
219 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_enable_txdma_channel"));
220 
221 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
222 	/*
223 	 * Use configuration data composed at init time. Write to hardware the
224 	 * transmit ring configurations.
225 	 */
226 	rs = hpi_txdma_ring_config(handle, OP_SET, channel,
227 	    (uint64_t *)&(tx_desc_p->tx_ring_cfig.value));
228 
229 	if (rs != HPI_SUCCESS) {
230 		return (HXGE_ERROR | rs);
231 	}
232 
233 	/* Write to hardware the mailbox */
234 	rs = hpi_txdma_mbox_config(handle, OP_SET, channel,
235 	    (uint64_t *)&mbox_p->tx_mbox.dma_cookie.dmac_laddress);
236 
237 	if (rs != HPI_SUCCESS) {
238 		return (HXGE_ERROR | rs);
239 	}
240 
241 	/* Start the DMA engine. */
242 	rs = hpi_txdma_channel_init_enable(handle, channel);
243 	if (rs != HPI_SUCCESS) {
244 		return (HXGE_ERROR | rs);
245 	}
246 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_enable_txdma_channel"));
247 	return (status);
248 }
249 
250 void
hxge_fill_tx_hdr(p_mblk_t mp,boolean_t fill_len,boolean_t l4_cksum,int pkt_len,uint8_t npads,p_tx_pkt_hdr_all_t pkthdrp)251 hxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len, boolean_t l4_cksum,
252     int pkt_len, uint8_t npads, p_tx_pkt_hdr_all_t pkthdrp)
253 {
254 	p_tx_pkt_header_t	hdrp;
255 	p_mblk_t		nmp;
256 	uint64_t		tmp;
257 	size_t			mblk_len;
258 	size_t			iph_len;
259 	size_t			hdrs_size;
260 	uint8_t			*ip_buf;
261 	uint16_t		eth_type;
262 	uint8_t			ipproto;
263 	boolean_t		is_vlan = B_FALSE;
264 	size_t			eth_hdr_size;
265 	uint8_t hdrs_buf[sizeof (struct ether_header) + 64 + sizeof (uint32_t)];
266 
267 	HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: mp $%p", mp));
268 
269 	/*
270 	 * Caller should zero out the headers first.
271 	 */
272 	hdrp = (p_tx_pkt_header_t)&pkthdrp->pkthdr;
273 
274 	if (fill_len) {
275 		HXGE_DEBUG_MSG((NULL, TX_CTL,
276 		    "==> hxge_fill_tx_hdr: pkt_len %d npads %d",
277 		    pkt_len, npads));
278 		tmp = (uint64_t)pkt_len;
279 		hdrp->value |= (tmp << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
280 
281 		goto fill_tx_header_done;
282 	}
283 	tmp = (uint64_t)npads;
284 	hdrp->value |= (tmp << TX_PKT_HEADER_PAD_SHIFT);
285 
286 	/*
287 	 * mp is the original data packet (does not include the Neptune
288 	 * transmit header).
289 	 */
290 	nmp = mp;
291 	mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
292 	HXGE_DEBUG_MSG((NULL, TX_CTL,
293 	    "==> hxge_fill_tx_hdr: mp $%p b_rptr $%p len %d",
294 	    mp, nmp->b_rptr, mblk_len));
295 	ip_buf = NULL;
296 	bcopy(nmp->b_rptr, &hdrs_buf[0], sizeof (struct ether_vlan_header));
297 	eth_type = ntohs(((p_ether_header_t)hdrs_buf)->ether_type);
298 	HXGE_DEBUG_MSG((NULL, TX_CTL,
299 	    "==> : hxge_fill_tx_hdr: (value 0x%llx) ether type 0x%x",
300 	    eth_type, hdrp->value));
301 
302 	if (eth_type < ETHERMTU) {
303 		tmp = 1ull;
304 		hdrp->value |= (tmp << TX_PKT_HEADER_LLC_SHIFT);
305 		HXGE_DEBUG_MSG((NULL, TX_CTL,
306 		    "==> hxge_tx_pkt_hdr_init: LLC value 0x%llx", hdrp->value));
307 		if (*(hdrs_buf + sizeof (struct ether_header)) ==
308 		    LLC_SNAP_SAP) {
309 			eth_type = ntohs(*((uint16_t *)(hdrs_buf +
310 			    sizeof (struct ether_header) + 6)));
311 			HXGE_DEBUG_MSG((NULL, TX_CTL,
312 			    "==> hxge_tx_pkt_hdr_init: LLC ether type 0x%x",
313 			    eth_type));
314 		} else {
315 			goto fill_tx_header_done;
316 		}
317 	} else if (eth_type == VLAN_ETHERTYPE) {
318 		tmp = 1ull;
319 		hdrp->value |= (tmp << TX_PKT_HEADER_VLAN__SHIFT);
320 
321 		eth_type = ntohs(((struct ether_vlan_header *)
322 		    hdrs_buf)->ether_type);
323 		is_vlan = B_TRUE;
324 		HXGE_DEBUG_MSG((NULL, TX_CTL,
325 		    "==> hxge_tx_pkt_hdr_init: VLAN value 0x%llx",
326 		    hdrp->value));
327 	}
328 	if (!is_vlan) {
329 		eth_hdr_size = sizeof (struct ether_header);
330 	} else {
331 		eth_hdr_size = sizeof (struct ether_vlan_header);
332 	}
333 
334 	switch (eth_type) {
335 	case ETHERTYPE_IP:
336 		if (mblk_len > eth_hdr_size + sizeof (uint8_t)) {
337 			ip_buf = nmp->b_rptr + eth_hdr_size;
338 			mblk_len -= eth_hdr_size;
339 			iph_len = ((*ip_buf) & 0x0f);
340 			if (mblk_len > (iph_len + sizeof (uint32_t))) {
341 				ip_buf = nmp->b_rptr;
342 				ip_buf += eth_hdr_size;
343 			} else {
344 				ip_buf = NULL;
345 			}
346 		}
347 		if (ip_buf == NULL) {
348 			hdrs_size = 0;
349 			((p_ether_header_t)hdrs_buf)->ether_type = 0;
350 			while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
351 				mblk_len = (size_t)nmp->b_wptr -
352 				    (size_t)nmp->b_rptr;
353 				if (mblk_len >=
354 				    (sizeof (hdrs_buf) - hdrs_size))
355 					mblk_len = sizeof (hdrs_buf) -
356 					    hdrs_size;
357 				bcopy(nmp->b_rptr,
358 				    &hdrs_buf[hdrs_size], mblk_len);
359 				hdrs_size += mblk_len;
360 				nmp = nmp->b_cont;
361 			}
362 			ip_buf = hdrs_buf;
363 			ip_buf += eth_hdr_size;
364 			iph_len = ((*ip_buf) & 0x0f);
365 		}
366 		ipproto = ip_buf[9];
367 
368 		tmp = (uint64_t)iph_len;
369 		hdrp->value |= (tmp << TX_PKT_HEADER_IHL_SHIFT);
370 		tmp = (uint64_t)(eth_hdr_size >> 1);
371 		hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
372 
373 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv4 "
374 		    " iph_len %d l3start %d eth_hdr_size %d proto 0x%x"
375 		    "tmp 0x%x", iph_len, hdrp->bits.l3start, eth_hdr_size,
376 		    ipproto, tmp));
377 		HXGE_DEBUG_MSG((NULL, TX_CTL,
378 		    "==> hxge_tx_pkt_hdr_init: IP value 0x%llx", hdrp->value));
379 		break;
380 
381 	case ETHERTYPE_IPV6:
382 		hdrs_size = 0;
383 		((p_ether_header_t)hdrs_buf)->ether_type = 0;
384 		while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
385 			mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
386 			if (mblk_len >= (sizeof (hdrs_buf) - hdrs_size))
387 				mblk_len = sizeof (hdrs_buf) - hdrs_size;
388 			bcopy(nmp->b_rptr, &hdrs_buf[hdrs_size], mblk_len);
389 			hdrs_size += mblk_len;
390 			nmp = nmp->b_cont;
391 		}
392 		ip_buf = hdrs_buf;
393 		ip_buf += eth_hdr_size;
394 
395 		tmp = 1ull;
396 		hdrp->value |= (tmp << TX_PKT_HEADER_IP_VER_SHIFT);
397 
398 		tmp = (eth_hdr_size >> 1);
399 		hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
400 
401 		/* byte 6 is the next header protocol */
402 		ipproto = ip_buf[6];
403 
404 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv6 "
405 		    " iph_len %d l3start %d eth_hdr_size %d proto 0x%x",
406 		    iph_len, hdrp->bits.l3start, eth_hdr_size, ipproto));
407 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_hdr_init: IPv6 "
408 		    "value 0x%llx", hdrp->value));
409 		break;
410 
411 	default:
412 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: non-IP"));
413 		goto fill_tx_header_done;
414 	}
415 
416 	switch (ipproto) {
417 	case IPPROTO_TCP:
418 		HXGE_DEBUG_MSG((NULL, TX_CTL,
419 		    "==> hxge_fill_tx_hdr: TCP (cksum flag %d)", l4_cksum));
420 		if (l4_cksum) {
421 			tmp = 1ull;
422 			hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
423 			HXGE_DEBUG_MSG((NULL, TX_CTL,
424 			    "==> hxge_tx_pkt_hdr_init: TCP CKSUM"
425 			    "value 0x%llx", hdrp->value));
426 		}
427 		HXGE_DEBUG_MSG((NULL, TX_CTL,
428 		    "==> hxge_tx_pkt_hdr_init: TCP value 0x%llx", hdrp->value));
429 		break;
430 
431 	case IPPROTO_UDP:
432 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: UDP"));
433 		if (l4_cksum) {
434 			tmp = 0x2ull;
435 			hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
436 		}
437 		HXGE_DEBUG_MSG((NULL, TX_CTL,
438 		    "==> hxge_tx_pkt_hdr_init: UDP value 0x%llx",
439 		    hdrp->value));
440 		break;
441 
442 	default:
443 		goto fill_tx_header_done;
444 	}
445 
446 fill_tx_header_done:
447 	HXGE_DEBUG_MSG((NULL, TX_CTL,
448 	    "==> hxge_fill_tx_hdr: pkt_len %d npads %d value 0x%llx",
449 	    pkt_len, npads, hdrp->value));
450 	HXGE_DEBUG_MSG((NULL, TX_CTL, "<== hxge_fill_tx_hdr"));
451 }
452 
453 /*ARGSUSED*/
454 p_mblk_t
hxge_tx_pkt_header_reserve(p_mblk_t mp,uint8_t * npads)455 hxge_tx_pkt_header_reserve(p_mblk_t mp, uint8_t *npads)
456 {
457 	p_mblk_t newmp = NULL;
458 
459 	if ((newmp = allocb(TX_PKT_HEADER_SIZE, BPRI_MED)) == NULL) {
460 		HXGE_DEBUG_MSG((NULL, TX_CTL,
461 		    "<== hxge_tx_pkt_header_reserve: allocb failed"));
462 		return (NULL);
463 	}
464 	HXGE_DEBUG_MSG((NULL, TX_CTL,
465 	    "==> hxge_tx_pkt_header_reserve: get new mp"));
466 	DB_TYPE(newmp) = M_DATA;
467 	newmp->b_rptr = newmp->b_wptr = DB_LIM(newmp);
468 	linkb(newmp, mp);
469 	newmp->b_rptr -= TX_PKT_HEADER_SIZE;
470 
471 	HXGE_DEBUG_MSG((NULL, TX_CTL,
472 	    "==>hxge_tx_pkt_header_reserve: b_rptr $%p b_wptr $%p",
473 	    newmp->b_rptr, newmp->b_wptr));
474 	HXGE_DEBUG_MSG((NULL, TX_CTL,
475 	    "<== hxge_tx_pkt_header_reserve: use new mp"));
476 	return (newmp);
477 }
478 
479 int
hxge_tx_pkt_nmblocks(p_mblk_t mp,int * tot_xfer_len_p)480 hxge_tx_pkt_nmblocks(p_mblk_t mp, int *tot_xfer_len_p)
481 {
482 	uint_t		nmblks;
483 	ssize_t		len;
484 	uint_t		pkt_len;
485 	p_mblk_t	nmp, bmp, tmp;
486 	uint8_t		*b_wptr;
487 
488 	HXGE_DEBUG_MSG((NULL, TX_CTL,
489 	    "==> hxge_tx_pkt_nmblocks: mp $%p rptr $%p wptr $%p len %d",
490 	    mp, mp->b_rptr, mp->b_wptr, MBLKL(mp)));
491 
492 	nmp = mp;
493 	bmp = mp;
494 	nmblks = 0;
495 	pkt_len = 0;
496 	*tot_xfer_len_p = 0;
497 
498 	while (nmp) {
499 		len = MBLKL(nmp);
500 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
501 		    "len %d pkt_len %d nmblks %d tot_xfer_len %d",
502 		    len, pkt_len, nmblks, *tot_xfer_len_p));
503 
504 		if (len <= 0) {
505 			bmp = nmp;
506 			nmp = nmp->b_cont;
507 			HXGE_DEBUG_MSG((NULL, TX_CTL,
508 			    "==> hxge_tx_pkt_nmblocks:"
509 			    " len (0) pkt_len %d nmblks %d", pkt_len, nmblks));
510 			continue;
511 		}
512 		*tot_xfer_len_p += len;
513 		HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
514 		    "len %d pkt_len %d nmblks %d tot_xfer_len %d",
515 		    len, pkt_len, nmblks, *tot_xfer_len_p));
516 
517 		if (len < hxge_bcopy_thresh) {
518 			HXGE_DEBUG_MSG((NULL, TX_CTL,
519 			    "==> hxge_tx_pkt_nmblocks: "
520 			    "len %d (< thresh) pkt_len %d nmblks %d",
521 			    len, pkt_len, nmblks));
522 			if (pkt_len == 0)
523 				nmblks++;
524 			pkt_len += len;
525 			if (pkt_len >= hxge_bcopy_thresh) {
526 				pkt_len = 0;
527 				len = 0;
528 				nmp = bmp;
529 			}
530 		} else {
531 			HXGE_DEBUG_MSG((NULL, TX_CTL,
532 			    "==> hxge_tx_pkt_nmblocks: "
533 			    "len %d (> thresh) pkt_len %d nmblks %d",
534 			    len, pkt_len, nmblks));
535 			pkt_len = 0;
536 			nmblks++;
537 			/*
538 			 * Hardware limits the transfer length to 4K. If len is
539 			 * more than 4K, we need to break it up to at most 2
540 			 * more blocks.
541 			 */
542 			if (len > TX_MAX_TRANSFER_LENGTH) {
543 				uint32_t nsegs;
544 
545 				HXGE_DEBUG_MSG((NULL, TX_CTL,
546 				    "==> hxge_tx_pkt_nmblocks: "
547 				    "len %d pkt_len %d nmblks %d nsegs %d",
548 				    len, pkt_len, nmblks, nsegs));
549 				nsegs = 1;
550 				if (len % (TX_MAX_TRANSFER_LENGTH * 2)) {
551 					++nsegs;
552 				}
553 				do {
554 					b_wptr = nmp->b_rptr +
555 					    TX_MAX_TRANSFER_LENGTH;
556 					nmp->b_wptr = b_wptr;
557 					if ((tmp = dupb(nmp)) == NULL) {
558 						return (0);
559 					}
560 					tmp->b_rptr = b_wptr;
561 					tmp->b_wptr = nmp->b_wptr;
562 					tmp->b_cont = nmp->b_cont;
563 					nmp->b_cont = tmp;
564 					nmblks++;
565 					if (--nsegs) {
566 						nmp = tmp;
567 					}
568 				} while (nsegs);
569 				nmp = tmp;
570 			}
571 		}
572 
573 		/*
574 		 * Hardware limits the transmit gather pointers to 15.
575 		 */
576 		if (nmp->b_cont && (nmblks + TX_GATHER_POINTERS_THRESHOLD) >
577 		    TX_MAX_GATHER_POINTERS) {
578 			HXGE_DEBUG_MSG((NULL, TX_CTL,
579 			    "==> hxge_tx_pkt_nmblocks: pull msg - "
580 			    "len %d pkt_len %d nmblks %d",
581 			    len, pkt_len, nmblks));
582 			/* Pull all message blocks from b_cont */
583 			if ((tmp = msgpullup(nmp->b_cont, -1)) == NULL) {
584 				return (0);
585 			}
586 			freemsg(nmp->b_cont);
587 			nmp->b_cont = tmp;
588 			pkt_len = 0;
589 		}
590 		bmp = nmp;
591 		nmp = nmp->b_cont;
592 	}
593 
594 	HXGE_DEBUG_MSG((NULL, TX_CTL,
595 	    "<== hxge_tx_pkt_nmblocks: rptr $%p wptr $%p "
596 	    "nmblks %d len %d tot_xfer_len %d",
597 	    mp->b_rptr, mp->b_wptr, nmblks, MBLKL(mp), *tot_xfer_len_p));
598 	return (nmblks);
599 }
600 
601 boolean_t
hxge_txdma_reclaim(p_hxge_t hxgep,p_tx_ring_t tx_ring_p,int nmblks)602 hxge_txdma_reclaim(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, int nmblks)
603 {
604 	boolean_t		status = B_TRUE;
605 	p_hxge_dma_common_t	tx_desc_dma_p;
606 	hxge_dma_common_t	desc_area;
607 	p_tx_desc_t		tx_desc_ring_vp;
608 	p_tx_desc_t		tx_desc_p;
609 	p_tx_desc_t		tx_desc_pp;
610 	tx_desc_t		r_tx_desc;
611 	p_tx_msg_t		tx_msg_ring;
612 	p_tx_msg_t		tx_msg_p;
613 	hpi_handle_t		handle;
614 	tdc_tdr_head_t		tx_head;
615 	uint32_t		pkt_len;
616 	uint_t			tx_rd_index;
617 	uint16_t		head_index, tail_index;
618 	uint8_t			tdc;
619 	boolean_t		head_wrap, tail_wrap;
620 	p_hxge_tx_ring_stats_t	tdc_stats;
621 	tdc_byte_cnt_t		byte_cnt;
622 	tdc_tdr_qlen_t		qlen;
623 	int			rc;
624 
625 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_reclaim"));
626 
627 	status = ((tx_ring_p->descs_pending < hxge_reclaim_pending) &&
628 	    (nmblks != 0));
629 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
630 	    "==> hxge_txdma_reclaim: pending %d  reclaim %d nmblks %d",
631 	    tx_ring_p->descs_pending, hxge_reclaim_pending, nmblks));
632 
633 	if (!status) {
634 		tx_desc_dma_p = &tx_ring_p->tdc_desc;
635 		desc_area = tx_ring_p->tdc_desc;
636 		tx_desc_ring_vp = tx_desc_dma_p->kaddrp;
637 		tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
638 		tx_rd_index = tx_ring_p->rd_index;
639 		tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
640 		tx_msg_ring = tx_ring_p->tx_msg_ring;
641 		tx_msg_p = &tx_msg_ring[tx_rd_index];
642 		tdc = tx_ring_p->tdc;
643 		tdc_stats = tx_ring_p->tdc_stats;
644 		if (tx_ring_p->descs_pending > tdc_stats->tx_max_pend) {
645 			tdc_stats->tx_max_pend = tx_ring_p->descs_pending;
646 		}
647 		tail_index = tx_ring_p->wr_index;
648 		tail_wrap = tx_ring_p->wr_index_wrap;
649 
650 		/*
651 		 * tdc_byte_cnt reg can be used to get bytes transmitted. It
652 		 * includes padding too in case of runt packets.
653 		 */
654 		handle = HXGE_DEV_HPI_HANDLE(hxgep);
655 		TXDMA_REG_READ64(handle, TDC_BYTE_CNT, tdc, &byte_cnt.value);
656 		tdc_stats->obytes_with_pad += byte_cnt.bits.byte_count;
657 
658 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
659 		    "==> hxge_txdma_reclaim: tdc %d tx_rd_index %d "
660 		    "tail_index %d tail_wrap %d tx_desc_p $%p ($%p) ",
661 		    tdc, tx_rd_index, tail_index, tail_wrap,
662 		    tx_desc_p, (*(uint64_t *)tx_desc_p)));
663 
664 		/*
665 		 * Read the hardware maintained transmit head and wrap around
666 		 * bit.
667 		 */
668 		TXDMA_REG_READ64(handle, TDC_TDR_HEAD, tdc, &tx_head.value);
669 		head_index = tx_head.bits.head;
670 		head_wrap = tx_head.bits.wrap;
671 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
672 		    "==> hxge_txdma_reclaim: "
673 		    "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
674 		    tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
675 
676 		/*
677 		 * For debug only. This can be used to verify the qlen and make
678 		 * sure the hardware is wrapping the Tdr correctly.
679 		 */
680 		TXDMA_REG_READ64(handle, TDC_TDR_QLEN, tdc, &qlen.value);
681 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
682 		    "==> hxge_txdma_reclaim: tdr_qlen %d tdr_pref_qlen %d",
683 		    qlen.bits.tdr_qlen, qlen.bits.tdr_pref_qlen));
684 
685 		if (head_index == tail_index) {
686 			if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index,
687 			    tail_wrap) && (head_index == tx_rd_index)) {
688 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
689 				    "==> hxge_txdma_reclaim: EMPTY"));
690 				return (B_TRUE);
691 			}
692 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
693 			    "==> hxge_txdma_reclaim: Checking if ring full"));
694 			if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
695 			    tail_wrap)) {
696 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
697 				    "==> hxge_txdma_reclaim: full"));
698 				return (B_FALSE);
699 			}
700 		}
701 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
702 		    "==> hxge_txdma_reclaim: tx_rd_index and head_index"));
703 
704 		/* XXXX: limit the # of reclaims */
705 		tx_desc_pp = &r_tx_desc;
706 		while ((tx_rd_index != head_index) &&
707 		    (tx_ring_p->descs_pending != 0)) {
708 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
709 			    "==> hxge_txdma_reclaim: Checking if pending"));
710 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
711 			    "==> hxge_txdma_reclaim: descs_pending %d ",
712 			    tx_ring_p->descs_pending));
713 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
714 			    "==> hxge_txdma_reclaim: "
715 			    "(tx_rd_index %d head_index %d (tx_desc_p $%p)",
716 			    tx_rd_index, head_index, tx_desc_p));
717 
718 			tx_desc_pp->value = tx_desc_p->value;
719 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
720 			    "==> hxge_txdma_reclaim: "
721 			    "(tx_rd_index %d head_index %d "
722 			    "tx_desc_p $%p (desc value 0x%llx) ",
723 			    tx_rd_index, head_index,
724 			    tx_desc_pp, (*(uint64_t *)tx_desc_pp)));
725 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
726 			    "==> hxge_txdma_reclaim: dump desc:"));
727 
728 			/*
729 			 * tdc_byte_cnt reg can be used to get bytes
730 			 * transmitted
731 			 */
732 			pkt_len = tx_desc_pp->bits.tr_len;
733 			tdc_stats->obytes += pkt_len;
734 			tdc_stats->opackets += tx_desc_pp->bits.sop;
735 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
736 			    "==> hxge_txdma_reclaim: pkt_len %d "
737 			    "tdc channel %d opackets %d",
738 			    pkt_len, tdc, tdc_stats->opackets));
739 
740 			if (tx_msg_p->flags.dma_type == USE_DVMA) {
741 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
742 				    "tx_desc_p = $%p tx_desc_pp = $%p "
743 				    "index = %d",
744 				    tx_desc_p, tx_desc_pp,
745 				    tx_ring_p->rd_index));
746 				(void) dvma_unload(tx_msg_p->dvma_handle,
747 				    0, -1);
748 				tx_msg_p->dvma_handle = NULL;
749 				if (tx_ring_p->dvma_wr_index ==
750 				    tx_ring_p->dvma_wrap_mask) {
751 					tx_ring_p->dvma_wr_index = 0;
752 				} else {
753 					tx_ring_p->dvma_wr_index++;
754 				}
755 				tx_ring_p->dvma_pending--;
756 			} else if (tx_msg_p->flags.dma_type == USE_DMA) {
757 				HXGE_DEBUG_MSG((hxgep, TX_CTL,
758 				    "==> hxge_txdma_reclaim: USE DMA"));
759 				if (rc = ddi_dma_unbind_handle
760 				    (tx_msg_p->dma_handle)) {
761 					cmn_err(CE_WARN, "hxge_reclaim: "
762 					    "ddi_dma_unbind_handle "
763 					    "failed. status %d", rc);
764 				}
765 			}
766 
767 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
768 			    "==> hxge_txdma_reclaim: count packets"));
769 
770 			/*
771 			 * count a chained packet only once.
772 			 */
773 			if (tx_msg_p->tx_message != NULL) {
774 				freemsg(tx_msg_p->tx_message);
775 				tx_msg_p->tx_message = NULL;
776 			}
777 			tx_msg_p->flags.dma_type = USE_NONE;
778 			tx_rd_index = tx_ring_p->rd_index;
779 			tx_rd_index = (tx_rd_index + 1) &
780 			    tx_ring_p->tx_wrap_mask;
781 			tx_ring_p->rd_index = tx_rd_index;
782 			tx_ring_p->descs_pending--;
783 			tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
784 			tx_msg_p = &tx_msg_ring[tx_rd_index];
785 		}
786 
787 		status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
788 		    (int)tx_ring_p->descs_pending - TX_FULL_MARK));
789 		if (status) {
790 			(void) atomic_cas_32((uint32_t *)&tx_ring_p->queueing,
791 			    1, 0);
792 		}
793 	} else {
794 		status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
795 		    (int)tx_ring_p->descs_pending - TX_FULL_MARK));
796 	}
797 
798 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
799 	    "<== hxge_txdma_reclaim status = 0x%08x", status));
800 	return (status);
801 }
802 
803 uint_t
hxge_tx_intr(caddr_t arg1,caddr_t arg2)804 hxge_tx_intr(caddr_t arg1, caddr_t arg2)
805 {
806 	p_hxge_ldv_t	ldvp = (p_hxge_ldv_t)arg1;
807 	p_hxge_t	hxgep = (p_hxge_t)arg2;
808 	p_hxge_ldg_t	ldgp;
809 	uint8_t		channel;
810 	uint32_t	vindex;
811 	hpi_handle_t	handle;
812 	tdc_stat_t	cs;
813 	p_tx_ring_t	*tx_rings;
814 	p_tx_ring_t	tx_ring_p;
815 	hpi_status_t	rs = HPI_SUCCESS;
816 	uint_t		serviced = DDI_INTR_UNCLAIMED;
817 	hxge_status_t	status = HXGE_OK;
818 
819 	if (ldvp == NULL) {
820 		HXGE_DEBUG_MSG((NULL, INT_CTL,
821 		    "<== hxge_tx_intr: hxgep $%p ldvp $%p", hxgep, ldvp));
822 		return (DDI_INTR_UNCLAIMED);
823 	}
824 
825 	if (arg2 == NULL || (void *) ldvp->hxgep != arg2) {
826 		hxgep = ldvp->hxgep;
827 	}
828 
829 	/*
830 	 * If the interface is not started, just swallow the interrupt
831 	 * and don't rearm the logical device.
832 	 */
833 	if (hxgep->hxge_mac_state != HXGE_MAC_STARTED)
834 		return (DDI_INTR_CLAIMED);
835 
836 	HXGE_DEBUG_MSG((hxgep, INT_CTL,
837 	    "==> hxge_tx_intr: hxgep(arg2) $%p ldvp(arg1) $%p", hxgep, ldvp));
838 
839 	/*
840 	 * This interrupt handler is for a specific transmit dma channel.
841 	 */
842 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
843 
844 	/* Get the control and status for this channel. */
845 	channel = ldvp->channel;
846 	ldgp = ldvp->ldgp;
847 	HXGE_DEBUG_MSG((hxgep, INT_CTL,
848 	    "==> hxge_tx_intr: hxgep $%p ldvp (ldvp) $%p channel %d",
849 	    hxgep, ldvp, channel));
850 
851 	rs = hpi_txdma_control_status(handle, OP_GET, channel, &cs);
852 	vindex = ldvp->vdma_index;
853 	HXGE_DEBUG_MSG((hxgep, INT_CTL,
854 	    "==> hxge_tx_intr:channel %d ring index %d status 0x%08x",
855 	    channel, vindex, rs));
856 
857 	if (!rs && cs.bits.marked) {
858 		HXGE_DEBUG_MSG((hxgep, INT_CTL,
859 		    "==> hxge_tx_intr:channel %d ring index %d "
860 		    "status 0x%08x (marked bit set)", channel, vindex, rs));
861 		tx_rings = hxgep->tx_rings->rings;
862 		tx_ring_p = tx_rings[vindex];
863 		HXGE_DEBUG_MSG((hxgep, INT_CTL,
864 		    "==> hxge_tx_intr:channel %d ring index %d "
865 		    "status 0x%08x (marked bit set, calling reclaim)",
866 		    channel, vindex, rs));
867 
868 		MUTEX_ENTER(&tx_ring_p->lock);
869 		(void) hxge_txdma_reclaim(hxgep, tx_rings[vindex], 0);
870 		MUTEX_EXIT(&tx_ring_p->lock);
871 		mac_tx_update(hxgep->mach);
872 	}
873 
874 	/*
875 	 * Process other transmit control and status. Check the ldv state.
876 	 */
877 	status = hxge_tx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs);
878 
879 	/* Clear the error bits */
880 	RXDMA_REG_WRITE64(handle, TDC_STAT, channel, cs.value);
881 
882 	/*
883 	 * Rearm this logical group if this is a single device group.
884 	 */
885 	if (ldgp->nldvs == 1) {
886 		HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_tx_intr: rearm"));
887 		if (status == HXGE_OK) {
888 			(void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg,
889 			    B_TRUE, ldgp->ldg_timer);
890 		}
891 	}
892 	HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_tx_intr"));
893 	serviced = DDI_INTR_CLAIMED;
894 	return (serviced);
895 }
896 
897 void
hxge_txdma_stop(p_hxge_t hxgep)898 hxge_txdma_stop(p_hxge_t hxgep)
899 {
900 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop"));
901 
902 	(void) hxge_tx_vmac_disable(hxgep);
903 	(void) hxge_txdma_hw_mode(hxgep, HXGE_DMA_STOP);
904 
905 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop"));
906 }
907 
908 hxge_status_t
hxge_txdma_hw_mode(p_hxge_t hxgep,boolean_t enable)909 hxge_txdma_hw_mode(p_hxge_t hxgep, boolean_t enable)
910 {
911 	int		i, ndmas;
912 	uint16_t	channel;
913 	p_tx_rings_t	tx_rings;
914 	p_tx_ring_t	*tx_desc_rings;
915 	hpi_handle_t	handle;
916 	hpi_status_t	rs = HPI_SUCCESS;
917 	hxge_status_t	status = HXGE_OK;
918 
919 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
920 	    "==> hxge_txdma_hw_mode: enable mode %d", enable));
921 
922 	if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) {
923 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
924 		    "<== hxge_txdma_mode: not initialized"));
925 		return (HXGE_ERROR);
926 	}
927 	tx_rings = hxgep->tx_rings;
928 	if (tx_rings == NULL) {
929 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
930 		    "<== hxge_txdma_hw_mode: NULL global ring pointer"));
931 		return (HXGE_ERROR);
932 	}
933 	tx_desc_rings = tx_rings->rings;
934 	if (tx_desc_rings == NULL) {
935 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
936 		    "<== hxge_txdma_hw_mode: NULL rings pointer"));
937 		return (HXGE_ERROR);
938 	}
939 	ndmas = tx_rings->ndmas;
940 	if (!ndmas) {
941 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
942 		    "<== hxge_txdma_hw_mode: no dma channel allocated"));
943 		return (HXGE_ERROR);
944 	}
945 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_mode: "
946 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
947 	    tx_rings, tx_desc_rings, ndmas));
948 
949 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
950 	for (i = 0; i < ndmas; i++) {
951 		if (tx_desc_rings[i] == NULL) {
952 			continue;
953 		}
954 		channel = tx_desc_rings[i]->tdc;
955 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
956 		    "==> hxge_txdma_hw_mode: channel %d", channel));
957 		if (enable) {
958 			rs = hpi_txdma_channel_enable(handle, channel);
959 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
960 			    "==> hxge_txdma_hw_mode: channel %d (enable) "
961 			    "rs 0x%x", channel, rs));
962 		} else {
963 			/*
964 			 * Stop the dma channel and waits for the stop done. If
965 			 * the stop done bit is not set, then force an error so
966 			 * TXC will stop. All channels bound to this port need
967 			 * to be stopped and reset after injecting an interrupt
968 			 * error.
969 			 */
970 			rs = hpi_txdma_channel_disable(handle, channel);
971 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
972 			    "==> hxge_txdma_hw_mode: channel %d (disable) "
973 			    "rs 0x%x", channel, rs));
974 		}
975 	}
976 
977 	status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
978 
979 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
980 	    "<== hxge_txdma_hw_mode: status 0x%x", status));
981 
982 	return (status);
983 }
984 
985 void
hxge_txdma_enable_channel(p_hxge_t hxgep,uint16_t channel)986 hxge_txdma_enable_channel(p_hxge_t hxgep, uint16_t channel)
987 {
988 	hpi_handle_t handle;
989 
990 	HXGE_DEBUG_MSG((hxgep, DMA_CTL,
991 	    "==> hxge_txdma_enable_channel: channel %d", channel));
992 
993 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
994 	/* enable the transmit dma channels */
995 	(void) hpi_txdma_channel_enable(handle, channel);
996 
997 	HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_txdma_enable_channel"));
998 }
999 
1000 void
hxge_txdma_disable_channel(p_hxge_t hxgep,uint16_t channel)1001 hxge_txdma_disable_channel(p_hxge_t hxgep, uint16_t channel)
1002 {
1003 	hpi_handle_t handle;
1004 
1005 	HXGE_DEBUG_MSG((hxgep, DMA_CTL,
1006 	    "==> hxge_txdma_disable_channel: channel %d", channel));
1007 
1008 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1009 	/* stop the transmit dma channels */
1010 	(void) hpi_txdma_channel_disable(handle, channel);
1011 
1012 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_disable_channel"));
1013 }
1014 
1015 int
hxge_txdma_stop_inj_err(p_hxge_t hxgep,int channel)1016 hxge_txdma_stop_inj_err(p_hxge_t hxgep, int channel)
1017 {
1018 	hpi_handle_t	handle;
1019 	int		status;
1020 	hpi_status_t	rs = HPI_SUCCESS;
1021 
1022 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop_inj_err"));
1023 
1024 	/*
1025 	 * Stop the dma channel waits for the stop done. If the stop done bit
1026 	 * is not set, then create an error.
1027 	 */
1028 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1029 	rs = hpi_txdma_channel_disable(handle, channel);
1030 	status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
1031 	if (status == HXGE_OK) {
1032 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1033 		    "<== hxge_txdma_stop_inj_err (channel %d): "
1034 		    "stopped OK", channel));
1035 		return (status);
1036 	}
1037 
1038 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1039 	    "==> hxge_txdma_stop_inj_err (channel): stop failed (0x%x) "
1040 	    " (injected error but still not stopped)", channel, rs));
1041 
1042 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop_inj_err"));
1043 
1044 	return (status);
1045 }
1046 
1047 /*ARGSUSED*/
1048 void
hxge_fixup_txdma_rings(p_hxge_t hxgep)1049 hxge_fixup_txdma_rings(p_hxge_t hxgep)
1050 {
1051 	int		index, ndmas;
1052 	uint16_t	channel;
1053 	p_tx_rings_t	tx_rings;
1054 
1055 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_txdma_rings"));
1056 
1057 	/*
1058 	 * For each transmit channel, reclaim each descriptor and free buffers.
1059 	 */
1060 	tx_rings = hxgep->tx_rings;
1061 	if (tx_rings == NULL) {
1062 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1063 		    "<== hxge_fixup_txdma_rings: NULL ring pointer"));
1064 		return;
1065 	}
1066 
1067 	ndmas = tx_rings->ndmas;
1068 	if (!ndmas) {
1069 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1070 		    "<== hxge_fixup_txdma_rings: no channel allocated"));
1071 		return;
1072 	}
1073 
1074 	if (tx_rings->rings == NULL) {
1075 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1076 		    "<== hxge_fixup_txdma_rings: NULL rings pointer"));
1077 		return;
1078 	}
1079 
1080 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_fixup_txdma_rings: "
1081 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1082 	    tx_rings, tx_rings->rings, ndmas));
1083 
1084 	for (index = 0; index < ndmas; index++) {
1085 		channel = tx_rings->rings[index]->tdc;
1086 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1087 		    "==> hxge_fixup_txdma_rings: channel %d", channel));
1088 		hxge_txdma_fixup_channel(hxgep, tx_rings->rings[index],
1089 		    channel);
1090 	}
1091 
1092 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_txdma_rings"));
1093 }
1094 
1095 /*ARGSUSED*/
1096 void
hxge_txdma_fix_channel(p_hxge_t hxgep,uint16_t channel)1097 hxge_txdma_fix_channel(p_hxge_t hxgep, uint16_t channel)
1098 {
1099 	p_tx_ring_t ring_p;
1100 
1101 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_channel"));
1102 
1103 	ring_p = hxge_txdma_get_ring(hxgep, channel);
1104 	if (ring_p == NULL) {
1105 		HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1106 		return;
1107 	}
1108 
1109 	if (ring_p->tdc != channel) {
1110 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1111 		    "<== hxge_txdma_fix_channel: channel not matched "
1112 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1113 		return;
1114 	}
1115 
1116 	hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1117 
1118 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1119 }
1120 
1121 /*ARGSUSED*/
1122 void
hxge_txdma_fixup_channel(p_hxge_t hxgep,p_tx_ring_t ring_p,uint16_t channel)1123 hxge_txdma_fixup_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1124 {
1125 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_channel"));
1126 
1127 	if (ring_p == NULL) {
1128 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1129 		    "<== hxge_txdma_fixup_channel: NULL ring pointer"));
1130 		return;
1131 	}
1132 	if (ring_p->tdc != channel) {
1133 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1134 		    "<== hxge_txdma_fixup_channel: channel not matched "
1135 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1136 		return;
1137 	}
1138 	MUTEX_ENTER(&ring_p->lock);
1139 	(void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1140 
1141 	ring_p->rd_index = 0;
1142 	ring_p->wr_index = 0;
1143 	ring_p->ring_head.value = 0;
1144 	ring_p->ring_kick_tail.value = 0;
1145 	ring_p->descs_pending = 0;
1146 	MUTEX_EXIT(&ring_p->lock);
1147 
1148 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_channel"));
1149 }
1150 
1151 /*ARGSUSED*/
1152 void
hxge_txdma_hw_kick(p_hxge_t hxgep)1153 hxge_txdma_hw_kick(p_hxge_t hxgep)
1154 {
1155 	int		index, ndmas;
1156 	uint16_t	channel;
1157 	p_tx_rings_t	tx_rings;
1158 
1159 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick"));
1160 
1161 	tx_rings = hxgep->tx_rings;
1162 	if (tx_rings == NULL) {
1163 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1164 		    "<== hxge_txdma_hw_kick: NULL ring pointer"));
1165 		return;
1166 	}
1167 	ndmas = tx_rings->ndmas;
1168 	if (!ndmas) {
1169 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1170 		    "<== hxge_txdma_hw_kick: no channel allocated"));
1171 		return;
1172 	}
1173 	if (tx_rings->rings == NULL) {
1174 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1175 		    "<== hxge_txdma_hw_kick: NULL rings pointer"));
1176 		return;
1177 	}
1178 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_kick: "
1179 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1180 	    tx_rings, tx_rings->rings, ndmas));
1181 
1182 	for (index = 0; index < ndmas; index++) {
1183 		channel = tx_rings->rings[index]->tdc;
1184 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1185 		    "==> hxge_txdma_hw_kick: channel %d", channel));
1186 		hxge_txdma_hw_kick_channel(hxgep, tx_rings->rings[index],
1187 		    channel);
1188 	}
1189 
1190 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick"));
1191 }
1192 
1193 /*ARGSUSED*/
1194 void
hxge_txdma_kick_channel(p_hxge_t hxgep,uint16_t channel)1195 hxge_txdma_kick_channel(p_hxge_t hxgep, uint16_t channel)
1196 {
1197 	p_tx_ring_t ring_p;
1198 
1199 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_kick_channel"));
1200 
1201 	ring_p = hxge_txdma_get_ring(hxgep, channel);
1202 	if (ring_p == NULL) {
1203 		HXGE_DEBUG_MSG((hxgep, TX_CTL, " hxge_txdma_kick_channel"));
1204 		return;
1205 	}
1206 
1207 	if (ring_p->tdc != channel) {
1208 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1209 		    "<== hxge_txdma_kick_channel: channel not matched "
1210 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1211 		return;
1212 	}
1213 
1214 	hxge_txdma_hw_kick_channel(hxgep, ring_p, channel);
1215 
1216 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_kick_channel"));
1217 }
1218 
1219 /*ARGSUSED*/
1220 void
hxge_txdma_hw_kick_channel(p_hxge_t hxgep,p_tx_ring_t ring_p,uint16_t channel)1221 hxge_txdma_hw_kick_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1222 {
1223 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick_channel"));
1224 
1225 	if (ring_p == NULL) {
1226 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1227 		    "<== hxge_txdma_hw_kick_channel: NULL ring pointer"));
1228 		return;
1229 	}
1230 
1231 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick_channel"));
1232 }
1233 
1234 /*ARGSUSED*/
1235 void
hxge_check_tx_hang(p_hxge_t hxgep)1236 hxge_check_tx_hang(p_hxge_t hxgep)
1237 {
1238 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_check_tx_hang"));
1239 
1240 	/*
1241 	 * Needs inputs from hardware for regs: head index had not moved since
1242 	 * last timeout. packets not transmitted or stuffed registers.
1243 	 */
1244 	if (hxge_txdma_hung(hxgep)) {
1245 		hxge_fixup_hung_txdma_rings(hxgep);
1246 	}
1247 
1248 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_check_tx_hang"));
1249 }
1250 
1251 int
hxge_txdma_hung(p_hxge_t hxgep)1252 hxge_txdma_hung(p_hxge_t hxgep)
1253 {
1254 	int		index, ndmas;
1255 	uint16_t	channel;
1256 	p_tx_rings_t	tx_rings;
1257 	p_tx_ring_t	tx_ring_p;
1258 
1259 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hung"));
1260 
1261 	tx_rings = hxgep->tx_rings;
1262 	if (tx_rings == NULL) {
1263 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1264 		    "<== hxge_txdma_hung: NULL ring pointer"));
1265 		return (B_FALSE);
1266 	}
1267 
1268 	ndmas = tx_rings->ndmas;
1269 	if (!ndmas) {
1270 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1271 		    "<== hxge_txdma_hung: no channel allocated"));
1272 		return (B_FALSE);
1273 	}
1274 
1275 	if (tx_rings->rings == NULL) {
1276 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1277 		    "<== hxge_txdma_hung: NULL rings pointer"));
1278 		return (B_FALSE);
1279 	}
1280 
1281 	for (index = 0; index < ndmas; index++) {
1282 		channel = tx_rings->rings[index]->tdc;
1283 		tx_ring_p = tx_rings->rings[index];
1284 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1285 		    "==> hxge_txdma_hung: channel %d", channel));
1286 		if (hxge_txdma_channel_hung(hxgep, tx_ring_p, channel)) {
1287 			return (B_TRUE);
1288 		}
1289 	}
1290 
1291 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hung"));
1292 
1293 	return (B_FALSE);
1294 }
1295 
1296 int
hxge_txdma_channel_hung(p_hxge_t hxgep,p_tx_ring_t tx_ring_p,uint16_t channel)1297 hxge_txdma_channel_hung(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, uint16_t channel)
1298 {
1299 	uint16_t	head_index, tail_index;
1300 	boolean_t	head_wrap, tail_wrap;
1301 	hpi_handle_t	handle;
1302 	tdc_tdr_head_t	tx_head;
1303 	uint_t		tx_rd_index;
1304 
1305 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung"));
1306 
1307 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1308 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1309 	    "==> hxge_txdma_channel_hung: channel %d", channel));
1310 	MUTEX_ENTER(&tx_ring_p->lock);
1311 	(void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
1312 
1313 	tail_index = tx_ring_p->wr_index;
1314 	tail_wrap = tx_ring_p->wr_index_wrap;
1315 	tx_rd_index = tx_ring_p->rd_index;
1316 	MUTEX_EXIT(&tx_ring_p->lock);
1317 
1318 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1319 	    "==> hxge_txdma_channel_hung: tdc %d tx_rd_index %d "
1320 	    "tail_index %d tail_wrap %d ",
1321 	    channel, tx_rd_index, tail_index, tail_wrap));
1322 	/*
1323 	 * Read the hardware maintained transmit head and wrap around bit.
1324 	 */
1325 	(void) hpi_txdma_ring_head_get(handle, channel, &tx_head);
1326 	head_index = tx_head.bits.head;
1327 	head_wrap = tx_head.bits.wrap;
1328 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung: "
1329 	    "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
1330 	    tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
1331 
1332 	if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index, tail_wrap) &&
1333 	    (head_index == tx_rd_index)) {
1334 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1335 		    "==> hxge_txdma_channel_hung: EMPTY"));
1336 		return (B_FALSE);
1337 	}
1338 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1339 	    "==> hxge_txdma_channel_hung: Checking if ring full"));
1340 	if (TXDMA_RING_FULL(head_index, head_wrap, tail_index, tail_wrap)) {
1341 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1342 		    "==> hxge_txdma_channel_hung: full"));
1343 		return (B_TRUE);
1344 	}
1345 
1346 	/* If not full, check with hardware to see if it is hung */
1347 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_channel_hung"));
1348 
1349 	return (B_FALSE);
1350 }
1351 
1352 /*ARGSUSED*/
1353 void
hxge_fixup_hung_txdma_rings(p_hxge_t hxgep)1354 hxge_fixup_hung_txdma_rings(p_hxge_t hxgep)
1355 {
1356 	int		index, ndmas;
1357 	uint16_t	channel;
1358 	p_tx_rings_t	tx_rings;
1359 
1360 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings"));
1361 	tx_rings = hxgep->tx_rings;
1362 	if (tx_rings == NULL) {
1363 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1364 		    "<== hxge_fixup_hung_txdma_rings: NULL ring pointer"));
1365 		return;
1366 	}
1367 	ndmas = tx_rings->ndmas;
1368 	if (!ndmas) {
1369 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1370 		    "<== hxge_fixup_hung_txdma_rings: no channel allocated"));
1371 		return;
1372 	}
1373 	if (tx_rings->rings == NULL) {
1374 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1375 		    "<== hxge_fixup_hung_txdma_rings: NULL rings pointer"));
1376 		return;
1377 	}
1378 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings: "
1379 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1380 	    tx_rings, tx_rings->rings, ndmas));
1381 
1382 	for (index = 0; index < ndmas; index++) {
1383 		channel = tx_rings->rings[index]->tdc;
1384 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1385 		    "==> hxge_fixup_hung_txdma_rings: channel %d", channel));
1386 		hxge_txdma_fixup_hung_channel(hxgep, tx_rings->rings[index],
1387 		    channel);
1388 	}
1389 
1390 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_hung_txdma_rings"));
1391 }
1392 
1393 /*ARGSUSED*/
1394 void
hxge_txdma_fix_hung_channel(p_hxge_t hxgep,uint16_t channel)1395 hxge_txdma_fix_hung_channel(p_hxge_t hxgep, uint16_t channel)
1396 {
1397 	p_tx_ring_t ring_p;
1398 
1399 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_hung_channel"));
1400 	ring_p = hxge_txdma_get_ring(hxgep, channel);
1401 	if (ring_p == NULL) {
1402 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1403 		    "<== hxge_txdma_fix_hung_channel"));
1404 		return;
1405 	}
1406 	if (ring_p->tdc != channel) {
1407 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1408 		    "<== hxge_txdma_fix_hung_channel: channel not matched "
1409 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1410 		return;
1411 	}
1412 	hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1413 
1414 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_hung_channel"));
1415 }
1416 
1417 /*ARGSUSED*/
1418 void
hxge_txdma_fixup_hung_channel(p_hxge_t hxgep,p_tx_ring_t ring_p,uint16_t channel)1419 hxge_txdma_fixup_hung_channel(p_hxge_t hxgep, p_tx_ring_t ring_p,
1420     uint16_t channel)
1421 {
1422 	hpi_handle_t	handle;
1423 	int		status = HXGE_OK;
1424 
1425 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_hung_channel"));
1426 
1427 	if (ring_p == NULL) {
1428 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1429 		    "<== hxge_txdma_fixup_hung_channel: NULL ring pointer"));
1430 		return;
1431 	}
1432 	if (ring_p->tdc != channel) {
1433 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1434 		    "<== hxge_txdma_fixup_hung_channel: channel "
1435 		    "not matched ring tdc %d passed channel",
1436 		    ring_p->tdc, channel));
1437 		return;
1438 	}
1439 	/* Reclaim descriptors */
1440 	MUTEX_ENTER(&ring_p->lock);
1441 	(void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1442 	MUTEX_EXIT(&ring_p->lock);
1443 
1444 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
1445 	/*
1446 	 * Stop the dma channel waits for the stop done. If the stop done bit
1447 	 * is not set, then force an error.
1448 	 */
1449 	status = hpi_txdma_channel_disable(handle, channel);
1450 	if (!(status & HPI_TXDMA_STOP_FAILED)) {
1451 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1452 		    "<== hxge_txdma_fixup_hung_channel: stopped OK "
1453 		    "ring tdc %d passed channel %d", ring_p->tdc, channel));
1454 		return;
1455 	}
1456 	/* Stop done bit will be set as a result of error injection */
1457 	status = hpi_txdma_channel_disable(handle, channel);
1458 	if (!(status & HPI_TXDMA_STOP_FAILED)) {
1459 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1460 		    "<== hxge_txdma_fixup_hung_channel: stopped again"
1461 		    "ring tdc %d passed channel", ring_p->tdc, channel));
1462 		return;
1463 	}
1464 
1465 	HXGE_DEBUG_MSG((hxgep, TX_CTL,
1466 	    "<== hxge_txdma_fixup_hung_channel: stop done still not set!! "
1467 	    "ring tdc %d passed channel", ring_p->tdc, channel));
1468 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_hung_channel"));
1469 }
1470 
1471 /*ARGSUSED*/
1472 void
hxge_reclaim_rings(p_hxge_t hxgep)1473 hxge_reclaim_rings(p_hxge_t hxgep)
1474 {
1475 	int		index, ndmas;
1476 	uint16_t	channel;
1477 	p_tx_rings_t	tx_rings;
1478 	p_tx_ring_t	tx_ring_p;
1479 
1480 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclaim_ring"));
1481 	tx_rings = hxgep->tx_rings;
1482 	if (tx_rings == NULL) {
1483 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1484 		    "<== hxge_reclain_rimgs: NULL ring pointer"));
1485 		return;
1486 	}
1487 	ndmas = tx_rings->ndmas;
1488 	if (!ndmas) {
1489 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1490 		    "<== hxge_reclain_rimgs: no channel allocated"));
1491 		return;
1492 	}
1493 	if (tx_rings->rings == NULL) {
1494 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1495 		    "<== hxge_reclain_rimgs: NULL rings pointer"));
1496 		return;
1497 	}
1498 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclain_rimgs: "
1499 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1500 	    tx_rings, tx_rings->rings, ndmas));
1501 
1502 	for (index = 0; index < ndmas; index++) {
1503 		channel = tx_rings->rings[index]->tdc;
1504 		HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> reclain_rimgs: channel %d",
1505 		    channel));
1506 		tx_ring_p = tx_rings->rings[index];
1507 		MUTEX_ENTER(&tx_ring_p->lock);
1508 		(void) hxge_txdma_reclaim(hxgep, tx_ring_p, channel);
1509 		MUTEX_EXIT(&tx_ring_p->lock);
1510 	}
1511 
1512 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_reclaim_rings"));
1513 }
1514 
1515 /*
1516  * Static functions start here.
1517  */
1518 static hxge_status_t
hxge_map_txdma(p_hxge_t hxgep)1519 hxge_map_txdma(p_hxge_t hxgep)
1520 {
1521 	int			i, ndmas;
1522 	uint16_t		channel;
1523 	p_tx_rings_t		tx_rings;
1524 	p_tx_ring_t		*tx_desc_rings;
1525 	p_tx_mbox_areas_t	tx_mbox_areas_p;
1526 	p_tx_mbox_t		*tx_mbox_p;
1527 	p_hxge_dma_pool_t	dma_buf_poolp;
1528 	p_hxge_dma_pool_t	dma_cntl_poolp;
1529 	p_hxge_dma_common_t	*dma_buf_p;
1530 	p_hxge_dma_common_t	*dma_cntl_p;
1531 	hxge_status_t		status = HXGE_OK;
1532 
1533 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma"));
1534 
1535 	dma_buf_poolp = hxgep->tx_buf_pool_p;
1536 	dma_cntl_poolp = hxgep->tx_cntl_pool_p;
1537 
1538 	if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) {
1539 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1540 		    "==> hxge_map_txdma: buf not allocated"));
1541 		return (HXGE_ERROR);
1542 	}
1543 	ndmas = dma_buf_poolp->ndmas;
1544 	if (!ndmas) {
1545 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1546 		    "<== hxge_map_txdma: no dma allocated"));
1547 		return (HXGE_ERROR);
1548 	}
1549 	dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
1550 	dma_cntl_p = dma_cntl_poolp->dma_buf_pool_p;
1551 
1552 	tx_rings = (p_tx_rings_t)KMEM_ZALLOC(sizeof (tx_rings_t), KM_SLEEP);
1553 	tx_desc_rings = (p_tx_ring_t *)KMEM_ZALLOC(
1554 	    sizeof (p_tx_ring_t) * ndmas, KM_SLEEP);
1555 
1556 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1557 	    "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
1558 
1559 	tx_mbox_areas_p = (p_tx_mbox_areas_t)
1560 	    KMEM_ZALLOC(sizeof (tx_mbox_areas_t), KM_SLEEP);
1561 	tx_mbox_p = (p_tx_mbox_t *)KMEM_ZALLOC(
1562 	    sizeof (p_tx_mbox_t) * ndmas, KM_SLEEP);
1563 
1564 	/*
1565 	 * Map descriptors from the buffer pools for each dma channel.
1566 	 */
1567 	for (i = 0; i < ndmas; i++) {
1568 		/*
1569 		 * Set up and prepare buffer blocks, descriptors and mailbox.
1570 		 */
1571 		channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1572 		status = hxge_map_txdma_channel(hxgep, channel,
1573 		    (p_hxge_dma_common_t *)&dma_buf_p[i],
1574 		    (p_tx_ring_t *)&tx_desc_rings[i],
1575 		    dma_buf_poolp->num_chunks[i],
1576 		    (p_hxge_dma_common_t *)&dma_cntl_p[i],
1577 		    (p_tx_mbox_t *)&tx_mbox_p[i]);
1578 		if (status != HXGE_OK) {
1579 			goto hxge_map_txdma_fail1;
1580 		}
1581 		tx_desc_rings[i]->index = (uint16_t)i;
1582 		tx_desc_rings[i]->tdc_stats = &hxgep->statsp->tdc_stats[i];
1583 	}
1584 
1585 	tx_rings->ndmas = ndmas;
1586 	tx_rings->rings = tx_desc_rings;
1587 	hxgep->tx_rings = tx_rings;
1588 	tx_mbox_areas_p->txmbox_areas_p = tx_mbox_p;
1589 	hxgep->tx_mbox_areas_p = tx_mbox_areas_p;
1590 
1591 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1592 	    "tx_rings $%p rings $%p", hxgep->tx_rings, hxgep->tx_rings->rings));
1593 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1594 	    "tx_rings $%p tx_desc_rings $%p",
1595 	    hxgep->tx_rings, tx_desc_rings));
1596 
1597 	goto hxge_map_txdma_exit;
1598 
1599 hxge_map_txdma_fail1:
1600 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1601 	    "==> hxge_map_txdma: uninit tx desc "
1602 	    "(status 0x%x channel %d i %d)", hxgep, status, channel, i));
1603 	i--;
1604 	for (; i >= 0; i--) {
1605 		channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1606 		hxge_unmap_txdma_channel(hxgep, channel, tx_desc_rings[i],
1607 		    tx_mbox_p[i]);
1608 	}
1609 
1610 	KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1611 	KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1612 	KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1613 	KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1614 
1615 hxge_map_txdma_exit:
1616 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1617 	    "==> hxge_map_txdma: (status 0x%x channel %d)", status, channel));
1618 
1619 	return (status);
1620 }
1621 
1622 static void
hxge_unmap_txdma(p_hxge_t hxgep)1623 hxge_unmap_txdma(p_hxge_t hxgep)
1624 {
1625 	int			i, ndmas;
1626 	uint8_t			channel;
1627 	p_tx_rings_t		tx_rings;
1628 	p_tx_ring_t		*tx_desc_rings;
1629 	p_tx_mbox_areas_t	tx_mbox_areas_p;
1630 	p_tx_mbox_t		*tx_mbox_p;
1631 	p_hxge_dma_pool_t	dma_buf_poolp;
1632 
1633 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma"));
1634 
1635 	dma_buf_poolp = hxgep->tx_buf_pool_p;
1636 	if (!dma_buf_poolp->buf_allocated) {
1637 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1638 		    "==> hxge_unmap_txdma: buf not allocated"));
1639 		return;
1640 	}
1641 	ndmas = dma_buf_poolp->ndmas;
1642 	if (!ndmas) {
1643 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1644 		    "<== hxge_unmap_txdma: no dma allocated"));
1645 		return;
1646 	}
1647 	tx_rings = hxgep->tx_rings;
1648 	tx_desc_rings = tx_rings->rings;
1649 	if (tx_rings == NULL) {
1650 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1651 		    "<== hxge_unmap_txdma: NULL ring pointer"));
1652 		return;
1653 	}
1654 	tx_desc_rings = tx_rings->rings;
1655 	if (tx_desc_rings == NULL) {
1656 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
1657 		    "<== hxge_unmap_txdma: NULL ring pointers"));
1658 		return;
1659 	}
1660 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma: "
1661 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
1662 	    tx_rings, tx_desc_rings, ndmas));
1663 
1664 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
1665 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
1666 
1667 	for (i = 0; i < ndmas; i++) {
1668 		channel = tx_desc_rings[i]->tdc;
1669 		(void) hxge_unmap_txdma_channel(hxgep, channel,
1670 		    (p_tx_ring_t)tx_desc_rings[i],
1671 		    (p_tx_mbox_t)tx_mbox_p[i]);
1672 	}
1673 
1674 	KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1675 	KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1676 	KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1677 	KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1678 
1679 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma"));
1680 }
1681 
1682 static hxge_status_t
hxge_map_txdma_channel(p_hxge_t hxgep,uint16_t channel,p_hxge_dma_common_t * dma_buf_p,p_tx_ring_t * tx_desc_p,uint32_t num_chunks,p_hxge_dma_common_t * dma_cntl_p,p_tx_mbox_t * tx_mbox_p)1683 hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1684     p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
1685     uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
1686     p_tx_mbox_t *tx_mbox_p)
1687 {
1688 	int status = HXGE_OK;
1689 
1690 	/*
1691 	 * Set up and prepare buffer blocks, descriptors and mailbox.
1692 	 */
1693 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1694 	    "==> hxge_map_txdma_channel (channel %d)", channel));
1695 
1696 	/*
1697 	 * Transmit buffer blocks
1698 	 */
1699 	status = hxge_map_txdma_channel_buf_ring(hxgep, channel,
1700 	    dma_buf_p, tx_desc_p, num_chunks);
1701 	if (status != HXGE_OK) {
1702 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1703 		    "==> hxge_map_txdma_channel (channel %d): "
1704 		    "map buffer failed 0x%x", channel, status));
1705 		goto hxge_map_txdma_channel_exit;
1706 	}
1707 	/*
1708 	 * Transmit block ring, and mailbox.
1709 	 */
1710 	hxge_map_txdma_channel_cfg_ring(hxgep, channel, dma_cntl_p, *tx_desc_p,
1711 	    tx_mbox_p);
1712 
1713 	goto hxge_map_txdma_channel_exit;
1714 
1715 hxge_map_txdma_channel_fail1:
1716 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1717 	    "==> hxge_map_txdma_channel: unmap buf"
1718 	    "(status 0x%x channel %d)", status, channel));
1719 	hxge_unmap_txdma_channel_buf_ring(hxgep, *tx_desc_p);
1720 
1721 hxge_map_txdma_channel_exit:
1722 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1723 	    "<== hxge_map_txdma_channel: (status 0x%x channel %d)",
1724 	    status, channel));
1725 
1726 	return (status);
1727 }
1728 
1729 /*ARGSUSED*/
1730 static void
hxge_unmap_txdma_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)1731 hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1732     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1733 {
1734 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1735 	    "==> hxge_unmap_txdma_channel (channel %d)", channel));
1736 
1737 	/* unmap tx block ring, and mailbox.  */
1738 	(void) hxge_unmap_txdma_channel_cfg_ring(hxgep, tx_ring_p, tx_mbox_p);
1739 
1740 	/* unmap buffer blocks */
1741 	(void) hxge_unmap_txdma_channel_buf_ring(hxgep, tx_ring_p);
1742 
1743 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma_channel"));
1744 }
1745 
1746 /*ARGSUSED*/
1747 static void
hxge_map_txdma_channel_cfg_ring(p_hxge_t hxgep,uint16_t dma_channel,p_hxge_dma_common_t * dma_cntl_p,p_tx_ring_t tx_ring_p,p_tx_mbox_t * tx_mbox_p)1748 hxge_map_txdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel,
1749     p_hxge_dma_common_t *dma_cntl_p, p_tx_ring_t tx_ring_p,
1750     p_tx_mbox_t *tx_mbox_p)
1751 {
1752 	p_tx_mbox_t		mboxp;
1753 	p_hxge_dma_common_t	cntl_dmap;
1754 	p_hxge_dma_common_t	dmap;
1755 	tdc_tdr_cfg_t		*tx_ring_cfig_p;
1756 	tdc_tdr_kick_t		*tx_ring_kick_p;
1757 	tdc_tdr_cfg_t		*tx_cs_p;
1758 	tdc_int_mask_t		*tx_evmask_p;
1759 	tdc_mbh_t		*mboxh_p;
1760 	tdc_mbl_t		*mboxl_p;
1761 	uint64_t		tx_desc_len;
1762 
1763 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1764 	    "==> hxge_map_txdma_channel_cfg_ring"));
1765 
1766 	cntl_dmap = *dma_cntl_p;
1767 
1768 	dmap = (p_hxge_dma_common_t)&tx_ring_p->tdc_desc;
1769 	hxge_setup_dma_common(dmap, cntl_dmap, tx_ring_p->tx_ring_size,
1770 	    sizeof (tx_desc_t));
1771 
1772 	/*
1773 	 * Zero out transmit ring descriptors.
1774 	 */
1775 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
1776 	tx_ring_cfig_p = &(tx_ring_p->tx_ring_cfig);
1777 	tx_ring_kick_p = &(tx_ring_p->tx_ring_kick);
1778 	tx_cs_p = &(tx_ring_p->tx_cs);
1779 	tx_evmask_p = &(tx_ring_p->tx_evmask);
1780 	tx_ring_cfig_p->value = 0;
1781 	tx_ring_kick_p->value = 0;
1782 	tx_cs_p->value = 0;
1783 	tx_evmask_p->value = 0;
1784 
1785 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1786 	    "==> hxge_map_txdma_channel_cfg_ring: channel %d des $%p",
1787 	    dma_channel, dmap->dma_cookie.dmac_laddress));
1788 
1789 	tx_ring_cfig_p->value = 0;
1790 
1791 	/* Hydra len is 11 bits and the lower 5 bits are 0s */
1792 	tx_desc_len = (uint64_t)(tx_ring_p->tx_ring_size >> 5);
1793 	tx_ring_cfig_p->value =
1794 	    (dmap->dma_cookie.dmac_laddress & TDC_TDR_CFG_ADDR_MASK) |
1795 	    (tx_desc_len << TDC_TDR_CFG_LEN_SHIFT);
1796 
1797 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1798 	    "==> hxge_map_txdma_channel_cfg_ring: channel %d cfg 0x%llx",
1799 	    dma_channel, tx_ring_cfig_p->value));
1800 
1801 	tx_cs_p->bits.reset = 1;
1802 
1803 	/* Map in mailbox */
1804 	mboxp = (p_tx_mbox_t)KMEM_ZALLOC(sizeof (tx_mbox_t), KM_SLEEP);
1805 	dmap = (p_hxge_dma_common_t)&mboxp->tx_mbox;
1806 	hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (txdma_mailbox_t));
1807 	mboxh_p = (tdc_mbh_t *)&tx_ring_p->tx_mbox_mbh;
1808 	mboxl_p = (tdc_mbl_t *)&tx_ring_p->tx_mbox_mbl;
1809 	mboxh_p->value = mboxl_p->value = 0;
1810 
1811 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1812 	    "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1813 	    dmap->dma_cookie.dmac_laddress));
1814 
1815 	mboxh_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress >>
1816 	    TDC_MBH_ADDR_SHIFT) & TDC_MBH_MASK);
1817 	mboxl_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress &
1818 	    TDC_MBL_MASK) >> TDC_MBL_SHIFT);
1819 
1820 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1821 	    "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1822 	    dmap->dma_cookie.dmac_laddress));
1823 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1824 	    "==> hxge_map_txdma_channel_cfg_ring: hmbox $%p mbox $%p",
1825 	    mboxh_p->bits.mbaddr, mboxl_p->bits.mbaddr));
1826 
1827 	/*
1828 	 * Set page valid and no mask
1829 	 */
1830 	tx_ring_p->page_hdl.value = 0;
1831 
1832 	*tx_mbox_p = mboxp;
1833 
1834 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1835 	    "<== hxge_map_txdma_channel_cfg_ring"));
1836 }
1837 
1838 /*ARGSUSED*/
1839 static void
hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)1840 hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
1841     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1842 {
1843 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1844 	    "==> hxge_unmap_txdma_channel_cfg_ring: channel %d",
1845 	    tx_ring_p->tdc));
1846 
1847 	KMEM_FREE(tx_mbox_p, sizeof (tx_mbox_t));
1848 
1849 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1850 	    "<== hxge_unmap_txdma_channel_cfg_ring"));
1851 }
1852 
1853 static hxge_status_t
hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep,uint16_t channel,p_hxge_dma_common_t * dma_buf_p,p_tx_ring_t * tx_desc_p,uint32_t num_chunks)1854 hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel,
1855     p_hxge_dma_common_t *dma_buf_p,
1856     p_tx_ring_t *tx_desc_p, uint32_t num_chunks)
1857 {
1858 	p_hxge_dma_common_t	dma_bufp, tmp_bufp;
1859 	p_hxge_dma_common_t	dmap;
1860 	hxge_os_dma_handle_t	tx_buf_dma_handle;
1861 	p_tx_ring_t		tx_ring_p;
1862 	p_tx_msg_t		tx_msg_ring;
1863 	hxge_status_t		status = HXGE_OK;
1864 	int			ddi_status = DDI_SUCCESS;
1865 	int			i, j, index;
1866 	uint32_t		size, bsize;
1867 	uint32_t		nblocks, nmsgs;
1868 	char			qname[TASKQ_NAMELEN];
1869 
1870 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1871 	    "==> hxge_map_txdma_channel_buf_ring"));
1872 
1873 	dma_bufp = tmp_bufp = *dma_buf_p;
1874 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1875 	    " hxge_map_txdma_channel_buf_ring: channel %d to map %d "
1876 	    "chunks bufp $%p", channel, num_chunks, dma_bufp));
1877 
1878 	nmsgs = 0;
1879 	for (i = 0; i < num_chunks; i++, tmp_bufp++) {
1880 		nmsgs += tmp_bufp->nblocks;
1881 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1882 		    "==> hxge_map_txdma_channel_buf_ring: channel %d "
1883 		    "bufp $%p nblocks %d nmsgs %d",
1884 		    channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
1885 	}
1886 	if (!nmsgs) {
1887 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1888 		    "<== hxge_map_txdma_channel_buf_ring: channel %d "
1889 		    "no msg blocks", channel));
1890 		status = HXGE_ERROR;
1891 
1892 		goto hxge_map_txdma_channel_buf_ring_exit;
1893 	}
1894 
1895 	tx_ring_p = (p_tx_ring_t)KMEM_ZALLOC(sizeof (tx_ring_t), KM_SLEEP);
1896 	tx_ring_p->hxgep = hxgep;
1897 	(void) snprintf(qname, TASKQ_NAMELEN, "hxge_%d_%d",
1898 	    hxgep->instance, channel);
1899 	tx_ring_p->taskq = ddi_taskq_create(hxgep->dip, qname, 1,
1900 	    TASKQ_DEFAULTPRI, 0);
1901 	if (tx_ring_p->taskq == NULL) {
1902 		goto hxge_map_txdma_channel_buf_ring_fail1;
1903 	}
1904 
1905 	MUTEX_INIT(&tx_ring_p->lock, NULL, MUTEX_DRIVER,
1906 	    (void *) hxgep->interrupt_cookie);
1907 	/*
1908 	 * Allocate transmit message rings and handles for packets not to be
1909 	 * copied to premapped buffers.
1910 	 */
1911 	size = nmsgs * sizeof (tx_msg_t);
1912 	tx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
1913 	for (i = 0; i < nmsgs; i++) {
1914 		ddi_status = ddi_dma_alloc_handle(hxgep->dip, &hxge_tx_dma_attr,
1915 		    DDI_DMA_DONTWAIT, 0, &tx_msg_ring[i].dma_handle);
1916 		if (ddi_status != DDI_SUCCESS) {
1917 			status |= HXGE_DDI_FAILED;
1918 			break;
1919 		}
1920 	}
1921 
1922 	if (i < nmsgs) {
1923 		HXGE_DEBUG_MSG((hxgep, HXGE_ERR_CTL,
1924 		    "Allocate handles failed."));
1925 
1926 		goto hxge_map_txdma_channel_buf_ring_fail1;
1927 	}
1928 	tx_ring_p->tdc = channel;
1929 	tx_ring_p->tx_msg_ring = tx_msg_ring;
1930 	tx_ring_p->tx_ring_size = nmsgs;
1931 	tx_ring_p->num_chunks = num_chunks;
1932 	if (!hxge_tx_intr_thres) {
1933 		hxge_tx_intr_thres = tx_ring_p->tx_ring_size / 4;
1934 	}
1935 	tx_ring_p->tx_wrap_mask = tx_ring_p->tx_ring_size - 1;
1936 	tx_ring_p->rd_index = 0;
1937 	tx_ring_p->wr_index = 0;
1938 	tx_ring_p->ring_head.value = 0;
1939 	tx_ring_p->ring_kick_tail.value = 0;
1940 	tx_ring_p->descs_pending = 0;
1941 
1942 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1943 	    "==> hxge_map_txdma_channel_buf_ring: channel %d "
1944 	    "actual tx desc max %d nmsgs %d (config hxge_tx_ring_size %d)",
1945 	    channel, tx_ring_p->tx_ring_size, nmsgs, hxge_tx_ring_size));
1946 
1947 	/*
1948 	 * Map in buffers from the buffer pool.
1949 	 */
1950 	index = 0;
1951 	bsize = dma_bufp->block_size;
1952 
1953 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma_channel_buf_ring: "
1954 	    "dma_bufp $%p tx_rng_p $%p tx_msg_rng_p $%p bsize %d",
1955 	    dma_bufp, tx_ring_p, tx_msg_ring, bsize));
1956 
1957 	for (i = 0; i < num_chunks; i++, dma_bufp++) {
1958 		bsize = dma_bufp->block_size;
1959 		nblocks = dma_bufp->nblocks;
1960 		tx_buf_dma_handle = dma_bufp->dma_handle;
1961 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1962 		    "==> hxge_map_txdma_channel_buf_ring: dma chunk %d "
1963 		    "size %d dma_bufp $%p",
1964 		    i, sizeof (hxge_dma_common_t), dma_bufp));
1965 
1966 		for (j = 0; j < nblocks; j++) {
1967 			tx_msg_ring[index].buf_dma_handle = tx_buf_dma_handle;
1968 			tx_msg_ring[index].offset_index = j;
1969 			dmap = &tx_msg_ring[index++].buf_dma;
1970 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1971 			    "==> hxge_map_txdma_channel_buf_ring: j %d"
1972 			    "dmap $%p", i, dmap));
1973 			hxge_setup_dma_common(dmap, dma_bufp, 1, bsize);
1974 		}
1975 	}
1976 
1977 	if (i < num_chunks) {
1978 		status = HXGE_ERROR;
1979 
1980 		goto hxge_map_txdma_channel_buf_ring_fail1;
1981 	}
1982 
1983 	*tx_desc_p = tx_ring_p;
1984 
1985 	goto hxge_map_txdma_channel_buf_ring_exit;
1986 
1987 hxge_map_txdma_channel_buf_ring_fail1:
1988 	if (tx_ring_p->taskq) {
1989 		ddi_taskq_destroy(tx_ring_p->taskq);
1990 		tx_ring_p->taskq = NULL;
1991 	}
1992 
1993 	index--;
1994 	for (; index >= 0; index--) {
1995 		if (tx_msg_ring[index].dma_handle != NULL) {
1996 			ddi_dma_free_handle(&tx_msg_ring[index].dma_handle);
1997 		}
1998 	}
1999 	MUTEX_DESTROY(&tx_ring_p->lock);
2000 	KMEM_FREE(tx_msg_ring, size);
2001 	KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2002 
2003 	status = HXGE_ERROR;
2004 
2005 hxge_map_txdma_channel_buf_ring_exit:
2006 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2007 	    "<== hxge_map_txdma_channel_buf_ring status 0x%x", status));
2008 
2009 	return (status);
2010 }
2011 
2012 /*ARGSUSED*/
2013 static void
hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep,p_tx_ring_t tx_ring_p)2014 hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep, p_tx_ring_t tx_ring_p)
2015 {
2016 	p_tx_msg_t	tx_msg_ring;
2017 	p_tx_msg_t	tx_msg_p;
2018 	int		i;
2019 
2020 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2021 	    "==> hxge_unmap_txdma_channel_buf_ring"));
2022 	if (tx_ring_p == NULL) {
2023 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2024 		    "<== hxge_unmap_txdma_channel_buf_ring: NULL ringp"));
2025 		return;
2026 	}
2027 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2028 	    "==> hxge_unmap_txdma_channel_buf_ring: channel %d",
2029 	    tx_ring_p->tdc));
2030 
2031 	MUTEX_ENTER(&tx_ring_p->lock);
2032 	tx_msg_ring = tx_ring_p->tx_msg_ring;
2033 	for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2034 		tx_msg_p = &tx_msg_ring[i];
2035 		if (tx_msg_p->flags.dma_type == USE_DVMA) {
2036 			HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "entry = %d", i));
2037 			(void) dvma_unload(tx_msg_p->dvma_handle, 0, -1);
2038 			tx_msg_p->dvma_handle = NULL;
2039 			if (tx_ring_p->dvma_wr_index ==
2040 			    tx_ring_p->dvma_wrap_mask) {
2041 				tx_ring_p->dvma_wr_index = 0;
2042 			} else {
2043 				tx_ring_p->dvma_wr_index++;
2044 			}
2045 			tx_ring_p->dvma_pending--;
2046 		} else if (tx_msg_p->flags.dma_type == USE_DMA) {
2047 			if (ddi_dma_unbind_handle(tx_msg_p->dma_handle)) {
2048 				cmn_err(CE_WARN, "hxge_unmap_tx_bug_ring: "
2049 				    "ddi_dma_unbind_handle failed.");
2050 			}
2051 		}
2052 		if (tx_msg_p->tx_message != NULL) {
2053 			freemsg(tx_msg_p->tx_message);
2054 			tx_msg_p->tx_message = NULL;
2055 		}
2056 	}
2057 
2058 	for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2059 		if (tx_msg_ring[i].dma_handle != NULL) {
2060 			ddi_dma_free_handle(&tx_msg_ring[i].dma_handle);
2061 		}
2062 	}
2063 	MUTEX_EXIT(&tx_ring_p->lock);
2064 
2065 	if (tx_ring_p->taskq) {
2066 		ddi_taskq_destroy(tx_ring_p->taskq);
2067 		tx_ring_p->taskq = NULL;
2068 	}
2069 
2070 	MUTEX_DESTROY(&tx_ring_p->lock);
2071 	KMEM_FREE(tx_msg_ring, sizeof (tx_msg_t) * tx_ring_p->tx_ring_size);
2072 	KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2073 
2074 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2075 	    "<== hxge_unmap_txdma_channel_buf_ring"));
2076 }
2077 
2078 static hxge_status_t
hxge_txdma_hw_start(p_hxge_t hxgep)2079 hxge_txdma_hw_start(p_hxge_t hxgep)
2080 {
2081 	int			i, ndmas;
2082 	uint16_t		channel;
2083 	p_tx_rings_t		tx_rings;
2084 	p_tx_ring_t		*tx_desc_rings;
2085 	p_tx_mbox_areas_t	tx_mbox_areas_p;
2086 	p_tx_mbox_t		*tx_mbox_p;
2087 	hxge_status_t		status = HXGE_OK;
2088 	uint64_t		tmp;
2089 
2090 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start"));
2091 
2092 	/*
2093 	 * Initialize REORD Table 1. Disable VMAC 2. Reset the FIFO Err Stat.
2094 	 * 3. Scrub memory and check for errors.
2095 	 */
2096 	(void) hxge_tx_vmac_disable(hxgep);
2097 
2098 	/*
2099 	 * Clear the error status
2100 	 */
2101 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2102 
2103 	/*
2104 	 * Scrub the rtab memory for the TDC and reset the TDC.
2105 	 */
2106 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, 0x0ULL);
2107 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, 0x0ULL);
2108 
2109 	for (i = 0; i < 256; i++) {
2110 		HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2111 		    (uint64_t)i);
2112 
2113 		/*
2114 		 * Write the command register with an indirect read instruction
2115 		 */
2116 		tmp = (0x1ULL << 30) | i;
2117 		HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2118 
2119 		/*
2120 		 * Wait for status done
2121 		 */
2122 		tmp = 0;
2123 		do {
2124 			HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2125 			    &tmp);
2126 		} while (((tmp >> 31) & 0x1ULL) == 0x0);
2127 	}
2128 
2129 	for (i = 0; i < 256; i++) {
2130 		/*
2131 		 * Write the command register with an indirect read instruction
2132 		 */
2133 		tmp = (0x1ULL << 30) | i;
2134 		HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2135 
2136 		/*
2137 		 * Wait for status done
2138 		 */
2139 		tmp = 0;
2140 		do {
2141 			HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2142 			    &tmp);
2143 		} while (((tmp >> 31) & 0x1ULL) == 0x0);
2144 
2145 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, &tmp);
2146 		if (0x1ff00ULL != (0x1ffffULL & tmp)) {
2147 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2148 			    "unexpected data (hi), entry: %x, value: 0x%0llx\n",
2149 			    i, (unsigned long long)tmp));
2150 			status = HXGE_ERROR;
2151 		}
2152 
2153 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, &tmp);
2154 		if (tmp != 0) {
2155 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2156 			    "unexpected data (lo), entry: %x\n", i));
2157 			status = HXGE_ERROR;
2158 		}
2159 
2160 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2161 		if (tmp != 0) {
2162 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2163 			    "parity error, entry: %x, val 0x%llx\n",
2164 			    i, (unsigned long long)tmp));
2165 			status = HXGE_ERROR;
2166 		}
2167 
2168 		HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2169 		if (tmp != 0) {
2170 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2171 			    "parity error, entry: %x\n", i));
2172 			status = HXGE_ERROR;
2173 		}
2174 	}
2175 
2176 	if (status != HXGE_OK)
2177 		goto hxge_txdma_hw_start_exit;
2178 
2179 	/*
2180 	 * Reset FIFO Error Status for the TDC and enable FIFO error events.
2181 	 */
2182 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2183 	HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_MASK, 0x0);
2184 
2185 	/*
2186 	 * Initialize the Transmit DMAs.
2187 	 */
2188 	tx_rings = hxgep->tx_rings;
2189 	if (tx_rings == NULL) {
2190 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2191 		    "<== hxge_txdma_hw_start: NULL ring pointer"));
2192 		return (HXGE_ERROR);
2193 	}
2194 
2195 	tx_desc_rings = tx_rings->rings;
2196 	if (tx_desc_rings == NULL) {
2197 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2198 		    "<== hxge_txdma_hw_start: NULL ring pointers"));
2199 		return (HXGE_ERROR);
2200 	}
2201 	ndmas = tx_rings->ndmas;
2202 	if (!ndmas) {
2203 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2204 		    "<== hxge_txdma_hw_start: no dma channel allocated"));
2205 		return (HXGE_ERROR);
2206 	}
2207 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start: "
2208 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
2209 	    tx_rings, tx_desc_rings, ndmas));
2210 
2211 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2212 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2213 
2214 	/*
2215 	 * Init the DMAs.
2216 	 */
2217 	for (i = 0; i < ndmas; i++) {
2218 		channel = tx_desc_rings[i]->tdc;
2219 		status = hxge_txdma_start_channel(hxgep, channel,
2220 		    (p_tx_ring_t)tx_desc_rings[i],
2221 		    (p_tx_mbox_t)tx_mbox_p[i]);
2222 		if (status != HXGE_OK) {
2223 			goto hxge_txdma_hw_start_fail1;
2224 		}
2225 	}
2226 
2227 	(void) hxge_tx_vmac_enable(hxgep);
2228 
2229 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2230 	    "==> hxge_txdma_hw_start: tx_rings $%p rings $%p",
2231 	    hxgep->tx_rings, hxgep->tx_rings->rings));
2232 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2233 	    "==> hxge_txdma_hw_start: tx_rings $%p tx_desc_rings $%p",
2234 	    hxgep->tx_rings, tx_desc_rings));
2235 
2236 	goto hxge_txdma_hw_start_exit;
2237 
2238 hxge_txdma_hw_start_fail1:
2239 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2240 	    "==> hxge_txdma_hw_start: disable (status 0x%x channel %d i %d)",
2241 	    status, channel, i));
2242 
2243 	for (; i >= 0; i--) {
2244 		channel = tx_desc_rings[i]->tdc,
2245 		    (void) hxge_txdma_stop_channel(hxgep, channel,
2246 		    (p_tx_ring_t)tx_desc_rings[i],
2247 		    (p_tx_mbox_t)tx_mbox_p[i]);
2248 	}
2249 
2250 hxge_txdma_hw_start_exit:
2251 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2252 	    "==> hxge_txdma_hw_start: (status 0x%x)", status));
2253 
2254 	return (status);
2255 }
2256 
2257 static void
hxge_txdma_hw_stop(p_hxge_t hxgep)2258 hxge_txdma_hw_stop(p_hxge_t hxgep)
2259 {
2260 	int			i, ndmas;
2261 	uint16_t		channel;
2262 	p_tx_rings_t		tx_rings;
2263 	p_tx_ring_t		*tx_desc_rings;
2264 	p_tx_mbox_areas_t	tx_mbox_areas_p;
2265 	p_tx_mbox_t		*tx_mbox_p;
2266 
2267 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop"));
2268 
2269 	tx_rings = hxgep->tx_rings;
2270 	if (tx_rings == NULL) {
2271 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2272 		    "<== hxge_txdma_hw_stop: NULL ring pointer"));
2273 		return;
2274 	}
2275 
2276 	tx_desc_rings = tx_rings->rings;
2277 	if (tx_desc_rings == NULL) {
2278 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2279 		    "<== hxge_txdma_hw_stop: NULL ring pointers"));
2280 		return;
2281 	}
2282 
2283 	ndmas = tx_rings->ndmas;
2284 	if (!ndmas) {
2285 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2286 		    "<== hxge_txdma_hw_stop: no dma channel allocated"));
2287 		return;
2288 	}
2289 
2290 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2291 	    "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2292 
2293 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2294 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2295 
2296 	for (i = 0; i < ndmas; i++) {
2297 		channel = tx_desc_rings[i]->tdc;
2298 		(void) hxge_txdma_stop_channel(hxgep, channel,
2299 		    (p_tx_ring_t)tx_desc_rings[i],
2300 		    (p_tx_mbox_t)tx_mbox_p[i]);
2301 	}
2302 
2303 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2304 	    "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2305 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_hw_stop"));
2306 }
2307 
2308 static hxge_status_t
hxge_txdma_start_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)2309 hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
2310     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2311 {
2312 	hxge_status_t status = HXGE_OK;
2313 
2314 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2315 	    "==> hxge_txdma_start_channel (channel %d)", channel));
2316 	/*
2317 	 * TXDMA/TXC must be in stopped state.
2318 	 */
2319 	(void) hxge_txdma_stop_inj_err(hxgep, channel);
2320 
2321 	/*
2322 	 * Reset TXDMA channel
2323 	 */
2324 	tx_ring_p->tx_cs.value = 0;
2325 	tx_ring_p->tx_cs.bits.reset = 1;
2326 	status = hxge_reset_txdma_channel(hxgep, channel,
2327 	    tx_ring_p->tx_cs.value);
2328 	if (status != HXGE_OK) {
2329 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2330 		    "==> hxge_txdma_start_channel (channel %d)"
2331 		    " reset channel failed 0x%x", channel, status));
2332 
2333 		goto hxge_txdma_start_channel_exit;
2334 	}
2335 
2336 	/*
2337 	 * Initialize the TXDMA channel specific FZC control configurations.
2338 	 * These FZC registers are pertaining to each TX channel (i.e. logical
2339 	 * pages).
2340 	 */
2341 	status = hxge_init_fzc_txdma_channel(hxgep, channel,
2342 	    tx_ring_p, tx_mbox_p);
2343 	if (status != HXGE_OK) {
2344 		goto hxge_txdma_start_channel_exit;
2345 	}
2346 
2347 	/*
2348 	 * Initialize the event masks.
2349 	 */
2350 	tx_ring_p->tx_evmask.value = 0;
2351 	status = hxge_init_txdma_channel_event_mask(hxgep,
2352 	    channel, &tx_ring_p->tx_evmask);
2353 	if (status != HXGE_OK) {
2354 		goto hxge_txdma_start_channel_exit;
2355 	}
2356 
2357 	/*
2358 	 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2359 	 * channels and enable each DMA channel.
2360 	 */
2361 	status = hxge_enable_txdma_channel(hxgep, channel,
2362 	    tx_ring_p, tx_mbox_p);
2363 	if (status != HXGE_OK) {
2364 		goto hxge_txdma_start_channel_exit;
2365 	}
2366 
2367 hxge_txdma_start_channel_exit:
2368 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_start_channel"));
2369 
2370 	return (status);
2371 }
2372 
2373 /*ARGSUSED*/
2374 static hxge_status_t
hxge_txdma_stop_channel(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p,p_tx_mbox_t tx_mbox_p)2375 hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
2376     p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2377 {
2378 	int status = HXGE_OK;
2379 
2380 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2381 	    "==> hxge_txdma_stop_channel: channel %d", channel));
2382 
2383 	/*
2384 	 * Stop (disable) TXDMA and TXC (if stop bit is set and STOP_N_GO bit
2385 	 * not set, the TXDMA reset state will not be set if reset TXDMA.
2386 	 */
2387 	(void) hxge_txdma_stop_inj_err(hxgep, channel);
2388 
2389 	/*
2390 	 * Reset TXDMA channel
2391 	 */
2392 	tx_ring_p->tx_cs.value = 0;
2393 	tx_ring_p->tx_cs.bits.reset = 1;
2394 	status = hxge_reset_txdma_channel(hxgep, channel,
2395 	    tx_ring_p->tx_cs.value);
2396 	if (status != HXGE_OK) {
2397 		goto hxge_txdma_stop_channel_exit;
2398 	}
2399 
2400 hxge_txdma_stop_channel_exit:
2401 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_stop_channel"));
2402 
2403 	return (status);
2404 }
2405 
2406 static p_tx_ring_t
hxge_txdma_get_ring(p_hxge_t hxgep,uint16_t channel)2407 hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel)
2408 {
2409 	int		index, ndmas;
2410 	uint16_t	tdc;
2411 	p_tx_rings_t	tx_rings;
2412 
2413 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_ring"));
2414 
2415 	tx_rings = hxgep->tx_rings;
2416 	if (tx_rings == NULL) {
2417 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2418 		    "<== hxge_txdma_get_ring: NULL ring pointer"));
2419 		return (NULL);
2420 	}
2421 	ndmas = tx_rings->ndmas;
2422 	if (!ndmas) {
2423 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2424 		    "<== hxge_txdma_get_ring: no channel allocated"));
2425 		return (NULL);
2426 	}
2427 	if (tx_rings->rings == NULL) {
2428 		HXGE_DEBUG_MSG((hxgep, TX_CTL,
2429 		    "<== hxge_txdma_get_ring: NULL rings pointer"));
2430 		return (NULL);
2431 	}
2432 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_ring: "
2433 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
2434 	    tx_rings, tx_rings, ndmas));
2435 
2436 	for (index = 0; index < ndmas; index++) {
2437 		tdc = tx_rings->rings[index]->tdc;
2438 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2439 		    "==> hxge_fixup_txdma_rings: channel %d", tdc));
2440 		if (channel == tdc) {
2441 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
2442 			    "<== hxge_txdma_get_ring: tdc %d ring $%p",
2443 			    tdc, tx_rings->rings[index]));
2444 			return (p_tx_ring_t)(tx_rings->rings[index]);
2445 		}
2446 	}
2447 
2448 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_ring"));
2449 
2450 	return (NULL);
2451 }
2452 
2453 static p_tx_mbox_t
hxge_txdma_get_mbox(p_hxge_t hxgep,uint16_t channel)2454 hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel)
2455 {
2456 	int			index, tdc, ndmas;
2457 	p_tx_rings_t		tx_rings;
2458 	p_tx_mbox_areas_t	tx_mbox_areas_p;
2459 	p_tx_mbox_t		*tx_mbox_p;
2460 
2461 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_mbox"));
2462 
2463 	tx_rings = hxgep->tx_rings;
2464 	if (tx_rings == NULL) {
2465 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2466 		    "<== hxge_txdma_get_mbox: NULL ring pointer"));
2467 		return (NULL);
2468 	}
2469 	tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2470 	if (tx_mbox_areas_p == NULL) {
2471 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2472 		    "<== hxge_txdma_get_mbox: NULL mbox pointer"));
2473 		return (NULL);
2474 	}
2475 	tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2476 
2477 	ndmas = tx_rings->ndmas;
2478 	if (!ndmas) {
2479 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2480 		    "<== hxge_txdma_get_mbox: no channel allocated"));
2481 		return (NULL);
2482 	}
2483 	if (tx_rings->rings == NULL) {
2484 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2485 		    "<== hxge_txdma_get_mbox: NULL rings pointer"));
2486 		return (NULL);
2487 	}
2488 	HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_mbox: "
2489 	    "tx_rings $%p tx_desc_rings $%p ndmas %d",
2490 	    tx_rings, tx_rings, ndmas));
2491 
2492 	for (index = 0; index < ndmas; index++) {
2493 		tdc = tx_rings->rings[index]->tdc;
2494 		HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2495 		    "==> hxge_txdma_get_mbox: channel %d", tdc));
2496 		if (channel == tdc) {
2497 			HXGE_DEBUG_MSG((hxgep, TX_CTL,
2498 			    "<== hxge_txdma_get_mbox: tdc %d ring $%p",
2499 			    tdc, tx_rings->rings[index]));
2500 			return (p_tx_mbox_t)(tx_mbox_p[index]);
2501 		}
2502 	}
2503 
2504 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_mbox"));
2505 
2506 	return (NULL);
2507 }
2508 
2509 /*ARGSUSED*/
2510 static hxge_status_t
hxge_tx_err_evnts(p_hxge_t hxgep,uint_t index,p_hxge_ldv_t ldvp,tdc_stat_t cs)2511 hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp,
2512     tdc_stat_t cs)
2513 {
2514 	hpi_handle_t		handle;
2515 	uint8_t			channel;
2516 	p_tx_ring_t		*tx_rings;
2517 	p_tx_ring_t		tx_ring_p;
2518 	p_hxge_tx_ring_stats_t	tdc_stats;
2519 	boolean_t		txchan_fatal = B_FALSE;
2520 	hxge_status_t		status = HXGE_OK;
2521 	tdc_drop_cnt_t		drop_cnt;
2522 
2523 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_tx_err_evnts"));
2524 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2525 	channel = ldvp->channel;
2526 
2527 	tx_rings = hxgep->tx_rings->rings;
2528 	tx_ring_p = tx_rings[index];
2529 	tdc_stats = tx_ring_p->tdc_stats;
2530 
2531 	/* Get the error counts if any */
2532 	TXDMA_REG_READ64(handle, TDC_DROP_CNT, channel, &drop_cnt.value);
2533 	tdc_stats->count_hdr_size_err += drop_cnt.bits.hdr_size_error_count;
2534 	tdc_stats->count_runt += drop_cnt.bits.runt_count;
2535 	tdc_stats->count_abort += drop_cnt.bits.abort_count;
2536 
2537 	if (cs.bits.peu_resp_err) {
2538 		tdc_stats->peu_resp_err++;
2539 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2540 		    HXGE_FM_EREPORT_TDMC_PEU_RESP_ERR);
2541 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2542 		    "==> hxge_tx_err_evnts(channel %d): "
2543 		    "fatal error: peu_resp_err", channel));
2544 		txchan_fatal = B_TRUE;
2545 	}
2546 
2547 	if (cs.bits.pkt_size_hdr_err) {
2548 		tdc_stats->pkt_size_hdr_err++;
2549 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2550 		    HXGE_FM_EREPORT_TDMC_PKT_SIZE_HDR_ERR);
2551 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2552 		    "==> hxge_tx_err_evnts(channel %d): "
2553 		    "fatal error: pkt_size_hdr_err", channel));
2554 		txchan_fatal = B_TRUE;
2555 	}
2556 
2557 	if (cs.bits.runt_pkt_drop_err) {
2558 		tdc_stats->runt_pkt_drop_err++;
2559 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2560 		    HXGE_FM_EREPORT_TDMC_RUNT_PKT_DROP_ERR);
2561 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2562 		    "==> hxge_tx_err_evnts(channel %d): "
2563 		    "fatal error: runt_pkt_drop_err", channel));
2564 		txchan_fatal = B_TRUE;
2565 	}
2566 
2567 	if (cs.bits.pkt_size_err) {
2568 		tdc_stats->pkt_size_err++;
2569 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2570 		    HXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR);
2571 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2572 		    "==> hxge_tx_err_evnts(channel %d): "
2573 		    "fatal error: pkt_size_err", channel));
2574 		txchan_fatal = B_TRUE;
2575 	}
2576 
2577 	if (cs.bits.tx_rng_oflow) {
2578 		tdc_stats->tx_rng_oflow++;
2579 		if (tdc_stats->tx_rng_oflow)
2580 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2581 			    "==> hxge_tx_err_evnts(channel %d): "
2582 			    "fatal error: tx_rng_oflow", channel));
2583 	}
2584 
2585 	if (cs.bits.pref_par_err) {
2586 		tdc_stats->pref_par_err++;
2587 
2588 		/* Get the address of parity error read data */
2589 		TXDMA_REG_READ64(hxgep->hpi_handle, TDC_PREF_PAR_LOG,
2590 		    channel, &tdc_stats->errlog.value);
2591 
2592 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2593 		    HXGE_FM_EREPORT_TDMC_PREF_PAR_ERR);
2594 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2595 		    "==> hxge_tx_err_evnts(channel %d): "
2596 		    "fatal error: pref_par_err", channel));
2597 		txchan_fatal = B_TRUE;
2598 	}
2599 
2600 	if (cs.bits.tdr_pref_cpl_to) {
2601 		tdc_stats->tdr_pref_cpl_to++;
2602 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2603 		    HXGE_FM_EREPORT_TDMC_TDR_PREF_CPL_TO);
2604 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2605 		    "==> hxge_tx_err_evnts(channel %d): "
2606 		    "fatal error: tdr_pref_cpl_to", channel));
2607 		txchan_fatal = B_TRUE;
2608 	}
2609 
2610 	if (cs.bits.pkt_cpl_to) {
2611 		tdc_stats->pkt_cpl_to++;
2612 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2613 		    HXGE_FM_EREPORT_TDMC_PKT_CPL_TO);
2614 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2615 		    "==> hxge_tx_err_evnts(channel %d): "
2616 		    "fatal error: pkt_cpl_to", channel));
2617 		txchan_fatal = B_TRUE;
2618 	}
2619 
2620 	if (cs.bits.invalid_sop) {
2621 		tdc_stats->invalid_sop++;
2622 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2623 		    HXGE_FM_EREPORT_TDMC_INVALID_SOP);
2624 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2625 		    "==> hxge_tx_err_evnts(channel %d): "
2626 		    "fatal error: invalid_sop", channel));
2627 		txchan_fatal = B_TRUE;
2628 	}
2629 
2630 	if (cs.bits.unexpected_sop) {
2631 		tdc_stats->unexpected_sop++;
2632 		HXGE_FM_REPORT_ERROR(hxgep, channel,
2633 		    HXGE_FM_EREPORT_TDMC_UNEXPECTED_SOP);
2634 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2635 		    "==> hxge_tx_err_evnts(channel %d): "
2636 		    "fatal error: unexpected_sop", channel));
2637 		txchan_fatal = B_TRUE;
2638 	}
2639 
2640 	/* Clear error injection source in case this is an injected error */
2641 	TXDMA_REG_WRITE64(hxgep->hpi_handle, TDC_STAT_INT_DBG, channel, 0);
2642 
2643 	if (txchan_fatal) {
2644 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2645 		    " hxge_tx_err_evnts: "
2646 		    " fatal error on channel %d cs 0x%llx\n",
2647 		    channel, cs.value));
2648 		status = hxge_txdma_fatal_err_recover(hxgep, channel,
2649 		    tx_ring_p);
2650 		if (status == HXGE_OK) {
2651 			FM_SERVICE_RESTORED(hxgep);
2652 		}
2653 	}
2654 
2655 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "<== hxge_tx_err_evnts"));
2656 
2657 	return (status);
2658 }
2659 
2660 hxge_status_t
hxge_txdma_handle_sys_errors(p_hxge_t hxgep)2661 hxge_txdma_handle_sys_errors(p_hxge_t hxgep)
2662 {
2663 	hpi_handle_t		handle;
2664 	hxge_status_t		status = HXGE_OK;
2665 	tdc_fifo_err_stat_t	fifo_stat;
2666 	hxge_tdc_sys_stats_t	*tdc_sys_stats;
2667 
2668 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_handle_sys_errors"));
2669 
2670 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2671 
2672 	/*
2673 	 * The FIFO is shared by all channels.
2674 	 * Get the status of Reorder Buffer and Reorder Table Buffer Errors
2675 	 */
2676 	HXGE_REG_RD64(handle, TDC_FIFO_ERR_STAT, &fifo_stat.value);
2677 
2678 	/*
2679 	 * Clear the error bits. Note that writing a 1 clears the bit. Writing
2680 	 * a 0 does nothing.
2681 	 */
2682 	HXGE_REG_WR64(handle, TDC_FIFO_ERR_STAT, fifo_stat.value);
2683 
2684 	tdc_sys_stats = &hxgep->statsp->tdc_sys_stats;
2685 	if (fifo_stat.bits.reord_tbl_par_err) {
2686 		tdc_sys_stats->reord_tbl_par_err++;
2687 		HXGE_FM_REPORT_ERROR(hxgep, 0,
2688 		    HXGE_FM_EREPORT_TDMC_REORD_TBL_PAR);
2689 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2690 		    "==> hxge_txdma_handle_sys_errors: fatal error: "
2691 		    "reord_tbl_par_err"));
2692 	}
2693 
2694 	if (fifo_stat.bits.reord_buf_ded_err) {
2695 		tdc_sys_stats->reord_buf_ded_err++;
2696 		HXGE_FM_REPORT_ERROR(hxgep, 0,
2697 		    HXGE_FM_EREPORT_TDMC_REORD_BUF_DED);
2698 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2699 		    "==> hxge_txdma_handle_sys_errors: "
2700 		    "fatal error: reord_buf_ded_err"));
2701 	}
2702 
2703 	if (fifo_stat.bits.reord_buf_sec_err) {
2704 		tdc_sys_stats->reord_buf_sec_err++;
2705 		if (tdc_sys_stats->reord_buf_sec_err == 1)
2706 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2707 			    "==> hxge_txdma_handle_sys_errors: "
2708 			    "reord_buf_sec_err"));
2709 	}
2710 
2711 	if (fifo_stat.bits.reord_tbl_par_err ||
2712 	    fifo_stat.bits.reord_buf_ded_err) {
2713 		status = hxge_tx_port_fatal_err_recover(hxgep);
2714 		if (status == HXGE_OK) {
2715 			FM_SERVICE_RESTORED(hxgep);
2716 		}
2717 	}
2718 
2719 	HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_handle_sys_errors"));
2720 
2721 	return (status);
2722 }
2723 
2724 static hxge_status_t
hxge_txdma_fatal_err_recover(p_hxge_t hxgep,uint16_t channel,p_tx_ring_t tx_ring_p)2725 hxge_txdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel,
2726     p_tx_ring_t tx_ring_p)
2727 {
2728 	hpi_handle_t	handle;
2729 	hpi_status_t	rs = HPI_SUCCESS;
2730 	p_tx_mbox_t	tx_mbox_p;
2731 	hxge_status_t	status = HXGE_OK;
2732 
2733 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2734 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2735 	    "Recovering from TxDMAChannel#%d error...", channel));
2736 
2737 	/*
2738 	 * Stop the dma channel waits for the stop done. If the stop done bit
2739 	 * is not set, then create an error.
2740 	 */
2741 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2742 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping txdma channel(%d)",
2743 	    channel));
2744 	MUTEX_ENTER(&tx_ring_p->lock);
2745 	rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2746 	if (rs != HPI_SUCCESS) {
2747 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2748 		    "==> hxge_txdma_fatal_err_recover (channel %d): "
2749 		    "stop failed ", channel));
2750 
2751 		goto fail;
2752 	}
2753 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming txdma channel(%d)",
2754 	    channel));
2755 	(void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2756 
2757 	/*
2758 	 * Reset TXDMA channel
2759 	 */
2760 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "resetting txdma channel(%d)",
2761 	    channel));
2762 	if ((rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel)) !=
2763 	    HPI_SUCCESS) {
2764 		HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2765 		    "==> hxge_txdma_fatal_err_recover (channel %d)"
2766 		    " reset channel failed 0x%x", channel, rs));
2767 
2768 		goto fail;
2769 	}
2770 	/*
2771 	 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
2772 	 * overflow fatal error if tail is not set to 0 after reset!
2773 	 */
2774 	TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
2775 
2776 	/*
2777 	 * Restart TXDMA channel
2778 	 *
2779 	 * Initialize the TXDMA channel specific FZC control configurations.
2780 	 * These FZC registers are pertaining to each TX channel (i.e. logical
2781 	 * pages).
2782 	 */
2783 	tx_mbox_p = hxge_txdma_get_mbox(hxgep, channel);
2784 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "restarting txdma channel(%d)",
2785 	    channel));
2786 	status = hxge_init_fzc_txdma_channel(hxgep, channel,
2787 	    tx_ring_p, tx_mbox_p);
2788 	if (status != HXGE_OK)
2789 		goto fail;
2790 
2791 	/*
2792 	 * Initialize the event masks.
2793 	 */
2794 	tx_ring_p->tx_evmask.value = 0;
2795 	status = hxge_init_txdma_channel_event_mask(hxgep, channel,
2796 	    &tx_ring_p->tx_evmask);
2797 	if (status != HXGE_OK)
2798 		goto fail;
2799 
2800 	tx_ring_p->wr_index_wrap = B_FALSE;
2801 	tx_ring_p->wr_index = 0;
2802 	tx_ring_p->rd_index = 0;
2803 
2804 	/*
2805 	 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2806 	 * channels and enable each DMA channel.
2807 	 */
2808 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "enabling txdma channel(%d)",
2809 	    channel));
2810 	status = hxge_enable_txdma_channel(hxgep, channel,
2811 	    tx_ring_p, tx_mbox_p);
2812 	MUTEX_EXIT(&tx_ring_p->lock);
2813 	if (status != HXGE_OK)
2814 		goto fail;
2815 
2816 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2817 	    "Recovery Successful, TxDMAChannel#%d Restored", channel));
2818 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2819 
2820 	return (HXGE_OK);
2821 
2822 fail:
2823 	MUTEX_EXIT(&tx_ring_p->lock);
2824 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2825 	    "hxge_txdma_fatal_err_recover (channel %d): "
2826 	    "failed to recover this txdma channel", channel));
2827 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2828 
2829 	return (status);
2830 }
2831 
2832 static hxge_status_t
hxge_tx_port_fatal_err_recover(p_hxge_t hxgep)2833 hxge_tx_port_fatal_err_recover(p_hxge_t hxgep)
2834 {
2835 	hpi_handle_t	handle;
2836 	hpi_status_t	rs = HPI_SUCCESS;
2837 	hxge_status_t	status = HXGE_OK;
2838 	p_tx_ring_t	*tx_desc_rings;
2839 	p_tx_rings_t	tx_rings;
2840 	p_tx_ring_t	tx_ring_p;
2841 	int		i, ndmas;
2842 	uint16_t	channel;
2843 	block_reset_t	reset_reg;
2844 
2845 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2846 	    "==> hxge_tx_port_fatal_err_recover"));
2847 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2848 	    "Recovering from TxPort error..."));
2849 
2850 	handle = HXGE_DEV_HPI_HANDLE(hxgep);
2851 
2852 	/* Reset TDC block from PEU for this fatal error */
2853 	reset_reg.value = 0;
2854 	reset_reg.bits.tdc_rst = 1;
2855 	HXGE_REG_WR32(handle, BLOCK_RESET, reset_reg.value);
2856 
2857 	HXGE_DELAY(1000);
2858 
2859 	/*
2860 	 * Stop the dma channel waits for the stop done. If the stop done bit
2861 	 * is not set, then create an error.
2862 	 */
2863 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping all DMA channels..."));
2864 
2865 	tx_rings = hxgep->tx_rings;
2866 	tx_desc_rings = tx_rings->rings;
2867 	ndmas = tx_rings->ndmas;
2868 
2869 	for (i = 0; i < ndmas; i++) {
2870 		if (tx_desc_rings[i] == NULL) {
2871 			continue;
2872 		}
2873 		tx_ring_p = tx_rings->rings[i];
2874 		MUTEX_ENTER(&tx_ring_p->lock);
2875 	}
2876 
2877 	for (i = 0; i < ndmas; i++) {
2878 		if (tx_desc_rings[i] == NULL) {
2879 			continue;
2880 		}
2881 		channel = tx_desc_rings[i]->tdc;
2882 		tx_ring_p = tx_rings->rings[i];
2883 		rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2884 		if (rs != HPI_SUCCESS) {
2885 			HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2886 			    "==> hxge_txdma_fatal_err_recover (channel %d): "
2887 			    "stop failed ", channel));
2888 
2889 			goto fail;
2890 		}
2891 	}
2892 
2893 	/*
2894 	 * Do reclaim on all of th DMAs.
2895 	 */
2896 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming all DMA channels..."));
2897 	for (i = 0; i < ndmas; i++) {
2898 		if (tx_desc_rings[i] == NULL) {
2899 			continue;
2900 		}
2901 		tx_ring_p = tx_rings->rings[i];
2902 		(void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2903 	}
2904 
2905 	/* Restart the TDC */
2906 	if ((status = hxge_txdma_hw_start(hxgep)) != HXGE_OK)
2907 		goto fail;
2908 
2909 	for (i = 0; i < ndmas; i++) {
2910 		if (tx_desc_rings[i] == NULL) {
2911 			continue;
2912 		}
2913 		tx_ring_p = tx_rings->rings[i];
2914 		MUTEX_EXIT(&tx_ring_p->lock);
2915 	}
2916 
2917 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2918 	    "Recovery Successful, TxPort Restored"));
2919 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2920 	    "<== hxge_tx_port_fatal_err_recover"));
2921 	return (HXGE_OK);
2922 
2923 fail:
2924 	for (i = 0; i < ndmas; i++) {
2925 		if (tx_desc_rings[i] == NULL) {
2926 			continue;
2927 		}
2928 		tx_ring_p = tx_rings->rings[i];
2929 		MUTEX_EXIT(&tx_ring_p->lock);
2930 	}
2931 
2932 	HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2933 	HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2934 	    "hxge_txdma_fatal_err_recover (channel %d): "
2935 	    "failed to recover this txdma channel"));
2936 
2937 	return (status);
2938 }
2939