144961713Sgirish /*
244961713Sgirish  * CDDL HEADER START
344961713Sgirish  *
444961713Sgirish  * The contents of this file are subject to the terms of the
544961713Sgirish  * Common Development and Distribution License (the "License").
644961713Sgirish  * You may not use this file except in compliance with the License.
744961713Sgirish  *
844961713Sgirish  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
944961713Sgirish  * or http://www.opensolaris.org/os/licensing.
1044961713Sgirish  * See the License for the specific language governing permissions
1144961713Sgirish  * and limitations under the License.
1244961713Sgirish  *
1344961713Sgirish  * When distributing Covered Code, include this CDDL HEADER in each
1444961713Sgirish  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1544961713Sgirish  * If applicable, add the following below this CDDL HEADER, with the
1644961713Sgirish  * fields enclosed by brackets "[]" replaced with your own identifying
1744961713Sgirish  * information: Portions Copyright [yyyy] [name of copyright owner]
1844961713Sgirish  *
1944961713Sgirish  * CDDL HEADER END
2044961713Sgirish  */
2144961713Sgirish /*
223e82a89eSmisaki  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
2344961713Sgirish  * Use is subject to license terms.
2444961713Sgirish  */
2544961713Sgirish 
2644961713Sgirish #pragma ident	"%Z%%M%	%I%	%E% SMI"
2744961713Sgirish 
2844961713Sgirish #include <sys/nxge/nxge_impl.h>
2944961713Sgirish #include <sys/nxge/nxge_rxdma.h>
30678453a8Sspeer #include <sys/nxge/nxge_hio.h>
31678453a8Sspeer 
32678453a8Sspeer #if !defined(_BIG_ENDIAN)
33678453a8Sspeer #include <npi_rx_rd32.h>
34678453a8Sspeer #endif
35678453a8Sspeer #include <npi_rx_rd64.h>
36678453a8Sspeer #include <npi_rx_wr64.h>
3744961713Sgirish 
3844961713Sgirish #define	NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp)	\
39678453a8Sspeer 	(rdcgrp + nxgep->pt_config.hw_config.def_mac_rxdma_grpid)
4044961713Sgirish #define	NXGE_ACTUAL_RDC(nxgep, rdc)	\
4144961713Sgirish 	(rdc + nxgep->pt_config.hw_config.start_rdc)
4244961713Sgirish 
4344961713Sgirish /*
4444961713Sgirish  * Globals: tunable parameters (/etc/system or adb)
4544961713Sgirish  *
4644961713Sgirish  */
4744961713Sgirish extern uint32_t nxge_rbr_size;
4844961713Sgirish extern uint32_t nxge_rcr_size;
4944961713Sgirish extern uint32_t	nxge_rbr_spare_size;
5044961713Sgirish 
5144961713Sgirish extern uint32_t nxge_mblks_pending;
5244961713Sgirish 
5344961713Sgirish /*
5444961713Sgirish  * Tunable to reduce the amount of time spent in the
5544961713Sgirish  * ISR doing Rx Processing.
5644961713Sgirish  */
5744961713Sgirish extern uint32_t nxge_max_rx_pkts;
5844961713Sgirish boolean_t nxge_jumbo_enable;
5944961713Sgirish 
6044961713Sgirish /*
6144961713Sgirish  * Tunables to manage the receive buffer blocks.
6244961713Sgirish  *
6344961713Sgirish  * nxge_rx_threshold_hi: copy all buffers.
6444961713Sgirish  * nxge_rx_bcopy_size_type: receive buffer block size type.
6544961713Sgirish  * nxge_rx_threshold_lo: copy only up to tunable block size type.
6644961713Sgirish  */
6744961713Sgirish extern nxge_rxbuf_threshold_t nxge_rx_threshold_hi;
6844961713Sgirish extern nxge_rxbuf_type_t nxge_rx_buf_size_type;
6944961713Sgirish extern nxge_rxbuf_threshold_t nxge_rx_threshold_lo;
7044961713Sgirish 
71*b4d05839Sml extern uint32_t	nxge_cksum_offload;
72678453a8Sspeer 
73678453a8Sspeer static nxge_status_t nxge_map_rxdma(p_nxge_t, int);
74678453a8Sspeer static void nxge_unmap_rxdma(p_nxge_t, int);
7544961713Sgirish 
7644961713Sgirish static nxge_status_t nxge_rxdma_hw_start_common(p_nxge_t);
7744961713Sgirish 
78678453a8Sspeer static nxge_status_t nxge_rxdma_hw_start(p_nxge_t, int);
79678453a8Sspeer static void nxge_rxdma_hw_stop(p_nxge_t, int);
8044961713Sgirish 
8144961713Sgirish static nxge_status_t nxge_map_rxdma_channel(p_nxge_t, uint16_t,
8244961713Sgirish     p_nxge_dma_common_t *,  p_rx_rbr_ring_t *,
8344961713Sgirish     uint32_t,
8444961713Sgirish     p_nxge_dma_common_t *, p_rx_rcr_ring_t *,
8544961713Sgirish     p_rx_mbox_t *);
8644961713Sgirish static void nxge_unmap_rxdma_channel(p_nxge_t, uint16_t,
8744961713Sgirish     p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t);
8844961713Sgirish 
8944961713Sgirish static nxge_status_t nxge_map_rxdma_channel_cfg_ring(p_nxge_t,
9044961713Sgirish     uint16_t,
9144961713Sgirish     p_nxge_dma_common_t *, p_rx_rbr_ring_t *,
9244961713Sgirish     p_rx_rcr_ring_t *, p_rx_mbox_t *);
9344961713Sgirish static void nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t,
9444961713Sgirish     p_rx_rcr_ring_t, p_rx_mbox_t);
9544961713Sgirish 
9644961713Sgirish static nxge_status_t nxge_map_rxdma_channel_buf_ring(p_nxge_t,
9744961713Sgirish     uint16_t,
9844961713Sgirish     p_nxge_dma_common_t *,
9944961713Sgirish     p_rx_rbr_ring_t *, uint32_t);
10044961713Sgirish static void nxge_unmap_rxdma_channel_buf_ring(p_nxge_t,
10144961713Sgirish     p_rx_rbr_ring_t);
10244961713Sgirish 
10344961713Sgirish static nxge_status_t nxge_rxdma_start_channel(p_nxge_t, uint16_t,
10444961713Sgirish     p_rx_rbr_ring_t, p_rx_rcr_ring_t, p_rx_mbox_t);
10544961713Sgirish static nxge_status_t nxge_rxdma_stop_channel(p_nxge_t, uint16_t);
10644961713Sgirish 
107678453a8Sspeer static mblk_t *
108678453a8Sspeer nxge_rx_pkts(p_nxge_t, p_rx_rcr_ring_t, rx_dma_ctl_stat_t, int);
10944961713Sgirish 
11044961713Sgirish static void nxge_receive_packet(p_nxge_t,
11144961713Sgirish 	p_rx_rcr_ring_t,
11244961713Sgirish 	p_rcr_entry_t,
11344961713Sgirish 	boolean_t *,
11444961713Sgirish 	mblk_t **, mblk_t **);
11544961713Sgirish 
11644961713Sgirish nxge_status_t nxge_disable_rxdma_channel(p_nxge_t, uint16_t);
11744961713Sgirish 
11844961713Sgirish static p_rx_msg_t nxge_allocb(size_t, uint32_t, p_nxge_dma_common_t);
11944961713Sgirish static void nxge_freeb(p_rx_msg_t);
120678453a8Sspeer static void nxge_rx_pkts_vring(p_nxge_t, uint_t, rx_dma_ctl_stat_t);
121678453a8Sspeer static nxge_status_t nxge_rx_err_evnts(p_nxge_t, int, rx_dma_ctl_stat_t);
12244961713Sgirish 
12344961713Sgirish static nxge_status_t nxge_rxdma_handle_port_errors(p_nxge_t,
12444961713Sgirish 				uint32_t, uint32_t);
12544961713Sgirish 
12644961713Sgirish static nxge_status_t nxge_rxbuf_index_info_init(p_nxge_t,
12744961713Sgirish     p_rx_rbr_ring_t);
12844961713Sgirish 
12944961713Sgirish 
13044961713Sgirish static nxge_status_t
13144961713Sgirish nxge_rxdma_fatal_err_recover(p_nxge_t, uint16_t);
13244961713Sgirish 
13344961713Sgirish nxge_status_t
13444961713Sgirish nxge_rx_port_fatal_err_recover(p_nxge_t);
13544961713Sgirish 
136678453a8Sspeer static void nxge_rxdma_databuf_free(p_rx_rbr_ring_t);
137678453a8Sspeer 
13844961713Sgirish nxge_status_t
13944961713Sgirish nxge_init_rxdma_channels(p_nxge_t nxgep)
14044961713Sgirish {
141678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
142678453a8Sspeer 	int i, count;
14344961713Sgirish 
14444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_init_rxdma_channels"));
14544961713Sgirish 
146678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
147678453a8Sspeer 		if (nxge_rxdma_hw_start_common(nxgep) != NXGE_OK) {
148678453a8Sspeer 			cmn_err(CE_NOTE, "hw_start_common");
149678453a8Sspeer 			return (NXGE_ERROR);
150678453a8Sspeer 		}
151678453a8Sspeer 	}
152678453a8Sspeer 
153678453a8Sspeer 	/*
154678453a8Sspeer 	 * NXGE_LOGICAL_GROUP_MAX > NXGE_MAX_RDC_GROUPS (8)
155678453a8Sspeer 	 * We only have 8 hardware RDC tables, but we may have
156678453a8Sspeer 	 * up to 16 logical (software-defined) groups of RDCS,
157678453a8Sspeer 	 * if we make use of layer 3 & 4 hardware classification.
158678453a8Sspeer 	 */
159678453a8Sspeer 	for (i = 0, count = 0; i < NXGE_LOGICAL_GROUP_MAX; i++) {
160678453a8Sspeer 		if ((1 << i) & set->lg.map) {
161678453a8Sspeer 			int channel;
162678453a8Sspeer 			nxge_grp_t *group = set->group[i];
163678453a8Sspeer 			for (channel = 0; channel < NXGE_MAX_RDCS; channel++) {
164678453a8Sspeer 				if ((1 << channel) & group->map) {
165678453a8Sspeer 					if ((nxge_grp_dc_add(nxgep,
166678453a8Sspeer 						(vr_handle_t)group,
167678453a8Sspeer 						VP_BOUND_RX, channel)))
168678453a8Sspeer 						return (NXGE_ERROR);
169678453a8Sspeer 				}
170678453a8Sspeer 			}
171678453a8Sspeer 		}
172678453a8Sspeer 		if (++count == set->lg.count)
173678453a8Sspeer 			break;
17444961713Sgirish 	}
17544961713Sgirish 
176678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_init_rxdma_channels"));
177678453a8Sspeer 
178678453a8Sspeer 	return (NXGE_OK);
179678453a8Sspeer }
180678453a8Sspeer 
181678453a8Sspeer nxge_status_t
182678453a8Sspeer nxge_init_rxdma_channel(p_nxge_t nxge, int channel)
183678453a8Sspeer {
184678453a8Sspeer 	nxge_status_t status;
185678453a8Sspeer 
186678453a8Sspeer 	NXGE_DEBUG_MSG((nxge, MEM2_CTL, "==> nxge_init_rxdma_channel"));
187678453a8Sspeer 
188678453a8Sspeer 	status = nxge_map_rxdma(nxge, channel);
18944961713Sgirish 	if (status != NXGE_OK) {
190678453a8Sspeer 		NXGE_ERROR_MSG((nxge, NXGE_ERR_CTL,
191678453a8Sspeer 		    "<== nxge_init_rxdma: status 0x%x", status));
192678453a8Sspeer 		return (status);
19344961713Sgirish 	}
19444961713Sgirish 
195678453a8Sspeer 	status = nxge_rxdma_hw_start(nxge, channel);
19644961713Sgirish 	if (status != NXGE_OK) {
197678453a8Sspeer 		nxge_unmap_rxdma(nxge, channel);
19844961713Sgirish 	}
19944961713Sgirish 
200678453a8Sspeer 	if (!nxge->statsp->rdc_ksp[channel])
201678453a8Sspeer 		nxge_setup_rdc_kstats(nxge, channel);
202678453a8Sspeer 
203678453a8Sspeer 	NXGE_DEBUG_MSG((nxge, MEM2_CTL,
204678453a8Sspeer 	    "<== nxge_init_rxdma_channel: status 0x%x", status));
20544961713Sgirish 
20644961713Sgirish 	return (status);
20744961713Sgirish }
20844961713Sgirish 
20944961713Sgirish void
21044961713Sgirish nxge_uninit_rxdma_channels(p_nxge_t nxgep)
21144961713Sgirish {
212678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
213678453a8Sspeer 	int rdc;
214678453a8Sspeer 
21544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channels"));
21644961713Sgirish 
217678453a8Sspeer 	if (set->owned.map == 0) {
218678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
219678453a8Sspeer 		    "nxge_uninit_rxdma_channels: no channels"));
220678453a8Sspeer 		return;
221678453a8Sspeer 	}
22244961713Sgirish 
223678453a8Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
224678453a8Sspeer 		if ((1 << rdc) & set->owned.map) {
225678453a8Sspeer 			nxge_grp_dc_remove(nxgep, VP_BOUND_RX, rdc);
226678453a8Sspeer 		}
227678453a8Sspeer 	}
228678453a8Sspeer 
229678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uninit_rxdma_channels"));
230678453a8Sspeer }
231678453a8Sspeer 
232678453a8Sspeer void
233678453a8Sspeer nxge_uninit_rxdma_channel(p_nxge_t nxgep, int channel)
234678453a8Sspeer {
235678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_uninit_rxdma_channel"));
236678453a8Sspeer 
237678453a8Sspeer 	if (nxgep->statsp->rdc_ksp[channel]) {
238678453a8Sspeer 		kstat_delete(nxgep->statsp->rdc_ksp[channel]);
239678453a8Sspeer 		nxgep->statsp->rdc_ksp[channel] = 0;
240678453a8Sspeer 	}
241678453a8Sspeer 
242678453a8Sspeer 	nxge_rxdma_hw_stop(nxgep, channel);
243678453a8Sspeer 	nxge_unmap_rxdma(nxgep, channel);
244678453a8Sspeer 
245678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_uinit_rxdma_channel"));
24644961713Sgirish }
24744961713Sgirish 
24844961713Sgirish nxge_status_t
24944961713Sgirish nxge_reset_rxdma_channel(p_nxge_t nxgep, uint16_t channel)
25044961713Sgirish {
25144961713Sgirish 	npi_handle_t		handle;
25244961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
25344961713Sgirish 	nxge_status_t		status = NXGE_OK;
25444961713Sgirish 
25544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_reset_rxdma_channel"));
25644961713Sgirish 
25744961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
25844961713Sgirish 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
25944961713Sgirish 
26044961713Sgirish 	if (rs != NPI_SUCCESS) {
26144961713Sgirish 		status = NXGE_ERROR | rs;
26244961713Sgirish 	}
26344961713Sgirish 
26444961713Sgirish 	return (status);
26544961713Sgirish }
26644961713Sgirish 
26744961713Sgirish void
26844961713Sgirish nxge_rxdma_regs_dump_channels(p_nxge_t nxgep)
26944961713Sgirish {
270678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
271678453a8Sspeer 	int rdc;
27244961713Sgirish 
27344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_regs_dump_channels"));
27444961713Sgirish 
275678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
276678453a8Sspeer 		npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep);
277678453a8Sspeer 		(void) npi_rxdma_dump_fzc_regs(handle);
27844961713Sgirish 	}
279678453a8Sspeer 
280678453a8Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
281678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
282678453a8Sspeer 		    "nxge_rxdma_regs_dump_channels: "
283678453a8Sspeer 		    "NULL ring pointer(s)"));
28444961713Sgirish 		return;
28544961713Sgirish 	}
28644961713Sgirish 
287678453a8Sspeer 	if (set->owned.map == 0) {
28844961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
289678453a8Sspeer 		    "nxge_rxdma_regs_dump_channels: no channels"));
29044961713Sgirish 		return;
29144961713Sgirish 	}
29244961713Sgirish 
293678453a8Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
294678453a8Sspeer 		if ((1 << rdc) & set->owned.map) {
295678453a8Sspeer 			rx_rbr_ring_t *ring =
296678453a8Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
297678453a8Sspeer 			if (ring) {
298678453a8Sspeer 				(void) nxge_dump_rxdma_channel(nxgep, rdc);
299678453a8Sspeer 			}
30044961713Sgirish 		}
30144961713Sgirish 	}
30244961713Sgirish 
30344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_regs_dump"));
30444961713Sgirish }
30544961713Sgirish 
30644961713Sgirish nxge_status_t
30744961713Sgirish nxge_dump_rxdma_channel(p_nxge_t nxgep, uint8_t channel)
30844961713Sgirish {
30944961713Sgirish 	npi_handle_t		handle;
31044961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
31144961713Sgirish 	nxge_status_t		status = NXGE_OK;
31244961713Sgirish 
31344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_dump_rxdma_channel"));
31444961713Sgirish 
31544961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
31644961713Sgirish 	rs = npi_rxdma_dump_rdc_regs(handle, channel);
31744961713Sgirish 
31844961713Sgirish 	if (rs != NPI_SUCCESS) {
31944961713Sgirish 		status = NXGE_ERROR | rs;
32044961713Sgirish 	}
32144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_dump_rxdma_channel"));
32244961713Sgirish 	return (status);
32344961713Sgirish }
32444961713Sgirish 
32544961713Sgirish nxge_status_t
32644961713Sgirish nxge_init_rxdma_channel_event_mask(p_nxge_t nxgep, uint16_t channel,
32744961713Sgirish     p_rx_dma_ent_msk_t mask_p)
32844961713Sgirish {
32944961713Sgirish 	npi_handle_t		handle;
33044961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
33144961713Sgirish 	nxge_status_t		status = NXGE_OK;
33244961713Sgirish 
33344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
33444961713Sgirish 		"<== nxge_init_rxdma_channel_event_mask"));
33544961713Sgirish 
33644961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
33744961713Sgirish 	rs = npi_rxdma_event_mask(handle, OP_SET, channel, mask_p);
33844961713Sgirish 	if (rs != NPI_SUCCESS) {
33944961713Sgirish 		status = NXGE_ERROR | rs;
34044961713Sgirish 	}
34144961713Sgirish 
34244961713Sgirish 	return (status);
34344961713Sgirish }
34444961713Sgirish 
34544961713Sgirish nxge_status_t
34644961713Sgirish nxge_init_rxdma_channel_cntl_stat(p_nxge_t nxgep, uint16_t channel,
34744961713Sgirish     p_rx_dma_ctl_stat_t cs_p)
34844961713Sgirish {
34944961713Sgirish 	npi_handle_t		handle;
35044961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
35144961713Sgirish 	nxge_status_t		status = NXGE_OK;
35244961713Sgirish 
35344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
35444961713Sgirish 		"<== nxge_init_rxdma_channel_cntl_stat"));
35544961713Sgirish 
35644961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
35744961713Sgirish 	rs = npi_rxdma_control_status(handle, OP_SET, channel, cs_p);
35844961713Sgirish 
35944961713Sgirish 	if (rs != NPI_SUCCESS) {
36044961713Sgirish 		status = NXGE_ERROR | rs;
36144961713Sgirish 	}
36244961713Sgirish 
36344961713Sgirish 	return (status);
36444961713Sgirish }
36544961713Sgirish 
366678453a8Sspeer /*
367678453a8Sspeer  * nxge_rxdma_cfg_rdcgrp_default_rdc
368678453a8Sspeer  *
369678453a8Sspeer  *	Set the default RDC for an RDC Group (Table)
370678453a8Sspeer  *
371678453a8Sspeer  * Arguments:
372678453a8Sspeer  * 	nxgep
373678453a8Sspeer  *	rdcgrp	The group to modify
374678453a8Sspeer  *	rdc	The new default RDC.
375678453a8Sspeer  *
376678453a8Sspeer  * Notes:
377678453a8Sspeer  *
378678453a8Sspeer  * NPI/NXGE function calls:
379678453a8Sspeer  *	npi_rxdma_cfg_rdc_table_default_rdc()
380678453a8Sspeer  *
381678453a8Sspeer  * Registers accessed:
382678453a8Sspeer  *	RDC_TBL_REG: FZC_ZCP + 0x10000
383678453a8Sspeer  *
384678453a8Sspeer  * Context:
385678453a8Sspeer  *	Service domain
386678453a8Sspeer  */
38744961713Sgirish nxge_status_t
388678453a8Sspeer nxge_rxdma_cfg_rdcgrp_default_rdc(
389678453a8Sspeer 	p_nxge_t nxgep,
390678453a8Sspeer 	uint8_t rdcgrp,
391678453a8Sspeer 	uint8_t rdc)
39244961713Sgirish {
39344961713Sgirish 	npi_handle_t		handle;
39444961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
39544961713Sgirish 	p_nxge_dma_pt_cfg_t	p_dma_cfgp;
39644961713Sgirish 	p_nxge_rdc_grp_t	rdc_grp_p;
39744961713Sgirish 	uint8_t actual_rdcgrp, actual_rdc;
39844961713Sgirish 
39944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
40044961713Sgirish 			    " ==> nxge_rxdma_cfg_rdcgrp_default_rdc"));
40144961713Sgirish 	p_dma_cfgp = (p_nxge_dma_pt_cfg_t)&nxgep->pt_config;
40244961713Sgirish 
40344961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
40444961713Sgirish 
405678453a8Sspeer 	/*
406678453a8Sspeer 	 * This has to be rewritten.  Do we even allow this anymore?
407678453a8Sspeer 	 */
40844961713Sgirish 	rdc_grp_p = &p_dma_cfgp->rdc_grps[rdcgrp];
409678453a8Sspeer 	RDC_MAP_IN(rdc_grp_p->map, rdc);
410678453a8Sspeer 	rdc_grp_p->def_rdc = rdc;
41144961713Sgirish 
41244961713Sgirish 	actual_rdcgrp = NXGE_ACTUAL_RDCGRP(nxgep, rdcgrp);
41344961713Sgirish 	actual_rdc = NXGE_ACTUAL_RDC(nxgep, rdc);
41444961713Sgirish 
415678453a8Sspeer 	rs = npi_rxdma_cfg_rdc_table_default_rdc(
416678453a8Sspeer 		handle, actual_rdcgrp, actual_rdc);
41744961713Sgirish 
41844961713Sgirish 	if (rs != NPI_SUCCESS) {
41944961713Sgirish 		return (NXGE_ERROR | rs);
42044961713Sgirish 	}
42144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
42244961713Sgirish 			    " <== nxge_rxdma_cfg_rdcgrp_default_rdc"));
42344961713Sgirish 	return (NXGE_OK);
42444961713Sgirish }
42544961713Sgirish 
42644961713Sgirish nxge_status_t
42744961713Sgirish nxge_rxdma_cfg_port_default_rdc(p_nxge_t nxgep, uint8_t port, uint8_t rdc)
42844961713Sgirish {
42944961713Sgirish 	npi_handle_t		handle;
43044961713Sgirish 
43144961713Sgirish 	uint8_t actual_rdc;
43244961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
43344961713Sgirish 
43444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
43544961713Sgirish 			    " ==> nxge_rxdma_cfg_port_default_rdc"));
43644961713Sgirish 
43744961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
438678453a8Sspeer 	actual_rdc = rdc;	/* XXX Hack! */
43944961713Sgirish 	rs = npi_rxdma_cfg_default_port_rdc(handle, port, actual_rdc);
44044961713Sgirish 
44144961713Sgirish 
44244961713Sgirish 	if (rs != NPI_SUCCESS) {
44344961713Sgirish 		return (NXGE_ERROR | rs);
44444961713Sgirish 	}
44544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
44644961713Sgirish 			    " <== nxge_rxdma_cfg_port_default_rdc"));
44744961713Sgirish 
44844961713Sgirish 	return (NXGE_OK);
44944961713Sgirish }
45044961713Sgirish 
45144961713Sgirish nxge_status_t
45244961713Sgirish nxge_rxdma_cfg_rcr_threshold(p_nxge_t nxgep, uint8_t channel,
45344961713Sgirish 				    uint16_t pkts)
45444961713Sgirish {
45544961713Sgirish 	npi_status_t	rs = NPI_SUCCESS;
45644961713Sgirish 	npi_handle_t	handle;
45744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
45844961713Sgirish 			    " ==> nxge_rxdma_cfg_rcr_threshold"));
45944961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
46044961713Sgirish 
46144961713Sgirish 	rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel, pkts);
46244961713Sgirish 
46344961713Sgirish 	if (rs != NPI_SUCCESS) {
46444961713Sgirish 		return (NXGE_ERROR | rs);
46544961713Sgirish 	}
46644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_threshold"));
46744961713Sgirish 	return (NXGE_OK);
46844961713Sgirish }
46944961713Sgirish 
47044961713Sgirish nxge_status_t
47144961713Sgirish nxge_rxdma_cfg_rcr_timeout(p_nxge_t nxgep, uint8_t channel,
47244961713Sgirish 			    uint16_t tout, uint8_t enable)
47344961713Sgirish {
47444961713Sgirish 	npi_status_t	rs = NPI_SUCCESS;
47544961713Sgirish 	npi_handle_t	handle;
47644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " ==> nxge_rxdma_cfg_rcr_timeout"));
47744961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
47844961713Sgirish 	if (enable == 0) {
47944961713Sgirish 		rs = npi_rxdma_cfg_rdc_rcr_timeout_disable(handle, channel);
48044961713Sgirish 	} else {
48144961713Sgirish 		rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
48244961713Sgirish 							    tout);
48344961713Sgirish 	}
48444961713Sgirish 
48544961713Sgirish 	if (rs != NPI_SUCCESS) {
48644961713Sgirish 		return (NXGE_ERROR | rs);
48744961713Sgirish 	}
48844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, " <== nxge_rxdma_cfg_rcr_timeout"));
48944961713Sgirish 	return (NXGE_OK);
49044961713Sgirish }
49144961713Sgirish 
49244961713Sgirish nxge_status_t
49344961713Sgirish nxge_enable_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
49444961713Sgirish     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p)
49544961713Sgirish {
49644961713Sgirish 	npi_handle_t		handle;
49744961713Sgirish 	rdc_desc_cfg_t 		rdc_desc;
49844961713Sgirish 	p_rcrcfig_b_t		cfgb_p;
49944961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
50044961713Sgirish 
50144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel"));
50244961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
50344961713Sgirish 	/*
50444961713Sgirish 	 * Use configuration data composed at init time.
50544961713Sgirish 	 * Write to hardware the receive ring configurations.
50644961713Sgirish 	 */
50744961713Sgirish 	rdc_desc.mbox_enable = 1;
50844961713Sgirish 	rdc_desc.mbox_addr = mbox_p->mbox_addr;
50944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
51044961713Sgirish 		"==> nxge_enable_rxdma_channel: mboxp $%p($%p)",
51144961713Sgirish 		mbox_p->mbox_addr, rdc_desc.mbox_addr));
51244961713Sgirish 
51344961713Sgirish 	rdc_desc.rbr_len = rbr_p->rbb_max;
51444961713Sgirish 	rdc_desc.rbr_addr = rbr_p->rbr_addr;
51544961713Sgirish 
51644961713Sgirish 	switch (nxgep->rx_bksize_code) {
51744961713Sgirish 	case RBR_BKSIZE_4K:
51844961713Sgirish 		rdc_desc.page_size = SIZE_4KB;
51944961713Sgirish 		break;
52044961713Sgirish 	case RBR_BKSIZE_8K:
52144961713Sgirish 		rdc_desc.page_size = SIZE_8KB;
52244961713Sgirish 		break;
52344961713Sgirish 	case RBR_BKSIZE_16K:
52444961713Sgirish 		rdc_desc.page_size = SIZE_16KB;
52544961713Sgirish 		break;
52644961713Sgirish 	case RBR_BKSIZE_32K:
52744961713Sgirish 		rdc_desc.page_size = SIZE_32KB;
52844961713Sgirish 		break;
52944961713Sgirish 	}
53044961713Sgirish 
53144961713Sgirish 	rdc_desc.size0 = rbr_p->npi_pkt_buf_size0;
53244961713Sgirish 	rdc_desc.valid0 = 1;
53344961713Sgirish 
53444961713Sgirish 	rdc_desc.size1 = rbr_p->npi_pkt_buf_size1;
53544961713Sgirish 	rdc_desc.valid1 = 1;
53644961713Sgirish 
53744961713Sgirish 	rdc_desc.size2 = rbr_p->npi_pkt_buf_size2;
53844961713Sgirish 	rdc_desc.valid2 = 1;
53944961713Sgirish 
54044961713Sgirish 	rdc_desc.full_hdr = rcr_p->full_hdr_flag;
54144961713Sgirish 	rdc_desc.offset = rcr_p->sw_priv_hdr_len;
54244961713Sgirish 
54344961713Sgirish 	rdc_desc.rcr_len = rcr_p->comp_size;
54444961713Sgirish 	rdc_desc.rcr_addr = rcr_p->rcr_addr;
54544961713Sgirish 
54644961713Sgirish 	cfgb_p = &(rcr_p->rcr_cfgb);
54744961713Sgirish 	rdc_desc.rcr_threshold = cfgb_p->bits.ldw.pthres;
548678453a8Sspeer 	/* For now, disable this timeout in a guest domain. */
549678453a8Sspeer 	if (isLDOMguest(nxgep)) {
550678453a8Sspeer 		rdc_desc.rcr_timeout = 0;
551678453a8Sspeer 		rdc_desc.rcr_timeout_enable = 0;
552678453a8Sspeer 	} else {
553678453a8Sspeer 		rdc_desc.rcr_timeout = cfgb_p->bits.ldw.timeout;
554678453a8Sspeer 		rdc_desc.rcr_timeout_enable = cfgb_p->bits.ldw.entout;
555678453a8Sspeer 	}
55644961713Sgirish 
55744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: "
55844961713Sgirish 		"rbr_len qlen %d pagesize code %d rcr_len %d",
55944961713Sgirish 		rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len));
56044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_enable_rxdma_channel: "
56144961713Sgirish 		"size 0 %d size 1 %d size 2 %d",
56244961713Sgirish 		rbr_p->npi_pkt_buf_size0, rbr_p->npi_pkt_buf_size1,
56344961713Sgirish 		rbr_p->npi_pkt_buf_size2));
56444961713Sgirish 
56544961713Sgirish 	rs = npi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc);
56644961713Sgirish 	if (rs != NPI_SUCCESS) {
56744961713Sgirish 		return (NXGE_ERROR | rs);
56844961713Sgirish 	}
56944961713Sgirish 
57044961713Sgirish 	/*
57144961713Sgirish 	 * Enable the timeout and threshold.
57244961713Sgirish 	 */
57344961713Sgirish 	rs = npi_rxdma_cfg_rdc_rcr_threshold(handle, channel,
57444961713Sgirish 			rdc_desc.rcr_threshold);
57544961713Sgirish 	if (rs != NPI_SUCCESS) {
57644961713Sgirish 		return (NXGE_ERROR | rs);
57744961713Sgirish 	}
57844961713Sgirish 
57944961713Sgirish 	rs = npi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
58044961713Sgirish 			rdc_desc.rcr_timeout);
58144961713Sgirish 	if (rs != NPI_SUCCESS) {
58244961713Sgirish 		return (NXGE_ERROR | rs);
58344961713Sgirish 	}
58444961713Sgirish 
58544961713Sgirish 	/* Enable the DMA */
58644961713Sgirish 	rs = npi_rxdma_cfg_rdc_enable(handle, channel);
58744961713Sgirish 	if (rs != NPI_SUCCESS) {
58844961713Sgirish 		return (NXGE_ERROR | rs);
58944961713Sgirish 	}
59044961713Sgirish 
59144961713Sgirish 	/* Kick the DMA engine. */
59244961713Sgirish 	npi_rxdma_rdc_rbr_kick(handle, channel, rbr_p->rbb_max);
59344961713Sgirish 	/* Clear the rbr empty bit */
59444961713Sgirish 	(void) npi_rxdma_channel_rbr_empty_clear(handle, channel);
59544961713Sgirish 
59644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_enable_rxdma_channel"));
59744961713Sgirish 
59844961713Sgirish 	return (NXGE_OK);
59944961713Sgirish }
60044961713Sgirish 
60144961713Sgirish nxge_status_t
60244961713Sgirish nxge_disable_rxdma_channel(p_nxge_t nxgep, uint16_t channel)
60344961713Sgirish {
60444961713Sgirish 	npi_handle_t		handle;
60544961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
60644961713Sgirish 
60744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_disable_rxdma_channel"));
60844961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
60944961713Sgirish 
61044961713Sgirish 	/* disable the DMA */
61144961713Sgirish 	rs = npi_rxdma_cfg_rdc_disable(handle, channel);
61244961713Sgirish 	if (rs != NPI_SUCCESS) {
61344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
61444961713Sgirish 			"<== nxge_disable_rxdma_channel:failed (0x%x)",
61544961713Sgirish 			rs));
61644961713Sgirish 		return (NXGE_ERROR | rs);
61744961713Sgirish 	}
61844961713Sgirish 
61944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_disable_rxdma_channel"));
62044961713Sgirish 	return (NXGE_OK);
62144961713Sgirish }
62244961713Sgirish 
62344961713Sgirish nxge_status_t
62444961713Sgirish nxge_rxdma_channel_rcrflush(p_nxge_t nxgep, uint8_t channel)
62544961713Sgirish {
62644961713Sgirish 	npi_handle_t		handle;
62744961713Sgirish 	nxge_status_t		status = NXGE_OK;
62844961713Sgirish 
62944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
63044961713Sgirish 		"<== nxge_init_rxdma_channel_rcrflush"));
63144961713Sgirish 
63244961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
63344961713Sgirish 	npi_rxdma_rdc_rcr_flush(handle, channel);
63444961713Sgirish 
63544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
63644961713Sgirish 		"<== nxge_init_rxdma_channel_rcrflsh"));
63744961713Sgirish 	return (status);
63844961713Sgirish 
63944961713Sgirish }
64044961713Sgirish 
64144961713Sgirish #define	MID_INDEX(l, r) ((r + l + 1) >> 1)
64244961713Sgirish 
64344961713Sgirish #define	TO_LEFT -1
64444961713Sgirish #define	TO_RIGHT 1
64544961713Sgirish #define	BOTH_RIGHT (TO_RIGHT + TO_RIGHT)
64644961713Sgirish #define	BOTH_LEFT (TO_LEFT + TO_LEFT)
64744961713Sgirish #define	IN_MIDDLE (TO_RIGHT + TO_LEFT)
64844961713Sgirish #define	NO_HINT 0xffffffff
64944961713Sgirish 
65044961713Sgirish /*ARGSUSED*/
65144961713Sgirish nxge_status_t
65244961713Sgirish nxge_rxbuf_pp_to_vp(p_nxge_t nxgep, p_rx_rbr_ring_t rbr_p,
653a3c5bd6dSspeer 	uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp,
654a3c5bd6dSspeer 	uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index)
65544961713Sgirish {
65644961713Sgirish 	int			bufsize;
65744961713Sgirish 	uint64_t		pktbuf_pp;
65844961713Sgirish 	uint64_t 		dvma_addr;
65944961713Sgirish 	rxring_info_t 		*ring_info;
66044961713Sgirish 	int 			base_side, end_side;
66144961713Sgirish 	int 			r_index, l_index, anchor_index;
66244961713Sgirish 	int 			found, search_done;
66344961713Sgirish 	uint32_t offset, chunk_size, block_size, page_size_mask;
66444961713Sgirish 	uint32_t chunk_index, block_index, total_index;
66544961713Sgirish 	int 			max_iterations, iteration;
66644961713Sgirish 	rxbuf_index_info_t 	*bufinfo;
66744961713Sgirish 
66844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_rxbuf_pp_to_vp"));
66944961713Sgirish 
67044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
67144961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: buf_pp $%p btype %d",
67244961713Sgirish 		pkt_buf_addr_pp,
67344961713Sgirish 		pktbufsz_type));
674adfcba55Sjoycey #if defined(__i386)
675adfcba55Sjoycey 	pktbuf_pp = (uint64_t)(uint32_t)pkt_buf_addr_pp;
676adfcba55Sjoycey #else
67744961713Sgirish 	pktbuf_pp = (uint64_t)pkt_buf_addr_pp;
678adfcba55Sjoycey #endif
67944961713Sgirish 
68044961713Sgirish 	switch (pktbufsz_type) {
68144961713Sgirish 	case 0:
68244961713Sgirish 		bufsize = rbr_p->pkt_buf_size0;
68344961713Sgirish 		break;
68444961713Sgirish 	case 1:
68544961713Sgirish 		bufsize = rbr_p->pkt_buf_size1;
68644961713Sgirish 		break;
68744961713Sgirish 	case 2:
68844961713Sgirish 		bufsize = rbr_p->pkt_buf_size2;
68944961713Sgirish 		break;
69044961713Sgirish 	case RCR_SINGLE_BLOCK:
69144961713Sgirish 		bufsize = 0;
69244961713Sgirish 		anchor_index = 0;
69344961713Sgirish 		break;
69444961713Sgirish 	default:
69544961713Sgirish 		return (NXGE_ERROR);
69644961713Sgirish 	}
69744961713Sgirish 
69844961713Sgirish 	if (rbr_p->num_blocks == 1) {
69944961713Sgirish 		anchor_index = 0;
70044961713Sgirish 		ring_info = rbr_p->ring_info;
70144961713Sgirish 		bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
70244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
70344961713Sgirish 			"==> nxge_rxbuf_pp_to_vp: (found, 1 block) "
70444961713Sgirish 			"buf_pp $%p btype %d anchor_index %d "
70544961713Sgirish 			"bufinfo $%p",
70644961713Sgirish 			pkt_buf_addr_pp,
70744961713Sgirish 			pktbufsz_type,
70844961713Sgirish 			anchor_index,
70944961713Sgirish 			bufinfo));
71044961713Sgirish 
71144961713Sgirish 		goto found_index;
71244961713Sgirish 	}
71344961713Sgirish 
71444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
71544961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: "
71644961713Sgirish 		"buf_pp $%p btype %d  anchor_index %d",
71744961713Sgirish 		pkt_buf_addr_pp,
71844961713Sgirish 		pktbufsz_type,
71944961713Sgirish 		anchor_index));
72044961713Sgirish 
72144961713Sgirish 	ring_info = rbr_p->ring_info;
72244961713Sgirish 	found = B_FALSE;
72344961713Sgirish 	bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
72444961713Sgirish 	iteration = 0;
72544961713Sgirish 	max_iterations = ring_info->max_iterations;
72644961713Sgirish 		/*
727a3c5bd6dSspeer 		 * First check if this block has been seen
72844961713Sgirish 		 * recently. This is indicated by a hint which
72944961713Sgirish 		 * is initialized when the first buffer of the block
73044961713Sgirish 		 * is seen. The hint is reset when the last buffer of
73144961713Sgirish 		 * the block has been processed.
73244961713Sgirish 		 * As three block sizes are supported, three hints
73344961713Sgirish 		 * are kept. The idea behind the hints is that once
73444961713Sgirish 		 * the hardware  uses a block for a buffer  of that
73544961713Sgirish 		 * size, it will use it exclusively for that size
73644961713Sgirish 		 * and will use it until it is exhausted. It is assumed
73744961713Sgirish 		 * that there would a single block being used for the same
73844961713Sgirish 		 * buffer sizes at any given time.
73944961713Sgirish 		 */
74044961713Sgirish 	if (ring_info->hint[pktbufsz_type] != NO_HINT) {
74144961713Sgirish 		anchor_index = ring_info->hint[pktbufsz_type];
74244961713Sgirish 		dvma_addr =  bufinfo[anchor_index].dvma_addr;
74344961713Sgirish 		chunk_size = bufinfo[anchor_index].buf_size;
74444961713Sgirish 		if ((pktbuf_pp >= dvma_addr) &&
74544961713Sgirish 			(pktbuf_pp < (dvma_addr + chunk_size))) {
74644961713Sgirish 			found = B_TRUE;
74744961713Sgirish 				/*
74844961713Sgirish 				 * check if this is the last buffer in the block
74944961713Sgirish 				 * If so, then reset the hint for the size;
75044961713Sgirish 				 */
75144961713Sgirish 
75244961713Sgirish 			if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size))
75344961713Sgirish 				ring_info->hint[pktbufsz_type] = NO_HINT;
75444961713Sgirish 		}
75544961713Sgirish 	}
75644961713Sgirish 
75744961713Sgirish 	if (found == B_FALSE) {
75844961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
75944961713Sgirish 			"==> nxge_rxbuf_pp_to_vp: (!found)"
76044961713Sgirish 			"buf_pp $%p btype %d anchor_index %d",
76144961713Sgirish 			pkt_buf_addr_pp,
76244961713Sgirish 			pktbufsz_type,
76344961713Sgirish 			anchor_index));
76444961713Sgirish 
76544961713Sgirish 			/*
76644961713Sgirish 			 * This is the first buffer of the block of this
76744961713Sgirish 			 * size. Need to search the whole information
76844961713Sgirish 			 * array.
76944961713Sgirish 			 * the search algorithm uses a binary tree search
77044961713Sgirish 			 * algorithm. It assumes that the information is
77144961713Sgirish 			 * already sorted with increasing order
77244961713Sgirish 			 * info[0] < info[1] < info[2]  .... < info[n-1]
77344961713Sgirish 			 * where n is the size of the information array
77444961713Sgirish 			 */
77544961713Sgirish 		r_index = rbr_p->num_blocks - 1;
77644961713Sgirish 		l_index = 0;
77744961713Sgirish 		search_done = B_FALSE;
77844961713Sgirish 		anchor_index = MID_INDEX(r_index, l_index);
77944961713Sgirish 		while (search_done == B_FALSE) {
78044961713Sgirish 			if ((r_index == l_index) ||
78144961713Sgirish 				(iteration >= max_iterations))
78244961713Sgirish 				search_done = B_TRUE;
78344961713Sgirish 			end_side = TO_RIGHT; /* to the right */
78444961713Sgirish 			base_side = TO_LEFT; /* to the left */
78544961713Sgirish 			/* read the DVMA address information and sort it */
78644961713Sgirish 			dvma_addr =  bufinfo[anchor_index].dvma_addr;
78744961713Sgirish 			chunk_size = bufinfo[anchor_index].buf_size;
78844961713Sgirish 			NXGE_DEBUG_MSG((nxgep, RX2_CTL,
78944961713Sgirish 				"==> nxge_rxbuf_pp_to_vp: (searching)"
79044961713Sgirish 				"buf_pp $%p btype %d "
79144961713Sgirish 				"anchor_index %d chunk_size %d dvmaaddr $%p",
79244961713Sgirish 				pkt_buf_addr_pp,
79344961713Sgirish 				pktbufsz_type,
79444961713Sgirish 				anchor_index,
79544961713Sgirish 				chunk_size,
79644961713Sgirish 				dvma_addr));
79744961713Sgirish 
79844961713Sgirish 			if (pktbuf_pp >= dvma_addr)
79944961713Sgirish 				base_side = TO_RIGHT; /* to the right */
80044961713Sgirish 			if (pktbuf_pp < (dvma_addr + chunk_size))
80144961713Sgirish 				end_side = TO_LEFT; /* to the left */
80244961713Sgirish 
80344961713Sgirish 			switch (base_side + end_side) {
80444961713Sgirish 				case IN_MIDDLE:
80544961713Sgirish 					/* found */
80644961713Sgirish 					found = B_TRUE;
80744961713Sgirish 					search_done = B_TRUE;
80844961713Sgirish 					if ((pktbuf_pp + bufsize) <
80944961713Sgirish 						(dvma_addr + chunk_size))
81044961713Sgirish 						ring_info->hint[pktbufsz_type] =
81144961713Sgirish 						bufinfo[anchor_index].buf_index;
81244961713Sgirish 					break;
81344961713Sgirish 				case BOTH_RIGHT:
81444961713Sgirish 						/* not found: go to the right */
81544961713Sgirish 					l_index = anchor_index + 1;
81644961713Sgirish 					anchor_index =
81744961713Sgirish 						MID_INDEX(r_index, l_index);
81844961713Sgirish 					break;
81944961713Sgirish 
82044961713Sgirish 				case  BOTH_LEFT:
82144961713Sgirish 						/* not found: go to the left */
82244961713Sgirish 					r_index = anchor_index - 1;
82344961713Sgirish 					anchor_index = MID_INDEX(r_index,
82444961713Sgirish 						l_index);
82544961713Sgirish 					break;
82644961713Sgirish 				default: /* should not come here */
82744961713Sgirish 					return (NXGE_ERROR);
82844961713Sgirish 			}
82944961713Sgirish 			iteration++;
83044961713Sgirish 		}
83144961713Sgirish 
83244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
83344961713Sgirish 			"==> nxge_rxbuf_pp_to_vp: (search done)"
83444961713Sgirish 			"buf_pp $%p btype %d anchor_index %d",
83544961713Sgirish 			pkt_buf_addr_pp,
83644961713Sgirish 			pktbufsz_type,
83744961713Sgirish 			anchor_index));
83844961713Sgirish 	}
83944961713Sgirish 
84044961713Sgirish 	if (found == B_FALSE) {
84144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
84244961713Sgirish 			"==> nxge_rxbuf_pp_to_vp: (search failed)"
84344961713Sgirish 			"buf_pp $%p btype %d anchor_index %d",
84444961713Sgirish 			pkt_buf_addr_pp,
84544961713Sgirish 			pktbufsz_type,
84644961713Sgirish 			anchor_index));
84744961713Sgirish 		return (NXGE_ERROR);
84844961713Sgirish 	}
84944961713Sgirish 
85044961713Sgirish found_index:
85144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
85244961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: (FOUND1)"
85344961713Sgirish 		"buf_pp $%p btype %d bufsize %d anchor_index %d",
85444961713Sgirish 		pkt_buf_addr_pp,
85544961713Sgirish 		pktbufsz_type,
85644961713Sgirish 		bufsize,
85744961713Sgirish 		anchor_index));
85844961713Sgirish 
85944961713Sgirish 	/* index of the first block in this chunk */
86044961713Sgirish 	chunk_index = bufinfo[anchor_index].start_index;
86144961713Sgirish 	dvma_addr =  bufinfo[anchor_index].dvma_addr;
86244961713Sgirish 	page_size_mask = ring_info->block_size_mask;
86344961713Sgirish 
86444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
86544961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: (FOUND3), get chunk)"
86644961713Sgirish 		"buf_pp $%p btype %d bufsize %d "
86744961713Sgirish 		"anchor_index %d chunk_index %d dvma $%p",
86844961713Sgirish 		pkt_buf_addr_pp,
86944961713Sgirish 		pktbufsz_type,
87044961713Sgirish 		bufsize,
87144961713Sgirish 		anchor_index,
87244961713Sgirish 		chunk_index,
87344961713Sgirish 		dvma_addr));
87444961713Sgirish 
87544961713Sgirish 	offset = pktbuf_pp - dvma_addr; /* offset within the chunk */
87644961713Sgirish 	block_size = rbr_p->block_size; /* System  block(page) size */
87744961713Sgirish 
87844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
87944961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: (FOUND4), get chunk)"
88044961713Sgirish 		"buf_pp $%p btype %d bufsize %d "
88144961713Sgirish 		"anchor_index %d chunk_index %d dvma $%p "
88244961713Sgirish 		"offset %d block_size %d",
88344961713Sgirish 		pkt_buf_addr_pp,
88444961713Sgirish 		pktbufsz_type,
88544961713Sgirish 		bufsize,
88644961713Sgirish 		anchor_index,
88744961713Sgirish 		chunk_index,
88844961713Sgirish 		dvma_addr,
88944961713Sgirish 		offset,
89044961713Sgirish 		block_size));
89144961713Sgirish 
89244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> getting total index"));
89344961713Sgirish 
89444961713Sgirish 	block_index = (offset / block_size); /* index within chunk */
89544961713Sgirish 	total_index = chunk_index + block_index;
89644961713Sgirish 
89744961713Sgirish 
89844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
89944961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: "
90044961713Sgirish 		"total_index %d dvma_addr $%p "
90144961713Sgirish 		"offset %d block_size %d "
90244961713Sgirish 		"block_index %d ",
90344961713Sgirish 		total_index, dvma_addr,
90444961713Sgirish 		offset, block_size,
90544961713Sgirish 		block_index));
906adfcba55Sjoycey #if defined(__i386)
907adfcba55Sjoycey 	*pkt_buf_addr_p = (uint64_t *)((uint32_t)bufinfo[anchor_index].kaddr +
908adfcba55Sjoycey 		(uint32_t)offset);
909adfcba55Sjoycey #else
910adfcba55Sjoycey 	*pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr +
911adfcba55Sjoycey 		(uint64_t)offset);
912adfcba55Sjoycey #endif
91344961713Sgirish 
91444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
91544961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: "
91644961713Sgirish 		"total_index %d dvma_addr $%p "
91744961713Sgirish 		"offset %d block_size %d "
91844961713Sgirish 		"block_index %d "
91944961713Sgirish 		"*pkt_buf_addr_p $%p",
92044961713Sgirish 		total_index, dvma_addr,
92144961713Sgirish 		offset, block_size,
92244961713Sgirish 		block_index,
92344961713Sgirish 		*pkt_buf_addr_p));
92444961713Sgirish 
92544961713Sgirish 
92644961713Sgirish 	*msg_index = total_index;
92744961713Sgirish 	*bufoffset =  (offset & page_size_mask);
92844961713Sgirish 
92944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
93044961713Sgirish 		"==> nxge_rxbuf_pp_to_vp: get msg index: "
93144961713Sgirish 		"msg_index %d bufoffset_index %d",
93244961713Sgirish 		*msg_index,
93344961713Sgirish 		*bufoffset));
93444961713Sgirish 
93544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rxbuf_pp_to_vp"));
93644961713Sgirish 
93744961713Sgirish 	return (NXGE_OK);
93844961713Sgirish }
93944961713Sgirish 
94044961713Sgirish /*
94144961713Sgirish  * used by quick sort (qsort) function
94244961713Sgirish  * to perform comparison
94344961713Sgirish  */
94444961713Sgirish static int
94544961713Sgirish nxge_sort_compare(const void *p1, const void *p2)
94644961713Sgirish {
94744961713Sgirish 
94844961713Sgirish 	rxbuf_index_info_t *a, *b;
94944961713Sgirish 
95044961713Sgirish 	a = (rxbuf_index_info_t *)p1;
95144961713Sgirish 	b = (rxbuf_index_info_t *)p2;
95244961713Sgirish 
95344961713Sgirish 	if (a->dvma_addr > b->dvma_addr)
95444961713Sgirish 		return (1);
95544961713Sgirish 	if (a->dvma_addr < b->dvma_addr)
95644961713Sgirish 		return (-1);
95744961713Sgirish 	return (0);
95844961713Sgirish }
95944961713Sgirish 
96044961713Sgirish 
96144961713Sgirish 
96244961713Sgirish /*
96344961713Sgirish  * grabbed this sort implementation from common/syscall/avl.c
96444961713Sgirish  *
96544961713Sgirish  */
96644961713Sgirish /*
96744961713Sgirish  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
96844961713Sgirish  * v = Ptr to array/vector of objs
96944961713Sgirish  * n = # objs in the array
97044961713Sgirish  * s = size of each obj (must be multiples of a word size)
97144961713Sgirish  * f = ptr to function to compare two objs
97244961713Sgirish  *	returns (-1 = less than, 0 = equal, 1 = greater than
97344961713Sgirish  */
97444961713Sgirish void
97544961713Sgirish nxge_ksort(caddr_t v, int n, int s, int (*f)())
97644961713Sgirish {
97744961713Sgirish 	int g, i, j, ii;
97844961713Sgirish 	unsigned int *p1, *p2;
97944961713Sgirish 	unsigned int tmp;
98044961713Sgirish 
98144961713Sgirish 	/* No work to do */
98244961713Sgirish 	if (v == NULL || n <= 1)
98344961713Sgirish 		return;
98444961713Sgirish 	/* Sanity check on arguments */
98544961713Sgirish 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
98644961713Sgirish 	ASSERT(s > 0);
98744961713Sgirish 
98844961713Sgirish 	for (g = n / 2; g > 0; g /= 2) {
98944961713Sgirish 		for (i = g; i < n; i++) {
99044961713Sgirish 			for (j = i - g; j >= 0 &&
99144961713Sgirish 				(*f)(v + j * s, v + (j + g) * s) == 1;
99244961713Sgirish 					j -= g) {
99344961713Sgirish 				p1 = (unsigned *)(v + j * s);
99444961713Sgirish 				p2 = (unsigned *)(v + (j + g) * s);
99544961713Sgirish 				for (ii = 0; ii < s / 4; ii++) {
99644961713Sgirish 					tmp = *p1;
99744961713Sgirish 					*p1++ = *p2;
99844961713Sgirish 					*p2++ = tmp;
99944961713Sgirish 				}
100044961713Sgirish 			}
100144961713Sgirish 		}
100244961713Sgirish 	}
100344961713Sgirish }
100444961713Sgirish 
100544961713Sgirish /*
100644961713Sgirish  * Initialize data structures required for rxdma
100744961713Sgirish  * buffer dvma->vmem address lookup
100844961713Sgirish  */
100944961713Sgirish /*ARGSUSED*/
101044961713Sgirish static nxge_status_t
101144961713Sgirish nxge_rxbuf_index_info_init(p_nxge_t nxgep, p_rx_rbr_ring_t rbrp)
101244961713Sgirish {
101344961713Sgirish 
101444961713Sgirish 	int index;
101544961713Sgirish 	rxring_info_t *ring_info;
101644961713Sgirish 	int max_iteration = 0, max_index = 0;
101744961713Sgirish 
101844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "==> nxge_rxbuf_index_info_init"));
101944961713Sgirish 
102044961713Sgirish 	ring_info = rbrp->ring_info;
102144961713Sgirish 	ring_info->hint[0] = NO_HINT;
102244961713Sgirish 	ring_info->hint[1] = NO_HINT;
102344961713Sgirish 	ring_info->hint[2] = NO_HINT;
102444961713Sgirish 	max_index = rbrp->num_blocks;
102544961713Sgirish 
102644961713Sgirish 		/* read the DVMA address information and sort it */
102744961713Sgirish 		/* do init of the information array */
102844961713Sgirish 
102944961713Sgirish 
103044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
103144961713Sgirish 		" nxge_rxbuf_index_info_init Sort ptrs"));
103244961713Sgirish 
103344961713Sgirish 		/* sort the array */
103444961713Sgirish 	nxge_ksort((void *)ring_info->buffer, max_index,
103544961713Sgirish 		sizeof (rxbuf_index_info_t), nxge_sort_compare);
103644961713Sgirish 
103744961713Sgirish 
103844961713Sgirish 
103944961713Sgirish 	for (index = 0; index < max_index; index++) {
104044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
104144961713Sgirish 			" nxge_rxbuf_index_info_init: sorted chunk %d "
104244961713Sgirish 			" ioaddr $%p kaddr $%p size %x",
104344961713Sgirish 			index, ring_info->buffer[index].dvma_addr,
104444961713Sgirish 			ring_info->buffer[index].kaddr,
104544961713Sgirish 			ring_info->buffer[index].buf_size));
104644961713Sgirish 	}
104744961713Sgirish 
104844961713Sgirish 	max_iteration = 0;
104944961713Sgirish 	while (max_index >= (1ULL << max_iteration))
105044961713Sgirish 		max_iteration++;
105144961713Sgirish 	ring_info->max_iterations = max_iteration + 1;
105244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA2_CTL,
105344961713Sgirish 		" nxge_rxbuf_index_info_init Find max iter %d",
105444961713Sgirish 					ring_info->max_iterations));
105544961713Sgirish 
105644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxbuf_index_info_init"));
105744961713Sgirish 	return (NXGE_OK);
105844961713Sgirish }
105944961713Sgirish 
10600a8e077aSspeer /* ARGSUSED */
106144961713Sgirish void
106244961713Sgirish nxge_dump_rcr_entry(p_nxge_t nxgep, p_rcr_entry_t entry_p)
106344961713Sgirish {
106444961713Sgirish #ifdef	NXGE_DEBUG
106544961713Sgirish 
106644961713Sgirish 	uint32_t bptr;
106744961713Sgirish 	uint64_t pp;
106844961713Sgirish 
106944961713Sgirish 	bptr = entry_p->bits.hdw.pkt_buf_addr;
107044961713Sgirish 
107144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
107244961713Sgirish 		"\trcr entry $%p "
107344961713Sgirish 		"\trcr entry 0x%0llx "
107444961713Sgirish 		"\trcr entry 0x%08x "
107544961713Sgirish 		"\trcr entry 0x%08x "
107644961713Sgirish 		"\tvalue 0x%0llx\n"
107744961713Sgirish 		"\tmulti = %d\n"
107844961713Sgirish 		"\tpkt_type = 0x%x\n"
107944961713Sgirish 		"\tzero_copy = %d\n"
108044961713Sgirish 		"\tnoport = %d\n"
108144961713Sgirish 		"\tpromis = %d\n"
108244961713Sgirish 		"\terror = 0x%04x\n"
108344961713Sgirish 		"\tdcf_err = 0x%01x\n"
108444961713Sgirish 		"\tl2_len = %d\n"
108544961713Sgirish 		"\tpktbufsize = %d\n"
108644961713Sgirish 		"\tpkt_buf_addr = $%p\n"
108744961713Sgirish 		"\tpkt_buf_addr (<< 6) = $%p\n",
108844961713Sgirish 		entry_p,
108944961713Sgirish 		*(int64_t *)entry_p,
109044961713Sgirish 		*(int32_t *)entry_p,
109144961713Sgirish 		*(int32_t *)((char *)entry_p + 32),
109244961713Sgirish 		entry_p->value,
109344961713Sgirish 		entry_p->bits.hdw.multi,
109444961713Sgirish 		entry_p->bits.hdw.pkt_type,
109544961713Sgirish 		entry_p->bits.hdw.zero_copy,
109644961713Sgirish 		entry_p->bits.hdw.noport,
109744961713Sgirish 		entry_p->bits.hdw.promis,
109844961713Sgirish 		entry_p->bits.hdw.error,
109944961713Sgirish 		entry_p->bits.hdw.dcf_err,
110044961713Sgirish 		entry_p->bits.hdw.l2_len,
110144961713Sgirish 		entry_p->bits.hdw.pktbufsz,
110244961713Sgirish 		bptr,
110344961713Sgirish 		entry_p->bits.ldw.pkt_buf_addr));
110444961713Sgirish 
110544961713Sgirish 	pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) <<
110644961713Sgirish 		RCR_PKT_BUF_ADDR_SHIFT;
110744961713Sgirish 
110844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "rcr pp 0x%llx l2 len %d",
110944961713Sgirish 		pp, (*(int64_t *)entry_p >> 40) & 0x3fff));
111044961713Sgirish #endif
111144961713Sgirish }
111244961713Sgirish 
111344961713Sgirish void
111444961713Sgirish nxge_rxdma_regs_dump(p_nxge_t nxgep, int rdc)
111544961713Sgirish {
111644961713Sgirish 	npi_handle_t		handle;
111744961713Sgirish 	rbr_stat_t 		rbr_stat;
111844961713Sgirish 	addr44_t 		hd_addr;
111944961713Sgirish 	addr44_t 		tail_addr;
112044961713Sgirish 	uint16_t 		qlen;
112144961713Sgirish 
112244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
112344961713Sgirish 		"==> nxge_rxdma_regs_dump: rdc channel %d", rdc));
112444961713Sgirish 
112544961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
112644961713Sgirish 
112744961713Sgirish 	/* RBR head */
112844961713Sgirish 	hd_addr.addr = 0;
112944961713Sgirish 	(void) npi_rxdma_rdc_rbr_head_get(handle, rdc, &hd_addr);
1130adfcba55Sjoycey #if defined(__i386)
113153f3d8ecSyc 	printf("nxge_rxdma_regs_dump: got hdptr $%p \n",
1132adfcba55Sjoycey 		(void *)(uint32_t)hd_addr.addr);
1133adfcba55Sjoycey #else
113453f3d8ecSyc 	printf("nxge_rxdma_regs_dump: got hdptr $%p \n",
113544961713Sgirish 		(void *)hd_addr.addr);
1136adfcba55Sjoycey #endif
113744961713Sgirish 
113844961713Sgirish 	/* RBR stats */
113944961713Sgirish 	(void) npi_rxdma_rdc_rbr_stat_get(handle, rdc, &rbr_stat);
114044961713Sgirish 	printf("nxge_rxdma_regs_dump: rbr len %d \n", rbr_stat.bits.ldw.qlen);
114144961713Sgirish 
114244961713Sgirish 	/* RCR tail */
114344961713Sgirish 	tail_addr.addr = 0;
114444961713Sgirish 	(void) npi_rxdma_rdc_rcr_tail_get(handle, rdc, &tail_addr);
1145adfcba55Sjoycey #if defined(__i386)
114653f3d8ecSyc 	printf("nxge_rxdma_regs_dump: got tail ptr $%p \n",
1147adfcba55Sjoycey 		(void *)(uint32_t)tail_addr.addr);
1148adfcba55Sjoycey #else
114953f3d8ecSyc 	printf("nxge_rxdma_regs_dump: got tail ptr $%p \n",
115044961713Sgirish 		(void *)tail_addr.addr);
1151adfcba55Sjoycey #endif
115244961713Sgirish 
115344961713Sgirish 	/* RCR qlen */
115444961713Sgirish 	(void) npi_rxdma_rdc_rcr_qlen_get(handle, rdc, &qlen);
115544961713Sgirish 	printf("nxge_rxdma_regs_dump: rcr len %x \n", qlen);
115644961713Sgirish 
115744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
115844961713Sgirish 		"<== nxge_rxdma_regs_dump: rdc rdc %d", rdc));
115944961713Sgirish }
116044961713Sgirish 
116144961713Sgirish void
116244961713Sgirish nxge_rxdma_stop(p_nxge_t nxgep)
116344961713Sgirish {
116444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop"));
116544961713Sgirish 
116644961713Sgirish 	(void) nxge_link_monitor(nxgep, LINK_MONITOR_STOP);
116744961713Sgirish 	(void) nxge_rx_mac_disable(nxgep);
116844961713Sgirish 	(void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_STOP);
116944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop"));
117044961713Sgirish }
117144961713Sgirish 
117244961713Sgirish void
117344961713Sgirish nxge_rxdma_stop_reinit(p_nxge_t nxgep)
117444961713Sgirish {
117544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_reinit"));
117644961713Sgirish 
117744961713Sgirish 	(void) nxge_rxdma_stop(nxgep);
117844961713Sgirish 	(void) nxge_uninit_rxdma_channels(nxgep);
117944961713Sgirish 	(void) nxge_init_rxdma_channels(nxgep);
118044961713Sgirish 
118144961713Sgirish #ifndef	AXIS_DEBUG_LB
118244961713Sgirish 	(void) nxge_xcvr_init(nxgep);
118344961713Sgirish 	(void) nxge_link_monitor(nxgep, LINK_MONITOR_START);
118444961713Sgirish #endif
118544961713Sgirish 	(void) nxge_rx_mac_enable(nxgep);
118644961713Sgirish 
118744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_reinit"));
118844961713Sgirish }
118944961713Sgirish 
119044961713Sgirish nxge_status_t
119144961713Sgirish nxge_rxdma_hw_mode(p_nxge_t nxgep, boolean_t enable)
119244961713Sgirish {
1193678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
1194678453a8Sspeer 	nxge_status_t status;
1195678453a8Sspeer 	npi_status_t rs;
1196678453a8Sspeer 	int rdc;
119744961713Sgirish 
119844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
119944961713Sgirish 		"==> nxge_rxdma_hw_mode: mode %d", enable));
120044961713Sgirish 
120144961713Sgirish 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
120244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1203678453a8Sspeer 		    "<== nxge_rxdma_mode: not initialized"));
120444961713Sgirish 		return (NXGE_ERROR);
120544961713Sgirish 	}
120644961713Sgirish 
1207678453a8Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
1208678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1209678453a8Sspeer 		    "<== nxge_tx_port_fatal_err_recover: "
1210678453a8Sspeer 		    "NULL ring pointer(s)"));
121144961713Sgirish 		return (NXGE_ERROR);
121244961713Sgirish 	}
121344961713Sgirish 
1214678453a8Sspeer 	if (set->owned.map == 0) {
121544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1216678453a8Sspeer 		    "nxge_rxdma_regs_dump_channels: no channels"));
1217678453a8Sspeer 		return (NULL);
121844961713Sgirish 	}
121944961713Sgirish 
1220678453a8Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1221678453a8Sspeer 		if ((1 << rdc) & set->owned.map) {
1222678453a8Sspeer 			rx_rbr_ring_t *ring =
1223678453a8Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
1224678453a8Sspeer 			npi_handle_t handle = NXGE_DEV_NPI_HANDLE(nxgep);
1225678453a8Sspeer 			if (ring) {
1226678453a8Sspeer 				if (enable) {
1227678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1228678453a8Sspeer 					    "==> nxge_rxdma_hw_mode: "
1229678453a8Sspeer 					    "channel %d (enable)", rdc));
1230678453a8Sspeer 					rs = npi_rxdma_cfg_rdc_enable
1231678453a8Sspeer 					    (handle, rdc);
1232678453a8Sspeer 				} else {
1233678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
1234678453a8Sspeer 					    "==> nxge_rxdma_hw_mode: "
1235678453a8Sspeer 					    "channel %d disable)", rdc));
1236678453a8Sspeer 					rs = npi_rxdma_cfg_rdc_disable
1237678453a8Sspeer 					    (handle, rdc);
1238678453a8Sspeer 				}
1239678453a8Sspeer 			}
124044961713Sgirish 		}
124144961713Sgirish 	}
124244961713Sgirish 
124344961713Sgirish 	status = ((rs == NPI_SUCCESS) ? NXGE_OK : NXGE_ERROR | rs);
124444961713Sgirish 
124544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
124644961713Sgirish 		"<== nxge_rxdma_hw_mode: status 0x%x", status));
124744961713Sgirish 
124844961713Sgirish 	return (status);
124944961713Sgirish }
125044961713Sgirish 
125144961713Sgirish void
125244961713Sgirish nxge_rxdma_enable_channel(p_nxge_t nxgep, uint16_t channel)
125344961713Sgirish {
125444961713Sgirish 	npi_handle_t		handle;
125544961713Sgirish 
125644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
125744961713Sgirish 		"==> nxge_rxdma_enable_channel: channel %d", channel));
125844961713Sgirish 
125944961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
126044961713Sgirish 	(void) npi_rxdma_cfg_rdc_enable(handle, channel);
126144961713Sgirish 
126244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_enable_channel"));
126344961713Sgirish }
126444961713Sgirish 
126544961713Sgirish void
126644961713Sgirish nxge_rxdma_disable_channel(p_nxge_t nxgep, uint16_t channel)
126744961713Sgirish {
126844961713Sgirish 	npi_handle_t		handle;
126944961713Sgirish 
127044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL,
127144961713Sgirish 		"==> nxge_rxdma_disable_channel: channel %d", channel));
127244961713Sgirish 
127344961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
127444961713Sgirish 	(void) npi_rxdma_cfg_rdc_disable(handle, channel);
127544961713Sgirish 
127644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DMA_CTL, "<== nxge_rxdma_disable_channel"));
127744961713Sgirish }
127844961713Sgirish 
127944961713Sgirish void
128044961713Sgirish nxge_hw_start_rx(p_nxge_t nxgep)
128144961713Sgirish {
128244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "==> nxge_hw_start_rx"));
128344961713Sgirish 
128444961713Sgirish 	(void) nxge_rxdma_hw_mode(nxgep, NXGE_DMA_START);
128544961713Sgirish 	(void) nxge_rx_mac_enable(nxgep);
128644961713Sgirish 
128744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, DDI_CTL, "<== nxge_hw_start_rx"));
128844961713Sgirish }
128944961713Sgirish 
129044961713Sgirish /*ARGSUSED*/
129144961713Sgirish void
129244961713Sgirish nxge_fixup_rxdma_rings(p_nxge_t nxgep)
129344961713Sgirish {
1294678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
1295678453a8Sspeer 	int rdc;
129644961713Sgirish 
129744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_fixup_rxdma_rings"));
129844961713Sgirish 
1299678453a8Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
1300678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1301678453a8Sspeer 		    "<== nxge_tx_port_fatal_err_recover: "
1302678453a8Sspeer 		    "NULL ring pointer(s)"));
130344961713Sgirish 		return;
130444961713Sgirish 	}
130544961713Sgirish 
1306678453a8Sspeer 	if (set->owned.map == 0) {
130744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1308678453a8Sspeer 		    "nxge_rxdma_regs_dump_channels: no channels"));
130944961713Sgirish 		return;
131044961713Sgirish 	}
131144961713Sgirish 
1312678453a8Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1313678453a8Sspeer 		if ((1 << rdc) & set->owned.map) {
1314678453a8Sspeer 			rx_rbr_ring_t *ring =
1315678453a8Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
1316678453a8Sspeer 			if (ring) {
1317678453a8Sspeer 				nxge_rxdma_hw_stop(nxgep, rdc);
1318678453a8Sspeer 				NXGE_DEBUG_MSG((nxgep, RX_CTL,
1319678453a8Sspeer 					"==> nxge_fixup_rxdma_rings: "
1320678453a8Sspeer 					"channel %d ring $%px",
1321678453a8Sspeer 					rdc, ring));
1322678453a8Sspeer 				(void) nxge_rxdma_fixup_channel
1323678453a8Sspeer 				    (nxgep, rdc, rdc);
1324678453a8Sspeer 			}
1325678453a8Sspeer 		}
132644961713Sgirish 	}
132744961713Sgirish 
132844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_fixup_rxdma_rings"));
132944961713Sgirish }
133044961713Sgirish 
133144961713Sgirish void
133244961713Sgirish nxge_rxdma_fix_channel(p_nxge_t nxgep, uint16_t channel)
133344961713Sgirish {
133444961713Sgirish 	int		i;
133544961713Sgirish 
133644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fix_channel"));
133744961713Sgirish 	i = nxge_rxdma_get_ring_index(nxgep, channel);
133844961713Sgirish 	if (i < 0) {
133944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
134044961713Sgirish 			"<== nxge_rxdma_fix_channel: no entry found"));
134144961713Sgirish 		return;
134244961713Sgirish 	}
134344961713Sgirish 
134444961713Sgirish 	nxge_rxdma_fixup_channel(nxgep, channel, i);
134544961713Sgirish 
1346678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fix_channel"));
134744961713Sgirish }
134844961713Sgirish 
134944961713Sgirish void
135044961713Sgirish nxge_rxdma_fixup_channel(p_nxge_t nxgep, uint16_t channel, int entry)
135144961713Sgirish {
135244961713Sgirish 	int			ndmas;
135344961713Sgirish 	p_rx_rbr_rings_t 	rx_rbr_rings;
135444961713Sgirish 	p_rx_rbr_ring_t		*rbr_rings;
135544961713Sgirish 	p_rx_rcr_rings_t 	rx_rcr_rings;
135644961713Sgirish 	p_rx_rcr_ring_t		*rcr_rings;
135744961713Sgirish 	p_rx_mbox_areas_t 	rx_mbox_areas_p;
135844961713Sgirish 	p_rx_mbox_t		*rx_mbox_p;
135944961713Sgirish 	p_nxge_dma_pool_t	dma_buf_poolp;
136044961713Sgirish 	p_nxge_dma_pool_t	dma_cntl_poolp;
136144961713Sgirish 	p_rx_rbr_ring_t 	rbrp;
136244961713Sgirish 	p_rx_rcr_ring_t 	rcrp;
136344961713Sgirish 	p_rx_mbox_t 		mboxp;
136444961713Sgirish 	p_nxge_dma_common_t 	dmap;
136544961713Sgirish 	nxge_status_t		status = NXGE_OK;
136644961713Sgirish 
136744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fixup_channel"));
136844961713Sgirish 
136944961713Sgirish 	(void) nxge_rxdma_stop_channel(nxgep, channel);
137044961713Sgirish 
137144961713Sgirish 	dma_buf_poolp = nxgep->rx_buf_pool_p;
137244961713Sgirish 	dma_cntl_poolp = nxgep->rx_cntl_pool_p;
137344961713Sgirish 
137444961713Sgirish 	if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) {
137544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
137644961713Sgirish 			"<== nxge_rxdma_fixup_channel: buf not allocated"));
137744961713Sgirish 		return;
137844961713Sgirish 	}
137944961713Sgirish 
138044961713Sgirish 	ndmas = dma_buf_poolp->ndmas;
138144961713Sgirish 	if (!ndmas) {
138244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
138344961713Sgirish 			"<== nxge_rxdma_fixup_channel: no dma allocated"));
138444961713Sgirish 		return;
138544961713Sgirish 	}
138644961713Sgirish 
1387a3c5bd6dSspeer 	rx_rbr_rings = nxgep->rx_rbr_rings;
138844961713Sgirish 	rx_rcr_rings = nxgep->rx_rcr_rings;
138944961713Sgirish 	rbr_rings = rx_rbr_rings->rbr_rings;
139044961713Sgirish 	rcr_rings = rx_rcr_rings->rcr_rings;
139144961713Sgirish 	rx_mbox_areas_p = nxgep->rx_mbox_areas_p;
139244961713Sgirish 	rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
139344961713Sgirish 
139444961713Sgirish 	/* Reinitialize the receive block and completion rings */
139544961713Sgirish 	rbrp = (p_rx_rbr_ring_t)rbr_rings[entry],
139644961713Sgirish 	rcrp = (p_rx_rcr_ring_t)rcr_rings[entry],
139744961713Sgirish 	mboxp = (p_rx_mbox_t)rx_mbox_p[entry];
139844961713Sgirish 
139944961713Sgirish 
140044961713Sgirish 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
140144961713Sgirish 	rbrp->rbr_rd_index = 0;
140244961713Sgirish 	rcrp->comp_rd_index = 0;
140344961713Sgirish 	rcrp->comp_wt_index = 0;
140444961713Sgirish 
140544961713Sgirish 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
140644961713Sgirish 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
140744961713Sgirish 
140844961713Sgirish 	status = nxge_rxdma_start_channel(nxgep, channel,
140944961713Sgirish 			rbrp, rcrp, mboxp);
141044961713Sgirish 	if (status != NXGE_OK) {
141144961713Sgirish 		goto nxge_rxdma_fixup_channel_fail;
141244961713Sgirish 	}
141344961713Sgirish 	if (status != NXGE_OK) {
141444961713Sgirish 		goto nxge_rxdma_fixup_channel_fail;
141544961713Sgirish 	}
141644961713Sgirish 
141744961713Sgirish nxge_rxdma_fixup_channel_fail:
141844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
141944961713Sgirish 		"==> nxge_rxdma_fixup_channel: failed (0x%08x)", status));
142044961713Sgirish 
142144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fixup_channel"));
142244961713Sgirish }
142344961713Sgirish 
1424678453a8Sspeer /* ARGSUSED */
142544961713Sgirish int
142644961713Sgirish nxge_rxdma_get_ring_index(p_nxge_t nxgep, uint16_t channel)
142744961713Sgirish {
1428678453a8Sspeer 	return (channel);
142944961713Sgirish }
143044961713Sgirish 
143144961713Sgirish p_rx_rbr_ring_t
143244961713Sgirish nxge_rxdma_get_rbr_ring(p_nxge_t nxgep, uint16_t channel)
143344961713Sgirish {
1434678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
1435678453a8Sspeer 	nxge_channel_t rdc;
143644961713Sgirish 
143744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
143844961713Sgirish 		"==> nxge_rxdma_get_rbr_ring: channel %d", channel));
143944961713Sgirish 
1440678453a8Sspeer 	if (nxgep->rx_rbr_rings == 0 || nxgep->rx_rbr_rings->rbr_rings == 0) {
1441678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1442678453a8Sspeer 		    "<== nxge_rxdma_get_rbr_ring: "
1443678453a8Sspeer 		    "NULL ring pointer(s)"));
144444961713Sgirish 		return (NULL);
144544961713Sgirish 	}
1446678453a8Sspeer 
1447678453a8Sspeer 	if (set->owned.map == 0) {
144844961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1449678453a8Sspeer 		    "<== nxge_rxdma_get_rbr_ring: no channels"));
145044961713Sgirish 		return (NULL);
145144961713Sgirish 	}
145244961713Sgirish 
1453678453a8Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1454678453a8Sspeer 		if ((1 << rdc) & set->owned.map) {
1455678453a8Sspeer 			rx_rbr_ring_t *ring =
1456678453a8Sspeer 			    nxgep->rx_rbr_rings->rbr_rings[rdc];
1457678453a8Sspeer 			if (ring) {
1458678453a8Sspeer 				if (channel == ring->rdc) {
1459678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, RX_CTL,
1460678453a8Sspeer 					    "==> nxge_rxdma_get_rbr_ring: "
1461678453a8Sspeer 					    "channel %d ring $%p", rdc, ring));
1462678453a8Sspeer 					return (ring);
1463678453a8Sspeer 				}
1464678453a8Sspeer 			}
146544961713Sgirish 		}
146644961713Sgirish 	}
146744961713Sgirish 
146844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
146944961713Sgirish 		"<== nxge_rxdma_get_rbr_ring: not found"));
147044961713Sgirish 
147144961713Sgirish 	return (NULL);
147244961713Sgirish }
147344961713Sgirish 
147444961713Sgirish p_rx_rcr_ring_t
147544961713Sgirish nxge_rxdma_get_rcr_ring(p_nxge_t nxgep, uint16_t channel)
147644961713Sgirish {
1477678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
1478678453a8Sspeer 	nxge_channel_t rdc;
147944961713Sgirish 
148044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
148144961713Sgirish 		"==> nxge_rxdma_get_rcr_ring: channel %d", channel));
148244961713Sgirish 
1483678453a8Sspeer 	if (nxgep->rx_rcr_rings == 0 || nxgep->rx_rcr_rings->rcr_rings == 0) {
1484678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, TX_CTL,
1485678453a8Sspeer 		    "<== nxge_rxdma_get_rcr_ring: "
1486678453a8Sspeer 		    "NULL ring pointer(s)"));
148744961713Sgirish 		return (NULL);
148844961713Sgirish 	}
1489678453a8Sspeer 
1490678453a8Sspeer 	if (set->owned.map == 0) {
149144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
1492678453a8Sspeer 		    "<== nxge_rxdma_get_rbr_ring: no channels"));
149344961713Sgirish 		return (NULL);
149444961713Sgirish 	}
149544961713Sgirish 
1496678453a8Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
1497678453a8Sspeer 		if ((1 << rdc) & set->owned.map) {
1498678453a8Sspeer 			rx_rcr_ring_t *ring =
1499678453a8Sspeer 			    nxgep->rx_rcr_rings->rcr_rings[rdc];
1500678453a8Sspeer 			if (ring) {
1501678453a8Sspeer 				if (channel == ring->rdc) {
1502678453a8Sspeer 					NXGE_DEBUG_MSG((nxgep, RX_CTL,
1503678453a8Sspeer 					    "==> nxge_rxdma_get_rcr_ring: "
1504678453a8Sspeer 					    "channel %d ring $%p", rdc, ring));
1505678453a8Sspeer 					return (ring);
1506678453a8Sspeer 				}
1507678453a8Sspeer 			}
150844961713Sgirish 		}
150944961713Sgirish 	}
151044961713Sgirish 
151144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
151244961713Sgirish 		"<== nxge_rxdma_get_rcr_ring: not found"));
151344961713Sgirish 
151444961713Sgirish 	return (NULL);
151544961713Sgirish }
151644961713Sgirish 
151744961713Sgirish /*
151844961713Sgirish  * Static functions start here.
151944961713Sgirish  */
152044961713Sgirish static p_rx_msg_t
152144961713Sgirish nxge_allocb(size_t size, uint32_t pri, p_nxge_dma_common_t dmabuf_p)
152244961713Sgirish {
152344961713Sgirish 	p_rx_msg_t nxge_mp 		= NULL;
152444961713Sgirish 	p_nxge_dma_common_t		dmamsg_p;
152544961713Sgirish 	uchar_t 			*buffer;
152644961713Sgirish 
152744961713Sgirish 	nxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP);
152844961713Sgirish 	if (nxge_mp == NULL) {
152956d930aeSspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
153044961713Sgirish 			"Allocation of a rx msg failed."));
153144961713Sgirish 		goto nxge_allocb_exit;
153244961713Sgirish 	}
153344961713Sgirish 
153444961713Sgirish 	nxge_mp->use_buf_pool = B_FALSE;
153544961713Sgirish 	if (dmabuf_p) {
153644961713Sgirish 		nxge_mp->use_buf_pool = B_TRUE;
153744961713Sgirish 		dmamsg_p = (p_nxge_dma_common_t)&nxge_mp->buf_dma;
153844961713Sgirish 		*dmamsg_p = *dmabuf_p;
153944961713Sgirish 		dmamsg_p->nblocks = 1;
154044961713Sgirish 		dmamsg_p->block_size = size;
154144961713Sgirish 		dmamsg_p->alength = size;
154244961713Sgirish 		buffer = (uchar_t *)dmabuf_p->kaddrp;
154344961713Sgirish 
154444961713Sgirish 		dmabuf_p->kaddrp = (void *)
154544961713Sgirish 				((char *)dmabuf_p->kaddrp + size);
154644961713Sgirish 		dmabuf_p->ioaddr_pp = (void *)
154744961713Sgirish 				((char *)dmabuf_p->ioaddr_pp + size);
154844961713Sgirish 		dmabuf_p->alength -= size;
154944961713Sgirish 		dmabuf_p->offset += size;
155044961713Sgirish 		dmabuf_p->dma_cookie.dmac_laddress += size;
155144961713Sgirish 		dmabuf_p->dma_cookie.dmac_size -= size;
155244961713Sgirish 
155344961713Sgirish 	} else {
155444961713Sgirish 		buffer = KMEM_ALLOC(size, KM_NOSLEEP);
155544961713Sgirish 		if (buffer == NULL) {
155656d930aeSspeer 			NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
155744961713Sgirish 				"Allocation of a receive page failed."));
155844961713Sgirish 			goto nxge_allocb_fail1;
155944961713Sgirish 		}
156044961713Sgirish 	}
156144961713Sgirish 
156244961713Sgirish 	nxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &nxge_mp->freeb);
156344961713Sgirish 	if (nxge_mp->rx_mblk_p == NULL) {
156456d930aeSspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL, "desballoc failed."));
156544961713Sgirish 		goto nxge_allocb_fail2;
156644961713Sgirish 	}
156744961713Sgirish 
156844961713Sgirish 	nxge_mp->buffer = buffer;
156944961713Sgirish 	nxge_mp->block_size = size;
157044961713Sgirish 	nxge_mp->freeb.free_func = (void (*)())nxge_freeb;
157144961713Sgirish 	nxge_mp->freeb.free_arg = (caddr_t)nxge_mp;
157244961713Sgirish 	nxge_mp->ref_cnt = 1;
157344961713Sgirish 	nxge_mp->free = B_TRUE;
157444961713Sgirish 	nxge_mp->rx_use_bcopy = B_FALSE;
157544961713Sgirish 
157614ea4bb7Ssd 	atomic_inc_32(&nxge_mblks_pending);
157744961713Sgirish 
157844961713Sgirish 	goto nxge_allocb_exit;
157944961713Sgirish 
158044961713Sgirish nxge_allocb_fail2:
158144961713Sgirish 	if (!nxge_mp->use_buf_pool) {
158244961713Sgirish 		KMEM_FREE(buffer, size);
158344961713Sgirish 	}
158444961713Sgirish 
158544961713Sgirish nxge_allocb_fail1:
158644961713Sgirish 	KMEM_FREE(nxge_mp, sizeof (rx_msg_t));
158744961713Sgirish 	nxge_mp = NULL;
158844961713Sgirish 
158944961713Sgirish nxge_allocb_exit:
159044961713Sgirish 	return (nxge_mp);
159144961713Sgirish }
159244961713Sgirish 
159344961713Sgirish p_mblk_t
159444961713Sgirish nxge_dupb(p_rx_msg_t nxge_mp, uint_t offset, size_t size)
159544961713Sgirish {
159644961713Sgirish 	p_mblk_t mp;
159744961713Sgirish 
159844961713Sgirish 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "==> nxge_dupb"));
159944961713Sgirish 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "nxge_mp = $%p "
160044961713Sgirish 		"offset = 0x%08X "
160144961713Sgirish 		"size = 0x%08X",
160244961713Sgirish 		nxge_mp, offset, size));
160344961713Sgirish 
160444961713Sgirish 	mp = desballoc(&nxge_mp->buffer[offset], size,
160544961713Sgirish 				0, &nxge_mp->freeb);
160644961713Sgirish 	if (mp == NULL) {
160744961713Sgirish 		NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
160844961713Sgirish 		goto nxge_dupb_exit;
160944961713Sgirish 	}
161044961713Sgirish 	atomic_inc_32(&nxge_mp->ref_cnt);
161144961713Sgirish 
161244961713Sgirish 
161344961713Sgirish nxge_dupb_exit:
161444961713Sgirish 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p",
161544961713Sgirish 		nxge_mp));
161644961713Sgirish 	return (mp);
161744961713Sgirish }
161844961713Sgirish 
161944961713Sgirish p_mblk_t
162044961713Sgirish nxge_dupb_bcopy(p_rx_msg_t nxge_mp, uint_t offset, size_t size)
162144961713Sgirish {
162244961713Sgirish 	p_mblk_t mp;
162344961713Sgirish 	uchar_t *dp;
162444961713Sgirish 
162544961713Sgirish 	mp = allocb(size + NXGE_RXBUF_EXTRA, 0);
162644961713Sgirish 	if (mp == NULL) {
162744961713Sgirish 		NXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
162844961713Sgirish 		goto nxge_dupb_bcopy_exit;
162944961713Sgirish 	}
163044961713Sgirish 	dp = mp->b_rptr = mp->b_rptr + NXGE_RXBUF_EXTRA;
163144961713Sgirish 	bcopy((void *)&nxge_mp->buffer[offset], dp, size);
163244961713Sgirish 	mp->b_wptr = dp + size;
163344961713Sgirish 
163444961713Sgirish nxge_dupb_bcopy_exit:
163544961713Sgirish 	NXGE_DEBUG_MSG((NULL, MEM_CTL, "<== nxge_dupb mp = $%p",
163644961713Sgirish 		nxge_mp));
163744961713Sgirish 	return (mp);
163844961713Sgirish }
163944961713Sgirish 
164044961713Sgirish void nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p,
164144961713Sgirish 	p_rx_msg_t rx_msg_p);
164244961713Sgirish 
164344961713Sgirish void
164444961713Sgirish nxge_post_page(p_nxge_t nxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p)
164544961713Sgirish {
164644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_post_page"));
164744961713Sgirish 
164844961713Sgirish 	/* Reuse this buffer */
164944961713Sgirish 	rx_msg_p->free = B_FALSE;
165044961713Sgirish 	rx_msg_p->cur_usage_cnt = 0;
165144961713Sgirish 	rx_msg_p->max_usage_cnt = 0;
165244961713Sgirish 	rx_msg_p->pkt_buf_size = 0;
165344961713Sgirish 
165444961713Sgirish 	if (rx_rbr_p->rbr_use_bcopy) {
165544961713Sgirish 		rx_msg_p->rx_use_bcopy = B_FALSE;
165644961713Sgirish 		atomic_dec_32(&rx_rbr_p->rbr_consumed);
165744961713Sgirish 	}
165844961713Sgirish 
165944961713Sgirish 	/*
166044961713Sgirish 	 * Get the rbr header pointer and its offset index.
166144961713Sgirish 	 */
166244961713Sgirish 	MUTEX_ENTER(&rx_rbr_p->post_lock);
166344961713Sgirish 	rx_rbr_p->rbr_wr_index =  ((rx_rbr_p->rbr_wr_index + 1) &
166444961713Sgirish 					    rx_rbr_p->rbr_wrap_mask);
166544961713Sgirish 	rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr;
166644961713Sgirish 	MUTEX_EXIT(&rx_rbr_p->post_lock);
166730ac2e7bSml 	npi_rxdma_rdc_rbr_kick(NXGE_DEV_NPI_HANDLE(nxgep),
166830ac2e7bSml 	    rx_rbr_p->rdc, 1);
166944961713Sgirish 
167044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
167144961713Sgirish 		"<== nxge_post_page (channel %d post_next_index %d)",
167244961713Sgirish 		rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index));
167344961713Sgirish 
167444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_post_page"));
167544961713Sgirish }
167644961713Sgirish 
167744961713Sgirish void
167844961713Sgirish nxge_freeb(p_rx_msg_t rx_msg_p)
167944961713Sgirish {
168044961713Sgirish 	size_t size;
168144961713Sgirish 	uchar_t *buffer = NULL;
168244961713Sgirish 	int ref_cnt;
1683958cea9eSml 	boolean_t free_state = B_FALSE;
168444961713Sgirish 
1685007969e0Stm 	rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p;
1686007969e0Stm 
168744961713Sgirish 	NXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> nxge_freeb"));
168844961713Sgirish 	NXGE_DEBUG_MSG((NULL, MEM2_CTL,
168944961713Sgirish 		"nxge_freeb:rx_msg_p = $%p (block pending %d)",
169044961713Sgirish 		rx_msg_p, nxge_mblks_pending));
169144961713Sgirish 
1692958cea9eSml 	/*
1693958cea9eSml 	 * First we need to get the free state, then
1694958cea9eSml 	 * atomic decrement the reference count to prevent
1695958cea9eSml 	 * the race condition with the interrupt thread that
1696958cea9eSml 	 * is processing a loaned up buffer block.
1697958cea9eSml 	 */
1698958cea9eSml 	free_state = rx_msg_p->free;
1699958cea9eSml 	ref_cnt = atomic_add_32_nv(&rx_msg_p->ref_cnt, -1);
170044961713Sgirish 	if (!ref_cnt) {
170130ac2e7bSml 		atomic_dec_32(&nxge_mblks_pending);
170244961713Sgirish 		buffer = rx_msg_p->buffer;
170344961713Sgirish 		size = rx_msg_p->block_size;
170444961713Sgirish 		NXGE_DEBUG_MSG((NULL, MEM2_CTL, "nxge_freeb: "
170544961713Sgirish 			"will free: rx_msg_p = $%p (block pending %d)",
170656d930aeSspeer 			rx_msg_p, nxge_mblks_pending));
170744961713Sgirish 
170844961713Sgirish 		if (!rx_msg_p->use_buf_pool) {
170944961713Sgirish 			KMEM_FREE(buffer, size);
171044961713Sgirish 		}
171114ea4bb7Ssd 
171214ea4bb7Ssd 		KMEM_FREE(rx_msg_p, sizeof (rx_msg_t));
1713007969e0Stm 
17143e82a89eSmisaki 		if (ring) {
17153e82a89eSmisaki 			/*
17163e82a89eSmisaki 			 * Decrement the receive buffer ring's reference
17173e82a89eSmisaki 			 * count, too.
17183e82a89eSmisaki 			 */
17193e82a89eSmisaki 			atomic_dec_32(&ring->rbr_ref_cnt);
1720007969e0Stm 
17213e82a89eSmisaki 			/*
1722678453a8Sspeer 			 * Free the receive buffer ring, if
17233e82a89eSmisaki 			 * 1. all the receive buffers have been freed
17243e82a89eSmisaki 			 * 2. and we are in the proper state (that is,
17253e82a89eSmisaki 			 *    we are not UNMAPPING).
17263e82a89eSmisaki 			 */
17273e82a89eSmisaki 			if (ring->rbr_ref_cnt == 0 &&
17283e82a89eSmisaki 			    ring->rbr_state == RBR_UNMAPPED) {
1729678453a8Sspeer 				/*
1730678453a8Sspeer 				 * Free receive data buffers,
1731678453a8Sspeer 				 * buffer index information
1732678453a8Sspeer 				 * (rxring_info) and
1733678453a8Sspeer 				 * the message block ring.
1734678453a8Sspeer 				 */
1735678453a8Sspeer 				NXGE_DEBUG_MSG((NULL, RX_CTL,
1736678453a8Sspeer 				    "nxge_freeb:rx_msg_p = $%p "
1737678453a8Sspeer 				    "(block pending %d) free buffers",
1738678453a8Sspeer 				    rx_msg_p, nxge_mblks_pending));
1739678453a8Sspeer 				nxge_rxdma_databuf_free(ring);
1740678453a8Sspeer 				if (ring->ring_info) {
1741678453a8Sspeer 					KMEM_FREE(ring->ring_info,
1742678453a8Sspeer 					    sizeof (rxring_info_t));
1743678453a8Sspeer 				}
1744678453a8Sspeer 
1745678453a8Sspeer 				if (ring->rx_msg_ring) {
1746678453a8Sspeer 					KMEM_FREE(ring->rx_msg_ring,
1747678453a8Sspeer 					    ring->tnblocks *
1748678453a8Sspeer 					    sizeof (p_rx_msg_t));
1749678453a8Sspeer 				}
17503e82a89eSmisaki 				KMEM_FREE(ring, sizeof (*ring));
17513e82a89eSmisaki 			}
1752007969e0Stm 		}
175314ea4bb7Ssd 		return;
175444961713Sgirish 	}
175544961713Sgirish 
175644961713Sgirish 	/*
175744961713Sgirish 	 * Repost buffer.
175844961713Sgirish 	 */
17593e82a89eSmisaki 	if (free_state && (ref_cnt == 1) && ring) {
176044961713Sgirish 		NXGE_DEBUG_MSG((NULL, RX_CTL,
176144961713Sgirish 		    "nxge_freeb: post page $%p:", rx_msg_p));
1762007969e0Stm 		if (ring->rbr_state == RBR_POSTING)
1763007969e0Stm 			nxge_post_page(rx_msg_p->nxgep, ring, rx_msg_p);
176444961713Sgirish 	}
176544961713Sgirish 
176644961713Sgirish 	NXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== nxge_freeb"));
176744961713Sgirish }
176844961713Sgirish 
176944961713Sgirish uint_t
177044961713Sgirish nxge_rx_intr(void *arg1, void *arg2)
177144961713Sgirish {
177244961713Sgirish 	p_nxge_ldv_t		ldvp = (p_nxge_ldv_t)arg1;
177344961713Sgirish 	p_nxge_t		nxgep = (p_nxge_t)arg2;
177444961713Sgirish 	p_nxge_ldg_t		ldgp;
177544961713Sgirish 	uint8_t			channel;
177644961713Sgirish 	npi_handle_t		handle;
177744961713Sgirish 	rx_dma_ctl_stat_t	cs;
177844961713Sgirish 
177944961713Sgirish #ifdef	NXGE_DEBUG
178044961713Sgirish 	rxdma_cfig1_t		cfg;
178144961713Sgirish #endif
178244961713Sgirish 	uint_t 			serviced = DDI_INTR_UNCLAIMED;
178344961713Sgirish 
178444961713Sgirish 	if (ldvp == NULL) {
178544961713Sgirish 		NXGE_DEBUG_MSG((NULL, INT_CTL,
178644961713Sgirish 			"<== nxge_rx_intr: arg2 $%p arg1 $%p",
178744961713Sgirish 			nxgep, ldvp));
178844961713Sgirish 
178944961713Sgirish 		return (DDI_INTR_CLAIMED);
179044961713Sgirish 	}
179144961713Sgirish 
179244961713Sgirish 	if (arg2 == NULL || (void *)ldvp->nxgep != arg2) {
179344961713Sgirish 		nxgep = ldvp->nxgep;
179444961713Sgirish 	}
17951d36aa9eSspeer 
17961d36aa9eSspeer 	if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
17971d36aa9eSspeer 	    (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
17981d36aa9eSspeer 		NXGE_DEBUG_MSG((nxgep, INT_CTL,
17991d36aa9eSspeer 		    "<== nxge_rx_intr: interface not started or intialized"));
18001d36aa9eSspeer 		return (DDI_INTR_CLAIMED);
18011d36aa9eSspeer 	}
18021d36aa9eSspeer 
180344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
180444961713Sgirish 		"==> nxge_rx_intr: arg2 $%p arg1 $%p",
180544961713Sgirish 		nxgep, ldvp));
180644961713Sgirish 
180744961713Sgirish 	/*
180844961713Sgirish 	 * This interrupt handler is for a specific
180944961713Sgirish 	 * receive dma channel.
181044961713Sgirish 	 */
181144961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
181244961713Sgirish 	/*
181344961713Sgirish 	 * Get the control and status for this channel.
181444961713Sgirish 	 */
181544961713Sgirish 	channel = ldvp->channel;
181644961713Sgirish 	ldgp = ldvp->ldgp;
181744961713Sgirish 	RXDMA_REG_READ64(handle, RX_DMA_CTL_STAT_REG, channel, &cs.value);
181844961713Sgirish 
181944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_intr:channel %d "
182044961713Sgirish 		"cs 0x%016llx rcrto 0x%x rcrthres %x",
182144961713Sgirish 		channel,
182244961713Sgirish 		cs.value,
182344961713Sgirish 		cs.bits.hdw.rcrto,
182444961713Sgirish 		cs.bits.hdw.rcrthres));
182544961713Sgirish 
1826678453a8Sspeer 	nxge_rx_pkts_vring(nxgep, ldvp->vdma_index, cs);
182744961713Sgirish 	serviced = DDI_INTR_CLAIMED;
182844961713Sgirish 
182944961713Sgirish 	/* error events. */
183044961713Sgirish 	if (cs.value & RX_DMA_CTL_STAT_ERROR) {
1831678453a8Sspeer 		(void) nxge_rx_err_evnts(nxgep, channel, cs);
183244961713Sgirish 	}
183344961713Sgirish 
183444961713Sgirish nxge_intr_exit:
183544961713Sgirish 	/*
183644961713Sgirish 	 * Enable the mailbox update interrupt if we want
183744961713Sgirish 	 * to use mailbox. We probably don't need to use
183844961713Sgirish 	 * mailbox as it only saves us one pio read.
183944961713Sgirish 	 * Also write 1 to rcrthres and rcrto to clear
184044961713Sgirish 	 * these two edge triggered bits.
184144961713Sgirish 	 */
184244961713Sgirish 
184344961713Sgirish 	cs.value &= RX_DMA_CTL_STAT_WR1C;
184444961713Sgirish 	cs.bits.hdw.mex = 1;
184544961713Sgirish 	RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG, channel,
184644961713Sgirish 			cs.value);
184744961713Sgirish 
184844961713Sgirish 	/*
184944961713Sgirish 	 * Rearm this logical group if this is a single device
185044961713Sgirish 	 * group.
185144961713Sgirish 	 */
185244961713Sgirish 	if (ldgp->nldvs == 1) {
185344961713Sgirish 		ldgimgm_t		mgm;
185444961713Sgirish 		mgm.value = 0;
185544961713Sgirish 		mgm.bits.ldw.arm = 1;
185644961713Sgirish 		mgm.bits.ldw.timer = ldgp->ldg_timer;
1857678453a8Sspeer 		if (isLDOMguest(nxgep)) {
1858678453a8Sspeer 			nxge_hio_ldgimgn(nxgep, ldgp);
1859678453a8Sspeer 		} else {
1860678453a8Sspeer 			NXGE_REG_WR64(handle,
186144961713Sgirish 			    LDGIMGN_REG + LDSV_OFFSET(ldgp->ldg),
186244961713Sgirish 			    mgm.value);
1863678453a8Sspeer 		}
186444961713Sgirish 	}
186544961713Sgirish 
186644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_intr: serviced %d",
186744961713Sgirish 		serviced));
186844961713Sgirish 	return (serviced);
186944961713Sgirish }
187044961713Sgirish 
187144961713Sgirish /*
187244961713Sgirish  * Process the packets received in the specified logical device
187344961713Sgirish  * and pass up a chain of message blocks to the upper layer.
187444961713Sgirish  */
187544961713Sgirish static void
1876678453a8Sspeer nxge_rx_pkts_vring(p_nxge_t nxgep, uint_t vindex, rx_dma_ctl_stat_t cs)
187744961713Sgirish {
187844961713Sgirish 	p_mblk_t		mp;
187944961713Sgirish 	p_rx_rcr_ring_t		rcrp;
188044961713Sgirish 
188144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts_vring"));
1882678453a8Sspeer 	rcrp = nxgep->rx_rcr_rings->rcr_rings[vindex];
1883678453a8Sspeer 	if (rcrp->poll_flag) {
1884678453a8Sspeer 		/* It is in the poll mode */
1885678453a8Sspeer 		return;
1886678453a8Sspeer 	}
1887678453a8Sspeer 
1888678453a8Sspeer 	if ((mp = nxge_rx_pkts(nxgep, rcrp, cs, -1)) == NULL) {
188944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
189044961713Sgirish 			"<== nxge_rx_pkts_vring: no mp"));
189144961713Sgirish 		return;
189244961713Sgirish 	}
189344961713Sgirish 
189444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts_vring: $%p",
189544961713Sgirish 		mp));
189644961713Sgirish 
189744961713Sgirish #ifdef  NXGE_DEBUG
189844961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
189944961713Sgirish 			"==> nxge_rx_pkts_vring:calling mac_rx "
190014ea4bb7Ssd 			"LEN %d mp $%p mp->b_cont $%p mp->b_next $%p rcrp $%p "
190144961713Sgirish 			"mac_handle $%p",
190214ea4bb7Ssd 			mp->b_wptr - mp->b_rptr,
190314ea4bb7Ssd 			mp, mp->b_cont, mp->b_next,
190444961713Sgirish 			rcrp, rcrp->rcr_mac_handle));
190544961713Sgirish 
190644961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
190744961713Sgirish 			"==> nxge_rx_pkts_vring: dump packets "
190844961713Sgirish 			"(mp $%p b_rptr $%p b_wptr $%p):\n %s",
190944961713Sgirish 			mp,
191044961713Sgirish 			mp->b_rptr,
191144961713Sgirish 			mp->b_wptr,
191214ea4bb7Ssd 			nxge_dump_packet((char *)mp->b_rptr,
191314ea4bb7Ssd 			mp->b_wptr - mp->b_rptr)));
191414ea4bb7Ssd 		if (mp->b_cont) {
191514ea4bb7Ssd 			NXGE_DEBUG_MSG((nxgep, RX_CTL,
191614ea4bb7Ssd 				"==> nxge_rx_pkts_vring: dump b_cont packets "
191714ea4bb7Ssd 				"(mp->b_cont $%p b_rptr $%p b_wptr $%p):\n %s",
191814ea4bb7Ssd 				mp->b_cont,
191914ea4bb7Ssd 				mp->b_cont->b_rptr,
192014ea4bb7Ssd 				mp->b_cont->b_wptr,
192114ea4bb7Ssd 				nxge_dump_packet((char *)mp->b_cont->b_rptr,
192214ea4bb7Ssd 				mp->b_cont->b_wptr - mp->b_cont->b_rptr)));
192314ea4bb7Ssd 		}
192444961713Sgirish 		if (mp->b_next) {
192544961713Sgirish 			NXGE_DEBUG_MSG((nxgep, RX_CTL,
192644961713Sgirish 				"==> nxge_rx_pkts_vring: dump next packets "
192744961713Sgirish 				"(b_rptr $%p): %s",
192844961713Sgirish 				mp->b_next->b_rptr,
192944961713Sgirish 				nxge_dump_packet((char *)mp->b_next->b_rptr,
193014ea4bb7Ssd 				mp->b_next->b_wptr - mp->b_next->b_rptr)));
193144961713Sgirish 		}
193244961713Sgirish #endif
193344961713Sgirish 
1934678453a8Sspeer 	if (!isLDOMguest(nxgep))
1935678453a8Sspeer 		mac_rx(nxgep->mach, rcrp->rcr_mac_handle, mp);
1936678453a8Sspeer #if defined(sun4v)
1937678453a8Sspeer 	else {			/* isLDOMguest(nxgep) */
1938678453a8Sspeer 		nxge_hio_data_t *nhd = (nxge_hio_data_t *)
1939678453a8Sspeer 		    nxgep->nxge_hw_p->hio;
1940678453a8Sspeer 		nx_vio_fp_t *vio = &nhd->hio.vio;
1941678453a8Sspeer 
1942678453a8Sspeer 		if (vio->cb.vio_net_rx_cb) {
1943678453a8Sspeer 			(*vio->cb.vio_net_rx_cb)
1944678453a8Sspeer 			    (nxgep->hio_vr->vhp, mp);
1945678453a8Sspeer 		}
1946678453a8Sspeer 	}
1947678453a8Sspeer #endif
194844961713Sgirish }
194944961713Sgirish 
195044961713Sgirish 
195144961713Sgirish /*
195244961713Sgirish  * This routine is the main packet receive processing function.
195344961713Sgirish  * It gets the packet type, error code, and buffer related
195444961713Sgirish  * information from the receive completion entry.
195544961713Sgirish  * How many completion entries to process is based on the number of packets
195644961713Sgirish  * queued by the hardware, a hardware maintained tail pointer
195744961713Sgirish  * and a configurable receive packet count.
195844961713Sgirish  *
195944961713Sgirish  * A chain of message blocks will be created as result of processing
196044961713Sgirish  * the completion entries. This chain of message blocks will be returned and
196144961713Sgirish  * a hardware control status register will be updated with the number of
196244961713Sgirish  * packets were removed from the hardware queue.
196344961713Sgirish  *
196444961713Sgirish  */
1965678453a8Sspeer static mblk_t *
1966678453a8Sspeer nxge_rx_pkts(p_nxge_t nxgep, p_rx_rcr_ring_t rcr_p, rx_dma_ctl_stat_t cs,
1967678453a8Sspeer     int bytes_to_pickup)
196844961713Sgirish {
196944961713Sgirish 	npi_handle_t		handle;
197044961713Sgirish 	uint8_t			channel;
197144961713Sgirish 	uint32_t		comp_rd_index;
197244961713Sgirish 	p_rcr_entry_t		rcr_desc_rd_head_p;
197344961713Sgirish 	p_rcr_entry_t		rcr_desc_rd_head_pp;
197444961713Sgirish 	p_mblk_t		nmp, mp_cont, head_mp, *tail_mp;
197544961713Sgirish 	uint16_t		qlen, nrcr_read, npkt_read;
1976678453a8Sspeer 	uint32_t		qlen_hw;
197744961713Sgirish 	boolean_t		multi;
1978678453a8Sspeer 	rcrcfig_b_t		rcr_cfg_b;
1979678453a8Sspeer 	int			totallen = 0;
1980a3c5bd6dSspeer #if defined(_BIG_ENDIAN)
198144961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
198244961713Sgirish #endif
198344961713Sgirish 
1984678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: "
1985678453a8Sspeer 		"channel %d", rcr_p->rdc));
198644961713Sgirish 
198744961713Sgirish 	if (!(nxgep->drv_state & STATE_HW_INITIALIZED)) {
198844961713Sgirish 		return (NULL);
198944961713Sgirish 	}
199044961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
199144961713Sgirish 	channel = rcr_p->rdc;
199244961713Sgirish 
199344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
199444961713Sgirish 		"==> nxge_rx_pkts: START: rcr channel %d "
199544961713Sgirish 		"head_p $%p head_pp $%p  index %d ",
199644961713Sgirish 		channel, rcr_p->rcr_desc_rd_head_p,
199744961713Sgirish 		rcr_p->rcr_desc_rd_head_pp,
199844961713Sgirish 		rcr_p->comp_rd_index));
199944961713Sgirish 
200044961713Sgirish 
2001a3c5bd6dSspeer #if !defined(_BIG_ENDIAN)
200244961713Sgirish 	qlen = RXDMA_REG_READ32(handle, RCRSTAT_A_REG, channel) & 0xffff;
200344961713Sgirish #else
200444961713Sgirish 	rs = npi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen);
200544961713Sgirish 	if (rs != NPI_SUCCESS) {
2006678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts: "
200744961713Sgirish 		"channel %d, get qlen failed 0x%08x",
2008678453a8Sspeer 		channel, rs));
200944961713Sgirish 		return (NULL);
201044961713Sgirish 	}
201144961713Sgirish #endif
201244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rx_pkts:rcr channel %d "
201344961713Sgirish 		"qlen %d", channel, qlen));
201444961713Sgirish 
201544961713Sgirish 
201644961713Sgirish 
201744961713Sgirish 	if (!qlen) {
201844961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
201944961713Sgirish 			"==> nxge_rx_pkts:rcr channel %d "
202044961713Sgirish 			"qlen %d (no pkts)", channel, qlen));
202144961713Sgirish 
202244961713Sgirish 		return (NULL);
202344961713Sgirish 	}
202444961713Sgirish 
202544961713Sgirish 	comp_rd_index = rcr_p->comp_rd_index;
202644961713Sgirish 
202744961713Sgirish 	rcr_desc_rd_head_p = rcr_p->rcr_desc_rd_head_p;
202844961713Sgirish 	rcr_desc_rd_head_pp = rcr_p->rcr_desc_rd_head_pp;
202944961713Sgirish 	nrcr_read = npkt_read = 0;
203044961713Sgirish 
203144961713Sgirish 	/*
203244961713Sgirish 	 * Number of packets queued
203344961713Sgirish 	 * (The jumbo or multi packet will be counted as only one
203444961713Sgirish 	 *  packets and it may take up more than one completion entry).
203544961713Sgirish 	 */
203644961713Sgirish 	qlen_hw = (qlen < nxge_max_rx_pkts) ?
203744961713Sgirish 		qlen : nxge_max_rx_pkts;
203844961713Sgirish 	head_mp = NULL;
203944961713Sgirish 	tail_mp = &head_mp;
204044961713Sgirish 	nmp = mp_cont = NULL;
204144961713Sgirish 	multi = B_FALSE;
204244961713Sgirish 
2043a3c5bd6dSspeer 	while (qlen_hw) {
204444961713Sgirish 
204544961713Sgirish #ifdef NXGE_DEBUG
204644961713Sgirish 		nxge_dump_rcr_entry(nxgep, rcr_desc_rd_head_p);
204744961713Sgirish #endif
204844961713Sgirish 		/*
204944961713Sgirish 		 * Process one completion ring entry.
205044961713Sgirish 		 */
205144961713Sgirish 		nxge_receive_packet(nxgep,
205244961713Sgirish 			rcr_p, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont);
205344961713Sgirish 
205444961713Sgirish 		/*
205544961713Sgirish 		 * message chaining modes
205644961713Sgirish 		 */
205714ea4bb7Ssd 		if (nmp) {
205844961713Sgirish 			nmp->b_next = NULL;
205914ea4bb7Ssd 			if (!multi && !mp_cont) { /* frame fits a partition */
206014ea4bb7Ssd 				*tail_mp = nmp;
206114ea4bb7Ssd 				tail_mp = &nmp->b_next;
2062678453a8Sspeer 				totallen += MBLKL(nmp);
206314ea4bb7Ssd 				nmp = NULL;
206414ea4bb7Ssd 			} else if (multi && !mp_cont) { /* first segment */
206514ea4bb7Ssd 				*tail_mp = nmp;
206614ea4bb7Ssd 				tail_mp = &nmp->b_cont;
2067678453a8Sspeer 				totallen += MBLKL(nmp);
206814ea4bb7Ssd 			} else if (multi && mp_cont) {	/* mid of multi segs */
206914ea4bb7Ssd 				*tail_mp = mp_cont;
207014ea4bb7Ssd 				tail_mp = &mp_cont->b_cont;
2071678453a8Sspeer 				totallen += MBLKL(mp_cont);
207214ea4bb7Ssd 			} else if (!multi && mp_cont) { /* last segment */
2073a3c5bd6dSspeer 				*tail_mp = mp_cont;
207414ea4bb7Ssd 				tail_mp = &nmp->b_next;
2075678453a8Sspeer 				totallen += MBLKL(mp_cont);
207614ea4bb7Ssd 				nmp = NULL;
207714ea4bb7Ssd 			}
207844961713Sgirish 		}
207944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
208044961713Sgirish 			"==> nxge_rx_pkts: loop: rcr channel %d "
208144961713Sgirish 			"before updating: multi %d "
208244961713Sgirish 			"nrcr_read %d "
208344961713Sgirish 			"npk read %d "
208444961713Sgirish 			"head_pp $%p  index %d ",
208544961713Sgirish 			channel,
208644961713Sgirish 			multi,
208744961713Sgirish 			nrcr_read, npkt_read, rcr_desc_rd_head_pp,
208844961713Sgirish 			comp_rd_index));
208944961713Sgirish 
209044961713Sgirish 		if (!multi) {
209144961713Sgirish 			qlen_hw--;
209244961713Sgirish 			npkt_read++;
209344961713Sgirish 		}
209444961713Sgirish 
209544961713Sgirish 		/*
209644961713Sgirish 		 * Update the next read entry.
209744961713Sgirish 		 */
209844961713Sgirish 		comp_rd_index = NEXT_ENTRY(comp_rd_index,
209944961713Sgirish 					rcr_p->comp_wrap_mask);
210044961713Sgirish 
210144961713Sgirish 		rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p,
210244961713Sgirish 				rcr_p->rcr_desc_first_p,
210344961713Sgirish 				rcr_p->rcr_desc_last_p);
210444961713Sgirish 
210544961713Sgirish 		nrcr_read++;
210644961713Sgirish 
210744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
210844961713Sgirish 			"<== nxge_rx_pkts: (SAM, process one packet) "
210944961713Sgirish 			"nrcr_read %d",
211044961713Sgirish 			nrcr_read));
211144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
211244961713Sgirish 			"==> nxge_rx_pkts: loop: rcr channel %d "
211344961713Sgirish 			"multi %d "
211444961713Sgirish 			"nrcr_read %d "
211544961713Sgirish 			"npk read %d "
211644961713Sgirish 			"head_pp $%p  index %d ",
211744961713Sgirish 			channel,
211844961713Sgirish 			multi,
211944961713Sgirish 			nrcr_read, npkt_read, rcr_desc_rd_head_pp,
212044961713Sgirish 			comp_rd_index));
212144961713Sgirish 
2122678453a8Sspeer 		if ((bytes_to_pickup != -1) &&
2123678453a8Sspeer 		    (totallen >= bytes_to_pickup)) {
2124678453a8Sspeer 			break;
2125678453a8Sspeer 		}
212644961713Sgirish 	}
212744961713Sgirish 
212844961713Sgirish 	rcr_p->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp;
212944961713Sgirish 	rcr_p->comp_rd_index = comp_rd_index;
213044961713Sgirish 	rcr_p->rcr_desc_rd_head_p = rcr_desc_rd_head_p;
213144961713Sgirish 
213214ea4bb7Ssd 	if ((nxgep->intr_timeout != rcr_p->intr_timeout) ||
213314ea4bb7Ssd 		(nxgep->intr_threshold != rcr_p->intr_threshold)) {
213414ea4bb7Ssd 		rcr_p->intr_timeout = nxgep->intr_timeout;
213514ea4bb7Ssd 		rcr_p->intr_threshold = nxgep->intr_threshold;
213614ea4bb7Ssd 		rcr_cfg_b.value = 0x0ULL;
213714ea4bb7Ssd 		if (rcr_p->intr_timeout)
213814ea4bb7Ssd 			rcr_cfg_b.bits.ldw.entout = 1;
213914ea4bb7Ssd 		rcr_cfg_b.bits.ldw.timeout = rcr_p->intr_timeout;
214014ea4bb7Ssd 		rcr_cfg_b.bits.ldw.pthres = rcr_p->intr_threshold;
214114ea4bb7Ssd 		RXDMA_REG_WRITE64(handle, RCRCFIG_B_REG,
214214ea4bb7Ssd 				    channel, rcr_cfg_b.value);
214314ea4bb7Ssd 	}
214444961713Sgirish 
214544961713Sgirish 	cs.bits.ldw.pktread = npkt_read;
214644961713Sgirish 	cs.bits.ldw.ptrread = nrcr_read;
214744961713Sgirish 	RXDMA_REG_WRITE64(handle, RX_DMA_CTL_STAT_REG,
214844961713Sgirish 			    channel, cs.value);
214944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
215044961713Sgirish 		"==> nxge_rx_pkts: EXIT: rcr channel %d "
215144961713Sgirish 		"head_pp $%p  index %016llx ",
215244961713Sgirish 		channel,
215344961713Sgirish 		rcr_p->rcr_desc_rd_head_pp,
215444961713Sgirish 		rcr_p->comp_rd_index));
215544961713Sgirish 	/*
215644961713Sgirish 	 * Update RCR buffer pointer read and number of packets
215744961713Sgirish 	 * read.
215844961713Sgirish 	 */
215944961713Sgirish 
216044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_pkts"));
216144961713Sgirish 	return (head_mp);
216244961713Sgirish }
216344961713Sgirish 
216444961713Sgirish void
216544961713Sgirish nxge_receive_packet(p_nxge_t nxgep,
216644961713Sgirish     p_rx_rcr_ring_t rcr_p, p_rcr_entry_t rcr_desc_rd_head_p,
216744961713Sgirish     boolean_t *multi_p, mblk_t **mp, mblk_t **mp_cont)
216844961713Sgirish {
216944961713Sgirish 	p_mblk_t		nmp = NULL;
217044961713Sgirish 	uint64_t		multi;
217144961713Sgirish 	uint64_t		dcf_err;
217244961713Sgirish 	uint8_t			channel;
217344961713Sgirish 
217444961713Sgirish 	boolean_t		first_entry = B_TRUE;
217544961713Sgirish 	boolean_t		is_tcp_udp = B_FALSE;
217644961713Sgirish 	boolean_t		buffer_free = B_FALSE;
217744961713Sgirish 	boolean_t		error_send_up = B_FALSE;
217844961713Sgirish 	uint8_t			error_type;
217944961713Sgirish 	uint16_t		l2_len;
218044961713Sgirish 	uint16_t		skip_len;
218144961713Sgirish 	uint8_t			pktbufsz_type;
218244961713Sgirish 	uint64_t		rcr_entry;
218344961713Sgirish 	uint64_t		*pkt_buf_addr_pp;
218444961713Sgirish 	uint64_t		*pkt_buf_addr_p;
218544961713Sgirish 	uint32_t		buf_offset;
218644961713Sgirish 	uint32_t		bsize;
218744961713Sgirish 	uint32_t		error_disp_cnt;
218844961713Sgirish 	uint32_t		msg_index;
218944961713Sgirish 	p_rx_rbr_ring_t		rx_rbr_p;
219044961713Sgirish 	p_rx_msg_t 		*rx_msg_ring_p;
219144961713Sgirish 	p_rx_msg_t		rx_msg_p;
219244961713Sgirish 	uint16_t		sw_offset_bytes = 0, hdr_size = 0;
219344961713Sgirish 	nxge_status_t		status = NXGE_OK;
219444961713Sgirish 	boolean_t		is_valid = B_FALSE;
219544961713Sgirish 	p_nxge_rx_ring_stats_t	rdc_stats;
2196a3c5bd6dSspeer 	uint32_t		bytes_read;
2197a3c5bd6dSspeer 	uint64_t		pkt_type;
2198a3c5bd6dSspeer 	uint64_t		frag;
21994202ea4bSsbehera 	boolean_t		pkt_too_long_err = B_FALSE;
220044961713Sgirish #ifdef	NXGE_DEBUG
220144961713Sgirish 	int			dump_len;
220244961713Sgirish #endif
220344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "==> nxge_receive_packet"));
220444961713Sgirish 	first_entry = (*mp == NULL) ? B_TRUE : B_FALSE;
220544961713Sgirish 
220644961713Sgirish 	rcr_entry = *((uint64_t *)rcr_desc_rd_head_p);
220744961713Sgirish 
220844961713Sgirish 	multi = (rcr_entry & RCR_MULTI_MASK);
220944961713Sgirish 	dcf_err = (rcr_entry & RCR_DCF_ERROR_MASK);
221044961713Sgirish 	pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK);
221144961713Sgirish 
221244961713Sgirish 	error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT);
221344961713Sgirish 	frag = (rcr_entry & RCR_FRAG_MASK);
221444961713Sgirish 
221544961713Sgirish 	l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT);
221644961713Sgirish 
221744961713Sgirish 	pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >>
221844961713Sgirish 				RCR_PKTBUFSZ_SHIFT);
2219adfcba55Sjoycey #if defined(__i386)
2220adfcba55Sjoycey 	pkt_buf_addr_pp = (uint64_t *)(uint32_t)((rcr_entry &
2221adfcba55Sjoycey 			RCR_PKT_BUF_ADDR_MASK) << RCR_PKT_BUF_ADDR_SHIFT);
2222adfcba55Sjoycey #else
222344961713Sgirish 	pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) <<
222444961713Sgirish 			RCR_PKT_BUF_ADDR_SHIFT);
2225adfcba55Sjoycey #endif
222644961713Sgirish 
222744961713Sgirish 	channel = rcr_p->rdc;
222844961713Sgirish 
222944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
223044961713Sgirish 		"==> nxge_receive_packet: entryp $%p entry 0x%0llx "
223114ea4bb7Ssd 		"pkt_buf_addr_pp $%p l2_len %d multi 0x%llx "
223244961713Sgirish 		"error_type 0x%x pkt_type 0x%x  "
223344961713Sgirish 		"pktbufsz_type %d ",
223444961713Sgirish 		rcr_desc_rd_head_p,
223544961713Sgirish 		rcr_entry, pkt_buf_addr_pp, l2_len,
223644961713Sgirish 		multi,
223744961713Sgirish 		error_type,
223844961713Sgirish 		pkt_type,
223944961713Sgirish 		pktbufsz_type));
224044961713Sgirish 
224144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
224244961713Sgirish 		"==> nxge_receive_packet: entryp $%p entry 0x%0llx "
224314ea4bb7Ssd 		"pkt_buf_addr_pp $%p l2_len %d multi 0x%llx "
224444961713Sgirish 		"error_type 0x%x pkt_type 0x%x ", rcr_desc_rd_head_p,
224544961713Sgirish 		rcr_entry, pkt_buf_addr_pp, l2_len,
224644961713Sgirish 		multi,
224744961713Sgirish 		error_type,
224844961713Sgirish 		pkt_type));
224944961713Sgirish 
225044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
225144961713Sgirish 		"==> (rbr) nxge_receive_packet: entry 0x%0llx "
225244961713Sgirish 		"full pkt_buf_addr_pp $%p l2_len %d",
225344961713Sgirish 		rcr_entry, pkt_buf_addr_pp, l2_len));
225444961713Sgirish 
225544961713Sgirish 	/* get the stats ptr */
225644961713Sgirish 	rdc_stats = rcr_p->rdc_stats;
225744961713Sgirish 
225844961713Sgirish 	if (!l2_len) {
225944961713Sgirish 
226044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
226144961713Sgirish 			"<== nxge_receive_packet: failed: l2 length is 0."));
226244961713Sgirish 		return;
226344961713Sgirish 	}
226444961713Sgirish 
22654202ea4bSsbehera 	/*
22664202ea4bSsbehera 	 * Sofware workaround for BMAC hardware limitation that allows
22674202ea4bSsbehera 	 * maxframe size of 1526, instead of 1522 for non-jumbo and 0x2406
22684202ea4bSsbehera 	 * instead of 0x2400 for jumbo.
22694202ea4bSsbehera 	 */
22704202ea4bSsbehera 	if (l2_len > nxgep->mac.maxframesize) {
22714202ea4bSsbehera 		pkt_too_long_err = B_TRUE;
22724202ea4bSsbehera 	}
22734202ea4bSsbehera 
227456d930aeSspeer 	/* Hardware sends us 4 bytes of CRC as no stripping is done.  */
227556d930aeSspeer 	l2_len -= ETHERFCSL;
227656d930aeSspeer 
227744961713Sgirish 	/* shift 6 bits to get the full io address */
2278adfcba55Sjoycey #if defined(__i386)
2279adfcba55Sjoycey 	pkt_buf_addr_pp = (uint64_t *)((uint32_t)pkt_buf_addr_pp <<
2280adfcba55Sjoycey 				RCR_PKT_BUF_ADDR_SHIFT_FULL);
2281adfcba55Sjoycey #else
228244961713Sgirish 	pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp <<
228344961713Sgirish 				RCR_PKT_BUF_ADDR_SHIFT_FULL);
2284adfcba55Sjoycey #endif
228544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
228644961713Sgirish 		"==> (rbr) nxge_receive_packet: entry 0x%0llx "
228744961713Sgirish 		"full pkt_buf_addr_pp $%p l2_len %d",
228844961713Sgirish 		rcr_entry, pkt_buf_addr_pp, l2_len));
228944961713Sgirish 
229044961713Sgirish 	rx_rbr_p = rcr_p->rx_rbr_p;
229144961713Sgirish 	rx_msg_ring_p = rx_rbr_p->rx_msg_ring;
229244961713Sgirish 
229344961713Sgirish 	if (first_entry) {
229444961713Sgirish 		hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL :
229544961713Sgirish 			RXDMA_HDR_SIZE_DEFAULT);
229644961713Sgirish 
229744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
229844961713Sgirish 			"==> nxge_receive_packet: first entry 0x%016llx "
229944961713Sgirish 			"pkt_buf_addr_pp $%p l2_len %d hdr %d",
230044961713Sgirish 			rcr_entry, pkt_buf_addr_pp, l2_len,
230144961713Sgirish 			hdr_size));
230244961713Sgirish 	}
230344961713Sgirish 
230444961713Sgirish 	MUTEX_ENTER(&rcr_p->lock);
230544961713Sgirish 	MUTEX_ENTER(&rx_rbr_p->lock);
230644961713Sgirish 
230744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
230844961713Sgirish 		"==> (rbr 1) nxge_receive_packet: entry 0x%0llx "
230944961713Sgirish 		"full pkt_buf_addr_pp $%p l2_len %d",
231044961713Sgirish 		rcr_entry, pkt_buf_addr_pp, l2_len));
231144961713Sgirish 
231244961713Sgirish 	/*
231344961713Sgirish 	 * Packet buffer address in the completion entry points
231444961713Sgirish 	 * to the starting buffer address (offset 0).
231544961713Sgirish 	 * Use the starting buffer address to locate the corresponding
231644961713Sgirish 	 * kernel address.
231744961713Sgirish 	 */
231844961713Sgirish 	status = nxge_rxbuf_pp_to_vp(nxgep, rx_rbr_p,
231944961713Sgirish 			pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p,
232044961713Sgirish 			&buf_offset,
232144961713Sgirish 			&msg_index);
232244961713Sgirish 
232344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
232444961713Sgirish 		"==> (rbr 2) nxge_receive_packet: entry 0x%0llx "
232544961713Sgirish 		"full pkt_buf_addr_pp $%p l2_len %d",
232644961713Sgirish 		rcr_entry, pkt_buf_addr_pp, l2_len));
232744961713Sgirish 
232844961713Sgirish 	if (status != NXGE_OK) {
232944961713Sgirish 		MUTEX_EXIT(&rx_rbr_p->lock);
233044961713Sgirish 		MUTEX_EXIT(&rcr_p->lock);
233144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
233244961713Sgirish 			"<== nxge_receive_packet: found vaddr failed %d",
233344961713Sgirish 				status));
233444961713Sgirish 		return;
233544961713Sgirish 	}
233644961713Sgirish 
233744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
233844961713Sgirish 		"==> (rbr 3) nxge_receive_packet: entry 0x%0llx "
233944961713Sgirish 		"full pkt_buf_addr_pp $%p l2_len %d",
234044961713Sgirish 		rcr_entry, pkt_buf_addr_pp, l2_len));
234144961713Sgirish 
234244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
234344961713Sgirish 		"==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx "
234444961713Sgirish 		"full pkt_buf_addr_pp $%p l2_len %d",
234544961713Sgirish 		msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
234644961713Sgirish 
234744961713Sgirish 	rx_msg_p = rx_msg_ring_p[msg_index];
234844961713Sgirish 
234944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
235044961713Sgirish 		"==> (rbr 4 msgindex %d) nxge_receive_packet: entry 0x%0llx "
235144961713Sgirish 		"full pkt_buf_addr_pp $%p l2_len %d",
235244961713Sgirish 		msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
235344961713Sgirish 
235444961713Sgirish 	switch (pktbufsz_type) {
235544961713Sgirish 	case RCR_PKTBUFSZ_0:
235644961713Sgirish 		bsize = rx_rbr_p->pkt_buf_size0_bytes;
235744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
235844961713Sgirish 			"==> nxge_receive_packet: 0 buf %d", bsize));
235944961713Sgirish 		break;
236044961713Sgirish 	case RCR_PKTBUFSZ_1:
236144961713Sgirish 		bsize = rx_rbr_p->pkt_buf_size1_bytes;
236244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
236344961713Sgirish 			"==> nxge_receive_packet: 1 buf %d", bsize));
236444961713Sgirish 		break;
236544961713Sgirish 	case RCR_PKTBUFSZ_2:
236644961713Sgirish 		bsize = rx_rbr_p->pkt_buf_size2_bytes;
236744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
236844961713Sgirish 			"==> nxge_receive_packet: 2 buf %d", bsize));
236944961713Sgirish 		break;
237044961713Sgirish 	case RCR_SINGLE_BLOCK:
237144961713Sgirish 		bsize = rx_msg_p->block_size;
237244961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
237344961713Sgirish 			"==> nxge_receive_packet: single %d", bsize));
237444961713Sgirish 
237544961713Sgirish 		break;
237644961713Sgirish 	default:
237744961713Sgirish 		MUTEX_EXIT(&rx_rbr_p->lock);
237844961713Sgirish 		MUTEX_EXIT(&rcr_p->lock);
237944961713Sgirish 		return;
238044961713Sgirish 	}
238144961713Sgirish 
238244961713Sgirish 	DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma,
238344961713Sgirish 		(buf_offset + sw_offset_bytes),
238444961713Sgirish 		(hdr_size + l2_len),
238544961713Sgirish 		DDI_DMA_SYNC_FORCPU);
238644961713Sgirish 
238744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
238844961713Sgirish 		"==> nxge_receive_packet: after first dump:usage count"));
238944961713Sgirish 
239044961713Sgirish 	if (rx_msg_p->cur_usage_cnt == 0) {
239144961713Sgirish 		if (rx_rbr_p->rbr_use_bcopy) {
239244961713Sgirish 			atomic_inc_32(&rx_rbr_p->rbr_consumed);
239344961713Sgirish 			if (rx_rbr_p->rbr_consumed <
239444961713Sgirish 					rx_rbr_p->rbr_threshold_hi) {
239544961713Sgirish 				if (rx_rbr_p->rbr_threshold_lo == 0 ||
239644961713Sgirish 					((rx_rbr_p->rbr_consumed >=
239744961713Sgirish 						rx_rbr_p->rbr_threshold_lo) &&
239844961713Sgirish 						(rx_rbr_p->rbr_bufsize_type >=
239944961713Sgirish 							pktbufsz_type))) {
240044961713Sgirish 					rx_msg_p->rx_use_bcopy = B_TRUE;
240144961713Sgirish 				}
240244961713Sgirish 			} else {
240344961713Sgirish 				rx_msg_p->rx_use_bcopy = B_TRUE;
240444961713Sgirish 			}
240544961713Sgirish 		}
240644961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX2_CTL,
240744961713Sgirish 			"==> nxge_receive_packet: buf %d (new block) ",
240844961713Sgirish 			bsize));
240944961713Sgirish 
241044961713Sgirish 		rx_msg_p->pkt_buf_size_code = pktbufsz_type;
241144961713Sgirish 		rx_msg_p->pkt_buf_size = bsize;
241244961713Sgirish 		rx_msg_p->cur_usage_cnt = 1;
241344961713Sgirish 		if (pktbufsz_type == RCR_SINGLE_BLOCK) {
241444961713Sgirish 			NXGE_DEBUG_MSG((nxgep, RX2_CTL,
241544961713Sgirish 				"==> nxge_receive_packet: buf %d "
241644961713Sgirish 				"(single block) ",
241744961713Sgirish 				bsize));
241844961713Sgirish 			/*
241944961713Sgirish 			 * Buffer can be reused once the free function
242044961713Sgirish 			 * is called.
242144961713Sgirish 			 */
242244961713Sgirish 			rx_msg_p->max_usage_cnt = 1;
242344961713Sgirish 			buffer_free = B_TRUE;
242444961713Sgirish 		} else {
242544961713Sgirish 			rx_msg_p->max_usage_cnt = rx_msg_p->block_size/bsize;
242644961713Sgirish 			if (rx_msg_p->max_usage_cnt == 1) {
242744961713Sgirish 				buffer_free = B_TRUE;
242844961713Sgirish 			}
242944961713Sgirish 		}
243044961713Sgirish 	} else {
243144961713Sgirish 		rx_msg_p->cur_usage_cnt++;
243244961713Sgirish 		if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) {
243344961713Sgirish 			buffer_free = B_TRUE;
243444961713Sgirish 		}
243544961713Sgirish 	}
243644961713Sgirish 
243744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
243844961713Sgirish 	    "msgbuf index = %d l2len %d bytes usage %d max_usage %d ",
243944961713Sgirish 		msg_index, l2_len,
244044961713Sgirish 		rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt));
244144961713Sgirish 
24424202ea4bSsbehera 	if ((error_type) || (dcf_err) || (pkt_too_long_err)) {
244344961713Sgirish 		rdc_stats->ierrors++;
244444961713Sgirish 		if (dcf_err) {
244544961713Sgirish 			rdc_stats->dcf_err++;
244644961713Sgirish #ifdef	NXGE_DEBUG
244744961713Sgirish 			if (!rdc_stats->dcf_err) {
244844961713Sgirish 				NXGE_DEBUG_MSG((nxgep, RX_CTL,
244944961713Sgirish 				"nxge_receive_packet: channel %d dcf_err rcr"
245044961713Sgirish 				" 0x%llx", channel, rcr_entry));
245144961713Sgirish 			}
245244961713Sgirish #endif
245344961713Sgirish 			NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL,
245444961713Sgirish 					NXGE_FM_EREPORT_RDMC_DCF_ERR);
24554202ea4bSsbehera 		} else if (pkt_too_long_err) {
24564202ea4bSsbehera 			rdc_stats->pkt_too_long_err++;
24574202ea4bSsbehera 			NXGE_DEBUG_MSG((nxgep, RX_CTL, " nxge_receive_packet:"
24584202ea4bSsbehera 			    " channel %d packet length [%d] > "
24594202ea4bSsbehera 			    "maxframesize [%d]", channel, l2_len + ETHERFCSL,
24604202ea4bSsbehera 			    nxgep->mac.maxframesize));
246144961713Sgirish 		} else {
246244961713Sgirish 				/* Update error stats */
246344961713Sgirish 			error_disp_cnt = NXGE_ERROR_SHOW_MAX;
246444961713Sgirish 			rdc_stats->errlog.compl_err_type = error_type;
246544961713Sgirish 
246644961713Sgirish 			switch (error_type) {
2467f6485eecSyc 			/*
2468f6485eecSyc 			 * Do not send FMA ereport for RCR_L2_ERROR and
2469f6485eecSyc 			 * RCR_L4_CSUM_ERROR because most likely they indicate
2470f6485eecSyc 			 * back pressure rather than HW failures.
2471f6485eecSyc 			 */
247253f3d8ecSyc 			case RCR_L2_ERROR:
247353f3d8ecSyc 				rdc_stats->l2_err++;
247453f3d8ecSyc 				if (rdc_stats->l2_err <
247553f3d8ecSyc 				    error_disp_cnt) {
247644961713Sgirish 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
247753f3d8ecSyc 					    " nxge_receive_packet:"
247853f3d8ecSyc 					    " channel %d RCR L2_ERROR",
247953f3d8ecSyc 					    channel));
248053f3d8ecSyc 				}
248153f3d8ecSyc 				break;
248253f3d8ecSyc 			case RCR_L4_CSUM_ERROR:
248353f3d8ecSyc 				error_send_up = B_TRUE;
248453f3d8ecSyc 				rdc_stats->l4_cksum_err++;
248553f3d8ecSyc 				if (rdc_stats->l4_cksum_err <
248653f3d8ecSyc 				    error_disp_cnt) {
248753f3d8ecSyc 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
248853f3d8ecSyc 					    " nxge_receive_packet:"
248953f3d8ecSyc 					    " channel %d"
249053f3d8ecSyc 					    " RCR L4_CSUM_ERROR", channel));
249153f3d8ecSyc 				}
249253f3d8ecSyc 				break;
2493f6485eecSyc 			/*
2494f6485eecSyc 			 * Do not send FMA ereport for RCR_FFLP_SOFT_ERROR and
2495f6485eecSyc 			 * RCR_ZCP_SOFT_ERROR because they reflect the same
2496f6485eecSyc 			 * FFLP and ZCP errors that have been reported by
2497f6485eecSyc 			 * nxge_fflp.c and nxge_zcp.c.
2498f6485eecSyc 			 */
249953f3d8ecSyc 			case RCR_FFLP_SOFT_ERROR:
250053f3d8ecSyc 				error_send_up = B_TRUE;
250153f3d8ecSyc 				rdc_stats->fflp_soft_err++;
250253f3d8ecSyc 				if (rdc_stats->fflp_soft_err <
250353f3d8ecSyc 				    error_disp_cnt) {
250453f3d8ecSyc 					NXGE_ERROR_MSG((nxgep,
250553f3d8ecSyc 					    NXGE_ERR_CTL,
250653f3d8ecSyc 					    " nxge_receive_packet:"
250753f3d8ecSyc 					    " channel %d"
250853f3d8ecSyc 					    " RCR FFLP_SOFT_ERROR", channel));
250953f3d8ecSyc 				}
251053f3d8ecSyc 				break;
251153f3d8ecSyc 			case RCR_ZCP_SOFT_ERROR:
251253f3d8ecSyc 				error_send_up = B_TRUE;
251353f3d8ecSyc 				rdc_stats->fflp_soft_err++;
251453f3d8ecSyc 				if (rdc_stats->zcp_soft_err <
251553f3d8ecSyc 				    error_disp_cnt)
251653f3d8ecSyc 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
251753f3d8ecSyc 					    " nxge_receive_packet: Channel %d"
251853f3d8ecSyc 					    " RCR ZCP_SOFT_ERROR", channel));
251953f3d8ecSyc 				break;
252053f3d8ecSyc 			default:
252153f3d8ecSyc 				rdc_stats->rcr_unknown_err++;
252253f3d8ecSyc 				if (rdc_stats->rcr_unknown_err
252353f3d8ecSyc 				    < error_disp_cnt) {
252453f3d8ecSyc 					NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
252553f3d8ecSyc 					    " nxge_receive_packet: Channel %d"
252653f3d8ecSyc 					    " RCR entry 0x%llx error 0x%x",
252753f3d8ecSyc 					    rcr_entry, channel, error_type));
252853f3d8ecSyc 				}
252953f3d8ecSyc 				break;
253044961713Sgirish 			}
253144961713Sgirish 		}
253244961713Sgirish 
253344961713Sgirish 		/*
253444961713Sgirish 		 * Update and repost buffer block if max usage
253544961713Sgirish 		 * count is reached.
253644961713Sgirish 		 */
253744961713Sgirish 		if (error_send_up == B_FALSE) {
2538958cea9eSml 			atomic_inc_32(&rx_msg_p->ref_cnt);
253944961713Sgirish 			if (buffer_free == B_TRUE) {
254044961713Sgirish 				rx_msg_p->free = B_TRUE;
254144961713Sgirish 			}
254244961713Sgirish 
254344961713Sgirish 			MUTEX_EXIT(&rx_rbr_p->lock);
254444961713Sgirish 			MUTEX_EXIT(&rcr_p->lock);
254544961713Sgirish 			nxge_freeb(rx_msg_p);
254644961713Sgirish 			return;
254744961713Sgirish 		}
254844961713Sgirish 	}
254944961713Sgirish 
255044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
255144961713Sgirish 		"==> nxge_receive_packet: DMA sync second "));
255244961713Sgirish 
255353f3d8ecSyc 	bytes_read = rcr_p->rcvd_pkt_bytes;
255444961713Sgirish 	skip_len = sw_offset_bytes + hdr_size;
255544961713Sgirish 	if (!rx_msg_p->rx_use_bcopy) {
2556958cea9eSml 		/*
2557958cea9eSml 		 * For loaned up buffers, the driver reference count
2558958cea9eSml 		 * will be incremented first and then the free state.
2559958cea9eSml 		 */
256053f3d8ecSyc 		if ((nmp = nxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) {
256114ea4bb7Ssd 			if (first_entry) {
256214ea4bb7Ssd 				nmp->b_rptr = &nmp->b_rptr[skip_len];
256353f3d8ecSyc 				if (l2_len < bsize - skip_len) {
256414ea4bb7Ssd 					nmp->b_wptr = &nmp->b_rptr[l2_len];
256553f3d8ecSyc 				} else {
256653f3d8ecSyc 					nmp->b_wptr = &nmp->b_rptr[bsize
256753f3d8ecSyc 					    - skip_len];
256853f3d8ecSyc 				}
256914ea4bb7Ssd 			} else {
257053f3d8ecSyc 				if (l2_len - bytes_read < bsize) {
257114ea4bb7Ssd 					nmp->b_wptr =
257214ea4bb7Ssd 					    &nmp->b_rptr[l2_len - bytes_read];
257353f3d8ecSyc 				} else {
257453f3d8ecSyc 					nmp->b_wptr = &nmp->b_rptr[bsize];
257553f3d8ecSyc 				}
257614ea4bb7Ssd 			}
257744961713Sgirish 		}
257853f3d8ecSyc 	} else {
257953f3d8ecSyc 		if (first_entry) {
258053f3d8ecSyc 			nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len,
258153f3d8ecSyc 			    l2_len < bsize - skip_len ?
258253f3d8ecSyc 			    l2_len : bsize - skip_len);
258353f3d8ecSyc 		} else {
258453f3d8ecSyc 			nmp = nxge_dupb_bcopy(rx_msg_p, buf_offset,
258553f3d8ecSyc 			    l2_len - bytes_read < bsize ?
258653f3d8ecSyc 			    l2_len - bytes_read : bsize);
258753f3d8ecSyc 		}
258853f3d8ecSyc 	}
258953f3d8ecSyc 	if (nmp != NULL) {
259053f3d8ecSyc 		if (first_entry)
259153f3d8ecSyc 			bytes_read  = nmp->b_wptr - nmp->b_rptr;
259253f3d8ecSyc 		else
259353f3d8ecSyc 			bytes_read += nmp->b_wptr - nmp->b_rptr;
259453f3d8ecSyc 
259553f3d8ecSyc 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
259653f3d8ecSyc 		    "==> nxge_receive_packet after dupb: "
259753f3d8ecSyc 		    "rbr consumed %d "
259853f3d8ecSyc 		    "pktbufsz_type %d "
259953f3d8ecSyc 		    "nmp $%p rptr $%p wptr $%p "
260053f3d8ecSyc 		    "buf_offset %d bzise %d l2_len %d skip_len %d",
260153f3d8ecSyc 		    rx_rbr_p->rbr_consumed,
260253f3d8ecSyc 		    pktbufsz_type,
260353f3d8ecSyc 		    nmp, nmp->b_rptr, nmp->b_wptr,
260453f3d8ecSyc 		    buf_offset, bsize, l2_len, skip_len));
260544961713Sgirish 	} else {
260644961713Sgirish 		cmn_err(CE_WARN, "!nxge_receive_packet: "
260744961713Sgirish 			"update stats (error)");
26082e59129aSraghus 		atomic_inc_32(&rx_msg_p->ref_cnt);
26092e59129aSraghus 		if (buffer_free == B_TRUE) {
26102e59129aSraghus 			rx_msg_p->free = B_TRUE;
26112e59129aSraghus 		}
26122e59129aSraghus 		MUTEX_EXIT(&rx_rbr_p->lock);
26132e59129aSraghus 		MUTEX_EXIT(&rcr_p->lock);
26142e59129aSraghus 		nxge_freeb(rx_msg_p);
26152e59129aSraghus 		return;
261644961713Sgirish 	}
2617ee5416c9Syc 
261844961713Sgirish 	if (buffer_free == B_TRUE) {
261944961713Sgirish 		rx_msg_p->free = B_TRUE;
262044961713Sgirish 	}
262144961713Sgirish 	/*
262244961713Sgirish 	 * ERROR, FRAG and PKT_TYPE are only reported
262344961713Sgirish 	 * in the first entry.
262444961713Sgirish 	 * If a packet is not fragmented and no error bit is set, then
262544961713Sgirish 	 * L4 checksum is OK.
262644961713Sgirish 	 */
262744961713Sgirish 	is_valid = (nmp != NULL);
262853f3d8ecSyc 	if (first_entry) {
262953f3d8ecSyc 		rdc_stats->ipackets++; /* count only 1st seg for jumbo */
263053f3d8ecSyc 		rdc_stats->ibytes += skip_len + l2_len < bsize ?
26317a2b8adfSyc 		    l2_len : bsize;
263253f3d8ecSyc 	} else {
263353f3d8ecSyc 		rdc_stats->ibytes += l2_len - bytes_read < bsize ?
263453f3d8ecSyc 		    l2_len - bytes_read : bsize;
263553f3d8ecSyc 	}
263653f3d8ecSyc 
263753f3d8ecSyc 	rcr_p->rcvd_pkt_bytes = bytes_read;
263853f3d8ecSyc 
263944961713Sgirish 	MUTEX_EXIT(&rx_rbr_p->lock);
264044961713Sgirish 	MUTEX_EXIT(&rcr_p->lock);
264144961713Sgirish 
264244961713Sgirish 	if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) {
264344961713Sgirish 		atomic_inc_32(&rx_msg_p->ref_cnt);
264444961713Sgirish 		nxge_freeb(rx_msg_p);
264544961713Sgirish 	}
264644961713Sgirish 
264744961713Sgirish 	if (is_valid) {
2648a3c5bd6dSspeer 		nmp->b_cont = NULL;
264944961713Sgirish 		if (first_entry) {
265044961713Sgirish 			*mp = nmp;
265144961713Sgirish 			*mp_cont = NULL;
265253f3d8ecSyc 		} else {
265344961713Sgirish 			*mp_cont = nmp;
265453f3d8ecSyc 		}
265544961713Sgirish 	}
265644961713Sgirish 
265744961713Sgirish 	/*
265844961713Sgirish 	 * Update stats and hardware checksuming.
265944961713Sgirish 	 */
266044961713Sgirish 	if (is_valid && !multi) {
2661678453a8Sspeer 		/*
2662*b4d05839Sml 		 * If the checksum flag nxge_chksum_offload
2663*b4d05839Sml 		 * is 1, TCP and UDP packets can be sent
2664678453a8Sspeer 		 * up with good checksum. If the checksum flag
2665*b4d05839Sml 		 * is set to 0, checksum reporting will apply to
2666678453a8Sspeer 		 * TCP packets only (workaround for a hardware bug).
2667*b4d05839Sml 		 * If the checksum flag nxge_cksum_offload is
2668*b4d05839Sml 		 * greater than 1, both TCP and UDP packets
2669*b4d05839Sml 		 * will not be reported its hardware checksum results.
2670678453a8Sspeer 		 */
2671*b4d05839Sml 		if (nxge_cksum_offload == 1) {
2672678453a8Sspeer 			is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP ||
267344961713Sgirish 				pkt_type == RCR_PKT_IS_UDP) ?
267444961713Sgirish 					B_TRUE: B_FALSE);
2675*b4d05839Sml 		} else if (!nxge_cksum_offload) {
2676678453a8Sspeer 			/* TCP checksum only. */
2677678453a8Sspeer 			is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP) ?
2678678453a8Sspeer 					B_TRUE: B_FALSE);
2679678453a8Sspeer 		}
268044961713Sgirish 
268144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_receive_packet: "
268214ea4bb7Ssd 			"is_valid 0x%x multi 0x%llx pkt %d frag %d error %d",
268344961713Sgirish 			is_valid, multi, is_tcp_udp, frag, error_type));
268444961713Sgirish 
268544961713Sgirish 		if (is_tcp_udp && !frag && !error_type) {
268644961713Sgirish 			(void) hcksum_assoc(nmp, NULL, NULL, 0, 0, 0, 0,
268744961713Sgirish 				HCK_FULLCKSUM_OK | HCK_FULLCKSUM, 0);
268844961713Sgirish 			NXGE_DEBUG_MSG((nxgep, RX_CTL,
268944961713Sgirish 				"==> nxge_receive_packet: Full tcp/udp cksum "
269014ea4bb7Ssd 				"is_valid 0x%x multi 0x%llx pkt %d frag %d "
269144961713Sgirish 				"error %d",
269244961713Sgirish 				is_valid, multi, is_tcp_udp, frag, error_type));
269344961713Sgirish 		}
269444961713Sgirish 	}
269544961713Sgirish 
269644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL,
269744961713Sgirish 		"==> nxge_receive_packet: *mp 0x%016llx", *mp));
269844961713Sgirish 
269944961713Sgirish 	*multi_p = (multi == RCR_MULTI_MASK);
270044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_receive_packet: "
270144961713Sgirish 		"multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx",
270244961713Sgirish 		*multi_p, nmp, *mp, *mp_cont));
270344961713Sgirish }
270444961713Sgirish 
270544961713Sgirish /*ARGSUSED*/
270644961713Sgirish static nxge_status_t
2707678453a8Sspeer nxge_rx_err_evnts(p_nxge_t nxgep, int channel, rx_dma_ctl_stat_t cs)
270844961713Sgirish {
270944961713Sgirish 	p_nxge_rx_ring_stats_t	rdc_stats;
271044961713Sgirish 	npi_handle_t		handle;
271144961713Sgirish 	npi_status_t		rs;
271244961713Sgirish 	boolean_t		rxchan_fatal = B_FALSE;
271344961713Sgirish 	boolean_t		rxport_fatal = B_FALSE;
271444961713Sgirish 	uint8_t			portn;
271544961713Sgirish 	nxge_status_t		status = NXGE_OK;
271644961713Sgirish 	uint32_t		error_disp_cnt = NXGE_ERROR_SHOW_MAX;
271744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, INT_CTL, "==> nxge_rx_err_evnts"));
271844961713Sgirish 
271944961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
272044961713Sgirish 	portn = nxgep->mac.portnum;
2721678453a8Sspeer 	rdc_stats = &nxgep->statsp->rdc_stats[channel];
272244961713Sgirish 
272344961713Sgirish 	if (cs.bits.hdw.rbr_tmout) {
272444961713Sgirish 		rdc_stats->rx_rbr_tmout++;
272544961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
272644961713Sgirish 					NXGE_FM_EREPORT_RDMC_RBR_TMOUT);
272744961713Sgirish 		rxchan_fatal = B_TRUE;
272844961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
272944961713Sgirish 			"==> nxge_rx_err_evnts: rx_rbr_timeout"));
273044961713Sgirish 	}
273144961713Sgirish 	if (cs.bits.hdw.rsp_cnt_err) {
273244961713Sgirish 		rdc_stats->rsp_cnt_err++;
273344961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
273444961713Sgirish 					NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR);
273544961713Sgirish 		rxchan_fatal = B_TRUE;
273644961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
273744961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
273844961713Sgirish 			"rsp_cnt_err", channel));
273944961713Sgirish 	}
274044961713Sgirish 	if (cs.bits.hdw.byte_en_bus) {
274144961713Sgirish 		rdc_stats->byte_en_bus++;
274244961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
274344961713Sgirish 					NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS);
274444961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
274544961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
274644961713Sgirish 			"fatal error: byte_en_bus", channel));
274744961713Sgirish 		rxchan_fatal = B_TRUE;
274844961713Sgirish 	}
274944961713Sgirish 	if (cs.bits.hdw.rsp_dat_err) {
275044961713Sgirish 		rdc_stats->rsp_dat_err++;
275144961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
275244961713Sgirish 					NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR);
275344961713Sgirish 		rxchan_fatal = B_TRUE;
275444961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
275544961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
275644961713Sgirish 			"fatal error: rsp_dat_err", channel));
275744961713Sgirish 	}
275844961713Sgirish 	if (cs.bits.hdw.rcr_ack_err) {
275944961713Sgirish 		rdc_stats->rcr_ack_err++;
276044961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
276144961713Sgirish 					NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR);
276244961713Sgirish 		rxchan_fatal = B_TRUE;
276344961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
276444961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
276544961713Sgirish 			"fatal error: rcr_ack_err", channel));
276644961713Sgirish 	}
276744961713Sgirish 	if (cs.bits.hdw.dc_fifo_err) {
276844961713Sgirish 		rdc_stats->dc_fifo_err++;
276944961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
277044961713Sgirish 					NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR);
277144961713Sgirish 		/* This is not a fatal error! */
277244961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
277344961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
277444961713Sgirish 			"dc_fifo_err", channel));
277544961713Sgirish 		rxport_fatal = B_TRUE;
277644961713Sgirish 	}
277744961713Sgirish 	if ((cs.bits.hdw.rcr_sha_par) || (cs.bits.hdw.rbr_pre_par)) {
277844961713Sgirish 		if ((rs = npi_rxdma_ring_perr_stat_get(handle,
277944961713Sgirish 				&rdc_stats->errlog.pre_par,
278044961713Sgirish 				&rdc_stats->errlog.sha_par))
278144961713Sgirish 				!= NPI_SUCCESS) {
278244961713Sgirish 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
278344961713Sgirish 				"==> nxge_rx_err_evnts(channel %d): "
278444961713Sgirish 				"rcr_sha_par: get perr", channel));
278544961713Sgirish 			return (NXGE_ERROR | rs);
278644961713Sgirish 		}
278744961713Sgirish 		if (cs.bits.hdw.rcr_sha_par) {
278844961713Sgirish 			rdc_stats->rcr_sha_par++;
278944961713Sgirish 			NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
279044961713Sgirish 					NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR);
279144961713Sgirish 			rxchan_fatal = B_TRUE;
279244961713Sgirish 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
279344961713Sgirish 				"==> nxge_rx_err_evnts(channel %d): "
279444961713Sgirish 				"fatal error: rcr_sha_par", channel));
279544961713Sgirish 		}
279644961713Sgirish 		if (cs.bits.hdw.rbr_pre_par) {
279744961713Sgirish 			rdc_stats->rbr_pre_par++;
279844961713Sgirish 			NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
279944961713Sgirish 					NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR);
280044961713Sgirish 			rxchan_fatal = B_TRUE;
280144961713Sgirish 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
280244961713Sgirish 				"==> nxge_rx_err_evnts(channel %d): "
280344961713Sgirish 				"fatal error: rbr_pre_par", channel));
280444961713Sgirish 		}
280544961713Sgirish 	}
280663e23a19Syc 	/*
280763e23a19Syc 	 * The Following 4 status bits are for information, the system
280863e23a19Syc 	 * is running fine. There is no need to send FMA ereports or
280963e23a19Syc 	 * log messages.
281063e23a19Syc 	 */
281144961713Sgirish 	if (cs.bits.hdw.port_drop_pkt) {
281244961713Sgirish 		rdc_stats->port_drop_pkt++;
281344961713Sgirish 	}
281444961713Sgirish 	if (cs.bits.hdw.wred_drop) {
281544961713Sgirish 		rdc_stats->wred_drop++;
281644961713Sgirish 	}
281744961713Sgirish 	if (cs.bits.hdw.rbr_pre_empty) {
281844961713Sgirish 		rdc_stats->rbr_pre_empty++;
281944961713Sgirish 	}
282044961713Sgirish 	if (cs.bits.hdw.rcr_shadow_full) {
282144961713Sgirish 		rdc_stats->rcr_shadow_full++;
282244961713Sgirish 	}
282344961713Sgirish 	if (cs.bits.hdw.config_err) {
282444961713Sgirish 		rdc_stats->config_err++;
282544961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
282644961713Sgirish 					NXGE_FM_EREPORT_RDMC_CONFIG_ERR);
282744961713Sgirish 		rxchan_fatal = B_TRUE;
282844961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
282944961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
283044961713Sgirish 			"config error", channel));
283144961713Sgirish 	}
283244961713Sgirish 	if (cs.bits.hdw.rcrincon) {
283344961713Sgirish 		rdc_stats->rcrincon++;
283444961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
283544961713Sgirish 					NXGE_FM_EREPORT_RDMC_RCRINCON);
283644961713Sgirish 		rxchan_fatal = B_TRUE;
283744961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
283844961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
283944961713Sgirish 			"fatal error: rcrincon error", channel));
284044961713Sgirish 	}
284144961713Sgirish 	if (cs.bits.hdw.rcrfull) {
284244961713Sgirish 		rdc_stats->rcrfull++;
284344961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
284444961713Sgirish 					NXGE_FM_EREPORT_RDMC_RCRFULL);
284544961713Sgirish 		rxchan_fatal = B_TRUE;
284644961713Sgirish 		if (rdc_stats->rcrfull < error_disp_cnt)
284744961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
284844961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
284944961713Sgirish 			"fatal error: rcrfull error", channel));
285044961713Sgirish 	}
285144961713Sgirish 	if (cs.bits.hdw.rbr_empty) {
285263e23a19Syc 		/*
285363e23a19Syc 		 * This bit is for information, there is no need
285463e23a19Syc 		 * send FMA ereport or log a message.
285563e23a19Syc 		 */
285644961713Sgirish 		rdc_stats->rbr_empty++;
285744961713Sgirish 	}
285844961713Sgirish 	if (cs.bits.hdw.rbrfull) {
285944961713Sgirish 		rdc_stats->rbrfull++;
286044961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
286144961713Sgirish 					NXGE_FM_EREPORT_RDMC_RBRFULL);
286244961713Sgirish 		rxchan_fatal = B_TRUE;
286344961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
286444961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
286544961713Sgirish 			"fatal error: rbr_full error", channel));
286644961713Sgirish 	}
286744961713Sgirish 	if (cs.bits.hdw.rbrlogpage) {
286844961713Sgirish 		rdc_stats->rbrlogpage++;
286944961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
287044961713Sgirish 					NXGE_FM_EREPORT_RDMC_RBRLOGPAGE);
287144961713Sgirish 		rxchan_fatal = B_TRUE;
287244961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
287344961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
287444961713Sgirish 			"fatal error: rbr logical page error", channel));
287544961713Sgirish 	}
287644961713Sgirish 	if (cs.bits.hdw.cfiglogpage) {
287744961713Sgirish 		rdc_stats->cfiglogpage++;
287844961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, channel,
287944961713Sgirish 					NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE);
288044961713Sgirish 		rxchan_fatal = B_TRUE;
288144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
288244961713Sgirish 			"==> nxge_rx_err_evnts(channel %d): "
288344961713Sgirish 			"fatal error: cfig logical page error", channel));
288444961713Sgirish 	}
288544961713Sgirish 
288644961713Sgirish 	if (rxport_fatal)  {
288744961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2888678453a8Sspeer 		    " nxge_rx_err_evnts: fatal error on Port #%d\n",
2889678453a8Sspeer 		    portn));
2890678453a8Sspeer 		if (isLDOMguest(nxgep)) {
2891678453a8Sspeer 			status = NXGE_ERROR;
2892678453a8Sspeer 		} else {
2893678453a8Sspeer 			status = nxge_ipp_fatal_err_recover(nxgep);
2894678453a8Sspeer 			if (status == NXGE_OK) {
2895678453a8Sspeer 				FM_SERVICE_RESTORED(nxgep);
2896678453a8Sspeer 			}
289744961713Sgirish 		}
289844961713Sgirish 	}
289944961713Sgirish 
290044961713Sgirish 	if (rxchan_fatal) {
290144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
2902678453a8Sspeer 		    " nxge_rx_err_evnts: fatal error on Channel #%d\n",
2903678453a8Sspeer 		    channel));
2904678453a8Sspeer 		if (isLDOMguest(nxgep)) {
2905678453a8Sspeer 			status = NXGE_ERROR;
2906678453a8Sspeer 		} else {
2907678453a8Sspeer 			status = nxge_rxdma_fatal_err_recover(nxgep, channel);
2908678453a8Sspeer 			if (status == NXGE_OK) {
2909678453a8Sspeer 				FM_SERVICE_RESTORED(nxgep);
2910678453a8Sspeer 			}
291144961713Sgirish 		}
291244961713Sgirish 	}
291344961713Sgirish 
291444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX2_CTL, "<== nxge_rx_err_evnts"));
291544961713Sgirish 
291644961713Sgirish 	return (status);
291744961713Sgirish }
291844961713Sgirish 
2919678453a8Sspeer /*
2920678453a8Sspeer  * nxge_rdc_hvio_setup
2921678453a8Sspeer  *
2922678453a8Sspeer  *	This code appears to setup some Hypervisor variables.
2923678453a8Sspeer  *
2924678453a8Sspeer  * Arguments:
2925678453a8Sspeer  * 	nxgep
2926678453a8Sspeer  * 	channel
2927678453a8Sspeer  *
2928678453a8Sspeer  * Notes:
2929678453a8Sspeer  *	What does NIU_LP_WORKAROUND mean?
2930678453a8Sspeer  *
2931678453a8Sspeer  * NPI/NXGE function calls:
2932678453a8Sspeer  *	na
2933678453a8Sspeer  *
2934678453a8Sspeer  * Context:
2935678453a8Sspeer  *	Any domain
2936678453a8Sspeer  */
2937678453a8Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
2938678453a8Sspeer static void
2939678453a8Sspeer nxge_rdc_hvio_setup(
2940678453a8Sspeer 	nxge_t *nxgep, int channel)
294144961713Sgirish {
2942678453a8Sspeer 	nxge_dma_common_t	*dma_common;
2943678453a8Sspeer 	nxge_dma_common_t	*dma_control;
2944678453a8Sspeer 	rx_rbr_ring_t		*ring;
2945678453a8Sspeer 
2946678453a8Sspeer 	ring = nxgep->rx_rbr_rings->rbr_rings[channel];
2947678453a8Sspeer 	dma_common = nxgep->rx_buf_pool_p->dma_buf_pool_p[channel];
2948678453a8Sspeer 
2949678453a8Sspeer 	ring->hv_set = B_FALSE;
2950678453a8Sspeer 
2951678453a8Sspeer 	ring->hv_rx_buf_base_ioaddr_pp = (uint64_t)
2952678453a8Sspeer 	    dma_common->orig_ioaddr_pp;
2953678453a8Sspeer 	ring->hv_rx_buf_ioaddr_size = (uint64_t)
2954678453a8Sspeer 	    dma_common->orig_alength;
2955678453a8Sspeer 
2956678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: "
2957678453a8Sspeer 	    "channel %d data buf base io $%lx ($%p) size 0x%lx (%ld 0x%lx)",
2958678453a8Sspeer 	    channel, ring->hv_rx_buf_base_ioaddr_pp,
2959678453a8Sspeer 	    dma_common->ioaddr_pp, ring->hv_rx_buf_ioaddr_size,
2960678453a8Sspeer 	    dma_common->orig_alength, dma_common->orig_alength));
2961678453a8Sspeer 
2962678453a8Sspeer 	dma_control = nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel];
2963678453a8Sspeer 
2964678453a8Sspeer 	ring->hv_rx_cntl_base_ioaddr_pp =
2965678453a8Sspeer 	    (uint64_t)dma_control->orig_ioaddr_pp;
2966678453a8Sspeer 	ring->hv_rx_cntl_ioaddr_size =
2967678453a8Sspeer 	    (uint64_t)dma_control->orig_alength;
2968678453a8Sspeer 
2969678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma_channel: "
2970678453a8Sspeer 	    "channel %d cntl base io $%p ($%p) size 0x%llx (%d 0x%x)",
2971678453a8Sspeer 	    channel, ring->hv_rx_cntl_base_ioaddr_pp,
2972678453a8Sspeer 	    dma_control->ioaddr_pp, ring->hv_rx_cntl_ioaddr_size,
2973678453a8Sspeer 	    dma_control->orig_alength, dma_control->orig_alength));
2974678453a8Sspeer }
297544961713Sgirish #endif
297644961713Sgirish 
2977678453a8Sspeer /*
2978678453a8Sspeer  * nxge_map_rxdma
2979678453a8Sspeer  *
2980678453a8Sspeer  *	Map an RDC into our kernel space.
2981678453a8Sspeer  *
2982678453a8Sspeer  * Arguments:
2983678453a8Sspeer  * 	nxgep
2984678453a8Sspeer  * 	channel	The channel to map.
2985678453a8Sspeer  *
2986678453a8Sspeer  * Notes:
2987678453a8Sspeer  *	1. Allocate & initialise a memory pool, if necessary.
2988678453a8Sspeer  *	2. Allocate however many receive buffers are required.
2989678453a8Sspeer  *	3. Setup buffers, descriptors, and mailbox.
2990678453a8Sspeer  *
2991678453a8Sspeer  * NPI/NXGE function calls:
2992678453a8Sspeer  *	nxge_alloc_rx_mem_pool()
2993678453a8Sspeer  *	nxge_alloc_rbb()
2994678453a8Sspeer  *	nxge_map_rxdma_channel()
2995678453a8Sspeer  *
2996678453a8Sspeer  * Registers accessed:
2997678453a8Sspeer  *
2998678453a8Sspeer  * Context:
2999678453a8Sspeer  *	Any domain
3000678453a8Sspeer  */
3001678453a8Sspeer static nxge_status_t
3002678453a8Sspeer nxge_map_rxdma(p_nxge_t nxgep, int channel)
3003678453a8Sspeer {
3004678453a8Sspeer 	nxge_dma_common_t	**data;
3005678453a8Sspeer 	nxge_dma_common_t	**control;
3006678453a8Sspeer 	rx_rbr_ring_t		**rbr_ring;
3007678453a8Sspeer 	rx_rcr_ring_t		**rcr_ring;
3008678453a8Sspeer 	rx_mbox_t		**mailbox;
3009678453a8Sspeer 	uint32_t		chunks;
301044961713Sgirish 
3011678453a8Sspeer 	nxge_status_t		status;
301244961713Sgirish 
3013678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_map_rxdma"));
301444961713Sgirish 
3015678453a8Sspeer 	if (!nxgep->rx_buf_pool_p) {
3016678453a8Sspeer 		if (nxge_alloc_rx_mem_pool(nxgep) != NXGE_OK) {
3017678453a8Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3018678453a8Sspeer 			    "<== nxge_map_rxdma: buf not allocated"));
3019678453a8Sspeer 			return (NXGE_ERROR);
3020678453a8Sspeer 		}
302144961713Sgirish 	}
302244961713Sgirish 
3023678453a8Sspeer 	if (nxge_alloc_rxb(nxgep, channel) != NXGE_OK)
3024678453a8Sspeer 		return (NXGE_ERROR);
302514ea4bb7Ssd 
302614ea4bb7Ssd 	/*
302714ea4bb7Ssd 	 * Timeout should be set based on the system clock divider.
302814ea4bb7Ssd 	 * The following timeout value of 1 assumes that the
302914ea4bb7Ssd 	 * granularity (1000) is 3 microseconds running at 300MHz.
303014ea4bb7Ssd 	 */
303114ea4bb7Ssd 
303214ea4bb7Ssd 	nxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT;
303314ea4bb7Ssd 	nxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT;
303444961713Sgirish 
303544961713Sgirish 	/*
3036678453a8Sspeer 	 * Map descriptors from the buffer polls for each dma channel.
303744961713Sgirish 	 */
303844961713Sgirish 
3039678453a8Sspeer 	/*
3040678453a8Sspeer 	 * Set up and prepare buffer blocks, descriptors
3041678453a8Sspeer 	 * and mailbox.
3042678453a8Sspeer 	 */
3043678453a8Sspeer 	data = &nxgep->rx_buf_pool_p->dma_buf_pool_p[channel];
3044678453a8Sspeer 	rbr_ring = &nxgep->rx_rbr_rings->rbr_rings[channel];
3045678453a8Sspeer 	chunks = nxgep->rx_buf_pool_p->num_chunks[channel];
304644961713Sgirish 
3047678453a8Sspeer 	control = &nxgep->rx_cntl_pool_p->dma_buf_pool_p[channel];
3048678453a8Sspeer 	rcr_ring = &nxgep->rx_rcr_rings->rcr_rings[channel];
304944961713Sgirish 
3050678453a8Sspeer 	mailbox = &nxgep->rx_mbox_areas_p->rxmbox_areas[channel];
305144961713Sgirish 
3052678453a8Sspeer 	status = nxge_map_rxdma_channel(nxgep, channel, data, rbr_ring,
3053678453a8Sspeer 	    chunks, control, rcr_ring, mailbox);
3054678453a8Sspeer 	if (status != NXGE_OK) {
3055678453a8Sspeer 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3056678453a8Sspeer 			"==> nxge_map_rxdma: nxge_map_rxdma_channel(%d) "
3057678453a8Sspeer 			"returned 0x%x",
3058678453a8Sspeer 			channel, status));
3059678453a8Sspeer 		return (status);
3060678453a8Sspeer 	}
3061678453a8Sspeer 	nxgep->rx_rbr_rings->rbr_rings[channel]->index = (uint16_t)channel;
3062678453a8Sspeer 	nxgep->rx_rcr_rings->rcr_rings[channel]->index = (uint16_t)channel;
3063678453a8Sspeer 	nxgep->rx_rcr_rings->rcr_rings[channel]->rdc_stats =
3064678453a8Sspeer 	    &nxgep->statsp->rdc_stats[channel];
306544961713Sgirish 
3066678453a8Sspeer #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
3067678453a8Sspeer 	if (!isLDOMguest(nxgep))
3068678453a8Sspeer 		nxge_rdc_hvio_setup(nxgep, channel);
3069678453a8Sspeer #endif
307044961713Sgirish 
307144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3072678453a8Sspeer 	    "<== nxge_map_rxdma: (status 0x%x channel %d)", status, channel));
307344961713Sgirish 
307444961713Sgirish 	return (status);
307544961713Sgirish }
307644961713Sgirish 
307744961713Sgirish static void
3078678453a8Sspeer nxge_unmap_rxdma(p_nxge_t nxgep, int channel)
307944961713Sgirish {
3080678453a8Sspeer 	rx_rbr_ring_t	*rbr_ring;
3081678453a8Sspeer 	rx_rcr_ring_t	*rcr_ring;
3082678453a8Sspeer 	rx_mbox_t	*mailbox;
308344961713Sgirish 
3084678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_unmap_rxdma(%d)", channel));
308544961713Sgirish 
3086678453a8Sspeer 	if (!nxgep->rx_rbr_rings || !nxgep->rx_rcr_rings ||
3087678453a8Sspeer 	    !nxgep->rx_mbox_areas_p)
308844961713Sgirish 		return;
308944961713Sgirish 
3090678453a8Sspeer 	rbr_ring = nxgep->rx_rbr_rings->rbr_rings[channel];
3091678453a8Sspeer 	rcr_ring = nxgep->rx_rcr_rings->rcr_rings[channel];
3092678453a8Sspeer 	mailbox = nxgep->rx_mbox_areas_p->rxmbox_areas[channel];
309344961713Sgirish 
3094678453a8Sspeer 	if (!rbr_ring || !rcr_ring || !mailbox)
3095678453a8Sspeer 		return;
309644961713Sgirish 
3097678453a8Sspeer 	(void) nxge_unmap_rxdma_channel(
3098678453a8Sspeer 		nxgep, channel, rbr_ring, rcr_ring, mailbox);
309944961713Sgirish 
3100678453a8Sspeer 	nxge_free_rxb(nxgep, channel);
310144961713Sgirish 
3102678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma"));
310344961713Sgirish }
310444961713Sgirish 
310544961713Sgirish nxge_status_t
310644961713Sgirish nxge_map_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
310744961713Sgirish     p_nxge_dma_common_t *dma_buf_p,  p_rx_rbr_ring_t *rbr_p,
310844961713Sgirish     uint32_t num_chunks,
310944961713Sgirish     p_nxge_dma_common_t *dma_cntl_p, p_rx_rcr_ring_t *rcr_p,
311044961713Sgirish     p_rx_mbox_t *rx_mbox_p)
311144961713Sgirish {
311244961713Sgirish 	int	status = NXGE_OK;
311344961713Sgirish 
311444961713Sgirish 	/*
311544961713Sgirish 	 * Set up and prepare buffer blocks, descriptors
311644961713Sgirish 	 * and mailbox.
311744961713Sgirish 	 */
311844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
311944961713Sgirish 		"==> nxge_map_rxdma_channel (channel %d)", channel));
312044961713Sgirish 	/*
312144961713Sgirish 	 * Receive buffer blocks
312244961713Sgirish 	 */
312344961713Sgirish 	status = nxge_map_rxdma_channel_buf_ring(nxgep, channel,
312444961713Sgirish 			dma_buf_p, rbr_p, num_chunks);
312544961713Sgirish 	if (status != NXGE_OK) {
312644961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
312744961713Sgirish 			"==> nxge_map_rxdma_channel (channel %d): "
312844961713Sgirish 			"map buffer failed 0x%x", channel, status));
312944961713Sgirish 		goto nxge_map_rxdma_channel_exit;
313044961713Sgirish 	}
313144961713Sgirish 
313244961713Sgirish 	/*
313344961713Sgirish 	 * Receive block ring, completion ring and mailbox.
313444961713Sgirish 	 */
313544961713Sgirish 	status = nxge_map_rxdma_channel_cfg_ring(nxgep, channel,
313644961713Sgirish 			dma_cntl_p, rbr_p, rcr_p, rx_mbox_p);
313744961713Sgirish 	if (status != NXGE_OK) {
313844961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
313944961713Sgirish 			"==> nxge_map_rxdma_channel (channel %d): "
314044961713Sgirish 			"map config failed 0x%x", channel, status));
314144961713Sgirish 		goto nxge_map_rxdma_channel_fail2;
314244961713Sgirish 	}
314344961713Sgirish 
314444961713Sgirish 	goto nxge_map_rxdma_channel_exit;
314544961713Sgirish 
314644961713Sgirish nxge_map_rxdma_channel_fail3:
314744961713Sgirish 	/* Free rbr, rcr */
314844961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
314944961713Sgirish 		"==> nxge_map_rxdma_channel: free rbr/rcr "
315044961713Sgirish 		"(status 0x%x channel %d)",
315144961713Sgirish 		status, channel));
315244961713Sgirish 	nxge_unmap_rxdma_channel_cfg_ring(nxgep,
315344961713Sgirish 		*rcr_p, *rx_mbox_p);
315444961713Sgirish 
315544961713Sgirish nxge_map_rxdma_channel_fail2:
315644961713Sgirish 	/* Free buffer blocks */
315744961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
315844961713Sgirish 		"==> nxge_map_rxdma_channel: free rx buffers"
315944961713Sgirish 		"(nxgep 0x%x status 0x%x channel %d)",
316044961713Sgirish 		nxgep, status, channel));
316144961713Sgirish 	nxge_unmap_rxdma_channel_buf_ring(nxgep, *rbr_p);
316244961713Sgirish 
316356d930aeSspeer 	status = NXGE_ERROR;
316456d930aeSspeer 
316544961713Sgirish nxge_map_rxdma_channel_exit:
316644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
316744961713Sgirish 		"<== nxge_map_rxdma_channel: "
316844961713Sgirish 		"(nxgep 0x%x status 0x%x channel %d)",
316944961713Sgirish 		nxgep, status, channel));
317044961713Sgirish 
317144961713Sgirish 	return (status);
317244961713Sgirish }
317344961713Sgirish 
317444961713Sgirish /*ARGSUSED*/
317544961713Sgirish static void
317644961713Sgirish nxge_unmap_rxdma_channel(p_nxge_t nxgep, uint16_t channel,
317744961713Sgirish     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
317844961713Sgirish {
317944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
318044961713Sgirish 		"==> nxge_unmap_rxdma_channel (channel %d)", channel));
318144961713Sgirish 
318244961713Sgirish 	/*
318344961713Sgirish 	 * unmap receive block ring, completion ring and mailbox.
318444961713Sgirish 	 */
318544961713Sgirish 	(void) nxge_unmap_rxdma_channel_cfg_ring(nxgep,
318644961713Sgirish 			rcr_p, rx_mbox_p);
318744961713Sgirish 
318844961713Sgirish 	/* unmap buffer blocks */
318944961713Sgirish 	(void) nxge_unmap_rxdma_channel_buf_ring(nxgep, rbr_p);
319044961713Sgirish 
319144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_unmap_rxdma_channel"));
319244961713Sgirish }
319344961713Sgirish 
319444961713Sgirish /*ARGSUSED*/
319544961713Sgirish static nxge_status_t
319644961713Sgirish nxge_map_rxdma_channel_cfg_ring(p_nxge_t nxgep, uint16_t dma_channel,
319744961713Sgirish     p_nxge_dma_common_t *dma_cntl_p, p_rx_rbr_ring_t *rbr_p,
319844961713Sgirish     p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p)
319944961713Sgirish {
320044961713Sgirish 	p_rx_rbr_ring_t 	rbrp;
320144961713Sgirish 	p_rx_rcr_ring_t 	rcrp;
320244961713Sgirish 	p_rx_mbox_t 		mboxp;
320344961713Sgirish 	p_nxge_dma_common_t 	cntl_dmap;
320444961713Sgirish 	p_nxge_dma_common_t 	dmap;
320544961713Sgirish 	p_rx_msg_t 		*rx_msg_ring;
320644961713Sgirish 	p_rx_msg_t 		rx_msg_p;
320744961713Sgirish 	p_rbr_cfig_a_t		rcfga_p;
320844961713Sgirish 	p_rbr_cfig_b_t		rcfgb_p;
320944961713Sgirish 	p_rcrcfig_a_t		cfga_p;
321044961713Sgirish 	p_rcrcfig_b_t		cfgb_p;
321144961713Sgirish 	p_rxdma_cfig1_t		cfig1_p;
321244961713Sgirish 	p_rxdma_cfig2_t		cfig2_p;
321344961713Sgirish 	p_rbr_kick_t		kick_p;
321444961713Sgirish 	uint32_t		dmaaddrp;
321544961713Sgirish 	uint32_t		*rbr_vaddrp;
321644961713Sgirish 	uint32_t		bkaddr;
321744961713Sgirish 	nxge_status_t		status = NXGE_OK;
321844961713Sgirish 	int			i;
321944961713Sgirish 	uint32_t 		nxge_port_rcr_size;
322044961713Sgirish 
322144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
322244961713Sgirish 		"==> nxge_map_rxdma_channel_cfg_ring"));
322344961713Sgirish 
322444961713Sgirish 	cntl_dmap = *dma_cntl_p;
322544961713Sgirish 
322644961713Sgirish 	/* Map in the receive block ring */
322744961713Sgirish 	rbrp = *rbr_p;
322844961713Sgirish 	dmap = (p_nxge_dma_common_t)&rbrp->rbr_desc;
322944961713Sgirish 	nxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4);
323044961713Sgirish 	/*
323144961713Sgirish 	 * Zero out buffer block ring descriptors.
323244961713Sgirish 	 */
323344961713Sgirish 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
323444961713Sgirish 
323544961713Sgirish 	rcfga_p = &(rbrp->rbr_cfga);
323644961713Sgirish 	rcfgb_p = &(rbrp->rbr_cfgb);
323744961713Sgirish 	kick_p = &(rbrp->rbr_kick);
323844961713Sgirish 	rcfga_p->value = 0;
323944961713Sgirish 	rcfgb_p->value = 0;
324044961713Sgirish 	kick_p->value = 0;
324144961713Sgirish 	rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress;
324244961713Sgirish 	rcfga_p->value = (rbrp->rbr_addr &
324344961713Sgirish 				(RBR_CFIG_A_STDADDR_MASK |
324444961713Sgirish 				RBR_CFIG_A_STDADDR_BASE_MASK));
324544961713Sgirish 	rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT);
324644961713Sgirish 
324744961713Sgirish 	rcfgb_p->bits.ldw.bufsz0 = rbrp->pkt_buf_size0;
324844961713Sgirish 	rcfgb_p->bits.ldw.vld0 = 1;
324944961713Sgirish 	rcfgb_p->bits.ldw.bufsz1 = rbrp->pkt_buf_size1;
325044961713Sgirish 	rcfgb_p->bits.ldw.vld1 = 1;
325144961713Sgirish 	rcfgb_p->bits.ldw.bufsz2 = rbrp->pkt_buf_size2;
325244961713Sgirish 	rcfgb_p->bits.ldw.vld2 = 1;
325344961713Sgirish 	rcfgb_p->bits.ldw.bksize = nxgep->rx_bksize_code;
325444961713Sgirish 
325544961713Sgirish 	/*
325644961713Sgirish 	 * For each buffer block, enter receive block address to the ring.
325744961713Sgirish 	 */
325844961713Sgirish 	rbr_vaddrp = (uint32_t *)dmap->kaddrp;
325944961713Sgirish 	rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp;
326044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
326144961713Sgirish 		"==> nxge_map_rxdma_channel_cfg_ring: channel %d "
326244961713Sgirish 		"rbr_vaddrp $%p", dma_channel, rbr_vaddrp));
326344961713Sgirish 
326444961713Sgirish 	rx_msg_ring = rbrp->rx_msg_ring;
326544961713Sgirish 	for (i = 0; i < rbrp->tnblocks; i++) {
326644961713Sgirish 		rx_msg_p = rx_msg_ring[i];
326744961713Sgirish 		rx_msg_p->nxgep = nxgep;
326844961713Sgirish 		rx_msg_p->rx_rbr_p = rbrp;
326944961713Sgirish 		bkaddr = (uint32_t)
327044961713Sgirish 			((rx_msg_p->buf_dma.dma_cookie.dmac_laddress
327144961713Sgirish 				>> RBR_BKADDR_SHIFT));
327244961713Sgirish 		rx_msg_p->free = B_FALSE;
327344961713Sgirish 		rx_msg_p->max_usage_cnt = 0xbaddcafe;
327444961713Sgirish 
327544961713Sgirish 		*rbr_vaddrp++ = bkaddr;
327644961713Sgirish 	}
327744961713Sgirish 
327844961713Sgirish 	kick_p->bits.ldw.bkadd = rbrp->rbb_max;
327944961713Sgirish 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
328044961713Sgirish 
328144961713Sgirish 	rbrp->rbr_rd_index = 0;
328244961713Sgirish 
328344961713Sgirish 	rbrp->rbr_consumed = 0;
328444961713Sgirish 	rbrp->rbr_use_bcopy = B_TRUE;
328544961713Sgirish 	rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0;
328644961713Sgirish 	/*
328744961713Sgirish 	 * Do bcopy on packets greater than bcopy size once
328844961713Sgirish 	 * the lo threshold is reached.
328944961713Sgirish 	 * This lo threshold should be less than the hi threshold.
329044961713Sgirish 	 *
329144961713Sgirish 	 * Do bcopy on every packet once the hi threshold is reached.
329244961713Sgirish 	 */
329344961713Sgirish 	if (nxge_rx_threshold_lo >= nxge_rx_threshold_hi) {
329444961713Sgirish 		/* default it to use hi */
329544961713Sgirish 		nxge_rx_threshold_lo = nxge_rx_threshold_hi;
329644961713Sgirish 	}
329744961713Sgirish 
329844961713Sgirish 	if (nxge_rx_buf_size_type > NXGE_RBR_TYPE2) {
329944961713Sgirish 		nxge_rx_buf_size_type = NXGE_RBR_TYPE2;
330044961713Sgirish 	}
330144961713Sgirish 	rbrp->rbr_bufsize_type = nxge_rx_buf_size_type;
330244961713Sgirish 
330344961713Sgirish 	switch (nxge_rx_threshold_hi) {
330444961713Sgirish 	default:
330544961713Sgirish 	case	NXGE_RX_COPY_NONE:
330644961713Sgirish 		/* Do not do bcopy at all */
330744961713Sgirish 		rbrp->rbr_use_bcopy = B_FALSE;
330844961713Sgirish 		rbrp->rbr_threshold_hi = rbrp->rbb_max;
330944961713Sgirish 		break;
331044961713Sgirish 
331144961713Sgirish 	case NXGE_RX_COPY_1:
331244961713Sgirish 	case NXGE_RX_COPY_2:
331344961713Sgirish 	case NXGE_RX_COPY_3:
331444961713Sgirish 	case NXGE_RX_COPY_4:
331544961713Sgirish 	case NXGE_RX_COPY_5:
331644961713Sgirish 	case NXGE_RX_COPY_6:
331744961713Sgirish 	case NXGE_RX_COPY_7:
331844961713Sgirish 		rbrp->rbr_threshold_hi =
331944961713Sgirish 			rbrp->rbb_max *
332044961713Sgirish 			(nxge_rx_threshold_hi)/NXGE_RX_BCOPY_SCALE;
332144961713Sgirish 		break;
332244961713Sgirish 
332344961713Sgirish 	case NXGE_RX_COPY_ALL:
332444961713Sgirish 		rbrp->rbr_threshold_hi = 0;
332544961713Sgirish 		break;
332644961713Sgirish 	}
332744961713Sgirish 
332844961713Sgirish 	switch (nxge_rx_threshold_lo) {
332944961713Sgirish 	default:
333044961713Sgirish 	case	NXGE_RX_COPY_NONE:
333144961713Sgirish 		/* Do not do bcopy at all */
333244961713Sgirish 		if (rbrp->rbr_use_bcopy) {
333344961713Sgirish 			rbrp->rbr_use_bcopy = B_FALSE;
333444961713Sgirish 		}
333544961713Sgirish 		rbrp->rbr_threshold_lo = rbrp->rbb_max;
333644961713Sgirish 		break;
333744961713Sgirish 
333844961713Sgirish 	case NXGE_RX_COPY_1:
333944961713Sgirish 	case NXGE_RX_COPY_2:
334044961713Sgirish 	case NXGE_RX_COPY_3:
334144961713Sgirish 	case NXGE_RX_COPY_4:
334244961713Sgirish 	case NXGE_RX_COPY_5:
334344961713Sgirish 	case NXGE_RX_COPY_6:
334444961713Sgirish 	case NXGE_RX_COPY_7:
334544961713Sgirish 		rbrp->rbr_threshold_lo =
334644961713Sgirish 			rbrp->rbb_max *
334744961713Sgirish 			(nxge_rx_threshold_lo)/NXGE_RX_BCOPY_SCALE;
334844961713Sgirish 		break;
334944961713Sgirish 
335044961713Sgirish 	case NXGE_RX_COPY_ALL:
335144961713Sgirish 		rbrp->rbr_threshold_lo = 0;
335244961713Sgirish 		break;
335344961713Sgirish 	}
335444961713Sgirish 
335544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
335644961713Sgirish 		"nxge_map_rxdma_channel_cfg_ring: channel %d "
335744961713Sgirish 		"rbb_max %d "
335844961713Sgirish 		"rbrp->rbr_bufsize_type %d "
335944961713Sgirish 		"rbb_threshold_hi %d "
336044961713Sgirish 		"rbb_threshold_lo %d",
336144961713Sgirish 		dma_channel,
336244961713Sgirish 		rbrp->rbb_max,
336344961713Sgirish 		rbrp->rbr_bufsize_type,
336444961713Sgirish 		rbrp->rbr_threshold_hi,
336544961713Sgirish 		rbrp->rbr_threshold_lo));
336644961713Sgirish 
336744961713Sgirish 	rbrp->page_valid.value = 0;
336844961713Sgirish 	rbrp->page_mask_1.value = rbrp->page_mask_2.value = 0;
336944961713Sgirish 	rbrp->page_value_1.value = rbrp->page_value_2.value = 0;
337044961713Sgirish 	rbrp->page_reloc_1.value = rbrp->page_reloc_2.value = 0;
337144961713Sgirish 	rbrp->page_hdl.value = 0;
337244961713Sgirish 
337344961713Sgirish 	rbrp->page_valid.bits.ldw.page0 = 1;
337444961713Sgirish 	rbrp->page_valid.bits.ldw.page1 = 1;
337544961713Sgirish 
337644961713Sgirish 	/* Map in the receive completion ring */
337744961713Sgirish 	rcrp = (p_rx_rcr_ring_t)
337814ea4bb7Ssd 		KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP);
337944961713Sgirish 	rcrp->rdc = dma_channel;
338044961713Sgirish 
338144961713Sgirish 	nxge_port_rcr_size = nxgep->nxge_port_rcr_size;
338244961713Sgirish 	rcrp->comp_size = nxge_port_rcr_size;
338344961713Sgirish 	rcrp->comp_wrap_mask = nxge_port_rcr_size - 1;
338444961713Sgirish 
338544961713Sgirish 	rcrp->max_receive_pkts = nxge_max_rx_pkts;
338644961713Sgirish 
338744961713Sgirish 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
338844961713Sgirish 	nxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size,
338944961713Sgirish 			sizeof (rcr_entry_t));
339044961713Sgirish 	rcrp->comp_rd_index = 0;
339144961713Sgirish 	rcrp->comp_wt_index = 0;
339244961713Sgirish 	rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
339344961713Sgirish 		(p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
339444961713Sgirish 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
3395adfcba55Sjoycey #if defined(__i386)
3396adfcba55Sjoycey 		(p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
3397adfcba55Sjoycey #else
339844961713Sgirish 		(p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
3399adfcba55Sjoycey #endif
340044961713Sgirish 
340144961713Sgirish 	rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
340244961713Sgirish 			(nxge_port_rcr_size - 1);
340344961713Sgirish 	rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
340444961713Sgirish 			(nxge_port_rcr_size - 1);
340544961713Sgirish 
340644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
340744961713Sgirish 		"==> nxge_map_rxdma_channel_cfg_ring: "
340844961713Sgirish 		"channel %d "
340944961713Sgirish 		"rbr_vaddrp $%p "
341044961713Sgirish 		"rcr_desc_rd_head_p $%p "
341144961713Sgirish 		"rcr_desc_rd_head_pp $%p "
341244961713Sgirish 		"rcr_desc_rd_last_p $%p "
341344961713Sgirish 		"rcr_desc_rd_last_pp $%p ",
341444961713Sgirish 		dma_channel,
341544961713Sgirish 		rbr_vaddrp,
341644961713Sgirish 		rcrp->rcr_desc_rd_head_p,
341744961713Sgirish 		rcrp->rcr_desc_rd_head_pp,
341844961713Sgirish 		rcrp->rcr_desc_last_p,
341944961713Sgirish 		rcrp->rcr_desc_last_pp));
342044961713Sgirish 
342144961713Sgirish 	/*
342244961713Sgirish 	 * Zero out buffer block ring descriptors.
342344961713Sgirish 	 */
342444961713Sgirish 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
342514ea4bb7Ssd 	rcrp->intr_timeout = nxgep->intr_timeout;
342614ea4bb7Ssd 	rcrp->intr_threshold = nxgep->intr_threshold;
342744961713Sgirish 	rcrp->full_hdr_flag = B_FALSE;
342844961713Sgirish 	rcrp->sw_priv_hdr_len = 0;
342944961713Sgirish 
343044961713Sgirish 	cfga_p = &(rcrp->rcr_cfga);
343144961713Sgirish 	cfgb_p = &(rcrp->rcr_cfgb);
343244961713Sgirish 	cfga_p->value = 0;
343344961713Sgirish 	cfgb_p->value = 0;
343444961713Sgirish 	rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress;
343544961713Sgirish 	cfga_p->value = (rcrp->rcr_addr &
343644961713Sgirish 			    (RCRCFIG_A_STADDR_MASK |
343744961713Sgirish 			    RCRCFIG_A_STADDR_BASE_MASK));
343844961713Sgirish 
343944961713Sgirish 	rcfga_p->value |= ((uint64_t)rcrp->comp_size <<
344044961713Sgirish 				RCRCFIG_A_LEN_SHIF);
344144961713Sgirish 
344244961713Sgirish 	/*
344344961713Sgirish 	 * Timeout should be set based on the system clock divider.
344444961713Sgirish 	 * The following timeout value of 1 assumes that the
344544961713Sgirish 	 * granularity (1000) is 3 microseconds running at 300MHz.
344644961713Sgirish 	 */
344714ea4bb7Ssd 	cfgb_p->bits.ldw.pthres = rcrp->intr_threshold;
344814ea4bb7Ssd 	cfgb_p->bits.ldw.timeout = rcrp->intr_timeout;
344944961713Sgirish 	cfgb_p->bits.ldw.entout = 1;
345044961713Sgirish 
345144961713Sgirish 	/* Map in the mailbox */
345244961713Sgirish 	mboxp = (p_rx_mbox_t)
345344961713Sgirish 			KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP);
345444961713Sgirish 	dmap = (p_nxge_dma_common_t)&mboxp->rx_mbox;
345544961713Sgirish 	nxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t));
345644961713Sgirish 	cfig1_p = (p_rxdma_cfig1_t)&mboxp->rx_cfg1;
345744961713Sgirish 	cfig2_p = (p_rxdma_cfig2_t)&mboxp->rx_cfg2;
345844961713Sgirish 	cfig1_p->value = cfig2_p->value = 0;
345944961713Sgirish 
346044961713Sgirish 	mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress;
346144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
346244961713Sgirish 		"==> nxge_map_rxdma_channel_cfg_ring: "
346344961713Sgirish 		"channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx",
346444961713Sgirish 		dma_channel, cfig1_p->value, cfig2_p->value,
346544961713Sgirish 		mboxp->mbox_addr));
346644961713Sgirish 
346744961713Sgirish 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress >> 32
346844961713Sgirish 			& 0xfff);
346944961713Sgirish 	cfig1_p->bits.ldw.mbaddr_h = dmaaddrp;
347044961713Sgirish 
347144961713Sgirish 
347244961713Sgirish 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff);
347344961713Sgirish 	dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress &
347444961713Sgirish 				RXDMA_CFIG2_MBADDR_L_MASK);
347544961713Sgirish 
347644961713Sgirish 	cfig2_p->bits.ldw.mbaddr = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT);
347744961713Sgirish 
347844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
347944961713Sgirish 		"==> nxge_map_rxdma_channel_cfg_ring: "
348044961713Sgirish 		"channel %d damaddrp $%p "
348144961713Sgirish 		"cfg1 0x%016llx cfig2 0x%016llx",
348244961713Sgirish 		dma_channel, dmaaddrp,
348344961713Sgirish 		cfig1_p->value, cfig2_p->value));
348444961713Sgirish 
348544961713Sgirish 	cfig2_p->bits.ldw.full_hdr = rcrp->full_hdr_flag;
348644961713Sgirish 	cfig2_p->bits.ldw.offset = rcrp->sw_priv_hdr_len;
348744961713Sgirish 
348844961713Sgirish 	rbrp->rx_rcr_p = rcrp;
348944961713Sgirish 	rcrp->rx_rbr_p = rbrp;
349044961713Sgirish 	*rcr_p = rcrp;
349144961713Sgirish 	*rx_mbox_p = mboxp;
349244961713Sgirish 
349344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
349444961713Sgirish 		"<== nxge_map_rxdma_channel_cfg_ring status 0x%08x", status));
349544961713Sgirish 
349644961713Sgirish 	return (status);
349744961713Sgirish }
349844961713Sgirish 
349944961713Sgirish /*ARGSUSED*/
350044961713Sgirish static void
350144961713Sgirish nxge_unmap_rxdma_channel_cfg_ring(p_nxge_t nxgep,
350244961713Sgirish     p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
350344961713Sgirish {
350444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
350544961713Sgirish 		"==> nxge_unmap_rxdma_channel_cfg_ring: channel %d",
350644961713Sgirish 		rcr_p->rdc));
350744961713Sgirish 
350844961713Sgirish 	KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t));
350944961713Sgirish 	KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t));
351044961713Sgirish 
351144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
351244961713Sgirish 		"<== nxge_unmap_rxdma_channel_cfg_ring"));
351344961713Sgirish }
351444961713Sgirish 
351544961713Sgirish static nxge_status_t
351644961713Sgirish nxge_map_rxdma_channel_buf_ring(p_nxge_t nxgep, uint16_t channel,
351744961713Sgirish     p_nxge_dma_common_t *dma_buf_p,
351844961713Sgirish     p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks)
351944961713Sgirish {
352044961713Sgirish 	p_rx_rbr_ring_t 	rbrp;
352144961713Sgirish 	p_nxge_dma_common_t 	dma_bufp, tmp_bufp;
352244961713Sgirish 	p_rx_msg_t 		*rx_msg_ring;
352344961713Sgirish 	p_rx_msg_t 		rx_msg_p;
352444961713Sgirish 	p_mblk_t 		mblk_p;
352544961713Sgirish 
352644961713Sgirish 	rxring_info_t *ring_info;
352744961713Sgirish 	nxge_status_t		status = NXGE_OK;
352844961713Sgirish 	int			i, j, index;
352944961713Sgirish 	uint32_t		size, bsize, nblocks, nmsgs;
353044961713Sgirish 
353144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
353244961713Sgirish 		"==> nxge_map_rxdma_channel_buf_ring: channel %d",
353344961713Sgirish 		channel));
353444961713Sgirish 
353544961713Sgirish 	dma_bufp = tmp_bufp = *dma_buf_p;
353644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
353744961713Sgirish 		" nxge_map_rxdma_channel_buf_ring: channel %d to map %d "
353844961713Sgirish 		"chunks bufp 0x%016llx",
353944961713Sgirish 		channel, num_chunks, dma_bufp));
354044961713Sgirish 
354144961713Sgirish 	nmsgs = 0;
354244961713Sgirish 	for (i = 0; i < num_chunks; i++, tmp_bufp++) {
354344961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
354444961713Sgirish 			"==> nxge_map_rxdma_channel_buf_ring: channel %d "
354544961713Sgirish 			"bufp 0x%016llx nblocks %d nmsgs %d",
354644961713Sgirish 			channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
354744961713Sgirish 		nmsgs += tmp_bufp->nblocks;
354844961713Sgirish 	}
354944961713Sgirish 	if (!nmsgs) {
355056d930aeSspeer 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
355144961713Sgirish 			"<== nxge_map_rxdma_channel_buf_ring: channel %d "
355244961713Sgirish 			"no msg blocks",
355344961713Sgirish 			channel));
355444961713Sgirish 		status = NXGE_ERROR;
355544961713Sgirish 		goto nxge_map_rxdma_channel_buf_ring_exit;
355644961713Sgirish 	}
355744961713Sgirish 
3558007969e0Stm 	rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (*rbrp), KM_SLEEP);
355944961713Sgirish 
356044961713Sgirish 	size = nmsgs * sizeof (p_rx_msg_t);
356144961713Sgirish 	rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
356244961713Sgirish 	ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t),
356344961713Sgirish 		KM_SLEEP);
356444961713Sgirish 
356544961713Sgirish 	MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER,
356644961713Sgirish 				(void *)nxgep->interrupt_cookie);
356744961713Sgirish 	MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER,
356844961713Sgirish 				(void *)nxgep->interrupt_cookie);
356944961713Sgirish 	rbrp->rdc = channel;
357044961713Sgirish 	rbrp->num_blocks = num_chunks;
357144961713Sgirish 	rbrp->tnblocks = nmsgs;
357244961713Sgirish 	rbrp->rbb_max = nmsgs;
357344961713Sgirish 	rbrp->rbr_max_size = nmsgs;
357444961713Sgirish 	rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1);
357544961713Sgirish 
357644961713Sgirish 	/*
357744961713Sgirish 	 * Buffer sizes suggested by NIU architect.
357844961713Sgirish 	 * 256, 512 and 2K.
357944961713Sgirish 	 */
358044961713Sgirish 
358144961713Sgirish 	rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B;
358244961713Sgirish 	rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES;
358344961713Sgirish 	rbrp->npi_pkt_buf_size0 = SIZE_256B;
358444961713Sgirish 
358544961713Sgirish 	rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K;
358644961713Sgirish 	rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES;
358744961713Sgirish 	rbrp->npi_pkt_buf_size1 = SIZE_1KB;
358844961713Sgirish 
358944961713Sgirish 	rbrp->block_size = nxgep->rx_default_block_size;
359044961713Sgirish 
359114ea4bb7Ssd 	if (!nxge_jumbo_enable && !nxgep->param_arr[param_accept_jumbo].value) {
359244961713Sgirish 		rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K;
359344961713Sgirish 		rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES;
359444961713Sgirish 		rbrp->npi_pkt_buf_size2 = SIZE_2KB;
359544961713Sgirish 	} else {
359644961713Sgirish 		if (rbrp->block_size >= 0x2000) {
359744961713Sgirish 			rbrp->pkt_buf_size2 = RBR_BUFSZ2_8K;
359844961713Sgirish 			rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_8K_BYTES;
359944961713Sgirish 			rbrp->npi_pkt_buf_size2 = SIZE_8KB;
360044961713Sgirish 		} else {
360144961713Sgirish 			rbrp->pkt_buf_size2 = RBR_BUFSZ2_4K;
360244961713Sgirish 			rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_4K_BYTES;
360344961713Sgirish 			rbrp->npi_pkt_buf_size2 = SIZE_4KB;
360444961713Sgirish 		}
360544961713Sgirish 	}
360644961713Sgirish 
360744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
360844961713Sgirish 		"==> nxge_map_rxdma_channel_buf_ring: channel %d "
360944961713Sgirish 		"actual rbr max %d rbb_max %d nmsgs %d "
361044961713Sgirish 		"rbrp->block_size %d default_block_size %d "
361144961713Sgirish 		"(config nxge_rbr_size %d nxge_rbr_spare_size %d)",
361244961713Sgirish 		channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs,
361344961713Sgirish 		rbrp->block_size, nxgep->rx_default_block_size,
361444961713Sgirish 		nxge_rbr_size, nxge_rbr_spare_size));
361544961713Sgirish 
361644961713Sgirish 	/* Map in buffers from the buffer pool.  */
361744961713Sgirish 	index = 0;
361844961713Sgirish 	for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) {
361944961713Sgirish 		bsize = dma_bufp->block_size;
362044961713Sgirish 		nblocks = dma_bufp->nblocks;
3621adfcba55Sjoycey #if defined(__i386)
3622adfcba55Sjoycey 		ring_info->buffer[i].dvma_addr = (uint32_t)dma_bufp->ioaddr_pp;
3623adfcba55Sjoycey #else
362444961713Sgirish 		ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp;
3625adfcba55Sjoycey #endif
362644961713Sgirish 		ring_info->buffer[i].buf_index = i;
362744961713Sgirish 		ring_info->buffer[i].buf_size = dma_bufp->alength;
362844961713Sgirish 		ring_info->buffer[i].start_index = index;
3629adfcba55Sjoycey #if defined(__i386)
3630adfcba55Sjoycey 		ring_info->buffer[i].kaddr = (uint32_t)dma_bufp->kaddrp;
3631adfcba55Sjoycey #else
363244961713Sgirish 		ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp;
3633adfcba55Sjoycey #endif
363444961713Sgirish 
363544961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
363644961713Sgirish 			" nxge_map_rxdma_channel_buf_ring: map channel %d "
363744961713Sgirish 			"chunk %d"
363844961713Sgirish 			" nblocks %d chunk_size %x block_size 0x%x "
363944961713Sgirish 			"dma_bufp $%p", channel, i,
364044961713Sgirish 			dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize,
364144961713Sgirish 			dma_bufp));
364244961713Sgirish 
364344961713Sgirish 		for (j = 0; j < nblocks; j++) {
364444961713Sgirish 			if ((rx_msg_p = nxge_allocb(bsize, BPRI_LO,
364544961713Sgirish 					dma_bufp)) == NULL) {
364656d930aeSspeer 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
364756d930aeSspeer 					"allocb failed (index %d i %d j %d)",
364856d930aeSspeer 					index, i, j));
364956d930aeSspeer 				goto nxge_map_rxdma_channel_buf_ring_fail1;
365044961713Sgirish 			}
365144961713Sgirish 			rx_msg_ring[index] = rx_msg_p;
365244961713Sgirish 			rx_msg_p->block_index = index;
365344961713Sgirish 			rx_msg_p->shifted_addr = (uint32_t)
365444961713Sgirish 				((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >>
365544961713Sgirish 					    RBR_BKADDR_SHIFT));
365644961713Sgirish 
365744961713Sgirish 			NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
365856d930aeSspeer 				"index %d j %d rx_msg_p $%p mblk %p",
365956d930aeSspeer 				index, j, rx_msg_p, rx_msg_p->rx_mblk_p));
366044961713Sgirish 
366144961713Sgirish 			mblk_p = rx_msg_p->rx_mblk_p;
366244961713Sgirish 			mblk_p->b_wptr = mblk_p->b_rptr + bsize;
3663007969e0Stm 
3664007969e0Stm 			rbrp->rbr_ref_cnt++;
366544961713Sgirish 			index++;
366644961713Sgirish 			rx_msg_p->buf_dma.dma_channel = channel;
366744961713Sgirish 		}
3668678453a8Sspeer 
3669678453a8Sspeer 		rbrp->rbr_alloc_type = DDI_MEM_ALLOC;
3670678453a8Sspeer 		if (dma_bufp->contig_alloc_type) {
3671678453a8Sspeer 			rbrp->rbr_alloc_type = CONTIG_MEM_ALLOC;
3672678453a8Sspeer 		}
3673678453a8Sspeer 
3674678453a8Sspeer 		if (dma_bufp->kmem_alloc_type) {
3675678453a8Sspeer 			rbrp->rbr_alloc_type = KMEM_ALLOC;
3676678453a8Sspeer 		}
3677678453a8Sspeer 
3678678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3679678453a8Sspeer 		    " nxge_map_rxdma_channel_buf_ring: map channel %d "
3680678453a8Sspeer 		    "chunk %d"
3681678453a8Sspeer 		    " nblocks %d chunk_size %x block_size 0x%x "
3682678453a8Sspeer 		    "dma_bufp $%p",
3683678453a8Sspeer 		    channel, i,
3684678453a8Sspeer 		    dma_bufp->nblocks, ring_info->buffer[i].buf_size, bsize,
3685678453a8Sspeer 		    dma_bufp));
368644961713Sgirish 	}
368744961713Sgirish 	if (i < rbrp->num_blocks) {
368844961713Sgirish 		goto nxge_map_rxdma_channel_buf_ring_fail1;
368944961713Sgirish 	}
369044961713Sgirish 
369144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
369244961713Sgirish 		"nxge_map_rxdma_channel_buf_ring: done buf init "
369344961713Sgirish 			"channel %d msg block entries %d",
369444961713Sgirish 			channel, index));
369544961713Sgirish 	ring_info->block_size_mask = bsize - 1;
369644961713Sgirish 	rbrp->rx_msg_ring = rx_msg_ring;
369744961713Sgirish 	rbrp->dma_bufp = dma_buf_p;
369844961713Sgirish 	rbrp->ring_info = ring_info;
369944961713Sgirish 
370044961713Sgirish 	status = nxge_rxbuf_index_info_init(nxgep, rbrp);
370144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
370244961713Sgirish 		" nxge_map_rxdma_channel_buf_ring: "
370344961713Sgirish 		"channel %d done buf info init", channel));
370444961713Sgirish 
3705007969e0Stm 	/*
3706007969e0Stm 	 * Finally, permit nxge_freeb() to call nxge_post_page().
3707007969e0Stm 	 */
3708007969e0Stm 	rbrp->rbr_state = RBR_POSTING;
3709007969e0Stm 
371044961713Sgirish 	*rbr_p = rbrp;
371144961713Sgirish 	goto nxge_map_rxdma_channel_buf_ring_exit;
371244961713Sgirish 
371344961713Sgirish nxge_map_rxdma_channel_buf_ring_fail1:
371444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
371544961713Sgirish 		" nxge_map_rxdma_channel_buf_ring: failed channel (0x%x)",
371644961713Sgirish 		channel, status));
371744961713Sgirish 
371844961713Sgirish 	index--;
371944961713Sgirish 	for (; index >= 0; index--) {
372044961713Sgirish 		rx_msg_p = rx_msg_ring[index];
372144961713Sgirish 		if (rx_msg_p != NULL) {
372214ea4bb7Ssd 			freeb(rx_msg_p->rx_mblk_p);
372344961713Sgirish 			rx_msg_ring[index] = NULL;
372444961713Sgirish 		}
372544961713Sgirish 	}
372644961713Sgirish nxge_map_rxdma_channel_buf_ring_fail:
372744961713Sgirish 	MUTEX_DESTROY(&rbrp->post_lock);
372844961713Sgirish 	MUTEX_DESTROY(&rbrp->lock);
372944961713Sgirish 	KMEM_FREE(ring_info, sizeof (rxring_info_t));
373044961713Sgirish 	KMEM_FREE(rx_msg_ring, size);
373144961713Sgirish 	KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t));
373244961713Sgirish 
373356d930aeSspeer 	status = NXGE_ERROR;
373456d930aeSspeer 
373544961713Sgirish nxge_map_rxdma_channel_buf_ring_exit:
373644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
373744961713Sgirish 		"<== nxge_map_rxdma_channel_buf_ring status 0x%08x", status));
373844961713Sgirish 
373944961713Sgirish 	return (status);
374044961713Sgirish }
374144961713Sgirish 
374244961713Sgirish /*ARGSUSED*/
374344961713Sgirish static void
374444961713Sgirish nxge_unmap_rxdma_channel_buf_ring(p_nxge_t nxgep,
374544961713Sgirish     p_rx_rbr_ring_t rbr_p)
374644961713Sgirish {
374744961713Sgirish 	p_rx_msg_t 		*rx_msg_ring;
374844961713Sgirish 	p_rx_msg_t 		rx_msg_p;
374944961713Sgirish 	rxring_info_t 		*ring_info;
375044961713Sgirish 	int			i;
375144961713Sgirish 	uint32_t		size;
375244961713Sgirish #ifdef	NXGE_DEBUG
375344961713Sgirish 	int			num_chunks;
375444961713Sgirish #endif
375544961713Sgirish 
375644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
375744961713Sgirish 		"==> nxge_unmap_rxdma_channel_buf_ring"));
375844961713Sgirish 	if (rbr_p == NULL) {
375944961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
376044961713Sgirish 			"<== nxge_unmap_rxdma_channel_buf_ring: NULL rbrp"));
376144961713Sgirish 		return;
376244961713Sgirish 	}
376344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
376444961713Sgirish 		"==> nxge_unmap_rxdma_channel_buf_ring: channel %d",
376544961713Sgirish 		rbr_p->rdc));
376644961713Sgirish 
376744961713Sgirish 	rx_msg_ring = rbr_p->rx_msg_ring;
376844961713Sgirish 	ring_info = rbr_p->ring_info;
376944961713Sgirish 
377044961713Sgirish 	if (rx_msg_ring == NULL || ring_info == NULL) {
377144961713Sgirish 			NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
377244961713Sgirish 		"<== nxge_unmap_rxdma_channel_buf_ring: "
377344961713Sgirish 		"rx_msg_ring $%p ring_info $%p",
377444961713Sgirish 		rx_msg_p, ring_info));
377544961713Sgirish 		return;
377644961713Sgirish 	}
377744961713Sgirish 
377844961713Sgirish #ifdef	NXGE_DEBUG
377944961713Sgirish 	num_chunks = rbr_p->num_blocks;
378044961713Sgirish #endif
378144961713Sgirish 	size = rbr_p->tnblocks * sizeof (p_rx_msg_t);
378244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
378344961713Sgirish 		" nxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d "
378444961713Sgirish 		"tnblocks %d (max %d) size ptrs %d ",
378544961713Sgirish 		rbr_p->rdc, num_chunks,
378644961713Sgirish 		rbr_p->tnblocks, rbr_p->rbr_max_size, size));
378744961713Sgirish 
378844961713Sgirish 	for (i = 0; i < rbr_p->tnblocks; i++) {
378944961713Sgirish 		rx_msg_p = rx_msg_ring[i];
379044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
379144961713Sgirish 			" nxge_unmap_rxdma_channel_buf_ring: "
379244961713Sgirish 			"rx_msg_p $%p",
379344961713Sgirish 			rx_msg_p));
379444961713Sgirish 		if (rx_msg_p != NULL) {
379514ea4bb7Ssd 			freeb(rx_msg_p->rx_mblk_p);
379644961713Sgirish 			rx_msg_ring[i] = NULL;
379744961713Sgirish 		}
379844961713Sgirish 	}
379944961713Sgirish 
3800007969e0Stm 	/*
3801007969e0Stm 	 * We no longer may use the mutex <post_lock>. By setting
3802007969e0Stm 	 * <rbr_state> to anything but POSTING, we prevent
3803007969e0Stm 	 * nxge_post_page() from accessing a dead mutex.
3804007969e0Stm 	 */
3805007969e0Stm 	rbr_p->rbr_state = RBR_UNMAPPING;
380644961713Sgirish 	MUTEX_DESTROY(&rbr_p->post_lock);
3807007969e0Stm 
380844961713Sgirish 	MUTEX_DESTROY(&rbr_p->lock);
3809007969e0Stm 
3810007969e0Stm 	if (rbr_p->rbr_ref_cnt == 0) {
3811678453a8Sspeer 		/*
3812678453a8Sspeer 		 * This is the normal state of affairs.
3813678453a8Sspeer 		 * Need to free the following buffers:
3814678453a8Sspeer 		 *  - data buffers
3815678453a8Sspeer 		 *  - rx_msg ring
3816678453a8Sspeer 		 *  - ring_info
3817678453a8Sspeer 		 *  - rbr ring
3818678453a8Sspeer 		 */
3819678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
3820678453a8Sspeer 		    "unmap_rxdma_buf_ring: No outstanding - freeing "));
3821678453a8Sspeer 		nxge_rxdma_databuf_free(rbr_p);
3822678453a8Sspeer 		KMEM_FREE(ring_info, sizeof (rxring_info_t));
3823678453a8Sspeer 		KMEM_FREE(rx_msg_ring, size);
3824007969e0Stm 		KMEM_FREE(rbr_p, sizeof (*rbr_p));
3825007969e0Stm 	} else {
3826007969e0Stm 		/*
3827007969e0Stm 		 * Some of our buffers are still being used.
3828007969e0Stm 		 * Therefore, tell nxge_freeb() this ring is
3829007969e0Stm 		 * unmapped, so it may free <rbr_p> for us.
3830007969e0Stm 		 */
3831007969e0Stm 		rbr_p->rbr_state = RBR_UNMAPPED;
3832007969e0Stm 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3833007969e0Stm 		    "unmap_rxdma_buf_ring: %d %s outstanding.",
3834007969e0Stm 		    rbr_p->rbr_ref_cnt,
3835007969e0Stm 		    rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs"));
3836007969e0Stm 	}
383744961713Sgirish 
383844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
383944961713Sgirish 		"<== nxge_unmap_rxdma_channel_buf_ring"));
384044961713Sgirish }
384144961713Sgirish 
3842678453a8Sspeer /*
3843678453a8Sspeer  * nxge_rxdma_hw_start_common
3844678453a8Sspeer  *
3845678453a8Sspeer  * Arguments:
3846678453a8Sspeer  * 	nxgep
3847678453a8Sspeer  *
3848678453a8Sspeer  * Notes:
3849678453a8Sspeer  *
3850678453a8Sspeer  * NPI/NXGE function calls:
3851678453a8Sspeer  *	nxge_init_fzc_rx_common();
3852678453a8Sspeer  *	nxge_init_fzc_rxdma_port();
3853678453a8Sspeer  *
3854678453a8Sspeer  * Registers accessed:
3855678453a8Sspeer  *
3856678453a8Sspeer  * Context:
3857678453a8Sspeer  *	Service domain
3858678453a8Sspeer  */
385944961713Sgirish static nxge_status_t
386044961713Sgirish nxge_rxdma_hw_start_common(p_nxge_t nxgep)
386144961713Sgirish {
386244961713Sgirish 	nxge_status_t		status = NXGE_OK;
386344961713Sgirish 
386444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common"));
386544961713Sgirish 
386644961713Sgirish 	/*
386744961713Sgirish 	 * Load the sharable parameters by writing to the
386844961713Sgirish 	 * function zero control registers. These FZC registers
386944961713Sgirish 	 * should be initialized only once for the entire chip.
387044961713Sgirish 	 */
387144961713Sgirish 	(void) nxge_init_fzc_rx_common(nxgep);
387244961713Sgirish 
387344961713Sgirish 	/*
387444961713Sgirish 	 * Initialize the RXDMA port specific FZC control configurations.
387544961713Sgirish 	 * These FZC registers are pertaining to each port.
387644961713Sgirish 	 */
387744961713Sgirish 	(void) nxge_init_fzc_rxdma_port(nxgep);
387844961713Sgirish 
387944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start_common"));
388044961713Sgirish 
388144961713Sgirish 	return (status);
388244961713Sgirish }
388344961713Sgirish 
388444961713Sgirish static nxge_status_t
3885678453a8Sspeer nxge_rxdma_hw_start(p_nxge_t nxgep, int channel)
388644961713Sgirish {
388744961713Sgirish 	int			i, ndmas;
388844961713Sgirish 	p_rx_rbr_rings_t 	rx_rbr_rings;
388944961713Sgirish 	p_rx_rbr_ring_t		*rbr_rings;
389044961713Sgirish 	p_rx_rcr_rings_t 	rx_rcr_rings;
389144961713Sgirish 	p_rx_rcr_ring_t		*rcr_rings;
389244961713Sgirish 	p_rx_mbox_areas_t 	rx_mbox_areas_p;
389344961713Sgirish 	p_rx_mbox_t		*rx_mbox_p;
389444961713Sgirish 	nxge_status_t		status = NXGE_OK;
389544961713Sgirish 
389644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start"));
389744961713Sgirish 
389844961713Sgirish 	rx_rbr_rings = nxgep->rx_rbr_rings;
389944961713Sgirish 	rx_rcr_rings = nxgep->rx_rcr_rings;
390044961713Sgirish 	if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
390144961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
390244961713Sgirish 			"<== nxge_rxdma_hw_start: NULL ring pointers"));
390344961713Sgirish 		return (NXGE_ERROR);
390444961713Sgirish 	}
390544961713Sgirish 	ndmas = rx_rbr_rings->ndmas;
390644961713Sgirish 	if (ndmas == 0) {
390744961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
390844961713Sgirish 			"<== nxge_rxdma_hw_start: no dma channel allocated"));
390944961713Sgirish 		return (NXGE_ERROR);
391044961713Sgirish 	}
391144961713Sgirish 
391244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
391344961713Sgirish 		"==> nxge_rxdma_hw_start (ndmas %d)", ndmas));
391444961713Sgirish 
391544961713Sgirish 	rbr_rings = rx_rbr_rings->rbr_rings;
391644961713Sgirish 	rcr_rings = rx_rcr_rings->rcr_rings;
391744961713Sgirish 	rx_mbox_areas_p = nxgep->rx_mbox_areas_p;
391844961713Sgirish 	if (rx_mbox_areas_p) {
391944961713Sgirish 		rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
392044961713Sgirish 	}
392144961713Sgirish 
3922678453a8Sspeer 	i = channel;
3923678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3924678453a8Sspeer 		"==> nxge_rxdma_hw_start (ndmas %d) channel %d",
3925678453a8Sspeer 		ndmas, channel));
3926678453a8Sspeer 	status = nxge_rxdma_start_channel(nxgep, channel,
3927678453a8Sspeer 	    (p_rx_rbr_ring_t)rbr_rings[i],
3928678453a8Sspeer 	    (p_rx_rcr_ring_t)rcr_rings[i],
3929678453a8Sspeer 	    (p_rx_mbox_t)rx_mbox_p[i]);
3930678453a8Sspeer 	if (status != NXGE_OK) {
3931678453a8Sspeer 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
3932678453a8Sspeer 		    "==> nxge_rxdma_hw_start: disable "
3933678453a8Sspeer 		    "(status 0x%x channel %d)", status, channel));
3934678453a8Sspeer 		return (status);
393544961713Sgirish 	}
393644961713Sgirish 
393744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_start: "
393844961713Sgirish 		"rx_rbr_rings 0x%016llx rings 0x%016llx",
393944961713Sgirish 		rx_rbr_rings, rx_rcr_rings));
394044961713Sgirish 
394144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
394244961713Sgirish 		"==> nxge_rxdma_hw_start: (status 0x%x)", status));
394344961713Sgirish 
394444961713Sgirish 	return (status);
394544961713Sgirish }
394644961713Sgirish 
394744961713Sgirish static void
3948678453a8Sspeer nxge_rxdma_hw_stop(p_nxge_t nxgep, int channel)
394944961713Sgirish {
395044961713Sgirish 	p_rx_rbr_rings_t 	rx_rbr_rings;
395144961713Sgirish 	p_rx_rcr_rings_t 	rx_rcr_rings;
395244961713Sgirish 
395344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop"));
395444961713Sgirish 
395544961713Sgirish 	rx_rbr_rings = nxgep->rx_rbr_rings;
395644961713Sgirish 	rx_rcr_rings = nxgep->rx_rcr_rings;
395744961713Sgirish 	if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
395844961713Sgirish 		NXGE_DEBUG_MSG((nxgep, RX_CTL,
395944961713Sgirish 			"<== nxge_rxdma_hw_stop: NULL ring pointers"));
396044961713Sgirish 		return;
396144961713Sgirish 	}
396244961713Sgirish 
396344961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
3964678453a8Sspeer 		"==> nxge_rxdma_hw_stop(channel %d)",
3965678453a8Sspeer 		channel));
3966678453a8Sspeer 	(void) nxge_rxdma_stop_channel(nxgep, channel);
396744961713Sgirish 
396844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_hw_stop: "
396944961713Sgirish 		"rx_rbr_rings 0x%016llx rings 0x%016llx",
397044961713Sgirish 		rx_rbr_rings, rx_rcr_rings));
397144961713Sgirish 
397244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_hw_stop"));
397344961713Sgirish }
397444961713Sgirish 
397544961713Sgirish 
397644961713Sgirish static nxge_status_t
397744961713Sgirish nxge_rxdma_start_channel(p_nxge_t nxgep, uint16_t channel,
397844961713Sgirish     p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p)
397944961713Sgirish 
398044961713Sgirish {
398144961713Sgirish 	npi_handle_t		handle;
398244961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
398344961713Sgirish 	rx_dma_ctl_stat_t	cs;
398444961713Sgirish 	rx_dma_ent_msk_t	ent_mask;
398544961713Sgirish 	nxge_status_t		status = NXGE_OK;
398644961713Sgirish 
398744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel"));
398844961713Sgirish 
398944961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
399044961713Sgirish 
399144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "nxge_rxdma_start_channel: "
399244961713Sgirish 		"npi handle addr $%p acc $%p",
399344961713Sgirish 		nxgep->npi_handle.regp, nxgep->npi_handle.regh));
399444961713Sgirish 
3995678453a8Sspeer 	/* Reset RXDMA channel, but not if you're a guest. */
3996678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
3997678453a8Sspeer 		rs = npi_rxdma_cfg_rdc_reset(handle, channel);
3998678453a8Sspeer 		if (rs != NPI_SUCCESS) {
3999678453a8Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4000678453a8Sspeer 			    "==> nxge_init_fzc_rdc: "
4001678453a8Sspeer 			    "npi_rxdma_cfg_rdc_reset(%d) returned 0x%08x",
4002678453a8Sspeer 			    channel, rs));
4003678453a8Sspeer 			return (NXGE_ERROR | rs);
4004678453a8Sspeer 		}
4005678453a8Sspeer 
4006678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4007678453a8Sspeer 		    "==> nxge_rxdma_start_channel: reset done: channel %d",
4008678453a8Sspeer 		    channel));
400944961713Sgirish 	}
401044961713Sgirish 
4011678453a8Sspeer #if defined(sun4v) && defined(NIU_LP_WORKAROUND)
4012678453a8Sspeer 	if (isLDOMguest(nxgep))
4013678453a8Sspeer 		(void) nxge_rdc_lp_conf(nxgep, channel);
4014678453a8Sspeer #endif
401544961713Sgirish 
401644961713Sgirish 	/*
401744961713Sgirish 	 * Initialize the RXDMA channel specific FZC control
401844961713Sgirish 	 * configurations. These FZC registers are pertaining
401944961713Sgirish 	 * to each RX channel (logical pages).
402044961713Sgirish 	 */
4021678453a8Sspeer 	if (!isLDOMguest(nxgep)) {
4022678453a8Sspeer 		status = nxge_init_fzc_rxdma_channel(nxgep, channel);
4023678453a8Sspeer 		if (status != NXGE_OK) {
4024678453a8Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4025678453a8Sspeer 				"==> nxge_rxdma_start_channel: "
4026678453a8Sspeer 				"init fzc rxdma failed (0x%08x channel %d)",
4027678453a8Sspeer 				status, channel));
4028678453a8Sspeer 			return (status);
4029678453a8Sspeer 		}
403044961713Sgirish 
4031678453a8Sspeer 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4032678453a8Sspeer 			"==> nxge_rxdma_start_channel: fzc done"));
4033678453a8Sspeer 	}
403444961713Sgirish 
403544961713Sgirish 	/* Set up the interrupt event masks. */
403644961713Sgirish 	ent_mask.value = 0;
403744961713Sgirish 	ent_mask.value |= RX_DMA_ENT_MSK_RBREMPTY_MASK;
403844961713Sgirish 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
4039678453a8Sspeer 	    &ent_mask);
404044961713Sgirish 	if (rs != NPI_SUCCESS) {
404144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
404244961713Sgirish 			"==> nxge_rxdma_start_channel: "
4043678453a8Sspeer 			"init rxdma event masks failed "
4044678453a8Sspeer 			"(0x%08x channel %d)",
404544961713Sgirish 			status, channel));
404644961713Sgirish 		return (NXGE_ERROR | rs);
404744961713Sgirish 	}
404844961713Sgirish 
4049678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4050678453a8Sspeer 		"==> nxge_rxdma_start_channel: "
405144961713Sgirish 		"event done: channel %d (mask 0x%016llx)",
405244961713Sgirish 		channel, ent_mask.value));
405344961713Sgirish 
405444961713Sgirish 	/* Initialize the receive DMA control and status register */
405544961713Sgirish 	cs.value = 0;
405644961713Sgirish 	cs.bits.hdw.mex = 1;
405744961713Sgirish 	cs.bits.hdw.rcrthres = 1;
405844961713Sgirish 	cs.bits.hdw.rcrto = 1;
405944961713Sgirish 	cs.bits.hdw.rbr_empty = 1;
406044961713Sgirish 	status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel, &cs);
406144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
406244961713Sgirish 		"channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value));
406344961713Sgirish 	if (status != NXGE_OK) {
406444961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
406544961713Sgirish 			"==> nxge_rxdma_start_channel: "
406644961713Sgirish 			"init rxdma control register failed (0x%08x channel %d",
406744961713Sgirish 			status, channel));
406844961713Sgirish 		return (status);
406944961713Sgirish 	}
407044961713Sgirish 
407144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
407244961713Sgirish 		"control done - channel %d cs 0x%016llx", channel, cs.value));
407344961713Sgirish 
407444961713Sgirish 	/*
407544961713Sgirish 	 * Load RXDMA descriptors, buffers, mailbox,
407644961713Sgirish 	 * initialise the receive DMA channels and
407744961713Sgirish 	 * enable each DMA channel.
407844961713Sgirish 	 */
407944961713Sgirish 	status = nxge_enable_rxdma_channel(nxgep,
4080678453a8Sspeer 	    channel, rbr_p, rcr_p, mbox_p);
408144961713Sgirish 
408244961713Sgirish 	if (status != NXGE_OK) {
408344961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4084678453a8Sspeer 		    " nxge_rxdma_start_channel: "
4085678453a8Sspeer 		    " enable rxdma failed (0x%08x channel %d)",
4086678453a8Sspeer 		    status, channel));
408744961713Sgirish 		return (status);
408844961713Sgirish 	}
408944961713Sgirish 
4090678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
4091678453a8Sspeer 	    "==> nxge_rxdma_start_channel: enabled channel %d"));
4092678453a8Sspeer 
4093678453a8Sspeer 	if (isLDOMguest(nxgep)) {
4094678453a8Sspeer 		/* Add interrupt handler for this channel. */
4095678453a8Sspeer 		if (nxge_hio_intr_add(nxgep, VP_BOUND_RX, channel)
4096678453a8Sspeer 		    != NXGE_OK) {
4097678453a8Sspeer 			NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4098678453a8Sspeer 			    " nxge_rxdma_start_channel: "
4099678453a8Sspeer 			    " nxge_hio_intr_add failed (0x%08x channel %d)",
4100678453a8Sspeer 		    status, channel));
4101678453a8Sspeer 		}
4102678453a8Sspeer 	}
4103678453a8Sspeer 
410444961713Sgirish 	ent_mask.value = 0;
410544961713Sgirish 	ent_mask.value |= (RX_DMA_ENT_MSK_WRED_DROP_MASK |
410644961713Sgirish 				RX_DMA_ENT_MSK_PTDROP_PKT_MASK);
410744961713Sgirish 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
410844961713Sgirish 			&ent_mask);
410944961713Sgirish 	if (rs != NPI_SUCCESS) {
411044961713Sgirish 		NXGE_DEBUG_MSG((nxgep, MEM2_CTL,
411144961713Sgirish 			"==> nxge_rxdma_start_channel: "
411244961713Sgirish 			"init rxdma event masks failed (0x%08x channel %d)",
411344961713Sgirish 			status, channel));
411444961713Sgirish 		return (NXGE_ERROR | rs);
411544961713Sgirish 	}
411644961713Sgirish 
411744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "==> nxge_rxdma_start_channel: "
411844961713Sgirish 		"control done - channel %d cs 0x%016llx", channel, cs.value));
411944961713Sgirish 
412044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, MEM2_CTL, "<== nxge_rxdma_start_channel"));
412144961713Sgirish 
412244961713Sgirish 	return (NXGE_OK);
412344961713Sgirish }
412444961713Sgirish 
412544961713Sgirish static nxge_status_t
412644961713Sgirish nxge_rxdma_stop_channel(p_nxge_t nxgep, uint16_t channel)
412744961713Sgirish {
412844961713Sgirish 	npi_handle_t		handle;
412944961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
413044961713Sgirish 	rx_dma_ctl_stat_t	cs;
413144961713Sgirish 	rx_dma_ent_msk_t	ent_mask;
413244961713Sgirish 	nxge_status_t		status = NXGE_OK;
413344961713Sgirish 
413444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel"));
413544961713Sgirish 
413644961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
413744961713Sgirish 
413844961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "nxge_rxdma_stop_channel: "
413944961713Sgirish 		"npi handle addr $%p acc $%p",
414044961713Sgirish 		nxgep->npi_handle.regp, nxgep->npi_handle.regh));
414144961713Sgirish 
414244961713Sgirish 	/* Reset RXDMA channel */
414344961713Sgirish 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
414444961713Sgirish 	if (rs != NPI_SUCCESS) {
414544961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
414644961713Sgirish 			    " nxge_rxdma_stop_channel: "
414744961713Sgirish 			    " reset rxdma failed (0x%08x channel %d)",
414844961713Sgirish 			    rs, channel));
414944961713Sgirish 		return (NXGE_ERROR | rs);
415044961713Sgirish 	}
415144961713Sgirish 
415244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
415344961713Sgirish 		"==> nxge_rxdma_stop_channel: reset done"));
415444961713Sgirish 
415544961713Sgirish 	/* Set up the interrupt event masks. */
415644961713Sgirish 	ent_mask.value = RX_DMA_ENT_MSK_ALL;
415744961713Sgirish 	rs = npi_rxdma_event_mask(handle, OP_SET, channel,
415844961713Sgirish 			&ent_mask);
415944961713Sgirish 	if (rs != NPI_SUCCESS) {
416044961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
416144961713Sgirish 			    "==> nxge_rxdma_stop_channel: "
416244961713Sgirish 			    "set rxdma event masks failed (0x%08x channel %d)",
416344961713Sgirish 			    rs, channel));
416444961713Sgirish 		return (NXGE_ERROR | rs);
416544961713Sgirish 	}
416644961713Sgirish 
416744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
416844961713Sgirish 		"==> nxge_rxdma_stop_channel: event done"));
416944961713Sgirish 
417044961713Sgirish 	/* Initialize the receive DMA control and status register */
417144961713Sgirish 	cs.value = 0;
417244961713Sgirish 	status = nxge_init_rxdma_channel_cntl_stat(nxgep, channel,
417344961713Sgirish 			&cs);
417444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_stop_channel: control "
417544961713Sgirish 		" to default (all 0s) 0x%08x", cs.value));
417644961713Sgirish 	if (status != NXGE_OK) {
417744961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
417844961713Sgirish 			    " nxge_rxdma_stop_channel: init rxdma"
417944961713Sgirish 			    " control register failed (0x%08x channel %d",
418044961713Sgirish 			status, channel));
418144961713Sgirish 		return (status);
418244961713Sgirish 	}
418344961713Sgirish 
418444961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL,
418544961713Sgirish 		"==> nxge_rxdma_stop_channel: control done"));
418644961713Sgirish 
418744961713Sgirish 	/* disable dma channel */
418844961713Sgirish 	status = nxge_disable_rxdma_channel(nxgep, channel);
418944961713Sgirish 
419044961713Sgirish 	if (status != NXGE_OK) {
419144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
419244961713Sgirish 			    " nxge_rxdma_stop_channel: "
419344961713Sgirish 			    " init enable rxdma failed (0x%08x channel %d)",
419444961713Sgirish 			    status, channel));
419544961713Sgirish 		return (status);
419644961713Sgirish 	}
419744961713Sgirish 
419844961713Sgirish 	NXGE_DEBUG_MSG((nxgep,
419944961713Sgirish 		RX_CTL, "==> nxge_rxdma_stop_channel: disable done"));
420044961713Sgirish 
420144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_stop_channel"));
420244961713Sgirish 
420344961713Sgirish 	return (NXGE_OK);
420444961713Sgirish }
420544961713Sgirish 
420644961713Sgirish nxge_status_t
420744961713Sgirish nxge_rxdma_handle_sys_errors(p_nxge_t nxgep)
420844961713Sgirish {
420944961713Sgirish 	npi_handle_t		handle;
421044961713Sgirish 	p_nxge_rdc_sys_stats_t	statsp;
421144961713Sgirish 	rx_ctl_dat_fifo_stat_t	stat;
421244961713Sgirish 	uint32_t		zcp_err_status;
421344961713Sgirish 	uint32_t		ipp_err_status;
421444961713Sgirish 	nxge_status_t		status = NXGE_OK;
421544961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
421644961713Sgirish 	boolean_t		my_err = B_FALSE;
421744961713Sgirish 
421844961713Sgirish 	handle = nxgep->npi_handle;
421944961713Sgirish 	statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats;
422044961713Sgirish 
422144961713Sgirish 	rs = npi_rxdma_rxctl_fifo_error_intr_get(handle, &stat);
422244961713Sgirish 
422344961713Sgirish 	if (rs != NPI_SUCCESS)
422444961713Sgirish 		return (NXGE_ERROR | rs);
422544961713Sgirish 
422644961713Sgirish 	if (stat.bits.ldw.id_mismatch) {
422744961713Sgirish 		statsp->id_mismatch++;
422844961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, nxgep->mac.portnum, NULL,
422944961713Sgirish 					NXGE_FM_EREPORT_RDMC_ID_MISMATCH);
423044961713Sgirish 		/* Global fatal error encountered */
423144961713Sgirish 	}
423244961713Sgirish 
423344961713Sgirish 	if ((stat.bits.ldw.zcp_eop_err) || (stat.bits.ldw.ipp_eop_err)) {
423444961713Sgirish 		switch (nxgep->mac.portnum) {
423544961713Sgirish 		case 0:
423644961713Sgirish 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT0) ||
423744961713Sgirish 				(stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT0)) {
423844961713Sgirish 				my_err = B_TRUE;
423944961713Sgirish 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
424044961713Sgirish 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
424144961713Sgirish 			}
424244961713Sgirish 			break;
424344961713Sgirish 		case 1:
424444961713Sgirish 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT1) ||
424544961713Sgirish 				(stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT1)) {
424644961713Sgirish 				my_err = B_TRUE;
424744961713Sgirish 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
424844961713Sgirish 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
424944961713Sgirish 			}
425044961713Sgirish 			break;
425144961713Sgirish 		case 2:
425244961713Sgirish 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT2) ||
425344961713Sgirish 				(stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT2)) {
425444961713Sgirish 				my_err = B_TRUE;
425544961713Sgirish 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
425644961713Sgirish 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
425744961713Sgirish 			}
425844961713Sgirish 			break;
425944961713Sgirish 		case 3:
426044961713Sgirish 			if ((stat.bits.ldw.zcp_eop_err & FIFO_EOP_PORT3) ||
426144961713Sgirish 				(stat.bits.ldw.ipp_eop_err & FIFO_EOP_PORT3)) {
426244961713Sgirish 				my_err = B_TRUE;
426344961713Sgirish 				zcp_err_status = stat.bits.ldw.zcp_eop_err;
426444961713Sgirish 				ipp_err_status = stat.bits.ldw.ipp_eop_err;
426544961713Sgirish 			}
426644961713Sgirish 			break;
426744961713Sgirish 		default:
426844961713Sgirish 			return (NXGE_ERROR);
426944961713Sgirish 		}
427044961713Sgirish 	}
427144961713Sgirish 
427244961713Sgirish 	if (my_err) {
427344961713Sgirish 		status = nxge_rxdma_handle_port_errors(nxgep, ipp_err_status,
427444961713Sgirish 							zcp_err_status);
427544961713Sgirish 		if (status != NXGE_OK)
427644961713Sgirish 			return (status);
427744961713Sgirish 	}
427844961713Sgirish 
427944961713Sgirish 	return (NXGE_OK);
428044961713Sgirish }
428144961713Sgirish 
428244961713Sgirish static nxge_status_t
428344961713Sgirish nxge_rxdma_handle_port_errors(p_nxge_t nxgep, uint32_t ipp_status,
428444961713Sgirish 							uint32_t zcp_status)
428544961713Sgirish {
428644961713Sgirish 	boolean_t		rxport_fatal = B_FALSE;
428744961713Sgirish 	p_nxge_rdc_sys_stats_t	statsp;
428844961713Sgirish 	nxge_status_t		status = NXGE_OK;
428944961713Sgirish 	uint8_t			portn;
429044961713Sgirish 
429144961713Sgirish 	portn = nxgep->mac.portnum;
429244961713Sgirish 	statsp = (p_nxge_rdc_sys_stats_t)&nxgep->statsp->rdc_sys_stats;
429344961713Sgirish 
429444961713Sgirish 	if (ipp_status & (0x1 << portn)) {
429544961713Sgirish 		statsp->ipp_eop_err++;
429644961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, NULL,
429744961713Sgirish 					NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR);
429844961713Sgirish 		rxport_fatal = B_TRUE;
429944961713Sgirish 	}
430044961713Sgirish 
430144961713Sgirish 	if (zcp_status & (0x1 << portn)) {
430244961713Sgirish 		statsp->zcp_eop_err++;
430344961713Sgirish 		NXGE_FM_REPORT_ERROR(nxgep, portn, NULL,
430444961713Sgirish 					NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR);
430544961713Sgirish 		rxport_fatal = B_TRUE;
430644961713Sgirish 	}
430744961713Sgirish 
430844961713Sgirish 	if (rxport_fatal) {
430944961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
431044961713Sgirish 			    " nxge_rxdma_handle_port_error: "
431144961713Sgirish 			    " fatal error on Port #%d\n",
431244961713Sgirish 				portn));
431344961713Sgirish 		status = nxge_rx_port_fatal_err_recover(nxgep);
431444961713Sgirish 		if (status == NXGE_OK) {
431544961713Sgirish 			FM_SERVICE_RESTORED(nxgep);
431644961713Sgirish 		}
431744961713Sgirish 	}
431844961713Sgirish 
431944961713Sgirish 	return (status);
432044961713Sgirish }
432144961713Sgirish 
432244961713Sgirish static nxge_status_t
432344961713Sgirish nxge_rxdma_fatal_err_recover(p_nxge_t nxgep, uint16_t channel)
432444961713Sgirish {
432544961713Sgirish 	npi_handle_t		handle;
432644961713Sgirish 	npi_status_t		rs = NPI_SUCCESS;
432744961713Sgirish 	nxge_status_t		status = NXGE_OK;
432844961713Sgirish 	p_rx_rbr_ring_t		rbrp;
432944961713Sgirish 	p_rx_rcr_ring_t		rcrp;
433044961713Sgirish 	p_rx_mbox_t		mboxp;
433144961713Sgirish 	rx_dma_ent_msk_t	ent_mask;
433244961713Sgirish 	p_nxge_dma_common_t	dmap;
433344961713Sgirish 	int			ring_idx;
433444961713Sgirish 	uint32_t		ref_cnt;
433544961713Sgirish 	p_rx_msg_t		rx_msg_p;
433644961713Sgirish 	int			i;
433744961713Sgirish 	uint32_t		nxge_port_rcr_size;
433844961713Sgirish 
433944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rxdma_fatal_err_recover"));
434044961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
434144961713Sgirish 			"Recovering from RxDMAChannel#%d error...", channel));
434244961713Sgirish 
434344961713Sgirish 	/*
434444961713Sgirish 	 * Stop the dma channel waits for the stop done.
434544961713Sgirish 	 * If the stop done bit is not set, then create
434644961713Sgirish 	 * an error.
434744961713Sgirish 	 */
434844961713Sgirish 
434944961713Sgirish 	handle = NXGE_DEV_NPI_HANDLE(nxgep);
435044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Rx DMA stop..."));
435144961713Sgirish 
435244961713Sgirish 	ring_idx = nxge_rxdma_get_ring_index(nxgep, channel);
435344961713Sgirish 	rbrp = (p_rx_rbr_ring_t)nxgep->rx_rbr_rings->rbr_rings[ring_idx];
435444961713Sgirish 	rcrp = (p_rx_rcr_ring_t)nxgep->rx_rcr_rings->rcr_rings[ring_idx];
435544961713Sgirish 
435644961713Sgirish 	MUTEX_ENTER(&rcrp->lock);
435744961713Sgirish 	MUTEX_ENTER(&rbrp->lock);
435844961713Sgirish 	MUTEX_ENTER(&rbrp->post_lock);
435944961713Sgirish 
436044961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA channel..."));
436144961713Sgirish 
436244961713Sgirish 	rs = npi_rxdma_cfg_rdc_disable(handle, channel);
436344961713Sgirish 	if (rs != NPI_SUCCESS) {
436444961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
436544961713Sgirish 			"nxge_disable_rxdma_channel:failed"));
436644961713Sgirish 		goto fail;
436744961713Sgirish 	}
436844961713Sgirish 
436944961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disable RxDMA interrupt..."));
437044961713Sgirish 
437144961713Sgirish 	/* Disable interrupt */
437244961713Sgirish 	ent_mask.value = RX_DMA_ENT_MSK_ALL;
437344961713Sgirish 	rs = npi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
437444961713Sgirish 	if (rs != NPI_SUCCESS) {
437544961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
437644961713Sgirish 				"nxge_rxdma_stop_channel: "
437744961713Sgirish 				"set rxdma event masks failed (channel %d)",
437844961713Sgirish 				channel));
437944961713Sgirish 	}
438044961713Sgirish 
438144961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel reset..."));
438244961713Sgirish 
438344961713Sgirish 	/* Reset RXDMA channel */
438444961713Sgirish 	rs = npi_rxdma_cfg_rdc_reset(handle, channel);
438544961713Sgirish 	if (rs != NPI_SUCCESS) {
438644961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
438744961713Sgirish 			"nxge_rxdma_fatal_err_recover: "
438844961713Sgirish 				" reset rxdma failed (channel %d)", channel));
438944961713Sgirish 		goto fail;
439044961713Sgirish 	}
439144961713Sgirish 
439244961713Sgirish 	nxge_port_rcr_size = nxgep->nxge_port_rcr_size;
439344961713Sgirish 
439444961713Sgirish 	mboxp =
439544961713Sgirish 	(p_rx_mbox_t)nxgep->rx_mbox_areas_p->rxmbox_areas[ring_idx];
439644961713Sgirish 
439744961713Sgirish 	rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
439844961713Sgirish 	rbrp->rbr_rd_index = 0;
439944961713Sgirish 
440044961713Sgirish 	rcrp->comp_rd_index = 0;
440144961713Sgirish 	rcrp->comp_wt_index = 0;
440244961713Sgirish 	rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
440344961713Sgirish 		(p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
440444961713Sgirish 	rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
4405adfcba55Sjoycey #if defined(__i386)
4406adfcba55Sjoycey 		(p_rcr_entry_t)(uint32_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
4407adfcba55Sjoycey #else
440844961713Sgirish 		(p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
4409adfcba55Sjoycey #endif
441044961713Sgirish 
441144961713Sgirish 	rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
441244961713Sgirish 		(nxge_port_rcr_size - 1);
441344961713Sgirish 	rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
441444961713Sgirish 		(nxge_port_rcr_size - 1);
441544961713Sgirish 
441644961713Sgirish 	dmap = (p_nxge_dma_common_t)&rcrp->rcr_desc;
441744961713Sgirish 	bzero((caddr_t)dmap->kaddrp, dmap->alength);
441844961713Sgirish 
441944961713Sgirish 	cmn_err(CE_NOTE, "!rbr entries = %d\n", rbrp->rbr_max_size);
442044961713Sgirish 
442144961713Sgirish 	for (i = 0; i < rbrp->rbr_max_size; i++) {
442244961713Sgirish 		rx_msg_p = rbrp->rx_msg_ring[i];
442344961713Sgirish 		ref_cnt = rx_msg_p->ref_cnt;
442444961713Sgirish 		if (ref_cnt != 1) {
4425a3c5bd6dSspeer 			if (rx_msg_p->cur_usage_cnt !=
4426a3c5bd6dSspeer 					rx_msg_p->max_usage_cnt) {
442744961713Sgirish 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
442844961713Sgirish 						"buf[%d]: cur_usage_cnt = %d "
442944961713Sgirish 						"max_usage_cnt = %d\n", i,
443044961713Sgirish 						rx_msg_p->cur_usage_cnt,
443144961713Sgirish 						rx_msg_p->max_usage_cnt));
4432a3c5bd6dSspeer 			} else {
4433a3c5bd6dSspeer 				/* Buffer can be re-posted */
4434a3c5bd6dSspeer 				rx_msg_p->free = B_TRUE;
4435a3c5bd6dSspeer 				rx_msg_p->cur_usage_cnt = 0;
4436a3c5bd6dSspeer 				rx_msg_p->max_usage_cnt = 0xbaddcafe;
4437a3c5bd6dSspeer 				rx_msg_p->pkt_buf_size = 0;
4438a3c5bd6dSspeer 			}
443944961713Sgirish 		}
444044961713Sgirish 	}
444144961713Sgirish 
444244961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "RxDMA channel re-start..."));
444344961713Sgirish 
444444961713Sgirish 	status = nxge_rxdma_start_channel(nxgep, channel, rbrp, rcrp, mboxp);
444544961713Sgirish 	if (status != NXGE_OK) {
444644961713Sgirish 		goto fail;
444744961713Sgirish 	}
444844961713Sgirish 
444944961713Sgirish 	MUTEX_EXIT(&rbrp->post_lock);
445044961713Sgirish 	MUTEX_EXIT(&rbrp->lock);
445144961713Sgirish 	MUTEX_EXIT(&rcrp->lock);
445244961713Sgirish 
445344961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
445444961713Sgirish 			"Recovery Successful, RxDMAChannel#%d Restored",
445544961713Sgirish 			channel));
445644961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "==> nxge_rxdma_fatal_err_recover"));
445744961713Sgirish 
445844961713Sgirish 	return (NXGE_OK);
445944961713Sgirish fail:
446044961713Sgirish 	MUTEX_EXIT(&rbrp->post_lock);
446144961713Sgirish 	MUTEX_EXIT(&rbrp->lock);
446244961713Sgirish 	MUTEX_EXIT(&rcrp->lock);
446344961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
446444961713Sgirish 
446544961713Sgirish 	return (NXGE_ERROR | rs);
446644961713Sgirish }
446744961713Sgirish 
446844961713Sgirish nxge_status_t
446944961713Sgirish nxge_rx_port_fatal_err_recover(p_nxge_t nxgep)
447044961713Sgirish {
4471678453a8Sspeer 	nxge_grp_set_t *set = &nxgep->rx_set;
4472678453a8Sspeer 	nxge_status_t status = NXGE_OK;
4473678453a8Sspeer 	int rdc;
447444961713Sgirish 
447544961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "<== nxge_rx_port_fatal_err_recover"));
447644961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
447744961713Sgirish 				"Recovering from RxPort error..."));
4478678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Disabling RxMAC...\n"));
447944961713Sgirish 
448044961713Sgirish 	if (nxge_rx_mac_disable(nxgep) != NXGE_OK)
448144961713Sgirish 		goto fail;
448244961713Sgirish 
448344961713Sgirish 	NXGE_DELAY(1000);
448444961713Sgirish 
4485678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Stopping all RxDMA channels..."));
448644961713Sgirish 
4487678453a8Sspeer 	for (rdc = 0; rdc < NXGE_MAX_RDCS; rdc++) {
4488678453a8Sspeer 		if ((1 << rdc) & set->owned.map) {
4489678453a8Sspeer 			if (nxge_rxdma_fatal_err_recover(nxgep, rdc)
4490678453a8Sspeer 			    != NXGE_OK) {
4491678453a8Sspeer 				NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
4492678453a8Sspeer 				    "Could not recover channel %d", rdc));
4493678453a8Sspeer 			}
449444961713Sgirish 		}
449544961713Sgirish 	}
449644961713Sgirish 
4497678453a8Sspeer 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Resetting IPP..."));
449844961713Sgirish 
449944961713Sgirish 	/* Reset IPP */
450044961713Sgirish 	if (nxge_ipp_reset(nxgep) != NXGE_OK) {
450144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
450244961713Sgirish 			"nxge_rx_port_fatal_err_recover: "
450344961713Sgirish 			"Failed to reset IPP"));
450444961713Sgirish 		goto fail;
450544961713Sgirish 	}
450644961713Sgirish 
450744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Reset RxMAC..."));
450844961713Sgirish 
450944961713Sgirish 	/* Reset RxMAC */
451044961713Sgirish 	if (nxge_rx_mac_reset(nxgep) != NXGE_OK) {
451144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
451244961713Sgirish 			"nxge_rx_port_fatal_err_recover: "
451344961713Sgirish 			"Failed to reset RxMAC"));
451444961713Sgirish 		goto fail;
451544961713Sgirish 	}
451644961713Sgirish 
451744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize IPP..."));
451844961713Sgirish 
451944961713Sgirish 	/* Re-Initialize IPP */
452044961713Sgirish 	if (nxge_ipp_init(nxgep) != NXGE_OK) {
452144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
452244961713Sgirish 			"nxge_rx_port_fatal_err_recover: "
452344961713Sgirish 			"Failed to init IPP"));
452444961713Sgirish 		goto fail;
452544961713Sgirish 	}
452644961713Sgirish 
452744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-initialize RxMAC..."));
452844961713Sgirish 
452944961713Sgirish 	/* Re-Initialize RxMAC */
453044961713Sgirish 	if ((status = nxge_rx_mac_init(nxgep)) != NXGE_OK) {
453144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
453244961713Sgirish 			"nxge_rx_port_fatal_err_recover: "
453344961713Sgirish 			"Failed to reset RxMAC"));
453444961713Sgirish 		goto fail;
453544961713Sgirish 	}
453644961713Sgirish 
453744961713Sgirish 	NXGE_DEBUG_MSG((nxgep, RX_CTL, "Re-enable RxMAC..."));
453844961713Sgirish 
453944961713Sgirish 	/* Re-enable RxMAC */
454044961713Sgirish 	if ((status = nxge_rx_mac_enable(nxgep)) != NXGE_OK) {
454144961713Sgirish 		NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
454244961713Sgirish 			"nxge_rx_port_fatal_err_recover: "
454344961713Sgirish 			"Failed to enable RxMAC"));
454444961713Sgirish 		goto fail;
454544961713Sgirish 	}
454644961713Sgirish 
454744961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
454844961713Sgirish 			"Recovery Successful, RxPort Restored"));
454944961713Sgirish 
455044961713Sgirish 	return (NXGE_OK);
455144961713Sgirish fail:
455244961713Sgirish 	NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL, "Recovery failed"));
455344961713Sgirish 	return (status);
455444961713Sgirish }
455544961713Sgirish 
455644961713Sgirish void
455744961713Sgirish nxge_rxdma_inject_err(p_nxge_t nxgep, uint32_t err_id, uint8_t chan)
455844961713Sgirish {
455944961713Sgirish 	rx_dma_ctl_stat_t	cs;
456044961713Sgirish 	rx_ctl_dat_fifo_stat_t	cdfs;
456144961713Sgirish 
456244961713Sgirish 	switch (err_id) {
456344961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR:
456444961713Sgirish 	case NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR:
456544961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR:
456644961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR:
456744961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RBR_TMOUT:
456844961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR:
456944961713Sgirish 	case NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS:
457044961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR:
457144961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RCRINCON:
457244961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RCRFULL:
457344961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RBRFULL:
457444961713Sgirish 	case NXGE_FM_EREPORT_RDMC_RBRLOGPAGE:
457544961713Sgirish 	case NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE:
457644961713Sgirish 	case NXGE_FM_EREPORT_RDMC_CONFIG_ERR:
457744961713Sgirish 		RXDMA_REG_READ64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG,
457844961713Sgirish 			chan, &cs.value);
457944961713Sgirish 		if (err_id == NXGE_FM_EREPORT_RDMC_RCR_ACK_ERR)
458044961713Sgirish 			cs.bits.hdw.rcr_ack_err = 1;
458144961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_DC_FIFO_ERR)
458244961713Sgirish 			cs.bits.hdw.dc_fifo_err = 1;
458344961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCR_SHA_PAR)
458444961713Sgirish 			cs.bits.hdw.rcr_sha_par = 1;
458544961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_PRE_PAR)
458644961713Sgirish 			cs.bits.hdw.rbr_pre_par = 1;
458744961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBR_TMOUT)
458844961713Sgirish 			cs.bits.hdw.rbr_tmout = 1;
458944961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_CNT_ERR)
459044961713Sgirish 			cs.bits.hdw.rsp_cnt_err = 1;
459144961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_BYTE_EN_BUS)
459244961713Sgirish 			cs.bits.hdw.byte_en_bus = 1;
459344961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RSP_DAT_ERR)
459444961713Sgirish 			cs.bits.hdw.rsp_dat_err = 1;
459544961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_CONFIG_ERR)
459644961713Sgirish 			cs.bits.hdw.config_err = 1;
459744961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCRINCON)
459844961713Sgirish 			cs.bits.hdw.rcrincon = 1;
459944961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RCRFULL)
460044961713Sgirish 			cs.bits.hdw.rcrfull = 1;
460144961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBRFULL)
460244961713Sgirish 			cs.bits.hdw.rbrfull = 1;
460344961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_RBRLOGPAGE)
460444961713Sgirish 			cs.bits.hdw.rbrlogpage = 1;
460544961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_CFIGLOGPAGE)
460644961713Sgirish 			cs.bits.hdw.cfiglogpage = 1;
4607adfcba55Sjoycey #if defined(__i386)
4608adfcba55Sjoycey 		cmn_err(CE_NOTE, "!Write 0x%llx to RX_DMA_CTL_STAT_DBG_REG\n",
4609adfcba55Sjoycey 				cs.value);
4610adfcba55Sjoycey #else
461144961713Sgirish 		cmn_err(CE_NOTE, "!Write 0x%lx to RX_DMA_CTL_STAT_DBG_REG\n",
461244961713Sgirish 				cs.value);
4613adfcba55Sjoycey #endif
461444961713Sgirish 		RXDMA_REG_WRITE64(nxgep->npi_handle, RX_DMA_CTL_STAT_DBG_REG,
461544961713Sgirish 			chan, cs.value);
461644961713Sgirish 		break;
461744961713Sgirish 	case NXGE_FM_EREPORT_RDMC_ID_MISMATCH:
461844961713Sgirish 	case NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR:
461944961713Sgirish 	case NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR:
462044961713Sgirish 		cdfs.value = 0;
462144961713Sgirish 		if (err_id ==  NXGE_FM_EREPORT_RDMC_ID_MISMATCH)
462244961713Sgirish 			cdfs.bits.ldw.id_mismatch = (1 << nxgep->mac.portnum);
462344961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_ZCP_EOP_ERR)
462444961713Sgirish 			cdfs.bits.ldw.zcp_eop_err = (1 << nxgep->mac.portnum);
462544961713Sgirish 		else if (err_id == NXGE_FM_EREPORT_RDMC_IPP_EOP_ERR)
462644961713Sgirish 			cdfs.bits.ldw.ipp_eop_err = (1 << nxgep->mac.portnum);
4627adfcba55Sjoycey #if defined(__i386)
4628adfcba55Sjoycey 		cmn_err(CE_NOTE,
4629adfcba55Sjoycey 			"!Write 0x%llx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n",
4630adfcba55Sjoycey 			cdfs.value);
4631adfcba55Sjoycey #else
463244961713Sgirish 		cmn_err(CE_NOTE,
463344961713Sgirish 			"!Write 0x%lx to RX_CTL_DAT_FIFO_STAT_DBG_REG\n",
463444961713Sgirish 			cdfs.value);
4635adfcba55Sjoycey #endif
4636678453a8Sspeer 		NXGE_REG_WR64(nxgep->npi_handle,
4637678453a8Sspeer 		    RX_CTL_DAT_FIFO_STAT_DBG_REG, cdfs.value);
463844961713Sgirish 		break;
463944961713Sgirish 	case NXGE_FM_EREPORT_RDMC_DCF_ERR:
464044961713Sgirish 		break;
464153f3d8ecSyc 	case NXGE_FM_EREPORT_RDMC_RCR_ERR:
464244961713Sgirish 		break;
464344961713Sgirish 	}
464444961713Sgirish }
4645678453a8Sspeer 
4646678453a8Sspeer static void
4647678453a8Sspeer nxge_rxdma_databuf_free(p_rx_rbr_ring_t rbr_p)
4648678453a8Sspeer {
4649678453a8Sspeer 	rxring_info_t 		*ring_info;
4650678453a8Sspeer 	int			index;
4651678453a8Sspeer 	uint32_t		chunk_size;
4652678453a8Sspeer 	uint64_t		kaddr;
4653678453a8Sspeer 	uint_t			num_blocks;
4654678453a8Sspeer 
4655678453a8Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_rxdma_databuf_free"));
4656678453a8Sspeer 
4657678453a8Sspeer 	if (rbr_p == NULL) {
4658678453a8Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4659678453a8Sspeer 		    "==> nxge_rxdma_databuf_free: NULL rbr pointer"));
4660678453a8Sspeer 		return;
4661678453a8Sspeer 	}
4662678453a8Sspeer 
4663678453a8Sspeer 	if (rbr_p->rbr_alloc_type == DDI_MEM_ALLOC) {
4664678453a8Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4665678453a8Sspeer 		    "==> nxge_rxdma_databuf_free: DDI"));
4666678453a8Sspeer 		return;
4667678453a8Sspeer 	}
4668678453a8Sspeer 
4669678453a8Sspeer 	ring_info = rbr_p->ring_info;
4670678453a8Sspeer 	if (ring_info == NULL) {
4671678453a8Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4672678453a8Sspeer 		    "==> nxge_rxdma_databuf_free: NULL ring info"));
4673678453a8Sspeer 		return;
4674678453a8Sspeer 	}
4675678453a8Sspeer 	num_blocks = rbr_p->num_blocks;
4676678453a8Sspeer 	for (index = 0; index < num_blocks; index++) {
4677678453a8Sspeer 		kaddr = ring_info->buffer[index].kaddr;
4678678453a8Sspeer 		chunk_size = ring_info->buffer[index].buf_size;
4679678453a8Sspeer 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
4680678453a8Sspeer 		    "==> nxge_rxdma_databuf_free: free chunk %d "
4681678453a8Sspeer 		    "kaddrp $%p chunk size %d",
4682678453a8Sspeer 		    index, kaddr, chunk_size));
4683678453a8Sspeer 		if (kaddr == NULL) continue;
4684678453a8Sspeer 		nxge_free_buf(rbr_p->rbr_alloc_type, kaddr, chunk_size);
4685678453a8Sspeer 		ring_info->buffer[index].kaddr = NULL;
4686678453a8Sspeer 	}
4687678453a8Sspeer 
4688678453a8Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_rxdma_databuf_free"));
4689678453a8Sspeer }
4690678453a8Sspeer 
4691678453a8Sspeer #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
4692678453a8Sspeer extern void contig_mem_free(void *, size_t);
4693678453a8Sspeer #endif
4694678453a8Sspeer 
4695678453a8Sspeer void
4696678453a8Sspeer nxge_free_buf(buf_alloc_type_t alloc_type, uint64_t kaddr, uint32_t buf_size)
4697678453a8Sspeer {
4698678453a8Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "==> nxge_free_buf"));
4699678453a8Sspeer 
4700678453a8Sspeer 	if (kaddr == NULL || !buf_size) {
4701678453a8Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4702678453a8Sspeer 		    "==> nxge_free_buf: invalid kaddr $%p size to free %d",
4703678453a8Sspeer 		    kaddr, buf_size));
4704678453a8Sspeer 		return;
4705678453a8Sspeer 	}
4706678453a8Sspeer 
4707678453a8Sspeer 	switch (alloc_type) {
4708678453a8Sspeer 	case KMEM_ALLOC:
4709678453a8Sspeer 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
4710678453a8Sspeer 		    "==> nxge_free_buf: freeing kmem $%p size %d",
4711678453a8Sspeer 		    kaddr, buf_size));
4712678453a8Sspeer #if defined(__i386)
4713678453a8Sspeer 		KMEM_FREE((void *)(uint32_t)kaddr, buf_size);
4714678453a8Sspeer #else
4715678453a8Sspeer 		KMEM_FREE((void *)kaddr, buf_size);
4716678453a8Sspeer #endif
4717678453a8Sspeer 		break;
4718678453a8Sspeer 
4719678453a8Sspeer #if	defined(sun4v) && defined(NIU_LP_WORKAROUND)
4720678453a8Sspeer 	case CONTIG_MEM_ALLOC:
4721678453a8Sspeer 		NXGE_DEBUG_MSG((NULL, DMA_CTL,
4722678453a8Sspeer 		    "==> nxge_free_buf: freeing contig_mem kaddr $%p size %d",
4723678453a8Sspeer 		    kaddr, buf_size));
4724678453a8Sspeer 		contig_mem_free((void *)kaddr, buf_size);
4725678453a8Sspeer 		break;
4726678453a8Sspeer #endif
4727678453a8Sspeer 
4728678453a8Sspeer 	default:
4729678453a8Sspeer 		NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
4730678453a8Sspeer 		    "<== nxge_free_buf: unsupported alloc type %d",
4731678453a8Sspeer 		    alloc_type));
4732678453a8Sspeer 		return;
4733678453a8Sspeer 	}
4734678453a8Sspeer 
4735678453a8Sspeer 	NXGE_DEBUG_MSG((NULL, DMA_CTL, "<== nxge_free_buf"));
4736678453a8Sspeer }
4737