19da57d7bSbt /*
29da57d7bSbt * CDDL HEADER START
39da57d7bSbt *
49da57d7bSbt * The contents of this file are subject to the terms of the
59da57d7bSbt * Common Development and Distribution License (the "License").
69da57d7bSbt * You may not use this file except in compliance with the License.
79da57d7bSbt *
8da14cebeSEric Cheng * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9da14cebeSEric Cheng * or http://www.opensolaris.org/os/licensing.
109da57d7bSbt * See the License for the specific language governing permissions
119da57d7bSbt * and limitations under the License.
129da57d7bSbt *
13da14cebeSEric Cheng * When distributing Covered Code, include this CDDL HEADER in each
14da14cebeSEric Cheng * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159da57d7bSbt * If applicable, add the following below this CDDL HEADER, with the
169da57d7bSbt * fields enclosed by brackets "[]" replaced with your own identifying
179da57d7bSbt * information: Portions Copyright [yyyy] [name of copyright owner]
189da57d7bSbt *
199da57d7bSbt * CDDL HEADER END
209da57d7bSbt */
219da57d7bSbt
22da14cebeSEric Cheng /*
235b6dd21fSchenlu chen - Sun Microsystems - Beijing China * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
245b6dd21fSchenlu chen - Sun Microsystems - Beijing China */
255b6dd21fSchenlu chen - Sun Microsystems - Beijing China
265b6dd21fSchenlu chen - Sun Microsystems - Beijing China /*
275b6dd21fSchenlu chen - Sun Microsystems - Beijing China * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
28*a9bfd41dSRyan Zezeski * Copyright 2017 Joyent, Inc.
29da14cebeSEric Cheng */
309da57d7bSbt
319da57d7bSbt #include "ixgbe_sw.h"
329da57d7bSbt
339da57d7bSbt /* function prototypes */
34ea65739eSchenlu chen - Sun Microsystems - Beijing China static mblk_t *ixgbe_rx_bind(ixgbe_rx_data_t *, uint32_t, uint32_t);
35ea65739eSchenlu chen - Sun Microsystems - Beijing China static mblk_t *ixgbe_rx_copy(ixgbe_rx_data_t *, uint32_t, uint32_t);
369da57d7bSbt static void ixgbe_rx_assoc_hcksum(mblk_t *, uint32_t);
37ffd8e883SWinson Wang - Sun Microsystems - Beijing China static mblk_t *ixgbe_lro_bind(ixgbe_rx_data_t *, uint32_t, uint32_t, uint32_t);
38ffd8e883SWinson Wang - Sun Microsystems - Beijing China static mblk_t *ixgbe_lro_copy(ixgbe_rx_data_t *, uint32_t, uint32_t, uint32_t);
39ffd8e883SWinson Wang - Sun Microsystems - Beijing China static int ixgbe_lro_get_start(ixgbe_rx_data_t *, uint32_t);
40ffd8e883SWinson Wang - Sun Microsystems - Beijing China static uint32_t ixgbe_lro_get_first(ixgbe_rx_data_t *, uint32_t);
419da57d7bSbt
429da57d7bSbt #ifndef IXGBE_DEBUG
439da57d7bSbt #pragma inline(ixgbe_rx_assoc_hcksum)
44ffd8e883SWinson Wang - Sun Microsystems - Beijing China #pragma inline(ixgbe_lro_get_start)
45ffd8e883SWinson Wang - Sun Microsystems - Beijing China #pragma inline(ixgbe_lro_get_first)
469da57d7bSbt #endif
479da57d7bSbt
489da57d7bSbt /*
499da57d7bSbt * ixgbe_rx_recycle - The call-back function to reclaim rx buffer.
509da57d7bSbt *
519da57d7bSbt * This function is called when an mp is freed by the user thru
529da57d7bSbt * freeb call (Only for mp constructed through desballoc call).
539da57d7bSbt * It returns back the freed buffer to the free list.
549da57d7bSbt */
559da57d7bSbt void
ixgbe_rx_recycle(caddr_t arg)569da57d7bSbt ixgbe_rx_recycle(caddr_t arg)
579da57d7bSbt {
58ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_t *ixgbe;
599da57d7bSbt ixgbe_rx_ring_t *rx_ring;
60ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_rx_data_t *rx_data;
619da57d7bSbt rx_control_block_t *recycle_rcb;
629da57d7bSbt uint32_t free_index;
63ea65739eSchenlu chen - Sun Microsystems - Beijing China uint32_t ref_cnt;
649da57d7bSbt
659da57d7bSbt recycle_rcb = (rx_control_block_t *)(uintptr_t)arg;
66ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_data = recycle_rcb->rx_data;
67ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_ring = rx_data->rx_ring;
68ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe = rx_ring->ixgbe;
699da57d7bSbt
70ea65739eSchenlu chen - Sun Microsystems - Beijing China if (recycle_rcb->ref_cnt == 0) {
71ea65739eSchenlu chen - Sun Microsystems - Beijing China /*
72ea65739eSchenlu chen - Sun Microsystems - Beijing China * This case only happens when rx buffers are being freed
73ea65739eSchenlu chen - Sun Microsystems - Beijing China * in ixgbe_stop() and freemsg() is called.
74ea65739eSchenlu chen - Sun Microsystems - Beijing China */
759da57d7bSbt return;
76ea65739eSchenlu chen - Sun Microsystems - Beijing China }
779da57d7bSbt
789da57d7bSbt ASSERT(recycle_rcb->mp == NULL);
799da57d7bSbt
809da57d7bSbt /*
819da57d7bSbt * Using the recycled data buffer to generate a new mblk
829da57d7bSbt */
839da57d7bSbt recycle_rcb->mp = desballoc((unsigned char *)
8473cd555cSBin Tu - Sun Microsystems - Beijing China recycle_rcb->rx_buf.address,
8573cd555cSBin Tu - Sun Microsystems - Beijing China recycle_rcb->rx_buf.size,
869da57d7bSbt 0, &recycle_rcb->free_rtn);
879da57d7bSbt
889da57d7bSbt /*
899da57d7bSbt * Put the recycled rx control block into free list
909da57d7bSbt */
91ea65739eSchenlu chen - Sun Microsystems - Beijing China mutex_enter(&rx_data->recycle_lock);
929da57d7bSbt
93ea65739eSchenlu chen - Sun Microsystems - Beijing China free_index = rx_data->rcb_tail;
94ea65739eSchenlu chen - Sun Microsystems - Beijing China ASSERT(rx_data->free_list[free_index] == NULL);
959da57d7bSbt
96ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_data->free_list[free_index] = recycle_rcb;
97ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_data->rcb_tail = NEXT_INDEX(free_index, 1, rx_data->free_list_size);
989da57d7bSbt
99ea65739eSchenlu chen - Sun Microsystems - Beijing China mutex_exit(&rx_data->recycle_lock);
1009da57d7bSbt
1019da57d7bSbt /*
1029da57d7bSbt * The atomic operation on the number of the available rx control
1039da57d7bSbt * blocks in the free list is used to make the recycling mutual
1049da57d7bSbt * exclusive with the receiving.
1059da57d7bSbt */
106ea65739eSchenlu chen - Sun Microsystems - Beijing China atomic_inc_32(&rx_data->rcb_free);
107ea65739eSchenlu chen - Sun Microsystems - Beijing China ASSERT(rx_data->rcb_free <= rx_data->free_list_size);
108ea65739eSchenlu chen - Sun Microsystems - Beijing China
109ea65739eSchenlu chen - Sun Microsystems - Beijing China /*
110ea65739eSchenlu chen - Sun Microsystems - Beijing China * Considering the case that the interface is unplumbed
111ea65739eSchenlu chen - Sun Microsystems - Beijing China * and there are still some buffers held by the upper layer.
112ea65739eSchenlu chen - Sun Microsystems - Beijing China * When the buffer is returned back, we need to free it.
113ea65739eSchenlu chen - Sun Microsystems - Beijing China */
114ea65739eSchenlu chen - Sun Microsystems - Beijing China ref_cnt = atomic_dec_32_nv(&recycle_rcb->ref_cnt);
115ea65739eSchenlu chen - Sun Microsystems - Beijing China if (ref_cnt == 0) {
116ea65739eSchenlu chen - Sun Microsystems - Beijing China if (recycle_rcb->mp != NULL) {
117ea65739eSchenlu chen - Sun Microsystems - Beijing China freemsg(recycle_rcb->mp);
118ea65739eSchenlu chen - Sun Microsystems - Beijing China recycle_rcb->mp = NULL;
119ea65739eSchenlu chen - Sun Microsystems - Beijing China }
120ea65739eSchenlu chen - Sun Microsystems - Beijing China
121ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_free_dma_buffer(&recycle_rcb->rx_buf);
122ea65739eSchenlu chen - Sun Microsystems - Beijing China
123ea65739eSchenlu chen - Sun Microsystems - Beijing China mutex_enter(&ixgbe->rx_pending_lock);
124ea65739eSchenlu chen - Sun Microsystems - Beijing China atomic_dec_32(&rx_data->rcb_pending);
125ea65739eSchenlu chen - Sun Microsystems - Beijing China atomic_dec_32(&ixgbe->rcb_pending);
126ea65739eSchenlu chen - Sun Microsystems - Beijing China
127ea65739eSchenlu chen - Sun Microsystems - Beijing China /*
128ea65739eSchenlu chen - Sun Microsystems - Beijing China * When there is not any buffer belonging to this rx_data
129ea65739eSchenlu chen - Sun Microsystems - Beijing China * held by the upper layer, the rx_data can be freed.
130ea65739eSchenlu chen - Sun Microsystems - Beijing China */
131ea65739eSchenlu chen - Sun Microsystems - Beijing China if ((rx_data->flag & IXGBE_RX_STOPPED) &&
132ea65739eSchenlu chen - Sun Microsystems - Beijing China (rx_data->rcb_pending == 0))
133ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_free_rx_ring_data(rx_data);
134ea65739eSchenlu chen - Sun Microsystems - Beijing China
135ea65739eSchenlu chen - Sun Microsystems - Beijing China mutex_exit(&ixgbe->rx_pending_lock);
136ea65739eSchenlu chen - Sun Microsystems - Beijing China }
1379da57d7bSbt }
1389da57d7bSbt
1399da57d7bSbt /*
1409da57d7bSbt * ixgbe_rx_copy - Use copy to process the received packet.
1419da57d7bSbt *
1429da57d7bSbt * This function will use bcopy to process the packet
1439da57d7bSbt * and send the copied packet upstream.
1449da57d7bSbt */
1459da57d7bSbt static mblk_t *
ixgbe_rx_copy(ixgbe_rx_data_t * rx_data,uint32_t index,uint32_t pkt_len)146ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_rx_copy(ixgbe_rx_data_t *rx_data, uint32_t index, uint32_t pkt_len)
1479da57d7bSbt {
148ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_t *ixgbe;
1499da57d7bSbt rx_control_block_t *current_rcb;
1509da57d7bSbt mblk_t *mp;
1519da57d7bSbt
152ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe = rx_data->rx_ring->ixgbe;
153ea65739eSchenlu chen - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[index];
1549da57d7bSbt
1559da57d7bSbt DMA_SYNC(¤t_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
1569da57d7bSbt
1579da57d7bSbt if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
1589da57d7bSbt DDI_FM_OK) {
159ea65739eSchenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
16062e6e1adSPaul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
16162e6e1adSPaul Guo return (NULL);
1629da57d7bSbt }
1639da57d7bSbt
1649da57d7bSbt /*
1659da57d7bSbt * Allocate buffer to receive this packet
1669da57d7bSbt */
1679da57d7bSbt mp = allocb(pkt_len + IPHDR_ALIGN_ROOM, 0);
1689da57d7bSbt if (mp == NULL) {
169ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_log(ixgbe, "ixgbe_rx_copy: allocate buffer failed");
1709da57d7bSbt return (NULL);
1719da57d7bSbt }
1729da57d7bSbt
1739da57d7bSbt /*
1749da57d7bSbt * Copy the data received into the new cluster
1759da57d7bSbt */
1769da57d7bSbt mp->b_rptr += IPHDR_ALIGN_ROOM;
1779da57d7bSbt bcopy(current_rcb->rx_buf.address, mp->b_rptr, pkt_len);
1789da57d7bSbt mp->b_wptr = mp->b_rptr + pkt_len;
1799da57d7bSbt
1809da57d7bSbt return (mp);
1819da57d7bSbt }
1829da57d7bSbt
1839da57d7bSbt /*
1849da57d7bSbt * ixgbe_rx_bind - Use existing DMA buffer to build mblk for receiving.
1859da57d7bSbt *
1869da57d7bSbt * This function will use pre-bound DMA buffer to receive the packet
1879da57d7bSbt * and build mblk that will be sent upstream.
1889da57d7bSbt */
1899da57d7bSbt static mblk_t *
ixgbe_rx_bind(ixgbe_rx_data_t * rx_data,uint32_t index,uint32_t pkt_len)190ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_rx_bind(ixgbe_rx_data_t *rx_data, uint32_t index, uint32_t pkt_len)
1919da57d7bSbt {
1929da57d7bSbt rx_control_block_t *current_rcb;
1939da57d7bSbt rx_control_block_t *free_rcb;
1949da57d7bSbt uint32_t free_index;
1959da57d7bSbt mblk_t *mp;
196ea65739eSchenlu chen - Sun Microsystems - Beijing China ixgbe_t *ixgbe = rx_data->rx_ring->ixgbe;
1979da57d7bSbt
1989da57d7bSbt /*
1999da57d7bSbt * If the free list is empty, we cannot proceed to send
2009da57d7bSbt * the current DMA buffer upstream. We'll have to return
2019da57d7bSbt * and use bcopy to process the packet.
2029da57d7bSbt */
203ea65739eSchenlu chen - Sun Microsystems - Beijing China if (ixgbe_atomic_reserve(&rx_data->rcb_free, 1) < 0)
2049da57d7bSbt return (NULL);
2059da57d7bSbt
206ea65739eSchenlu chen - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[index];
2079da57d7bSbt /*
2089da57d7bSbt * If the mp of the rx control block is NULL, try to do
2099da57d7bSbt * desballoc again.
2109da57d7bSbt */
2119da57d7bSbt if (current_rcb->mp == NULL) {
2129da57d7bSbt current_rcb->mp = desballoc((unsigned char *)
21373cd555cSBin Tu - Sun Microsystems - Beijing China current_rcb->rx_buf.address,
21473cd555cSBin Tu - Sun Microsystems - Beijing China current_rcb->rx_buf.size,
2159da57d7bSbt 0, ¤t_rcb->free_rtn);
2169da57d7bSbt /*
2179da57d7bSbt * If it is failed to built a mblk using the current
2189da57d7bSbt * DMA buffer, we have to return and use bcopy to
2199da57d7bSbt * process the packet.
2209da57d7bSbt */
22173cd555cSBin Tu - Sun Microsystems - Beijing China if (current_rcb->mp == NULL) {
222ea65739eSchenlu chen - Sun Microsystems - Beijing China atomic_inc_32(&rx_data->rcb_free);
2239da57d7bSbt return (NULL);
2249da57d7bSbt }
2259da57d7bSbt }
2269da57d7bSbt /*
2279da57d7bSbt * Sync up the data received
2289da57d7bSbt */
2299da57d7bSbt DMA_SYNC(¤t_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
2309da57d7bSbt
2319da57d7bSbt if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
2329da57d7bSbt DDI_FM_OK) {
233ea65739eSchenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
23462e6e1adSPaul Guo atomic_inc_32(&rx_data->rcb_free);
23562e6e1adSPaul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
23662e6e1adSPaul Guo return (NULL);
2379da57d7bSbt }
2389da57d7bSbt
2399da57d7bSbt mp = current_rcb->mp;
2409da57d7bSbt current_rcb->mp = NULL;
241ea65739eSchenlu chen - Sun Microsystems - Beijing China atomic_inc_32(¤t_rcb->ref_cnt);
2429da57d7bSbt
2439da57d7bSbt mp->b_wptr = mp->b_rptr + pkt_len;
2449da57d7bSbt mp->b_next = mp->b_cont = NULL;
2459da57d7bSbt
2469da57d7bSbt /*
2479da57d7bSbt * Strip off one free rx control block from the free list
2489da57d7bSbt */
249ea65739eSchenlu chen - Sun Microsystems - Beijing China free_index = rx_data->rcb_head;
250ea65739eSchenlu chen - Sun Microsystems - Beijing China free_rcb = rx_data->free_list[free_index];
2519da57d7bSbt ASSERT(free_rcb != NULL);
252ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_data->free_list[free_index] = NULL;
253ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_data->rcb_head = NEXT_INDEX(free_index, 1, rx_data->free_list_size);
2549da57d7bSbt
2559da57d7bSbt /*
2569da57d7bSbt * Put the rx control block to the work list
2579da57d7bSbt */
258ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_data->work_list[index] = free_rcb;
2599da57d7bSbt
2609da57d7bSbt return (mp);
2619da57d7bSbt }
2629da57d7bSbt
263ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
264ffd8e883SWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_bind - Use existing DMA buffer to build LRO mblk for receiving.
265ffd8e883SWinson Wang - Sun Microsystems - Beijing China *
266ffd8e883SWinson Wang - Sun Microsystems - Beijing China * This function will use pre-bound DMA buffers to receive the packet
267ffd8e883SWinson Wang - Sun Microsystems - Beijing China * and build LRO mblk that will be sent upstream.
268ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
269ffd8e883SWinson Wang - Sun Microsystems - Beijing China static mblk_t *
ixgbe_lro_bind(ixgbe_rx_data_t * rx_data,uint32_t lro_start,uint32_t lro_num,uint32_t pkt_len)270ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_lro_bind(ixgbe_rx_data_t *rx_data, uint32_t lro_start,
271ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t lro_num, uint32_t pkt_len)
272ffd8e883SWinson Wang - Sun Microsystems - Beijing China {
273ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb;
274ffd8e883SWinson Wang - Sun Microsystems - Beijing China union ixgbe_adv_rx_desc *current_rbd;
275ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *free_rcb;
276ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t free_index;
277ffd8e883SWinson Wang - Sun Microsystems - Beijing China int lro_next;
278ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t last_pkt_len;
279ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t i;
280ffd8e883SWinson Wang - Sun Microsystems - Beijing China mblk_t *mp;
281ffd8e883SWinson Wang - Sun Microsystems - Beijing China mblk_t *mblk_head;
282ffd8e883SWinson Wang - Sun Microsystems - Beijing China mblk_t **mblk_tail;
283ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_t *ixgbe = rx_data->rx_ring->ixgbe;
284ffd8e883SWinson Wang - Sun Microsystems - Beijing China
285ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
286ffd8e883SWinson Wang - Sun Microsystems - Beijing China * If the free list is empty, we cannot proceed to send
287ffd8e883SWinson Wang - Sun Microsystems - Beijing China * the current DMA buffer upstream. We'll have to return
288ffd8e883SWinson Wang - Sun Microsystems - Beijing China * and use bcopy to process the packet.
289ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
290ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (ixgbe_atomic_reserve(&rx_data->rcb_free, lro_num) < 0)
291ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (NULL);
292ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_start];
293ffd8e883SWinson Wang - Sun Microsystems - Beijing China
294ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
295ffd8e883SWinson Wang - Sun Microsystems - Beijing China * If any one of the rx data blocks can not support
296ffd8e883SWinson Wang - Sun Microsystems - Beijing China * lro bind operation, We'll have to return and use
297ffd8e883SWinson Wang - Sun Microsystems - Beijing China * bcopy to process the lro packet.
298ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
299ffd8e883SWinson Wang - Sun Microsystems - Beijing China for (i = lro_num; i > 0; i--) {
300ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
301ffd8e883SWinson Wang - Sun Microsystems - Beijing China * Sync up the data received
302ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
303ffd8e883SWinson Wang - Sun Microsystems - Beijing China DMA_SYNC(¤t_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
304ffd8e883SWinson Wang - Sun Microsystems - Beijing China
305ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
306ffd8e883SWinson Wang - Sun Microsystems - Beijing China DDI_FM_OK) {
307ffd8e883SWinson Wang - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
308ffd8e883SWinson Wang - Sun Microsystems - Beijing China atomic_add_32(&rx_data->rcb_free, lro_num);
309ffd8e883SWinson Wang - Sun Microsystems - Beijing China atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
310ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (NULL);
311ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
312ffd8e883SWinson Wang - Sun Microsystems - Beijing China
313ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
314ffd8e883SWinson Wang - Sun Microsystems - Beijing China * If the mp of the rx control block is NULL, try to do
315ffd8e883SWinson Wang - Sun Microsystems - Beijing China * desballoc again.
316ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
317ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (current_rcb->mp == NULL) {
318ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->mp = desballoc((unsigned char *)
319ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.address,
320ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.size,
321ffd8e883SWinson Wang - Sun Microsystems - Beijing China 0, ¤t_rcb->free_rtn);
322ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
323ffd8e883SWinson Wang - Sun Microsystems - Beijing China * If it is failed to built a mblk using the current
324ffd8e883SWinson Wang - Sun Microsystems - Beijing China * DMA buffer, we have to return and use bcopy to
325ffd8e883SWinson Wang - Sun Microsystems - Beijing China * process the packet.
326ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
327ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (current_rcb->mp == NULL) {
328ffd8e883SWinson Wang - Sun Microsystems - Beijing China atomic_add_32(&rx_data->rcb_free, lro_num);
329ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (NULL);
330ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
331ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
332ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (current_rcb->lro_next != -1)
333ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
334ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
335ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
336ffd8e883SWinson Wang - Sun Microsystems - Beijing China
337ffd8e883SWinson Wang - Sun Microsystems - Beijing China mblk_head = NULL;
338ffd8e883SWinson Wang - Sun Microsystems - Beijing China mblk_tail = &mblk_head;
339ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next = lro_start;
340ffd8e883SWinson Wang - Sun Microsystems - Beijing China last_pkt_len = pkt_len - ixgbe->rx_buf_size * (lro_num - 1);
341ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
342ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
343ffd8e883SWinson Wang - Sun Microsystems - Beijing China while (lro_num --) {
344ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp = current_rcb->mp;
345ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->mp = NULL;
346ffd8e883SWinson Wang - Sun Microsystems - Beijing China atomic_inc_32(¤t_rcb->ref_cnt);
347ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (lro_num != 0)
348ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp->b_wptr = mp->b_rptr + ixgbe->rx_buf_size;
349ffd8e883SWinson Wang - Sun Microsystems - Beijing China else
350ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp->b_wptr = mp->b_rptr + last_pkt_len;
351ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp->b_next = mp->b_cont = NULL;
352ffd8e883SWinson Wang - Sun Microsystems - Beijing China *mblk_tail = mp;
353ffd8e883SWinson Wang - Sun Microsystems - Beijing China mblk_tail = &mp->b_cont;
354ffd8e883SWinson Wang - Sun Microsystems - Beijing China
355ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
356ffd8e883SWinson Wang - Sun Microsystems - Beijing China * Strip off one free rx control block from the free list
357ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
358ffd8e883SWinson Wang - Sun Microsystems - Beijing China free_index = rx_data->rcb_head;
359ffd8e883SWinson Wang - Sun Microsystems - Beijing China free_rcb = rx_data->free_list[free_index];
360ffd8e883SWinson Wang - Sun Microsystems - Beijing China ASSERT(free_rcb != NULL);
361ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->free_list[free_index] = NULL;
362ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->rcb_head = NEXT_INDEX(free_index, 1,
363ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->free_list_size);
364ffd8e883SWinson Wang - Sun Microsystems - Beijing China
365ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
366ffd8e883SWinson Wang - Sun Microsystems - Beijing China * Put the rx control block to the work list
367ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
368ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[lro_next] = free_rcb;
369ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
370ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_next = -1;
371ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_prev = -1;
372ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_pkt = B_FALSE;
373ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr = free_rcb->rx_buf.dma_address;
374ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
375ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (lro_next == -1)
376ffd8e883SWinson Wang - Sun Microsystems - Beijing China break;
377ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
378ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
379ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
380ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (mblk_head);
381ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
382ffd8e883SWinson Wang - Sun Microsystems - Beijing China
383ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
384ffd8e883SWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_copy - Use copy to process the received LRO packet.
385ffd8e883SWinson Wang - Sun Microsystems - Beijing China *
386ffd8e883SWinson Wang - Sun Microsystems - Beijing China * This function will use bcopy to process the LRO packet
387ffd8e883SWinson Wang - Sun Microsystems - Beijing China * and send the copied packet upstream.
388ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
389ffd8e883SWinson Wang - Sun Microsystems - Beijing China static mblk_t *
ixgbe_lro_copy(ixgbe_rx_data_t * rx_data,uint32_t lro_start,uint32_t lro_num,uint32_t pkt_len)390ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_lro_copy(ixgbe_rx_data_t *rx_data, uint32_t lro_start,
391ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t lro_num, uint32_t pkt_len)
392ffd8e883SWinson Wang - Sun Microsystems - Beijing China {
393ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_t *ixgbe;
394ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb;
395ffd8e883SWinson Wang - Sun Microsystems - Beijing China union ixgbe_adv_rx_desc *current_rbd;
396ffd8e883SWinson Wang - Sun Microsystems - Beijing China mblk_t *mp;
397ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t last_pkt_len;
398ffd8e883SWinson Wang - Sun Microsystems - Beijing China int lro_next;
399ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t i;
400ffd8e883SWinson Wang - Sun Microsystems - Beijing China
401ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe = rx_data->rx_ring->ixgbe;
402ffd8e883SWinson Wang - Sun Microsystems - Beijing China
403ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
404ffd8e883SWinson Wang - Sun Microsystems - Beijing China * Allocate buffer to receive this LRO packet
405ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
406ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp = allocb(pkt_len + IPHDR_ALIGN_ROOM, 0);
407ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (mp == NULL) {
408ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_log(ixgbe, "LRO copy MP alloc failed");
409ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (NULL);
410ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
411ffd8e883SWinson Wang - Sun Microsystems - Beijing China
412ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_start];
413ffd8e883SWinson Wang - Sun Microsystems - Beijing China
414ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
415ffd8e883SWinson Wang - Sun Microsystems - Beijing China * Sync up the LRO packet data received
416ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
417ffd8e883SWinson Wang - Sun Microsystems - Beijing China for (i = lro_num; i > 0; i--) {
418ffd8e883SWinson Wang - Sun Microsystems - Beijing China DMA_SYNC(¤t_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
419ffd8e883SWinson Wang - Sun Microsystems - Beijing China
420ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
421ffd8e883SWinson Wang - Sun Microsystems - Beijing China DDI_FM_OK) {
422ffd8e883SWinson Wang - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
423ffd8e883SWinson Wang - Sun Microsystems - Beijing China atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
424ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (NULL);
425ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
426ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (current_rcb->lro_next != -1)
427ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
428ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
429ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
430ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next = lro_start;
431ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
432ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
433ffd8e883SWinson Wang - Sun Microsystems - Beijing China last_pkt_len = pkt_len - ixgbe->rx_buf_size * (lro_num - 1);
434ffd8e883SWinson Wang - Sun Microsystems - Beijing China
435ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
436ffd8e883SWinson Wang - Sun Microsystems - Beijing China * Copy the data received into the new cluster
437ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
438ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp->b_rptr += IPHDR_ALIGN_ROOM;
439ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp->b_wptr += IPHDR_ALIGN_ROOM;
440ffd8e883SWinson Wang - Sun Microsystems - Beijing China while (lro_num --) {
441ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (lro_num != 0) {
442ffd8e883SWinson Wang - Sun Microsystems - Beijing China bcopy(current_rcb->rx_buf.address, mp->b_wptr,
443ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe->rx_buf_size);
444ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp->b_wptr += ixgbe->rx_buf_size;
445ffd8e883SWinson Wang - Sun Microsystems - Beijing China } else {
446ffd8e883SWinson Wang - Sun Microsystems - Beijing China bcopy(current_rcb->rx_buf.address, mp->b_wptr,
447ffd8e883SWinson Wang - Sun Microsystems - Beijing China last_pkt_len);
448ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp->b_wptr += last_pkt_len;
449ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
450ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
451ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_next = -1;
452ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_prev = -1;
453ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_pkt = B_FALSE;
454ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr = current_rcb->rx_buf.dma_address;
455ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
456ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (lro_next == -1)
457ffd8e883SWinson Wang - Sun Microsystems - Beijing China break;
458ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
459ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
460ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
461ffd8e883SWinson Wang - Sun Microsystems - Beijing China
462ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (mp);
463ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
464ffd8e883SWinson Wang - Sun Microsystems - Beijing China
465ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
466ffd8e883SWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_get_start - get the start rcb index in one LRO packet
467ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
468ffd8e883SWinson Wang - Sun Microsystems - Beijing China static int
ixgbe_lro_get_start(ixgbe_rx_data_t * rx_data,uint32_t rx_next)469ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_lro_get_start(ixgbe_rx_data_t *rx_data, uint32_t rx_next)
470ffd8e883SWinson Wang - Sun Microsystems - Beijing China {
471ffd8e883SWinson Wang - Sun Microsystems - Beijing China int lro_prev;
472ffd8e883SWinson Wang - Sun Microsystems - Beijing China int lro_start;
473ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t lro_num = 1;
474ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *prev_rcb;
475ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb = rx_data->work_list[rx_next];
476ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_prev = current_rcb->lro_prev;
477ffd8e883SWinson Wang - Sun Microsystems - Beijing China
478ffd8e883SWinson Wang - Sun Microsystems - Beijing China while (lro_prev != -1) {
479ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_num ++;
480ffd8e883SWinson Wang - Sun Microsystems - Beijing China prev_rcb = rx_data->work_list[lro_prev];
481ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_start = lro_prev;
482ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_prev = prev_rcb->lro_prev;
483ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
484ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num = lro_num;
485ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (lro_start);
486ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
487ffd8e883SWinson Wang - Sun Microsystems - Beijing China
488ffd8e883SWinson Wang - Sun Microsystems - Beijing China /*
489ffd8e883SWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_get_first - get the first LRO rcb index
490ffd8e883SWinson Wang - Sun Microsystems - Beijing China */
491ffd8e883SWinson Wang - Sun Microsystems - Beijing China static uint32_t
ixgbe_lro_get_first(ixgbe_rx_data_t * rx_data,uint32_t rx_next)492ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_lro_get_first(ixgbe_rx_data_t *rx_data, uint32_t rx_next)
493ffd8e883SWinson Wang - Sun Microsystems - Beijing China {
494ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb;
495ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t lro_first;
496ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_first = rx_data->lro_first;
497ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_first];
498ffd8e883SWinson Wang - Sun Microsystems - Beijing China while ((!current_rcb->lro_pkt) && (lro_first != rx_next)) {
499ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_first = NEXT_INDEX(lro_first, 1, rx_data->ring_size);
500ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_first];
501ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
502ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->lro_first = lro_first;
503ffd8e883SWinson Wang - Sun Microsystems - Beijing China return (lro_first);
504ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
505ffd8e883SWinson Wang - Sun Microsystems - Beijing China
5069da57d7bSbt /*
5079da57d7bSbt * ixgbe_rx_assoc_hcksum - Check the rx hardware checksum status and associate
5089da57d7bSbt * the hcksum flags.
5099da57d7bSbt */
5109da57d7bSbt static void
ixgbe_rx_assoc_hcksum(mblk_t * mp,uint32_t status_error)5119da57d7bSbt ixgbe_rx_assoc_hcksum(mblk_t *mp, uint32_t status_error)
5129da57d7bSbt {
5139da57d7bSbt uint32_t hcksum_flags = 0;
5149da57d7bSbt
5159da57d7bSbt /*
5169da57d7bSbt * Check TCP/UDP checksum
5179da57d7bSbt */
5189da57d7bSbt if ((status_error & IXGBE_RXD_STAT_L4CS) &&
5199da57d7bSbt !(status_error & IXGBE_RXDADV_ERR_TCPE))
5200dc2366fSVenugopal Iyer hcksum_flags |= HCK_FULLCKSUM_OK;
5219da57d7bSbt
5229da57d7bSbt /*
5239da57d7bSbt * Check IP Checksum
5249da57d7bSbt */
5259da57d7bSbt if ((status_error & IXGBE_RXD_STAT_IPCS) &&
5269da57d7bSbt !(status_error & IXGBE_RXDADV_ERR_IPE))
5270dc2366fSVenugopal Iyer hcksum_flags |= HCK_IPV4_HDRCKSUM_OK;
5289da57d7bSbt
5299da57d7bSbt if (hcksum_flags != 0) {
5300dc2366fSVenugopal Iyer mac_hcksum_set(mp, 0, 0, 0, 0, hcksum_flags);
5319da57d7bSbt }
5329da57d7bSbt }
5339da57d7bSbt
5349da57d7bSbt /*
535da14cebeSEric Cheng * ixgbe_ring_rx - Receive the data of one ring.
5369da57d7bSbt *
5379da57d7bSbt * This function goes throught h/w descriptor in one specified rx ring,
5389da57d7bSbt * receives the data if the descriptor status shows the data is ready.
5399da57d7bSbt * It returns a chain of mblks containing the received data, to be
5409da57d7bSbt * passed up to mac_rx().
5419da57d7bSbt */
5429da57d7bSbt mblk_t *
ixgbe_ring_rx(ixgbe_rx_ring_t * rx_ring,int poll_bytes)543da14cebeSEric Cheng ixgbe_ring_rx(ixgbe_rx_ring_t *rx_ring, int poll_bytes)
5449da57d7bSbt {
5459da57d7bSbt union ixgbe_adv_rx_desc *current_rbd;
5469da57d7bSbt rx_control_block_t *current_rcb;
5479da57d7bSbt mblk_t *mp;
5489da57d7bSbt mblk_t *mblk_head;
5499da57d7bSbt mblk_t **mblk_tail;
5509da57d7bSbt uint32_t rx_next;
5519da57d7bSbt uint32_t rx_tail;
5529da57d7bSbt uint32_t pkt_len;
5539da57d7bSbt uint32_t status_error;
5549da57d7bSbt uint32_t pkt_num;
555ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t rsc_cnt;
556ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t lro_first;
557ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t lro_start;
558ffd8e883SWinson Wang - Sun Microsystems - Beijing China uint32_t lro_next;
559ffd8e883SWinson Wang - Sun Microsystems - Beijing China boolean_t lro_eop;
560da14cebeSEric Cheng uint32_t received_bytes;
5619da57d7bSbt ixgbe_t *ixgbe = rx_ring->ixgbe;
562ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_rx_data_t *rx_data;
5639da57d7bSbt
56462e6e1adSPaul Guo if ((ixgbe->ixgbe_state & IXGBE_SUSPENDED) ||
56562e6e1adSPaul Guo (ixgbe->ixgbe_state & IXGBE_ERROR) ||
5665b6dd21fSchenlu chen - Sun Microsystems - Beijing China (ixgbe->ixgbe_state & IXGBE_OVERTEMP) ||
56762e6e1adSPaul Guo !(ixgbe->ixgbe_state & IXGBE_STARTED))
56862e6e1adSPaul Guo return (NULL);
56962e6e1adSPaul Guo
570ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data = rx_ring->rx_data;
571ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_eop = B_FALSE;
5729da57d7bSbt mblk_head = NULL;
5739da57d7bSbt mblk_tail = &mblk_head;
5749da57d7bSbt
5759da57d7bSbt /*
5769da57d7bSbt * Sync the receive descriptors before accepting the packets
5779da57d7bSbt */
578ea65739eSchenlu chen - Sun Microsystems - Beijing China DMA_SYNC(&rx_data->rbd_area, DDI_DMA_SYNC_FORKERNEL);
5799da57d7bSbt
580ea65739eSchenlu chen - Sun Microsystems - Beijing China if (ixgbe_check_dma_handle(rx_data->rbd_area.dma_handle) != DDI_FM_OK) {
581ea65739eSchenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
58262e6e1adSPaul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
58362e6e1adSPaul Guo return (NULL);
5849da57d7bSbt }
5859da57d7bSbt
5869da57d7bSbt /*
5879da57d7bSbt * Get the start point of rx bd ring which should be examined
5889da57d7bSbt * during this cycle.
5899da57d7bSbt */
590ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_next = rx_data->rbd_next;
591ea65739eSchenlu chen - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[rx_next];
592da14cebeSEric Cheng received_bytes = 0;
5939da57d7bSbt pkt_num = 0;
5949da57d7bSbt status_error = current_rbd->wb.upper.status_error;
5959da57d7bSbt while (status_error & IXGBE_RXD_STAT_DD) {
5969da57d7bSbt /*
5979da57d7bSbt * If adapter has found errors, but the error
5989da57d7bSbt * is hardware checksum error, this does not discard the
5999da57d7bSbt * packet: let upper layer compute the checksum;
6009da57d7bSbt * Otherwise discard the packet.
6019da57d7bSbt */
6029da57d7bSbt if ((status_error & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) ||
603ffd8e883SWinson Wang - Sun Microsystems - Beijing China ((!ixgbe->lro_enable) &&
604ffd8e883SWinson Wang - Sun Microsystems - Beijing China (!(status_error & IXGBE_RXD_STAT_EOP)))) {
605*a9bfd41dSRyan Zezeski rx_ring->stat_frame_error++;
6069da57d7bSbt goto rx_discard;
6079da57d7bSbt }
6089da57d7bSbt
609*a9bfd41dSRyan Zezeski if ((status_error & IXGBE_RXDADV_ERR_TCPE) ||
610*a9bfd41dSRyan Zezeski (status_error & IXGBE_RXDADV_ERR_IPE))
611*a9bfd41dSRyan Zezeski rx_ring->stat_cksum_error++;
6129da57d7bSbt
613ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (ixgbe->lro_enable) {
614ffd8e883SWinson Wang - Sun Microsystems - Beijing China rsc_cnt = (current_rbd->wb.lower.lo_dword.data &
615ffd8e883SWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_RSCCNT_MASK) >>
616ffd8e883SWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_RSCCNT_SHIFT;
617ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (rsc_cnt != 0) {
618ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (status_error & IXGBE_RXD_STAT_EOP) {
619ffd8e883SWinson Wang - Sun Microsystems - Beijing China pkt_len = current_rbd->wb.upper.length;
620ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (rx_data->work_list[rx_next]->
621ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_prev != -1) {
622ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_start =
623ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe_lro_get_start(rx_data,
624ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_next);
625ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe->lro_pkt_count++;
626ffd8e883SWinson Wang - Sun Microsystems - Beijing China pkt_len +=
627ffd8e883SWinson Wang - Sun Microsystems - Beijing China (rx_data->lro_num - 1) *
628ffd8e883SWinson Wang - Sun Microsystems - Beijing China ixgbe->rx_buf_size;
629ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_eop = B_TRUE;
630ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
631ffd8e883SWinson Wang - Sun Microsystems - Beijing China } else {
632ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next = (status_error &
633ffd8e883SWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_NEXTP_MASK) >>
634ffd8e883SWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_NEXTP_SHIFT;
635ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[lro_next]->lro_prev
636ffd8e883SWinson Wang - Sun Microsystems - Beijing China = rx_next;
637ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[rx_next]->lro_next =
638ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_next;
639ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[rx_next]->lro_pkt =
640ffd8e883SWinson Wang - Sun Microsystems - Beijing China B_TRUE;
641ffd8e883SWinson Wang - Sun Microsystems - Beijing China goto rx_discard;
642ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
643ffd8e883SWinson Wang - Sun Microsystems - Beijing China
644ffd8e883SWinson Wang - Sun Microsystems - Beijing China } else {
645ffd8e883SWinson Wang - Sun Microsystems - Beijing China pkt_len = current_rbd->wb.upper.length;
646ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
647ffd8e883SWinson Wang - Sun Microsystems - Beijing China } else {
648ffd8e883SWinson Wang - Sun Microsystems - Beijing China pkt_len = current_rbd->wb.upper.length;
649ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
650ffd8e883SWinson Wang - Sun Microsystems - Beijing China
651da14cebeSEric Cheng
652da14cebeSEric Cheng if ((poll_bytes != IXGBE_POLL_NULL) &&
653da14cebeSEric Cheng ((received_bytes + pkt_len) > poll_bytes))
654da14cebeSEric Cheng break;
655da14cebeSEric Cheng
656da14cebeSEric Cheng received_bytes += pkt_len;
6579da57d7bSbt mp = NULL;
658ffd8e883SWinson Wang - Sun Microsystems - Beijing China
6599da57d7bSbt /*
6609da57d7bSbt * For packets with length more than the copy threshold,
6619da57d7bSbt * we'll first try to use the existing DMA buffer to build
6629da57d7bSbt * an mblk and send the mblk upstream.
6639da57d7bSbt *
6649da57d7bSbt * If the first method fails, or the packet length is less
6659da57d7bSbt * than the copy threshold, we'll allocate a new mblk and
6669da57d7bSbt * copy the packet data to the new mblk.
6679da57d7bSbt */
668ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (lro_eop) {
669ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_lro_bind(rx_data, lro_start,
670ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num, pkt_len);
671ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (mp == NULL)
672ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_lro_copy(rx_data, lro_start,
673ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num, pkt_len);
674ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_eop = B_FALSE;
675ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num = 0;
676ffd8e883SWinson Wang - Sun Microsystems - Beijing China
677ffd8e883SWinson Wang - Sun Microsystems - Beijing China } else {
678ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (pkt_len > ixgbe->rx_copy_thresh)
679ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_rx_bind(rx_data, rx_next, pkt_len);
680ffd8e883SWinson Wang - Sun Microsystems - Beijing China
681ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (mp == NULL)
682ffd8e883SWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_rx_copy(rx_data, rx_next, pkt_len);
683ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
6849da57d7bSbt if (mp != NULL) {
6859da57d7bSbt /*
6869da57d7bSbt * Check h/w checksum offload status
6879da57d7bSbt */
6889da57d7bSbt if (ixgbe->rx_hcksum_enable)
6899da57d7bSbt ixgbe_rx_assoc_hcksum(mp, status_error);
6909da57d7bSbt
6919da57d7bSbt *mblk_tail = mp;
6929da57d7bSbt mblk_tail = &mp->b_next;
6939da57d7bSbt }
6949da57d7bSbt
6959da57d7bSbt rx_discard:
6969da57d7bSbt /*
6979da57d7bSbt * Reset rx descriptor read bits
6989da57d7bSbt */
699ea65739eSchenlu chen - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[rx_next];
700ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (ixgbe->lro_enable) {
701ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (!current_rcb->lro_pkt) {
702ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr =
703ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.dma_address;
704ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
705ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
706ffd8e883SWinson Wang - Sun Microsystems - Beijing China } else {
707ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr =
708ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.dma_address;
709ffd8e883SWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
710ffd8e883SWinson Wang - Sun Microsystems - Beijing China }
7119da57d7bSbt
712ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_next = NEXT_INDEX(rx_next, 1, rx_data->ring_size);
7139da57d7bSbt
7149da57d7bSbt /*
7159da57d7bSbt * The receive function is in interrupt context, so here
716ea65739eSchenlu chen - Sun Microsystems - Beijing China * rx_limit_per_intr is used to avoid doing receiving too long
7179da57d7bSbt * per interrupt.
7189da57d7bSbt */
719ea65739eSchenlu chen - Sun Microsystems - Beijing China if (++pkt_num > ixgbe->rx_limit_per_intr) {
720*a9bfd41dSRyan Zezeski rx_ring->stat_exceed_pkt++;
7219da57d7bSbt break;
7229da57d7bSbt }
7239da57d7bSbt
724ea65739eSchenlu chen - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[rx_next];
7259da57d7bSbt status_error = current_rbd->wb.upper.status_error;
7269da57d7bSbt }
7279da57d7bSbt
7280dc2366fSVenugopal Iyer rx_ring->stat_rbytes += received_bytes;
7290dc2366fSVenugopal Iyer rx_ring->stat_ipackets += pkt_num;
7300dc2366fSVenugopal Iyer
731ea65739eSchenlu chen - Sun Microsystems - Beijing China DMA_SYNC(&rx_data->rbd_area, DDI_DMA_SYNC_FORDEV);
7329da57d7bSbt
733ea65739eSchenlu chen - Sun Microsystems - Beijing China rx_data->rbd_next = rx_next;
7349da57d7bSbt
7359da57d7bSbt /*
7369da57d7bSbt * Update the h/w tail accordingly
7379da57d7bSbt */
738ffd8e883SWinson Wang - Sun Microsystems - Beijing China if (ixgbe->lro_enable) {
739ffd8e883SWinson Wang - Sun Microsystems - Beijing China lro_first = ixgbe_lro_get_first(rx_data, rx_next);
740ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_tail = PREV_INDEX(lro_first, 1, rx_data->ring_size);
741ffd8e883SWinson Wang - Sun Microsystems - Beijing China } else
742ffd8e883SWinson Wang - Sun Microsystems - Beijing China rx_tail = PREV_INDEX(rx_next, 1, rx_data->ring_size);
743ffd8e883SWinson Wang - Sun Microsystems - Beijing China
7440dc2366fSVenugopal Iyer IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_RDT(rx_ring->hw_index), rx_tail);
7459da57d7bSbt
7469da57d7bSbt if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
747ea65739eSchenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
74862e6e1adSPaul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
7499da57d7bSbt }
7509da57d7bSbt
7519da57d7bSbt return (mblk_head);
7529da57d7bSbt }
753da14cebeSEric Cheng
754da14cebeSEric Cheng mblk_t *
ixgbe_ring_rx_poll(void * arg,int n_bytes)755da14cebeSEric Cheng ixgbe_ring_rx_poll(void *arg, int n_bytes)
756da14cebeSEric Cheng {
757da14cebeSEric Cheng ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)arg;
758da14cebeSEric Cheng mblk_t *mp = NULL;
759da14cebeSEric Cheng
760da14cebeSEric Cheng ASSERT(n_bytes >= 0);
761da14cebeSEric Cheng
762da14cebeSEric Cheng if (n_bytes == 0)
76362e6e1adSPaul Guo return (NULL);
764da14cebeSEric Cheng
765da14cebeSEric Cheng mutex_enter(&rx_ring->rx_lock);
766da14cebeSEric Cheng mp = ixgbe_ring_rx(rx_ring, n_bytes);
767da14cebeSEric Cheng mutex_exit(&rx_ring->rx_lock);
768da14cebeSEric Cheng
769da14cebeSEric Cheng return (mp);
770da14cebeSEric Cheng }
771