xref: /illumos-gate/usr/src/uts/common/io/xge/hal/xgehal/xgehal-device-fp.c (revision 7eced415e5dd557aef2d78483b5a7785f0e13670)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  *
21  * Copyright (c) 2002-2006 Neterion, Inc.
22  */
23 
24 #ifdef XGE_DEBUG_FP
25 #include "xgehal-device.h"
26 #endif
27 
28 #include "xgehal-ring.h"
29 #include "xgehal-fifo.h"
30 
31 /**
32  * xge_hal_device_bar0 - Get BAR0 mapped address.
33  * @hldev: HAL device handle.
34  *
35  * Returns:	BAR0 address of	the	specified device.
36  */
37 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	char *
38 xge_hal_device_bar0(xge_hal_device_t *hldev)
39 {
40 	return hldev->bar0;
41 }
42 
43 /**
44  * xge_hal_device_isrbar0 -	Get	BAR0 mapped	address.
45  * @hldev: HAL device handle.
46  *
47  * Returns:	BAR0 address of	the	specified device.
48  */
49 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	char *
50 xge_hal_device_isrbar0(xge_hal_device_t	*hldev)
51 {
52 	return hldev->isrbar0;
53 }
54 
55 /**
56  * xge_hal_device_bar1 - Get BAR1 mapped address.
57  * @hldev: HAL device handle.
58  *
59  * Returns:	BAR1 address of	the	specified device.
60  */
61 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	char *
62 xge_hal_device_bar1(xge_hal_device_t *hldev)
63 {
64 	return hldev->bar1;
65 }
66 
67 /**
68  * xge_hal_device_bar0_set - Set BAR0 mapped address.
69  * @hldev: HAL device handle.
70  * @bar0: BAR0 mapped address.
71  * * Set BAR0 address in the HAL device	object.
72  */
73 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
74 xge_hal_device_bar0_set(xge_hal_device_t *hldev, char *bar0)
75 {
76 	xge_assert(bar0);
77 	hldev->bar0	= bar0;
78 }
79 
80 /**
81  * xge_hal_device_isrbar0_set -	Set	BAR0 mapped	address.
82  * @hldev: HAL device handle.
83  * @isrbar0: BAR0 mapped address.
84  * * Set BAR0 address in the HAL device	object.
85  */
86 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
87 xge_hal_device_isrbar0_set(xge_hal_device_t	*hldev,	char *isrbar0)
88 {
89 	xge_assert(isrbar0);
90 	hldev->isrbar0 = isrbar0;
91 }
92 
93 /**
94  * xge_hal_device_bar1_set - Set BAR1 mapped address.
95  * @hldev: HAL device handle.
96  * @channelh: Channel handle.
97  * @bar1: BAR1 mapped address.
98  *
99  * Set BAR1	address	for	the	given channel.
100  */
101 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
102 xge_hal_device_bar1_set(xge_hal_device_t *hldev, xge_hal_channel_h channelh,
103 			   char	*bar1)
104 {
105 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
106 
107 	xge_assert(bar1);
108 	xge_assert(fifo);
109 
110 	/* Initializing	the	BAR1 address as	the	start of
111 	 * the FIFO	queue pointer and as a location	of FIFO	control
112 	 * word. */
113 	fifo->hw_pair =
114 			(xge_hal_fifo_hw_pair_t	*) (bar1 +
115 				(fifo->channel.post_qid	* XGE_HAL_FIFO_HW_PAIR_OFFSET));
116 	hldev->bar1	= bar1;
117 }
118 
119 
120 /**
121  * xge_hal_device_rev -	Get	Device revision	number.
122  * @hldev: HAL device handle.
123  *
124  * Returns:	Device revision	number
125  */
126 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	int
127 xge_hal_device_rev(xge_hal_device_t	*hldev)
128 {
129 		return hldev->revision;
130 }
131 
132 
133 /**
134  * xge_hal_device_begin_irq	- Begin	IRQ	processing.
135  * @hldev: HAL device handle.
136  * @reason:	"Reason" for the interrupt,	the	value of Xframe's
137  *			general_int_status register.
138  *
139  * The function	performs two actions, It first checks whether (shared IRQ) the
140  * interrupt was raised	by the device. Next, it	masks the device interrupts.
141  *
142  * Note:
143  * xge_hal_device_begin_irq() does not flush MMIO writes through the
144  * bridge. Therefore, two back-to-back interrupts are potentially possible.
145  * It is the responsibility	of the ULD to make sure	that only one
146  * xge_hal_device_continue_irq() runs at a time.
147  *
148  * Returns:	0, if the interrupt	is not "ours" (note	that in	this case the
149  * device remain enabled).
150  * Otherwise, xge_hal_device_begin_irq() returns 64bit general adapter
151  * status.
152  * See also: xge_hal_device_handle_irq()
153  */
154 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	xge_hal_status_e
155 xge_hal_device_begin_irq(xge_hal_device_t *hldev, u64 *reason)
156 {
157 	u64	val64;
158 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
159 
160 	hldev->stats.sw_dev_info_stats.total_intr_cnt++;
161 
162 	val64 =	xge_os_pio_mem_read64(hldev->pdev,
163 				  hldev->regh0,	&isrbar0->general_int_status);
164 	if (xge_os_unlikely(!val64)) {
165 		/* not Xframe interrupt	*/
166 		hldev->stats.sw_dev_info_stats.not_xge_intr_cnt++;
167 		*reason	= 0;
168 			return XGE_HAL_ERR_WRONG_IRQ;
169 	}
170 
171 	if (xge_os_unlikely(val64 == XGE_HAL_ALL_FOXES)) {
172 				u64	adapter_status =
173 						xge_os_pio_mem_read64(hldev->pdev, hldev->regh0,
174 						  &isrbar0->adapter_status);
175 				if (adapter_status == XGE_HAL_ALL_FOXES)  {
176 					(void) xge_queue_produce(hldev->queueh,
177 						 XGE_HAL_EVENT_SLOT_FREEZE,
178 						 hldev,
179 						 1,	 /*	critical: slot freeze */
180 						 sizeof(u64),
181 						 (void*)&adapter_status);
182 			*reason	= 0;
183 			return XGE_HAL_ERR_CRITICAL;
184 		}
185 	}
186 
187 	*reason	= val64;
188 
189 	/* separate	fast path, i.e.	no errors */
190 	if (val64 &	XGE_HAL_GEN_INTR_RXTRAFFIC)	{
191 		hldev->stats.sw_dev_info_stats.rx_traffic_intr_cnt++;
192 		return XGE_HAL_OK;
193 	}
194 	if (val64 &	XGE_HAL_GEN_INTR_TXTRAFFIC)	{
195 		hldev->stats.sw_dev_info_stats.tx_traffic_intr_cnt++;
196 		return XGE_HAL_OK;
197 	}
198 
199 	hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
200 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_TXPIC)) {
201 		xge_hal_status_e status;
202 		hldev->stats.sw_dev_info_stats.txpic_intr_cnt++;
203 		status = __hal_device_handle_txpic(hldev, val64);
204 		if (status != XGE_HAL_OK) {
205 			return status;
206 		}
207 	}
208 
209 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_TXDMA)) {
210 		xge_hal_status_e status;
211 		hldev->stats.sw_dev_info_stats.txdma_intr_cnt++;
212 		status = __hal_device_handle_txdma(hldev, val64);
213 		if (status != XGE_HAL_OK) {
214 			return status;
215 		}
216 	}
217 
218 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_TXMAC)) {
219 		xge_hal_status_e status;
220 		hldev->stats.sw_dev_info_stats.txmac_intr_cnt++;
221 		status = __hal_device_handle_txmac(hldev, val64);
222 		if (status != XGE_HAL_OK) {
223 			return status;
224 		}
225 	}
226 
227 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_TXXGXS)) {
228 		xge_hal_status_e status;
229 		hldev->stats.sw_dev_info_stats.txxgxs_intr_cnt++;
230 		status = __hal_device_handle_txxgxs(hldev, val64);
231 		if (status != XGE_HAL_OK) {
232 			return status;
233 		}
234 	}
235 
236 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_RXPIC)) {
237 		xge_hal_status_e status;
238 		hldev->stats.sw_dev_info_stats.rxpic_intr_cnt++;
239 		status = __hal_device_handle_rxpic(hldev, val64);
240 		if (status != XGE_HAL_OK) {
241 			return status;
242 		}
243 	}
244 
245 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_RXDMA)) {
246 		xge_hal_status_e status;
247 		hldev->stats.sw_dev_info_stats.rxdma_intr_cnt++;
248 		status = __hal_device_handle_rxdma(hldev, val64);
249 		if (status != XGE_HAL_OK) {
250 			return status;
251 		}
252 	}
253 
254 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_RXMAC)) {
255 		xge_hal_status_e status;
256 		hldev->stats.sw_dev_info_stats.rxmac_intr_cnt++;
257 		status = __hal_device_handle_rxmac(hldev, val64);
258 		if (status != XGE_HAL_OK) {
259 			return status;
260 		}
261 	}
262 
263 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_RXXGXS)) {
264 		xge_hal_status_e status;
265 		hldev->stats.sw_dev_info_stats.rxxgxs_intr_cnt++;
266 		status = __hal_device_handle_rxxgxs(hldev, val64);
267 		if (status != XGE_HAL_OK) {
268 			return status;
269 		}
270 	}
271 
272 	if (xge_os_unlikely(val64 &	XGE_HAL_GEN_INTR_MC)) {
273 		xge_hal_status_e status;
274 		hldev->stats.sw_dev_info_stats.mc_intr_cnt++;
275 		status = __hal_device_handle_mc(hldev, val64);
276 		if (status != XGE_HAL_OK) {
277 			return status;
278 		}
279 	}
280 
281 	return XGE_HAL_OK;
282 }
283 
284 /**
285  * xge_hal_device_clear_rx - Acknowledge (that is, clear) the
286  * condition that has caused the RX	interrupt.
287  * @hldev: HAL device handle.
288  *
289  * Acknowledge (that is, clear)	the	condition that has caused
290  * the Rx interrupt.
291  * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
292  * xge_hal_device_clear_tx(), xge_hal_device_mask_rx().
293  */
294 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
295 xge_hal_device_clear_rx(xge_hal_device_t *hldev)
296 {
297 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
298 
299 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
300 				 0xFFFFFFFFFFFFFFFFULL,
301 				 &isrbar0->rx_traffic_int);
302 }
303 
304 /**
305  * xge_hal_device_clear_tx - Acknowledge (that is, clear) the
306  * condition that has caused the TX	interrupt.
307  * @hldev: HAL device handle.
308  *
309  * Acknowledge (that is, clear)	the	condition that has caused
310  * the Tx interrupt.
311  * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
312  * xge_hal_device_clear_rx(), xge_hal_device_mask_tx().
313  */
314 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
315 xge_hal_device_clear_tx(xge_hal_device_t *hldev)
316 {
317 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
318 
319 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
320 				 0xFFFFFFFFFFFFFFFFULL,
321 				 &isrbar0->tx_traffic_int);
322 }
323 
324 /**
325  * xge_hal_device_poll_rx_channel -	Poll Rx	channel	for	completed
326  * descriptors and process the same.
327  * @channel: HAL channel.
328  * @got_rx: Buffer to return the flag set if receive interrupt is occured
329  *
330  * The function	polls the Rx channel for the completed	descriptors	and	calls
331  * the upper-layer driver (ULD)	via	supplied completion	callback.
332  *
333  * Returns:	XGE_HAL_OK,	if the polling is completed	successful.
334  * XGE_HAL_COMPLETIONS_REMAIN: There are still more	completed
335  * descriptors available which are yet to be processed.
336  *
337  * See also: xge_hal_device_poll_tx_channel()
338  */
339 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE	xge_hal_status_e
340 xge_hal_device_poll_rx_channel(xge_hal_channel_t *channel, int *got_rx)
341 {
342 	xge_hal_status_e ret = XGE_HAL_OK;
343 	xge_hal_dtr_h first_dtrh;
344 	xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
345 	u8 t_code;
346 	int got_bytes;
347 
348 	/* for each opened rx channel */
349 	got_bytes = *got_rx = 0;
350 	((xge_hal_ring_t *)channel)->cmpl_cnt = 0;
351 	channel->poll_bytes = 0;
352 	if ((ret = xge_hal_ring_dtr_next_completed (channel, &first_dtrh,
353 		&t_code)) == XGE_HAL_OK) {
354 		if (channel->callback(channel, first_dtrh,
355 			t_code,	channel->userdata) != XGE_HAL_OK) {
356 			(*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
357 			got_bytes += channel->poll_bytes + 1;
358 			ret = XGE_HAL_COMPLETIONS_REMAIN;
359 		} else {
360 			(*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
361 			got_bytes += channel->poll_bytes + 1;
362 		}
363 	}
364 
365 	if (*got_rx) {
366 		hldev->irq_workload_rxd[channel->post_qid] += *got_rx;
367 		hldev->irq_workload_rxcnt[channel->post_qid] ++;
368 	}
369 	hldev->irq_workload_rxlen[channel->post_qid] += got_bytes;
370 
371 	return ret;
372 }
373 
374 /**
375  * xge_hal_device_poll_tx_channel -	Poll Tx	channel	for	completed
376  * descriptors and process the same.
377  * @channel: HAL channel.
378  * @got_tx: Buffer to return the flag set if transmit interrupt is occured
379  *
380  * The function	polls the Tx channel for the completed	descriptors	and	calls
381  * the upper-layer driver (ULD)	via	supplied completion	callback.
382  *
383  * Returns:	XGE_HAL_OK,	if the polling is completed	successful.
384  * XGE_HAL_COMPLETIONS_REMAIN: There are still more	completed
385  * descriptors available which are yet to be processed.
386  *
387  * See also: xge_hal_device_poll_rx_channel().
388  */
389 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE xge_hal_status_e
390 xge_hal_device_poll_tx_channel(xge_hal_channel_t *channel, int *got_tx)
391 {
392 	xge_hal_dtr_h first_dtrh;
393 	xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
394 	u8 t_code;
395 	int got_bytes;
396 
397 	/* for each opened tx channel */
398 	got_bytes = *got_tx = 0;
399 	channel->poll_bytes = 0;
400 	if (xge_hal_fifo_dtr_next_completed (channel, &first_dtrh,
401 		&t_code) ==	XGE_HAL_OK)	{
402 		if (channel->callback(channel, first_dtrh,
403 			t_code,	channel->userdata) != XGE_HAL_OK) {
404 			(*got_tx)++;
405 			got_bytes += channel->poll_bytes + 1;
406 			return XGE_HAL_COMPLETIONS_REMAIN;
407 		}
408 		(*got_tx)++;
409 		got_bytes += channel->poll_bytes + 1;
410 	}
411 
412 	if (*got_tx) {
413 		hldev->irq_workload_txd[channel->post_qid] += *got_tx;
414 		hldev->irq_workload_txcnt[channel->post_qid] ++;
415 	}
416 	hldev->irq_workload_txlen[channel->post_qid] += got_bytes;
417 
418 	return XGE_HAL_OK;
419 }
420 
421 /**
422  * xge_hal_device_poll_rx_channels - Poll Rx channels for completed
423  * descriptors and process the same.
424  * @hldev: HAL device handle.
425  * @got_rx: Buffer to return flag set if receive is ready
426  *
427  * The function	polls the Rx channels for the completed	descriptors	and	calls
428  * the upper-layer driver (ULD)	via	supplied completion	callback.
429  *
430  * Returns:	XGE_HAL_OK,	if the polling is completed	successful.
431  * XGE_HAL_COMPLETIONS_REMAIN: There are still more	completed
432  * descriptors available which are yet to be processed.
433  *
434  * See also: xge_hal_device_poll_tx_channels(),	xge_hal_device_continue_irq().
435  */
436 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	xge_hal_status_e
437 xge_hal_device_poll_rx_channels(xge_hal_device_t *hldev, int *got_rx)
438 {
439 	xge_list_t *item;
440 	xge_hal_channel_t *channel;
441 
442 	/* for each opened rx channel */
443 	xge_list_for_each(item,	&hldev->ring_channels) {
444 		if (hldev->terminating)
445 			return XGE_HAL_OK;
446 		channel	= xge_container_of(item, xge_hal_channel_t,	item);
447 		(void) xge_hal_device_poll_rx_channel(channel, got_rx);
448 	}
449 
450 	return XGE_HAL_OK;
451 }
452 
453 /**
454  * xge_hal_device_poll_tx_channels - Poll Tx channels for completed
455  * descriptors and process the same.
456  * @hldev: HAL device handle.
457  * @got_tx: Buffer to return flag set if transmit is ready
458  *
459  * The function	polls the Tx channels for the completed	descriptors	and	calls
460  * the upper-layer driver (ULD)	via	supplied completion	callback.
461  *
462  * Returns:	XGE_HAL_OK,	if the polling is completed	successful.
463  * XGE_HAL_COMPLETIONS_REMAIN: There are still more	completed
464  * descriptors available which are yet to be processed.
465  *
466  * See also: xge_hal_device_poll_rx_channels(),	xge_hal_device_continue_irq().
467  */
468 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	xge_hal_status_e
469 xge_hal_device_poll_tx_channels(xge_hal_device_t *hldev, int *got_tx)
470 {
471 	xge_list_t *item;
472 	xge_hal_channel_t *channel;
473 
474 	/* for each opened tx channel */
475 	xge_list_for_each(item,	&hldev->fifo_channels) {
476 		if (hldev->terminating)
477 			return XGE_HAL_OK;
478 		channel	= xge_container_of(item, xge_hal_channel_t, item);
479 		(void) xge_hal_device_poll_tx_channel(channel, got_tx);
480 	}
481 
482 	return XGE_HAL_OK;
483 }
484 
485 /**
486  * xge_hal_device_mask_tx -	Mask Tx	interrupts.
487  * @hldev: HAL device handle.
488  *
489  * Mask	Tx device interrupts.
490  *
491  * See also: xge_hal_device_unmask_tx(), xge_hal_device_mask_rx(),
492  * xge_hal_device_clear_tx().
493  */
494 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
495 xge_hal_device_mask_tx(xge_hal_device_t	*hldev)
496 {
497 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
498 
499 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
500 				   0xFFFFFFFFFFFFFFFFULL,
501 				   &isrbar0->tx_traffic_mask);
502 }
503 
504 /**
505  * xge_hal_device_mask_rx -	Mask Rx	interrupts.
506  * @hldev: HAL device handle.
507  *
508  * Mask	Rx device interrupts.
509  *
510  * See also: xge_hal_device_unmask_rx(), xge_hal_device_mask_tx(),
511  * xge_hal_device_clear_rx().
512  */
513 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
514 xge_hal_device_mask_rx(xge_hal_device_t	*hldev)
515 {
516 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
517 
518 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
519 				   0xFFFFFFFFFFFFFFFFULL,
520 				   &isrbar0->rx_traffic_mask);
521 }
522 
523 /**
524  * xge_hal_device_mask_all - Mask all device interrupts.
525  * @hldev: HAL device handle.
526  *
527  * Mask	all	device interrupts.
528  *
529  * See also: xge_hal_device_unmask_all()
530  */
531 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
532 xge_hal_device_mask_all(xge_hal_device_t *hldev)
533 {
534 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
535 
536 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
537 				   0xFFFFFFFFFFFFFFFFULL,
538 				   &isrbar0->general_int_mask);
539 }
540 
541 /**
542  * xge_hal_device_unmask_tx	- Unmask Tx	interrupts.
543  * @hldev: HAL device handle.
544  *
545  * Unmask Tx device	interrupts.
546  *
547  * See also: xge_hal_device_mask_tx(), xge_hal_device_clear_tx().
548  */
549 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
550 xge_hal_device_unmask_tx(xge_hal_device_t *hldev)
551 {
552 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
553 
554 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
555 				   0x0ULL,
556 				   &isrbar0->tx_traffic_mask);
557 }
558 
559 /**
560  * xge_hal_device_unmask_rx	- Unmask Rx	interrupts.
561  * @hldev: HAL device handle.
562  *
563  * Unmask Rx device	interrupts.
564  *
565  * See also: xge_hal_device_mask_rx(), xge_hal_device_clear_rx().
566  */
567 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
568 xge_hal_device_unmask_rx(xge_hal_device_t *hldev)
569 {
570 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
571 
572 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
573 				   0x0ULL,
574 				   &isrbar0->rx_traffic_mask);
575 }
576 
577 /**
578  * xge_hal_device_unmask_all - Unmask all device interrupts.
579  * @hldev: HAL device handle.
580  *
581  * Unmask all device interrupts.
582  *
583  * See also: xge_hal_device_mask_all()
584  */
585 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	void
586 xge_hal_device_unmask_all(xge_hal_device_t *hldev)
587 {
588 	xge_hal_pci_bar0_t *isrbar0	= (xge_hal_pci_bar0_t *)hldev->isrbar0;
589 
590 	xge_os_pio_mem_write64(hldev->pdev,	hldev->regh0,
591 				   0x0ULL,
592 				   &isrbar0->general_int_mask);
593 }
594 
595 
596 /**
597  * xge_hal_device_continue_irq - Continue handling IRQ:	process	all
598  * completed descriptors.
599  * @hldev: HAL device handle.
600  *
601  * Process completed descriptors and unmask	the	device interrupts.
602  *
603  * The xge_hal_device_continue_irq() walks all open	channels
604  * and calls upper-layer driver	(ULD) via supplied completion
605  * callback. Note that the completion callback is specified	at channel open
606  * time, see xge_hal_channel_open().
607  *
608  * Note	that the xge_hal_device_continue_irq is	part of	the	_fast_ path.
609  * To optimize the processing, the function	does _not_ check for
610  * errors and alarms.
611  *
612  * The latter is done in a polling fashion,	via	xge_hal_device_poll().
613  *
614  * Returns:	XGE_HAL_OK.
615  *
616  * See also: xge_hal_device_handle_irq(), xge_hal_device_poll(),
617  * xge_hal_ring_dtr_next_completed(),
618  * xge_hal_fifo_dtr_next_completed(), xge_hal_channel_callback_f{}.
619  */
620 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	xge_hal_status_e
621 xge_hal_device_continue_irq(xge_hal_device_t *hldev)
622 {
623 	int	got_rx = 1,	got_tx = 1;
624 	int	isr_polling_cnt	= hldev->config.isr_polling_cnt;
625 	int	count =	0;
626 
627 	do
628 	{
629 		if (got_rx)
630 			(void) xge_hal_device_poll_rx_channels(hldev, &got_rx);
631 		if (got_tx && hldev->tti_enabled)
632 			(void) xge_hal_device_poll_tx_channels(hldev, &got_tx);
633 
634 		if (!got_rx && !got_tx)
635 			break;
636 
637 		count += (got_rx + got_tx);
638 	}while (isr_polling_cnt--);
639 
640 	if (!count)
641 		hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
642 
643 	return XGE_HAL_OK;
644 }
645 
646 /**
647  * xge_hal_device_handle_irq - Handle device IRQ.
648  * @hldev: HAL device handle.
649  *
650  * Perform the complete	handling of	the	line interrupt.	The	function
651  * performs	two	calls.
652  * First it	uses xge_hal_device_begin_irq()	to	check the reason for
653  * the interrupt and mask the device interrupts.
654  * Second, it calls	xge_hal_device_continue_irq() to process all
655  * completed descriptors and re-enable the interrupts.
656  *
657  * Returns:	XGE_HAL_OK - success;
658  * XGE_HAL_ERR_WRONG_IRQ - (shared)	IRQ	produced by	other device.
659  *
660  * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq().
661  */
662 __HAL_STATIC_DEVICE	__HAL_INLINE_DEVICE	xge_hal_status_e
663 xge_hal_device_handle_irq(xge_hal_device_t *hldev)
664 {
665 	u64	reason;
666 	xge_hal_status_e status;
667 
668 	xge_hal_device_mask_all(hldev);
669 
670 	status = xge_hal_device_begin_irq(hldev, &reason);
671 	if (status != XGE_HAL_OK) {
672 		xge_hal_device_unmask_all(hldev);
673 		return status;
674 	}
675 
676 	if (reason & XGE_HAL_GEN_INTR_RXTRAFFIC) {
677 		xge_hal_device_clear_rx(hldev);
678 	}
679 
680 	status = xge_hal_device_continue_irq(hldev);
681 
682 	xge_hal_device_clear_tx(hldev);
683 
684 	xge_hal_device_unmask_all(hldev);
685 
686 	return status;
687 }
688 
689 #if	defined(XGE_HAL_CONFIG_LRO)
690 
691 
692 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
693 __hal_lro_check_for_session_match(lro_t	*lro, tcplro_t *tcp, iplro_t *ip)
694 {
695 
696 	/* Match Source	address	field */
697 	if ((lro->ip_hdr->saddr	!= ip->saddr))
698 		return XGE_HAL_FAIL;
699 
700 	/* Match Destination address field */
701 	if ((lro->ip_hdr->daddr	!= ip->daddr))
702 		return XGE_HAL_FAIL;
703 
704 	/* Match Source	Port field */
705 	if ((lro->tcp_hdr->source != tcp->source))
706 		return XGE_HAL_FAIL;
707 
708 	/* Match Destination Port field	*/
709 	if ((lro->tcp_hdr->dest	!= tcp->dest))
710 		return XGE_HAL_FAIL;
711 
712 	return XGE_HAL_OK;
713 }
714 
715 /*
716  * __hal_tcp_seg_len: Find the tcp seg len.
717  * @ip:	ip header.
718  * @tcp: tcp header.
719  * returns:	Tcp	seg	length.
720  */
721 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL u16
722 __hal_tcp_seg_len(iplro_t *ip, tcplro_t	*tcp)
723 {
724 	u16	ret;
725 
726 	ret	=  (xge_os_ntohs(ip->tot_len) -
727 		   ((ip->version_ihl & 0x0F)<<2) -
728 		   ((tcp->doff_res)>>2));
729 	return (ret);
730 }
731 
732 /*
733  * __hal_ip_lro_capable: Finds whether ip is lro capable.
734  * @ip:	ip header.
735  * @ext_info:  descriptor info.
736  */
737 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
738 __hal_ip_lro_capable(iplro_t *ip,
739 			 xge_hal_dtr_info_t	*ext_info)
740 {
741 
742 #ifdef XGE_LL_DEBUG_DUMP_PKT
743 		{
744 			u16	i;
745 			u8 ch, *iph	= (u8 *)ip;
746 
747 			xge_debug_ring(XGE_TRACE, "Dump	Ip:" );
748 			for	(i =0; i < 40; i++)	{
749 				ch = ntohs(*((u8 *)(iph	+ i)) );
750 				printf("i:%d %02x, ",i,ch);
751 			}
752 		}
753 #endif
754 
755 	if (ip->version_ihl	!= IP_FAST_PATH_HDR_MASK) {
756 		xge_debug_ring(XGE_ERR,	"iphdr !=45	:%d",ip->version_ihl);
757 		return XGE_HAL_FAIL;
758 	}
759 
760 	if (ext_info->proto	& XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) {
761 		xge_debug_ring(XGE_ERR,	"IP	fragmented");
762 		return XGE_HAL_FAIL;
763 	}
764 
765 	return XGE_HAL_OK;
766 }
767 
768 /*
769  * __hal_tcp_lro_capable: Finds	whether	tcp	is lro capable.
770  * @ip:	ip header.
771  * @tcp: tcp header.
772  */
773 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
774 __hal_tcp_lro_capable(iplro_t *ip, tcplro_t	*tcp, lro_t	*lro, int *ts_off)
775 {
776 #ifdef XGE_LL_DEBUG_DUMP_PKT
777 		{
778 			u8 ch;
779 			u16	i;
780 
781 			xge_debug_ring(XGE_TRACE, "Dump	Tcp:" );
782 			for	(i =0; i < 20; i++)	{
783 				ch = ntohs(*((u8 *)((u8	*)tcp +	i))	);
784 				xge_os_printf("i:%d	%02x, ",i,ch);
785 			}
786 		}
787 #endif
788 	if ((TCP_FAST_PATH_HDR_MASK2 !=	tcp->ctrl) &&
789 		(TCP_FAST_PATH_HDR_MASK3 !=	tcp->ctrl))
790 		goto _exit_fail;
791 
792 	*ts_off	= -1;
793 	if (TCP_FAST_PATH_HDR_MASK1	!= tcp->doff_res) {
794 		u16	tcp_hdr_len	= tcp->doff_res	>> 2; /* TCP header	len	*/
795 		u16	off	= 20; /* Start of tcp options */
796 		int	i, diff;
797 
798 		/* Does	Packet can contain time	stamp */
799 		if (tcp_hdr_len	< 32) {
800 			/*
801 			 * If the session is not opened, we	can	consider
802 			 * this	packet for LRO
803 			 */
804 			if (lro	== NULL)
805 				return XGE_HAL_OK;
806 
807 			goto _exit_fail;
808 		}
809 
810 		/* Ignore No-operation 0x1 */
811 		while (((u8	*)tcp)[off]	== 0x1)
812 			off++;
813 
814 		/* Next	option == Timestamp	*/
815 		if (((u8 *)tcp)[off] !=	0x8) {
816 			/*
817 			 * If the session ie not opened, we	can	consider
818 			 * this	packet for LRO
819 			 */
820 			if (lro	== NULL)
821 				return XGE_HAL_OK;
822 
823 			goto _exit_fail;
824 		}
825 
826 		*ts_off	= off;
827 		if (lro	== NULL)
828 			return XGE_HAL_OK;
829 
830 		/*
831 		 * Now the session is opened. If the LRO frame doesn't
832 		 * have	time stamp,	we cannot consider current packet for
833 		 * LRO.
834 		 */
835 		if (lro->ts_off	== -1) {
836 			xge_debug_ring(XGE_ERR,	"Pkt received with time	stamp after	session	opened with	no time	stamp :	%02x %02x", tcp->doff_res, tcp->ctrl);
837 			return XGE_HAL_FAIL;
838 		}
839 
840 		/*
841 		 * If the difference is	greater	than three,	then there are
842 		 * more	options	possible.
843 		 * else, there are two cases:
844 		 * case	1: remaining are padding bytes.
845 		 * case	2: remaining can contain options or	padding
846 		 */
847 		off	+= ((u8	*)tcp)[off+1];
848 		diff = tcp_hdr_len - off;
849 		if (diff > 3) {
850 			/*
851 			 * Probably	contains more options.
852 			 */
853 			xge_debug_ring(XGE_ERR,	"tcphdr	not	fastpth	: pkt received with	tcp	options	in addition	to time	stamp after	the	session	is opened %02x %02x	", tcp->doff_res,	tcp->ctrl);
854 			return XGE_HAL_FAIL;
855 		}
856 
857 		for	(i = 0;	i <	diff; i++) {
858 			u8 byte	= ((u8 *)tcp)[off+i];
859 
860 			/* Ignore No-operation 0x1 */
861 			if ((byte == 0x0) || (byte == 0x1))
862 				continue;
863 			xge_debug_ring(XGE_ERR,	"tcphdr	not	fastpth	: pkt received with	tcp	options	in addition	to time	stamp after	the	session	is opened %02x %02x	", tcp->doff_res,	tcp->ctrl);
864 			return XGE_HAL_FAIL;
865 		}
866 
867 		/*
868 		 * Update the time stamp of	LRO	frame.
869 		 */
870 		xge_os_memcpy(((char *)lro->tcp_hdr	+ lro->ts_off +	2),
871 				(char *)((char *)tcp + (*ts_off) + 2), 8);
872 	}
873 
874 	return XGE_HAL_OK;
875 
876 _exit_fail:
877 	xge_debug_ring(XGE_TRACE,	"tcphdr	not	fastpth	%02x %02x", tcp->doff_res, tcp->ctrl);
878 	return XGE_HAL_FAIL;
879 
880 }
881 
882 /*
883  * __hal_lro_capable: Finds	whether	frame is lro capable.
884  * @buffer:	Ethernet frame.
885  * @ip:	ip frame.
886  * @tcp: tcp frame.
887  * @ext_info: Descriptor info.
888  */
889 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
890 __hal_lro_capable( u8 *buffer,
891 		   iplro_t **ip,
892 		   tcplro_t	**tcp,
893        xge_hal_dtr_info_t *ext_info)
894 {
895 	u8 ip_off, ip_length;
896 
897 	if (!(ext_info->proto &	XGE_HAL_FRAME_PROTO_TCP)) {
898 		xge_debug_ring(XGE_ERR,	"Cant do lro %d", ext_info->proto);
899 		return XGE_HAL_FAIL;
900 	}
901 
902   if ( !*ip )
903   {
904 #ifdef XGE_LL_DEBUG_DUMP_PKT
905 		{
906 			u8 ch;
907 			u16	i;
908 
909 			xge_os_printf("Dump	Eth:" );
910 			for	(i =0; i < 60; i++)	{
911 				ch = ntohs(*((u8 *)(buffer + i)) );
912 				xge_os_printf("i:%d	%02x, ",i,ch);
913 			}
914 		}
915 #endif
916 
917     switch (ext_info->frame) {
918     case XGE_HAL_FRAME_TYPE_DIX:
919       ip_off = XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE;
920       break;
921     case XGE_HAL_FRAME_TYPE_LLC:
922       ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE	+
923                 XGE_HAL_HEADER_802_2_SIZE);
924       break;
925     case XGE_HAL_FRAME_TYPE_SNAP:
926       ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE	+
927                 XGE_HAL_HEADER_SNAP_SIZE);
928       break;
929     default: //	XGE_HAL_FRAME_TYPE_IPX,	etc.
930       return XGE_HAL_FAIL;
931     }
932 
933 
934     if (ext_info->proto	& XGE_HAL_FRAME_PROTO_VLAN_TAGGED) {
935       ip_off += XGE_HAL_HEADER_VLAN_SIZE;
936     }
937 
938     /* Grab	ip,	tcp	headers	*/
939     *ip	= (iplro_t *)((char*)buffer	+ ip_off);
940   } /* !*ip */
941 
942 	ip_length =	(u8)((*ip)->version_ihl	& 0x0F);
943 	ip_length =	ip_length <<2;
944 	*tcp = (tcplro_t *)((char *)*ip + ip_length);
945 
946 	xge_debug_ring(XGE_TRACE, "ip_length:%d	ip:"XGE_OS_LLXFMT
947 		   " tcp:"XGE_OS_LLXFMT"", (int)ip_length,
948 		(unsigned long long)(ulong_t)*ip, (unsigned long long)(ulong_t)*tcp);
949 
950 	return XGE_HAL_OK;
951 
952 }
953 
954 
955 /*
956  * __hal_open_lro_session: Open	a new LRO session.
957  * @buffer:	Ethernet frame.
958  * @ip:	ip header.
959  * @tcp: tcp header.
960  * @lro: lro pointer
961  * @ext_info: Descriptor info.
962  * @hldev: Hal context.
963  * @ring_lro: LRO descriptor per rx ring.
964  * @slot: Bucket no.
965  * @tcp_seg_len: Length	of tcp segment.
966  * @ts_off:	time stamp offset in the packet.
967  */
968 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
969 __hal_open_lro_session (u8 *buffer,	iplro_t	*ip, tcplro_t *tcp,	lro_t **lro,
970 			xge_hal_device_t *hldev, xge_hal_lro_desc_t	*ring_lro, int slot,
971       u32 tcp_seg_len, int	ts_off)
972 {
973 
974 	lro_t *lro_new = &ring_lro->lro_pool[slot];
975 
976 	lro_new->in_use			=	1;
977 	lro_new->ll_hdr			=	buffer;
978 	lro_new->ip_hdr			=	ip;
979 	lro_new->tcp_hdr		=	tcp;
980 	lro_new->tcp_next_seq_num	=	tcp_seg_len	+ xge_os_ntohl(
981 								tcp->seq);
982 	lro_new->tcp_seq_num		=	tcp->seq;
983 	lro_new->tcp_ack_num		=	tcp->ack_seq;
984 	lro_new->sg_num			=	1;
985 	lro_new->total_length		=	xge_os_ntohs(ip->tot_len);
986 	lro_new->frags_len		=	0;
987 	lro_new->ts_off			=	ts_off;
988 
989 	hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
990 	hldev->stats.sw_dev_info_stats.tot_lro_sessions++;
991 
992 	*lro = ring_lro->lro_recent = lro_new;
993 	return;
994 }
995 /*
996  * __hal_lro_get_free_slot:	Get	a free LRO bucket.
997  * @ring_lro: LRO descriptor per ring.
998  */
999 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
1000 __hal_lro_get_free_slot	(xge_hal_lro_desc_t	*ring_lro)
1001 {
1002 	int	i;
1003 
1004 	for	(i = 0;	i <	XGE_HAL_LRO_MAX_BUCKETS; i++) {
1005 		lro_t *lro_temp	= &ring_lro->lro_pool[i];
1006 
1007 		if (!lro_temp->in_use)
1008 			return i;
1009 	}
1010 	return -1;
1011 }
1012 
1013 /*
1014  * __hal_get_lro_session: Gets matching	LRO	session	or creates one.
1015  * @eth_hdr:	Ethernet header.
1016  * @ip:	ip header.
1017  * @tcp: tcp header.
1018  * @lro: lro pointer
1019  * @ext_info: Descriptor info.
1020  * @hldev: Hal context.
1021  * @ring_lro: LRO descriptor per rx ring
1022  */
1023 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1024 __hal_get_lro_session (u8 *eth_hdr,
1025 			   iplro_t *ip,
1026 			   tcplro_t	*tcp,
1027 			   lro_t **lro,
1028 			   xge_hal_dtr_info_t *ext_info,
1029 			   xge_hal_device_t	*hldev,
1030 			   xge_hal_lro_desc_t	*ring_lro,
1031 			   lro_t **lro_end3	/* Valid only when ret=END_3 */)
1032 {
1033 	lro_t *lro_match;
1034 	int	i, free_slot = -1;
1035 	u32	tcp_seg_len;
1036 	int	ts_off = -1;
1037 
1038 	*lro = lro_match = NULL;
1039 	/*
1040 	 * Compare the incoming	frame with the lro session left	from the
1041 	 * previous	call.  There is	a good chance that this	incoming frame
1042 	 * matches the lro session.
1043 	 */
1044 	if (ring_lro->lro_recent && ring_lro->lro_recent->in_use)	{
1045 		if (__hal_lro_check_for_session_match(ring_lro->lro_recent,
1046 							  tcp, ip)
1047 							== XGE_HAL_OK)
1048 			lro_match =	ring_lro->lro_recent;
1049 	}
1050 
1051 	if (!lro_match)	{
1052 		/*
1053 		 * Search in the pool of LROs for the session that matches
1054 		 * the incoming	frame.
1055 		 */
1056 		for	(i = 0;	i <	XGE_HAL_LRO_MAX_BUCKETS; i++) {
1057 			lro_t *lro_temp	= &ring_lro->lro_pool[i];
1058 
1059 			if (!lro_temp->in_use) {
1060 				if (free_slot == -1)
1061 					free_slot =	i;
1062 				continue;
1063 			}
1064 
1065 			if (__hal_lro_check_for_session_match(lro_temp,	tcp,
1066 							  ip) == XGE_HAL_OK) {
1067 				lro_match =	lro_temp;
1068 				break;
1069 			}
1070 		}
1071 	}
1072 
1073 
1074 	if (lro_match) {
1075 		/*
1076 		 * Matching	LRO	Session	found
1077 		 */
1078 		*lro = lro_match;
1079 
1080 		if (lro_match->tcp_next_seq_num	!= xge_os_ntohl(tcp->seq)) {
1081      xge_debug_ring(XGE_ERR,	"**retransmit  **"
1082 						"found***");
1083 			hldev->stats.sw_dev_info_stats.lro_out_of_seq_pkt_cnt++;
1084 			return XGE_HAL_INF_LRO_END_2;
1085 		}
1086 
1087 		if (XGE_HAL_OK != __hal_ip_lro_capable(ip, ext_info))
1088     {
1089 			return XGE_HAL_INF_LRO_END_2;
1090     }
1091 
1092 		if (XGE_HAL_OK != __hal_tcp_lro_capable(ip,	tcp, lro_match,
1093 							&ts_off)) {
1094 			/*
1095 			 * Close the current session and open a	new
1096 			 * LRO session with	this packet,
1097 			 * provided	it has tcp payload
1098 			 */
1099 			tcp_seg_len	= __hal_tcp_seg_len(ip,	tcp);
1100 			if (tcp_seg_len	== 0)
1101       {
1102 				return XGE_HAL_INF_LRO_END_2;
1103       }
1104 
1105 			/* Get a free bucket  */
1106 			free_slot =	__hal_lro_get_free_slot(ring_lro);
1107 			if (free_slot == -1)
1108       {
1109 				return XGE_HAL_INF_LRO_END_2;
1110       }
1111 
1112 			/*
1113 			 * Open	a new LRO session
1114 			 */
1115 			__hal_open_lro_session (eth_hdr,	ip,	tcp, lro_end3,
1116 						hldev, ring_lro, free_slot, tcp_seg_len,
1117 						ts_off);
1118 
1119 			return XGE_HAL_INF_LRO_END_3;
1120 		}
1121 
1122 				/*
1123 		 * The frame is	good, in-sequence, can be LRO-ed;
1124 		 * take	its	(latest) ACK - unless it is	a dupack.
1125 		 * Note: to	be exact need to check window size as well..
1126 		*/
1127 		if (lro_match->tcp_ack_num == tcp->ack_seq &&
1128 			lro_match->tcp_seq_num == tcp->seq)	{
1129 			hldev->stats.sw_dev_info_stats.lro_dup_pkt_cnt++;
1130 			return XGE_HAL_INF_LRO_END_2;
1131 		}
1132 
1133 		lro_match->tcp_seq_num = tcp->seq;
1134 		lro_match->tcp_ack_num = tcp->ack_seq;
1135 		lro_match->frags_len +=	__hal_tcp_seg_len(ip, tcp);
1136 
1137 		ring_lro->lro_recent =	lro_match;
1138 
1139 		return XGE_HAL_INF_LRO_CONT;
1140 	}
1141 
1142 	/* ********** New Session ***************/
1143 	if (free_slot == -1)
1144 		return XGE_HAL_INF_LRO_UNCAPABLE;
1145 
1146 	if (XGE_HAL_FAIL ==	__hal_ip_lro_capable(ip, ext_info))
1147 		return XGE_HAL_INF_LRO_UNCAPABLE;
1148 
1149 	if (XGE_HAL_FAIL ==	__hal_tcp_lro_capable(ip, tcp, NULL, &ts_off))
1150 		return XGE_HAL_INF_LRO_UNCAPABLE;
1151 
1152 	xge_debug_ring(XGE_TRACE, "Creating	lro	session.");
1153 
1154 	/*
1155 	 * Open	a LRO session, provided	the	packet contains	payload.
1156 	 */
1157 	tcp_seg_len	= __hal_tcp_seg_len(ip,	tcp);
1158 	if (tcp_seg_len	== 0)
1159 		return XGE_HAL_INF_LRO_UNCAPABLE;
1160 
1161 	__hal_open_lro_session (eth_hdr,	ip,	tcp, lro, hldev, ring_lro, free_slot,
1162 				tcp_seg_len, ts_off);
1163 
1164 	return XGE_HAL_INF_LRO_BEGIN;
1165 }
1166 
1167 /*
1168  * __hal_lro_under_optimal_thresh: Finds whether combined session is optimal.
1169  * @ip:	ip header.
1170  * @tcp: tcp header.
1171  * @lro: lro pointer
1172  * @hldev: Hal context.
1173  */
1174 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1175 __hal_lro_under_optimal_thresh (iplro_t	*ip,
1176 					tcplro_t *tcp,
1177 				lro_t *lro,
1178 				xge_hal_device_t *hldev)
1179 {
1180 	if (!lro) return XGE_HAL_FAIL;
1181 
1182 	if ((lro->total_length + __hal_tcp_seg_len(ip, tcp)	) >
1183 						hldev->config.lro_frm_len) {
1184 		xge_debug_ring(XGE_TRACE, "Max LRO frame len exceeded:"
1185 		 "max length %d	", hldev->config.lro_frm_len);
1186 		hldev->stats.sw_dev_info_stats.lro_frm_len_exceed_cnt++;
1187 		return XGE_HAL_FAIL;
1188 	}
1189 
1190 	if (lro->sg_num	== hldev->config.lro_sg_size) {
1191 		xge_debug_ring(XGE_TRACE, "Max sg count	exceeded:"
1192 				 "max sg %d	", hldev->config.lro_sg_size);
1193 		hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1194 		return XGE_HAL_FAIL;
1195 	}
1196 
1197 	return XGE_HAL_OK;
1198 }
1199 
1200 /*
1201  * __hal_collapse_ip_hdr: Collapses	ip header.
1202  * @ip:	ip header.
1203  * @tcp: tcp header.
1204  * @lro: lro pointer
1205  * @hldev: Hal context.
1206  */
1207 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1208 __hal_collapse_ip_hdr (	iplro_t	*ip,
1209 			tcplro_t *tcp,
1210 			lro_t *lro,
1211 			xge_hal_device_t *hldev)
1212 {
1213 
1214 	lro->total_length += __hal_tcp_seg_len(ip, tcp);
1215 
1216 	/* May be we have to handle	time stamps	or more	options	*/
1217 
1218 	return XGE_HAL_OK;
1219 
1220 }
1221 
1222 /*
1223  * __hal_collapse_tcp_hdr: Collapses tcp header.
1224  * @ip:	ip header.
1225  * @tcp: tcp header.
1226  * @lro: lro pointer
1227  * @hldev: Hal context.
1228  */
1229 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1230 __hal_collapse_tcp_hdr ( iplro_t *ip,
1231 			 tcplro_t *tcp,
1232 			 lro_t *lro,
1233 			 xge_hal_device_t *hldev)
1234 {
1235 	lro->tcp_next_seq_num += __hal_tcp_seg_len(ip, tcp);
1236 	return XGE_HAL_OK;
1237 
1238 }
1239 
1240 /*
1241  * __hal_append_lro: Appends new frame to existing LRO session.
1242  * @ip:	ip header.
1243  * @tcp: IN	tcp	header,	OUT	tcp	payload.
1244  * @seg_len: tcp payload length.
1245  * @lro: lro pointer
1246  * @hldev: Hal context.
1247  */
1248 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1249 __hal_append_lro(iplro_t *ip,
1250 		 tcplro_t **tcp,
1251 		 u32 *seg_len,
1252 		 lro_t *lro,
1253 		 xge_hal_device_t *hldev)
1254 {
1255 	(void) __hal_collapse_ip_hdr(ip, *tcp,	lro, hldev);
1256 	(void) __hal_collapse_tcp_hdr(ip, *tcp, lro, hldev);
1257 	// Update mbuf chain will be done in ll	driver.
1258 	// xge_hal_accumulate_large_rx on success of appending new frame to
1259 	// lro will	return to ll driver	tcpdata	pointer, and tcp payload length.
1260 	// along with return code lro frame	appended.
1261 
1262 	lro->sg_num++;
1263 	*seg_len = __hal_tcp_seg_len(ip, *tcp);
1264 	*tcp = (tcplro_t *)((char *)*tcp	+ (((*tcp)->doff_res)>>2));
1265 
1266 	return XGE_HAL_OK;
1267 
1268 }
1269 
1270 /**
1271  * __xge_hal_accumulate_large_rx:	LRO	a given	frame
1272  * frames
1273  * @ring: rx ring number
1274  * @eth_hdr: ethernet header.
1275  * @ip_hdr: ip header (optional)
1276  * @tcp: tcp header.
1277  * @seglen:	packet length.
1278  * @p_lro: lro pointer.
1279  * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1280  * @hldev: HAL device.
1281  * @lro_end3: for lro_end3 output
1282  *
1283  * LRO the newly received frame, i.e. attach it	(if	possible) to the
1284  * already accumulated (i.e., already LRO-ed) received frames (if any),
1285  * to form one super-sized frame for the subsequent	processing
1286  * by the stack.
1287  */
1288 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1289 xge_hal_lro_process_rx(int ring, u8 *eth_hdr, u8 *ip_hdr, tcplro_t **tcp,
1290                        u32 *seglen, lro_t **p_lro,
1291                        xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1292                        lro_t **lro_end3)
1293 {
1294 	iplro_t	*ip = (iplro_t *)ip_hdr;
1295 	xge_hal_status_e ret;
1296 	lro_t *lro;
1297 
1298 	xge_debug_ring(XGE_TRACE, "Entered accumu lro. ");
1299 	if (XGE_HAL_OK != __hal_lro_capable(eth_hdr, &ip, (tcplro_t **)tcp,
1300                                       ext_info))
1301 		return XGE_HAL_INF_LRO_UNCAPABLE;
1302 
1303 	/*
1304 	 * This	function shall get matching LRO or else
1305 	 * create one and return it
1306 	 */
1307 	ret = __hal_get_lro_session(eth_hdr, ip, (tcplro_t *)*tcp,
1308                               p_lro, ext_info, hldev,	&hldev->lro_desc[ring],
1309                               lro_end3);
1310 	xge_debug_ring(XGE_TRACE, "ret from get_lro:%d ",ret);
1311 	lro = *p_lro;
1312 	if (XGE_HAL_INF_LRO_CONT == ret) {
1313 		if (XGE_HAL_OK == __hal_lro_under_optimal_thresh(ip,
1314 						(tcplro_t *)*tcp, lro, hldev)) {
1315 			(void) __hal_append_lro(ip,(tcplro_t **) tcp, seglen,
1316 							 lro, hldev);
1317 			hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
1318 
1319 			if (lro->sg_num	>= hldev->config.lro_sg_size) {
1320 				hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1321 				ret = XGE_HAL_INF_LRO_END_1;
1322 			}
1323 
1324 		} else ret = XGE_HAL_INF_LRO_END_2;
1325 	}
1326 
1327 	/*
1328 	 * Since its time to flush,
1329 	 * update ip header	so that	it can be sent up
1330 	 */
1331 	if ((ret == XGE_HAL_INF_LRO_END_1) ||
1332 		(ret ==	XGE_HAL_INF_LRO_END_2) ||
1333 		(ret ==	XGE_HAL_INF_LRO_END_3))	{
1334 		lro->ip_hdr->tot_len = xge_os_htons((*p_lro)->total_length);
1335 		lro->ip_hdr->check = xge_os_htons(0);
1336 		lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1337 					(lro->ip_hdr->version_ihl & 0x0F));
1338 		lro->tcp_hdr->ack_seq =	lro->tcp_ack_num;
1339 	}
1340 
1341 	return (ret);
1342 }
1343 
1344 /**
1345  * xge_hal_accumulate_large_rx:	LRO	a given	frame
1346  * frames
1347  * @buffer:	Ethernet frame.
1348  * @tcp: tcp header.
1349  * @seglen:	packet length.
1350  * @p_lro: lro pointer.
1351  * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1352  * @hldev: HAL device.
1353  * @lro_end3: for lro_end3 output
1354  *
1355  * LRO the newly received frame, i.e. attach it	(if	possible) to the
1356  * already accumulated (i.e., already LRO-ed) received frames (if any),
1357  * to form one super-sized frame for the subsequent	processing
1358  * by the stack.
1359  */
1360 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1361 xge_hal_accumulate_large_rx(u8 *buffer, tcplro_t **tcp, u32 *seglen,
1362 lro_t **p_lro, xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1363 lro_t **lro_end3)
1364 {
1365   int ring = 0;
1366   return xge_hal_lro_process_rx(ring, buffer, NULL, tcp, seglen, p_lro,
1367                                 ext_info, hldev, lro_end3);
1368 }
1369 
1370 /**
1371  * xge_hal_lro_close_session: Close LRO session
1372  * @lro: LRO Session.
1373  * @hldev: HAL Context.
1374  */
1375 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
1376 xge_hal_lro_close_session (lro_t *lro)
1377 {
1378 	lro->in_use = 0;
1379 }
1380 
1381 /**
1382  * xge_hal_lro_next_session: Returns next LRO session in the list or NULL
1383  *					if none	exists.
1384  * @hldev: HAL Context.
1385  * @ring: rx ring number.
1386  */
1387 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t	*
1388 xge_hal_lro_next_session (xge_hal_device_t *hldev, int ring)
1389 {
1390 xge_hal_lro_desc_t *ring_lro = &hldev->lro_desc[ring];
1391 	int	i;
1392 	int	start_idx =	ring_lro->lro_next_idx;
1393 
1394 	for(i =	start_idx; i < XGE_HAL_LRO_MAX_BUCKETS;	i++) {
1395 		lro_t *lro = &ring_lro->lro_pool[i];
1396 
1397 		if (!lro->in_use)
1398 			continue;
1399 
1400 		lro->ip_hdr->tot_len = xge_os_htons(lro->total_length);
1401 		lro->ip_hdr->check = xge_os_htons(0);
1402 		lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1403 								(lro->ip_hdr->version_ihl &	0x0F));
1404 		ring_lro->lro_next_idx	= i	+ 1;
1405 		return lro;
1406 	}
1407 
1408 	ring_lro->lro_next_idx	= 0;
1409 	return NULL;
1410 
1411 }
1412 
1413 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
1414 xge_hal_lro_get_next_session(xge_hal_device_t *hldev)
1415 {
1416   int ring = 0; /* assume default ring=0 */
1417   return xge_hal_lro_next_session(hldev, ring);
1418 }
1419 #endif
1420