1a23fd118Syl /*
2a23fd118Syl  * CDDL HEADER START
3a23fd118Syl  *
4a23fd118Syl  * The contents of this file are subject to the terms of the
5a23fd118Syl  * Common Development and Distribution License (the "License").
6a23fd118Syl  * You may not use this file except in compliance with the License.
7a23fd118Syl  *
8a23fd118Syl  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9a23fd118Syl  * or http://www.opensolaris.org/os/licensing.
10a23fd118Syl  * See the License for the specific language governing permissions
11a23fd118Syl  * and limitations under the License.
12a23fd118Syl  *
13a23fd118Syl  * When distributing Covered Code, include this CDDL HEADER in each
14a23fd118Syl  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15a23fd118Syl  * If applicable, add the following below this CDDL HEADER, with the
16a23fd118Syl  * fields enclosed by brackets "[]" replaced with your own identifying
17a23fd118Syl  * information: Portions Copyright [yyyy] [name of copyright owner]
18a23fd118Syl  *
19a23fd118Syl  * CDDL HEADER END
20a23fd118Syl  *
218347601bSyl  * Copyright (c) 2002-2006 Neterion, Inc.
22a23fd118Syl  */
23a23fd118Syl 
24a23fd118Syl #include "xgehal-fifo.h"
25a23fd118Syl #include "xgehal-device.h"
26a23fd118Syl 
27a23fd118Syl static xge_hal_status_e
__hal_fifo_mempool_item_alloc(xge_hal_mempool_h mempoolh,void * memblock,int memblock_index,xge_hal_mempool_dma_t * dma_object,void * item,int index,int is_last,void * userdata)28a23fd118Syl __hal_fifo_mempool_item_alloc(xge_hal_mempool_h mempoolh,
29a23fd118Syl 			      void *memblock,
30a23fd118Syl 			      int memblock_index,
31a23fd118Syl 			      xge_hal_mempool_dma_t *dma_object,
32a23fd118Syl 			      void *item,
33a23fd118Syl 			      int index,
34a23fd118Syl 			      int is_last,
35a23fd118Syl 			      void *userdata)
36a23fd118Syl {
37a23fd118Syl 	int memblock_item_idx;
38a23fd118Syl 	xge_hal_fifo_txdl_priv_t *txdl_priv;
39a23fd118Syl 	xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)item;
40a23fd118Syl 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata;
41a23fd118Syl 
42a23fd118Syl 	xge_assert(item);
438347601bSyl 	txdl_priv = (xge_hal_fifo_txdl_priv_t *) \
448347601bSyl                 __hal_mempool_item_priv((xge_hal_mempool_t *) mempoolh,
458347601bSyl                                         memblock_index,
468347601bSyl                                         item,
478347601bSyl                                         &memblock_item_idx);
48a23fd118Syl 	xge_assert(txdl_priv);
49a23fd118Syl 
50a23fd118Syl 	/* pre-format HAL's TxDL's private */
51a23fd118Syl 	txdl_priv->dma_offset = (char*)item - (char*)memblock;
52a23fd118Syl 	txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset;
53a23fd118Syl 	txdl_priv->dma_handle = dma_object->handle;
54a23fd118Syl 	txdl_priv->memblock   = memblock;
55a23fd118Syl 	txdl_priv->first_txdp = (xge_hal_fifo_txd_t *)item;
56a23fd118Syl 	txdl_priv->next_txdl_priv = NULL;
57a23fd118Syl 	txdl_priv->dang_txdl = NULL;
58a23fd118Syl 	txdl_priv->dang_frags = 0;
59a23fd118Syl 	txdl_priv->alloc_frags = 0;
60a23fd118Syl 
61a23fd118Syl #ifdef XGE_DEBUG_ASSERT
62a23fd118Syl 	txdl_priv->dma_object = dma_object;
63a23fd118Syl #endif
64a23fd118Syl 	txdp->host_control = (u64)(ulong_t)txdl_priv;
65a23fd118Syl 
66a23fd118Syl #ifdef XGE_HAL_ALIGN_XMIT
67a23fd118Syl 	txdl_priv->align_vaddr = NULL;
68a23fd118Syl 	txdl_priv->align_dma_addr = (dma_addr_t)0;
69a23fd118Syl 
70a23fd118Syl #ifndef XGE_HAL_ALIGN_XMIT_ALLOC_RT
71a23fd118Syl 	{
72a23fd118Syl 	xge_hal_status_e status;
73a23fd118Syl 	if (fifo->config->alignment_size) {
74a23fd118Syl 	        status =__hal_fifo_dtr_align_alloc_map(fifo, txdp);
75a23fd118Syl 		if (status != XGE_HAL_OK)  {
76a23fd118Syl 		        xge_debug_mm(XGE_ERR,
77a23fd118Syl 		              "align buffer[%d] %d bytes, status %d",
78a23fd118Syl 			      index,
798347601bSyl 			      fifo->align_size,
80a23fd118Syl 			      status);
81a23fd118Syl 		        return status;
82a23fd118Syl 		}
83a23fd118Syl 	}
84a23fd118Syl 	}
85a23fd118Syl #endif
86a23fd118Syl #endif
87a23fd118Syl 
88a23fd118Syl 	if (fifo->channel.dtr_init) {
89a23fd118Syl 		fifo->channel.dtr_init(fifo, (xge_hal_dtr_h)txdp, index,
90a23fd118Syl 			   fifo->channel.userdata, XGE_HAL_CHANNEL_OC_NORMAL);
91a23fd118Syl 	}
92a23fd118Syl 
93a23fd118Syl 	return XGE_HAL_OK;
94a23fd118Syl }
95a23fd118Syl 
96a23fd118Syl 
97a23fd118Syl static xge_hal_status_e
__hal_fifo_mempool_item_free(xge_hal_mempool_h mempoolh,void * memblock,int memblock_index,xge_hal_mempool_dma_t * dma_object,void * item,int index,int is_last,void * userdata)98a23fd118Syl __hal_fifo_mempool_item_free(xge_hal_mempool_h mempoolh,
99a23fd118Syl 			      void *memblock,
100a23fd118Syl 			      int memblock_index,
101a23fd118Syl 			      xge_hal_mempool_dma_t *dma_object,
102a23fd118Syl 			      void *item,
103a23fd118Syl 			      int index,
104a23fd118Syl 			      int is_last,
105a23fd118Syl 			      void *userdata)
106a23fd118Syl {
107a23fd118Syl 	int memblock_item_idx;
108a23fd118Syl 	xge_hal_fifo_txdl_priv_t *txdl_priv;
109a23fd118Syl #ifdef XGE_HAL_ALIGN_XMIT
110a23fd118Syl 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata;
111a23fd118Syl #endif
112a23fd118Syl 
113a23fd118Syl 	xge_assert(item);
114a23fd118Syl 
1158347601bSyl 	txdl_priv = (xge_hal_fifo_txdl_priv_t *) \
1168347601bSyl                 __hal_mempool_item_priv((xge_hal_mempool_t *) mempoolh,
1178347601bSyl                                         memblock_index,
1188347601bSyl                                         item,
1198347601bSyl                                         &memblock_item_idx);
120a23fd118Syl 	xge_assert(txdl_priv);
121a23fd118Syl 
122a23fd118Syl #ifdef XGE_HAL_ALIGN_XMIT
123a23fd118Syl 	if (fifo->config->alignment_size) {
124a23fd118Syl 		if (txdl_priv->align_dma_addr != 0) {
125a23fd118Syl 			xge_os_dma_unmap(fifo->channel.pdev,
126a23fd118Syl 			       txdl_priv->align_dma_handle,
127a23fd118Syl 			       txdl_priv->align_dma_addr,
1288347601bSyl 			       fifo->align_size,
129a23fd118Syl 			       XGE_OS_DMA_DIR_TODEVICE);
130a23fd118Syl 
131a23fd118Syl 			txdl_priv->align_dma_addr = 0;
132a23fd118Syl 		}
133a23fd118Syl 
134a23fd118Syl 		if (txdl_priv->align_vaddr != NULL) {
135a23fd118Syl 			xge_os_dma_free(fifo->channel.pdev,
136a23fd118Syl 			      txdl_priv->align_vaddr,
1378347601bSyl 			      fifo->align_size,
138a23fd118Syl 			      &txdl_priv->align_dma_acch,
139a23fd118Syl 			      &txdl_priv->align_dma_handle);
140a23fd118Syl 
141a23fd118Syl 			txdl_priv->align_vaddr = NULL;
142a23fd118Syl 		}
143a23fd118Syl 	}
144a23fd118Syl #endif
145a23fd118Syl 
146a23fd118Syl 	return XGE_HAL_OK;
147a23fd118Syl }
148a23fd118Syl 
149a23fd118Syl xge_hal_status_e
__hal_fifo_open(xge_hal_channel_h channelh,xge_hal_channel_attr_t * attr)150a23fd118Syl __hal_fifo_open(xge_hal_channel_h channelh, xge_hal_channel_attr_t *attr)
151a23fd118Syl {
152a23fd118Syl 	xge_hal_device_t *hldev;
153a23fd118Syl 	xge_hal_status_e status;
154a23fd118Syl 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
155a23fd118Syl 	xge_hal_fifo_queue_t *queue;
156a23fd118Syl 	int i, txdl_size, max_arr_index, mid_point;
157a23fd118Syl 	xge_hal_dtr_h  dtrh;
158a23fd118Syl 
159a23fd118Syl 	hldev = (xge_hal_device_t *)fifo->channel.devh;
160a23fd118Syl 	fifo->config = &hldev->config.fifo;
161a23fd118Syl 	queue = &fifo->config->queue[attr->post_qid];
162a23fd118Syl 
163a23fd118Syl #if defined(XGE_HAL_TX_MULTI_RESERVE)
164a23fd118Syl 	xge_os_spin_lock_init(&fifo->channel.reserve_lock, hldev->pdev);
165a23fd118Syl #elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ)
166a23fd118Syl 	xge_os_spin_lock_init_irq(&fifo->channel.reserve_lock, hldev->irqh);
167a23fd118Syl #endif
168a23fd118Syl #if defined(XGE_HAL_TX_MULTI_POST)
169a23fd118Syl 	if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA)  {
170a23fd118Syl                 fifo->post_lock_ptr = &hldev->xena_post_lock;
171a23fd118Syl 	} else {
172a23fd118Syl 	        xge_os_spin_lock_init(&fifo->channel.post_lock, hldev->pdev);
173a23fd118Syl                 fifo->post_lock_ptr = &fifo->channel.post_lock;
174a23fd118Syl 	}
175a23fd118Syl #elif defined(XGE_HAL_TX_MULTI_POST_IRQ)
176a23fd118Syl 	if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA)  {
177a23fd118Syl                 fifo->post_lock_ptr = &hldev->xena_post_lock;
178a23fd118Syl 	} else {
179a23fd118Syl 	        xge_os_spin_lock_init_irq(&fifo->channel.post_lock,
180a23fd118Syl 					hldev->irqh);
181a23fd118Syl                 fifo->post_lock_ptr = &fifo->channel.post_lock;
182a23fd118Syl 	}
183a23fd118Syl #endif
184a23fd118Syl 
1858347601bSyl 	fifo->align_size =
1868347601bSyl 		fifo->config->alignment_size * fifo->config->max_aligned_frags;
1878347601bSyl 
188a23fd118Syl 	/* Initializing the BAR1 address as the start of
189a23fd118Syl 	 * the FIFO queue pointer and as a location of FIFO control
190a23fd118Syl 	 * word. */
191a23fd118Syl 	fifo->hw_pair =
192a23fd118Syl 	        (xge_hal_fifo_hw_pair_t *) (void *)(hldev->bar1 +
193a23fd118Syl 		        (attr->post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET));
194a23fd118Syl 
195a23fd118Syl 	/* apply "interrupts per txdl" attribute */
196a23fd118Syl 	fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_UTILZ;
197a23fd118Syl 	if (queue->intr) {
198a23fd118Syl 		fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_PER_LIST;
199a23fd118Syl 	}
200a23fd118Syl 	fifo->no_snoop_bits =
201a23fd118Syl 		(int)(XGE_HAL_TX_FIFO_NO_SNOOP(queue->no_snoop_bits));
202a23fd118Syl 
203a23fd118Syl 	/*
204a23fd118Syl 	 * FIFO memory management strategy:
205a23fd118Syl 	 *
206a23fd118Syl 	 * TxDL splitted into three independent parts:
207a23fd118Syl 	 *	- set of TxD's
208a23fd118Syl 	 *	- TxD HAL private part
209a23fd118Syl 	 *	- upper layer private part
210a23fd118Syl 	 *
211a23fd118Syl 	 * Adaptative memory allocation used. i.e. Memory allocated on
212a23fd118Syl 	 * demand with the size which will fit into one memory block.
213a23fd118Syl 	 * One memory block may contain more than one TxDL. In simple case
214a23fd118Syl 	 * memory block size can be equal to CPU page size. On more
215a23fd118Syl 	 * sophisticated OS's memory block can be contigious across
216a23fd118Syl 	 * several pages.
217a23fd118Syl 	 *
218a23fd118Syl 	 * During "reserve" operations more memory can be allocated on demand
219a23fd118Syl 	 * for example due to FIFO full condition.
220a23fd118Syl 	 *
221a23fd118Syl 	 * Pool of memory memblocks never shrinks except __hal_fifo_close
222a23fd118Syl 	 * routine which will essentially stop channel and free the resources.
223a23fd118Syl 	 */
224a23fd118Syl 
225a23fd118Syl 	/* TxDL common private size == TxDL private + ULD private */
226a23fd118Syl 	fifo->priv_size = sizeof(xge_hal_fifo_txdl_priv_t) +
227a23fd118Syl 	attr->per_dtr_space;
228a23fd118Syl 	fifo->priv_size = ((fifo->priv_size + __xge_os_cacheline_size -1) /
229a23fd118Syl                                __xge_os_cacheline_size) *
230a23fd118Syl                                __xge_os_cacheline_size;
231a23fd118Syl 
232a23fd118Syl 	/* recompute txdl size to be cacheline aligned */
233a23fd118Syl 	fifo->txdl_size = fifo->config->max_frags * sizeof(xge_hal_fifo_txd_t);
234a23fd118Syl 	txdl_size = ((fifo->txdl_size + __xge_os_cacheline_size - 1) /
235a23fd118Syl 			__xge_os_cacheline_size) * __xge_os_cacheline_size;
236a23fd118Syl 
237a23fd118Syl 	if (fifo->txdl_size != txdl_size)
238*7eced415Sxw 	        xge_debug_fifo(XGE_ERR, "cacheline > 128 ( ?? ): %d, %d, %d, %d",
239a23fd118Syl 		fifo->config->max_frags, fifo->txdl_size, txdl_size,
240a23fd118Syl 		__xge_os_cacheline_size);
241a23fd118Syl 
242a23fd118Syl 	fifo->txdl_size = txdl_size;
243a23fd118Syl 
244a23fd118Syl 	/* since dtr_init() callback will be called from item_alloc(),
245a23fd118Syl 	 * the same way channels userdata might be used prior to
246a23fd118Syl 	 * channel_initialize() */
247a23fd118Syl 	fifo->channel.dtr_init = attr->dtr_init;
248a23fd118Syl 	fifo->channel.userdata = attr->userdata;
249a23fd118Syl 	fifo->txdl_per_memblock = fifo->config->memblock_size /
250a23fd118Syl 		fifo->txdl_size;
251a23fd118Syl 
252a23fd118Syl 	fifo->mempool = __hal_mempool_create(hldev->pdev,
253a23fd118Syl 					     fifo->config->memblock_size,
254a23fd118Syl 					     fifo->txdl_size,
255a23fd118Syl 					     fifo->priv_size,
256a23fd118Syl 					     queue->initial,
257a23fd118Syl 					     queue->max,
258a23fd118Syl 					     __hal_fifo_mempool_item_alloc,
259a23fd118Syl 					     __hal_fifo_mempool_item_free,
260a23fd118Syl 					     fifo);
261a23fd118Syl 	if (fifo->mempool == NULL) {
262a23fd118Syl 		return XGE_HAL_ERR_OUT_OF_MEMORY;
263a23fd118Syl 	}
264a23fd118Syl 
265a23fd118Syl 	status = __hal_channel_initialize(channelh, attr,
2668347601bSyl 					(void **) __hal_mempool_items_arr(fifo->mempool),
267a23fd118Syl 					queue->initial, queue->max,
268a23fd118Syl 					fifo->config->reserve_threshold);
269a23fd118Syl 	if (status != XGE_HAL_OK) {
270a23fd118Syl 		__hal_fifo_close(channelh);
271a23fd118Syl 		return status;
272a23fd118Syl 	}
273a23fd118Syl 	xge_debug_fifo(XGE_TRACE,
274a23fd118Syl 		"DTR  reserve_length:%d reserve_top:%d\n"
275a23fd118Syl 		"max_frags:%d reserve_threshold:%d\n"
276*7eced415Sxw 		"memblock_size:%d alignment_size:%d max_aligned_frags:%d",
277a23fd118Syl 		fifo->channel.reserve_length, fifo->channel.reserve_top,
278a23fd118Syl 		fifo->config->max_frags, fifo->config->reserve_threshold,
279a23fd118Syl 		fifo->config->memblock_size, fifo->config->alignment_size,
280a23fd118Syl 		fifo->config->max_aligned_frags);
281a23fd118Syl 
282a23fd118Syl #ifdef XGE_DEBUG_ASSERT
283a23fd118Syl 	for ( i = 0; i < fifo->channel.reserve_length; i++) {
284a23fd118Syl 		xge_debug_fifo(XGE_TRACE, "DTR before reversing index:%d"
285*7eced415Sxw 		" handle:%p", i, fifo->channel.reserve_arr[i]);
286a23fd118Syl 	}
287a23fd118Syl #endif
288a23fd118Syl 
289a23fd118Syl 	xge_assert(fifo->channel.reserve_length);
290a23fd118Syl 	/* reverse the FIFO dtr array */
291a23fd118Syl 	max_arr_index	= fifo->channel.reserve_length - 1;
292a23fd118Syl 	max_arr_index	-=fifo->channel.reserve_top;
293a23fd118Syl 	xge_assert(max_arr_index);
294a23fd118Syl 	mid_point = (fifo->channel.reserve_length - fifo->channel.reserve_top)/2;
295a23fd118Syl 	for (i = 0; i < mid_point; i++) {
296*7eced415Sxw 		dtrh =	fifo->channel.reserve_arr[i];
297*7eced415Sxw 		fifo->channel.reserve_arr[i] =
298a23fd118Syl 			fifo->channel.reserve_arr[max_arr_index - i];
299a23fd118Syl 		fifo->channel.reserve_arr[max_arr_index  - i] = dtrh;
300a23fd118Syl 	}
301a23fd118Syl 
302a23fd118Syl #ifdef XGE_DEBUG_ASSERT
303a23fd118Syl 	for ( i = 0; i < fifo->channel.reserve_length; i++) {
304a23fd118Syl 		xge_debug_fifo(XGE_TRACE, "DTR after reversing index:%d"
305*7eced415Sxw 		" handle:%p", i, fifo->channel.reserve_arr[i]);
306a23fd118Syl 	}
307a23fd118Syl #endif
308a23fd118Syl 
309a23fd118Syl 	return XGE_HAL_OK;
310a23fd118Syl }
311a23fd118Syl 
312a23fd118Syl void
__hal_fifo_close(xge_hal_channel_h channelh)313a23fd118Syl __hal_fifo_close(xge_hal_channel_h channelh)
314a23fd118Syl {
315a23fd118Syl 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
316a23fd118Syl 	xge_hal_device_t *hldev = (xge_hal_device_t *)fifo->channel.devh;
317a23fd118Syl 
318a23fd118Syl 	if (fifo->mempool) {
319a23fd118Syl 		__hal_mempool_destroy(fifo->mempool);
320a23fd118Syl 	}
321a23fd118Syl 
322a23fd118Syl 	__hal_channel_terminate(channelh);
323a23fd118Syl 
324a23fd118Syl #if defined(XGE_HAL_TX_MULTI_RESERVE)
325a23fd118Syl 	xge_os_spin_lock_destroy(&fifo->channel.reserve_lock, hldev->pdev);
326a23fd118Syl #elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ)
327a23fd118Syl 	xge_os_spin_lock_destroy_irq(&fifo->channel.reserve_lock, hldev->pdev);
328a23fd118Syl #endif
329a23fd118Syl 	if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC)  {
330a23fd118Syl #if defined(XGE_HAL_TX_MULTI_POST)
331a23fd118Syl 		xge_os_spin_lock_destroy(&fifo->channel.post_lock, hldev->pdev);
332a23fd118Syl #elif defined(XGE_HAL_TX_MULTI_POST_IRQ)
333a23fd118Syl 		xge_os_spin_lock_destroy_irq(&fifo->channel.post_lock,
334a23fd118Syl 					     hldev->pdev);
335a23fd118Syl #endif
336a23fd118Syl 	}
337a23fd118Syl }
338a23fd118Syl 
339a23fd118Syl void
__hal_fifo_hw_initialize(xge_hal_device_h devh)340a23fd118Syl __hal_fifo_hw_initialize(xge_hal_device_h devh)
341a23fd118Syl {
342a23fd118Syl 	xge_hal_device_t *hldev = (xge_hal_device_t *)devh;
343*7eced415Sxw 	xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)(void *)hldev->bar0;
344a23fd118Syl 	u64* tx_fifo_partitions[4];
345a23fd118Syl 	u64* tx_fifo_wrr[5];
346*7eced415Sxw 	u64  tx_fifo_wrr_value[5];
347a23fd118Syl 	u64 val64, part0;
348a23fd118Syl 	int i;
349a23fd118Syl 
350a23fd118Syl 	/*  Tx DMA Initialization */
351a23fd118Syl 
352a23fd118Syl 	tx_fifo_partitions[0] = &bar0->tx_fifo_partition_0;
353a23fd118Syl 	tx_fifo_partitions[1] = &bar0->tx_fifo_partition_1;
354a23fd118Syl 	tx_fifo_partitions[2] = &bar0->tx_fifo_partition_2;
355a23fd118Syl 	tx_fifo_partitions[3] = &bar0->tx_fifo_partition_3;
356a23fd118Syl 
357a23fd118Syl 	tx_fifo_wrr[0] = &bar0->tx_w_round_robin_0;
358a23fd118Syl 	tx_fifo_wrr[1] = &bar0->tx_w_round_robin_1;
359a23fd118Syl 	tx_fifo_wrr[2] = &bar0->tx_w_round_robin_2;
360a23fd118Syl 	tx_fifo_wrr[3] = &bar0->tx_w_round_robin_3;
361a23fd118Syl 	tx_fifo_wrr[4] = &bar0->tx_w_round_robin_4;
362a23fd118Syl 
363*7eced415Sxw 	tx_fifo_wrr_value[0] = XGE_HAL_FIFO_WRR_0;
364*7eced415Sxw 	tx_fifo_wrr_value[1] = XGE_HAL_FIFO_WRR_1;
365*7eced415Sxw 	tx_fifo_wrr_value[2] = XGE_HAL_FIFO_WRR_2;
366*7eced415Sxw 	tx_fifo_wrr_value[3] = XGE_HAL_FIFO_WRR_3;
367*7eced415Sxw 	tx_fifo_wrr_value[4] = XGE_HAL_FIFO_WRR_4;
368*7eced415Sxw 
369a23fd118Syl 	/* Note: WRR calendar must be configured before the transmit
370*7eced415Sxw 	 *       FIFOs are enabled! page 6-77 user guide */
371a23fd118Syl 
372*7eced415Sxw 	if (!hldev->config.rts_qos_en) {
373*7eced415Sxw 		/* all zeroes for Round-Robin */
374*7eced415Sxw 		for (i = 0; i < XGE_HAL_FIFO_MAX_WRR; i++) {
375*7eced415Sxw 			xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0,
376*7eced415Sxw 					       tx_fifo_wrr[i]);
377*7eced415Sxw 		}
378*7eced415Sxw 
379*7eced415Sxw 		/* reset all of them but '0' */
380*7eced415Sxw 		for (i=1; i < XGE_HAL_FIFO_MAX_PARTITION; i++) {
381*7eced415Sxw 			xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0ULL,
382*7eced415Sxw 					       tx_fifo_partitions[i]);
383*7eced415Sxw 		}
384*7eced415Sxw 	} else { /* Change the default settings */
385a23fd118Syl 
386*7eced415Sxw 		for (i = 0; i < XGE_HAL_FIFO_MAX_WRR; i++) {
387*7eced415Sxw 			xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
388*7eced415Sxw 				       tx_fifo_wrr_value[i], tx_fifo_wrr[i]);
389*7eced415Sxw 		}
390a23fd118Syl 	}
391a23fd118Syl 
392a23fd118Syl 	/* configure only configured FIFOs */
393a23fd118Syl 	val64 = 0; part0 = 0;
394a23fd118Syl 	for (i = 0; i < XGE_HAL_MAX_FIFO_NUM; i++) {
395a23fd118Syl 		int reg_half = i % 2;
396a23fd118Syl 		int reg_num = i / 2;
397a23fd118Syl 
398a23fd118Syl 		if (hldev->config.fifo.queue[i].configured) {
3998347601bSyl 			int priority = hldev->config.fifo.queue[i].priority;
400a23fd118Syl 			val64 |=
401a23fd118Syl 			    vBIT((hldev->config.fifo.queue[i].max-1),
402a23fd118Syl 				(((reg_half) * 32) + 19),
403a23fd118Syl 				13) | vBIT(priority, (((reg_half)*32) + 5), 3);
404a23fd118Syl 		}
405a23fd118Syl 
406a23fd118Syl 		/* NOTE: do write operation for each second u64 half
407*7eced415Sxw 		 *       or force for first one if configured number
408*7eced415Sxw 		 *	 is even */
409a23fd118Syl 		if (reg_half) {
410a23fd118Syl 			if (reg_num == 0) {
411a23fd118Syl 				/* skip partition '0', must write it once at
412a23fd118Syl 				 * the end */
413a23fd118Syl 				part0 = val64;
414a23fd118Syl 			} else {
415a23fd118Syl 				xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
416a23fd118Syl 				     val64, tx_fifo_partitions[reg_num]);
417a23fd118Syl 				xge_debug_fifo(XGE_TRACE,
418a23fd118Syl 					"fifo partition_%d at: "
4198347601bSyl 					"0x"XGE_OS_LLXFMT" is: 0x"XGE_OS_LLXFMT,
4208347601bSyl 					reg_num, (unsigned long long)(ulong_t)
4218347601bSyl 					tx_fifo_partitions[reg_num],
422a23fd118Syl 					(unsigned long long)val64);
423a23fd118Syl 			}
424a23fd118Syl 			val64 = 0;
425a23fd118Syl 		}
426a23fd118Syl 	}
427a23fd118Syl 
428a23fd118Syl 	part0 |= BIT(0); /* to enable the FIFO partition. */
429a23fd118Syl 	__hal_pio_mem_write32_lower(hldev->pdev, hldev->regh0, (u32)part0,
430a23fd118Syl 	                     tx_fifo_partitions[0]);
431a23fd118Syl 	xge_os_wmb();
432a23fd118Syl 	__hal_pio_mem_write32_upper(hldev->pdev, hldev->regh0, (u32)(part0>>32),
433a23fd118Syl 	                     tx_fifo_partitions[0]);
434a23fd118Syl 	xge_debug_fifo(XGE_TRACE, "fifo partition_0 at: "
4358347601bSyl 			"0x"XGE_OS_LLXFMT" is: 0x"XGE_OS_LLXFMT,
436a23fd118Syl 			(unsigned long long)(ulong_t)
437a23fd118Syl 				tx_fifo_partitions[0],
438a23fd118Syl 			(unsigned long long) part0);
439a23fd118Syl 
440a23fd118Syl 	/*
441a23fd118Syl 	 * Initialization of Tx_PA_CONFIG register to ignore packet
442a23fd118Syl 	 * integrity checking.
443a23fd118Syl 	 */
444a23fd118Syl 	val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0,
445a23fd118Syl 	                            &bar0->tx_pa_cfg);
446a23fd118Syl 	val64 |= XGE_HAL_TX_PA_CFG_IGNORE_FRM_ERR |
447a23fd118Syl 		 XGE_HAL_TX_PA_CFG_IGNORE_SNAP_OUI |
448a23fd118Syl 		 XGE_HAL_TX_PA_CFG_IGNORE_LLC_CTRL |
449a23fd118Syl 		 XGE_HAL_TX_PA_CFG_IGNORE_L2_ERR;
450a23fd118Syl 	xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64,
451a23fd118Syl 	                     &bar0->tx_pa_cfg);
452*7eced415Sxw 
453*7eced415Sxw 	if (hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX)
454*7eced415Sxw 		return;
455*7eced415Sxw 
456*7eced415Sxw 	/*
457*7eced415Sxw 	 * Assign MSI-X vectors
458*7eced415Sxw 	 */
459*7eced415Sxw 	for (i = 0; i < XGE_HAL_MAX_FIFO_NUM; i++) {
460*7eced415Sxw 		xge_list_t *item;
461*7eced415Sxw 		xge_hal_channel_t *channel = NULL;
462*7eced415Sxw 
463*7eced415Sxw 		if (!hldev->config.fifo.queue[i].configured ||
464*7eced415Sxw 		    !hldev->config.fifo.queue[i].intr_vector)
465*7eced415Sxw 			continue;
466*7eced415Sxw 
467*7eced415Sxw 		/* find channel */
468*7eced415Sxw 		xge_list_for_each(item, &hldev->free_channels) {
469*7eced415Sxw 			xge_hal_channel_t *tmp;
470*7eced415Sxw 			tmp = xge_container_of(item, xge_hal_channel_t,
471*7eced415Sxw 						   item);
472*7eced415Sxw 			if (tmp->type == XGE_HAL_CHANNEL_TYPE_FIFO &&
473*7eced415Sxw 			    tmp->post_qid == i) {
474*7eced415Sxw 				channel = tmp;
475*7eced415Sxw 				break;
476*7eced415Sxw 			}
477*7eced415Sxw 		}
478*7eced415Sxw 
479*7eced415Sxw 		if (channel) {
480*7eced415Sxw 			(void) xge_hal_channel_msix_set(channel,
481*7eced415Sxw 				hldev->config.fifo.queue[i].intr_vector);
482*7eced415Sxw 		}
483*7eced415Sxw 	}
484*7eced415Sxw 
485a23fd118Syl 	xge_debug_fifo(XGE_TRACE, "%s", "fifo channels initialized");
486a23fd118Syl }
487a23fd118Syl 
488a23fd118Syl #ifdef XGE_HAL_ALIGN_XMIT
489a23fd118Syl void
__hal_fifo_dtr_align_free_unmap(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)490a23fd118Syl __hal_fifo_dtr_align_free_unmap(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
491a23fd118Syl {
492a23fd118Syl         xge_hal_fifo_txdl_priv_t *txdl_priv;
493a23fd118Syl 	xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh;
494a23fd118Syl 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
495a23fd118Syl 
496a23fd118Syl 	txdl_priv = __hal_fifo_txdl_priv(txdp);
497a23fd118Syl 
498a23fd118Syl 	if (txdl_priv->align_dma_addr != 0) {
499a23fd118Syl 		xge_os_dma_unmap(fifo->channel.pdev,
500a23fd118Syl 		       txdl_priv->align_dma_handle,
501a23fd118Syl 		       txdl_priv->align_dma_addr,
5028347601bSyl 		       fifo->align_size,
503a23fd118Syl 		       XGE_OS_DMA_DIR_TODEVICE);
504a23fd118Syl 
505a23fd118Syl                 txdl_priv->align_dma_addr = 0;
506a23fd118Syl 	}
507a23fd118Syl 
508a23fd118Syl         if (txdl_priv->align_vaddr != NULL) {
509a23fd118Syl 	        xge_os_dma_free(fifo->channel.pdev,
510a23fd118Syl 	              txdl_priv->align_vaddr,
5118347601bSyl 	              fifo->align_size,
512a23fd118Syl 	              &txdl_priv->align_dma_acch,
513a23fd118Syl 	              &txdl_priv->align_dma_handle);
514a23fd118Syl 
515a23fd118Syl 
516a23fd118Syl 	        txdl_priv->align_vaddr = NULL;
517a23fd118Syl         }
518a23fd118Syl  }
519a23fd118Syl 
520a23fd118Syl xge_hal_status_e
__hal_fifo_dtr_align_alloc_map(xge_hal_channel_h channelh,xge_hal_dtr_h dtrh)521a23fd118Syl __hal_fifo_dtr_align_alloc_map(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
522a23fd118Syl {
523a23fd118Syl         xge_hal_fifo_txdl_priv_t *txdl_priv;
524a23fd118Syl 	xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh;
525a23fd118Syl 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
526a23fd118Syl 
527a23fd118Syl 	xge_assert(txdp);
528a23fd118Syl 
529a23fd118Syl 	txdl_priv = __hal_fifo_txdl_priv(txdp);
530a23fd118Syl 
531a23fd118Syl 	/* allocate alignment DMA-buffer */
532*7eced415Sxw 	txdl_priv->align_vaddr = (char *)xge_os_dma_malloc(fifo->channel.pdev,
5338347601bSyl 				fifo->align_size,
534a23fd118Syl 				XGE_OS_DMA_CACHELINE_ALIGNED |
535a23fd118Syl 				XGE_OS_DMA_STREAMING,
536a23fd118Syl 				&txdl_priv->align_dma_handle,
537a23fd118Syl 				&txdl_priv->align_dma_acch);
538a23fd118Syl 	if (txdl_priv->align_vaddr == NULL) {
539a23fd118Syl 		return XGE_HAL_ERR_OUT_OF_MEMORY;
540a23fd118Syl 	}
541a23fd118Syl 
542a23fd118Syl 	/* map it */
543a23fd118Syl 	txdl_priv->align_dma_addr = xge_os_dma_map(fifo->channel.pdev,
544a23fd118Syl 		txdl_priv->align_dma_handle, txdl_priv->align_vaddr,
5458347601bSyl 		fifo->align_size,
546a23fd118Syl 		XGE_OS_DMA_DIR_TODEVICE, XGE_OS_DMA_STREAMING);
547a23fd118Syl 
548a23fd118Syl 	if (txdl_priv->align_dma_addr == XGE_OS_INVALID_DMA_ADDR) {
549a23fd118Syl                 __hal_fifo_dtr_align_free_unmap(channelh, dtrh);
550a23fd118Syl 		return XGE_HAL_ERR_OUT_OF_MAPPING;
551a23fd118Syl 	}
552a23fd118Syl 
553a23fd118Syl 	return XGE_HAL_OK;
554a23fd118Syl }
555a23fd118Syl #endif
556a23fd118Syl 
557a23fd118Syl 
558