xref: /illumos-gate/usr/src/uts/common/io/xge/hal/xgehal/xgehal-channel.c (revision 7eced415e5dd557aef2d78483b5a7785f0e13670)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  *
21  * Copyright (c) 2002-2006 Neterion, Inc.
22  */
23 
24 #include "xgehal-channel.h"
25 #include "xgehal-fifo.h"
26 #include "xgehal-ring.h"
27 #include "xgehal-device.h"
28 #include "xgehal-regs.h"
29 
30 /*
31  * __hal_channel_dtr_next_reservelist
32  *
33  * Walking through the all available DTRs.
34  */
35 static xge_hal_status_e
36 __hal_channel_dtr_next_reservelist(xge_hal_channel_h channelh,
37 		xge_hal_dtr_h *dtrh)
38 {
39 	xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
40 
41 	if (channel->reserve_top >= channel->reserve_length) {
42 		return XGE_HAL_INF_NO_MORE_FREED_DESCRIPTORS;
43 	}
44 
45 	*dtrh = channel->reserve_arr[channel->reserve_top++];
46 
47 	return XGE_HAL_OK;
48 }
49 
50 /*
51  * __hal_channel_dtr_next_freelist
52  *
53  * Walking through the "freed" DTRs.
54  */
55 static xge_hal_status_e
56 __hal_channel_dtr_next_freelist(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh)
57 {
58 	xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
59 
60 	if (channel->reserve_initial == channel->free_length) {
61 		return XGE_HAL_INF_NO_MORE_FREED_DESCRIPTORS;
62 	}
63 
64 	*dtrh = channel->free_arr[channel->free_length++];
65 
66 	return XGE_HAL_OK;
67 }
68 
69 /*
70  * __hal_channel_dtr_next_not_completed - Get the _next_ posted but
71  *                                     not completed descriptor.
72  *
73  * Walking through the "not completed" DTRs.
74  */
75 static xge_hal_status_e
76 __hal_channel_dtr_next_not_completed(xge_hal_channel_h channelh,
77 		xge_hal_dtr_h *dtrh)
78 {
79 	/* LINTED E_FUNC_SET_NOT_USED */
80 	xge_hal_ring_rxd_1_t *rxdp; /* doesn't matter 1, 3 or 5... */
81 
82 	__hal_channel_dtr_try_complete(channelh, dtrh);
83 	if (*dtrh == NULL) {
84 		return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
85 	}
86 
87 	rxdp = (xge_hal_ring_rxd_1_t *)*dtrh;
88 	xge_assert(rxdp->host_control!=0);
89 
90 	__hal_channel_dtr_complete(channelh);
91 
92 	return XGE_HAL_OK;
93 }
94 
95 xge_hal_channel_t*
96 __hal_channel_allocate(xge_hal_device_h devh, int post_qid,
97 		xge_hal_channel_type_e type)
98 {
99 	xge_hal_device_t *hldev = (xge_hal_device_t*)devh;
100 	xge_hal_channel_t *channel;
101 	int size = 0;
102 
103 	switch(type) {
104 		case XGE_HAL_CHANNEL_TYPE_FIFO:
105 			xge_assert(post_qid + 1 >= XGE_HAL_MIN_FIFO_NUM &&
106 				 post_qid + 1 <= XGE_HAL_MAX_FIFO_NUM);
107 			size = sizeof(xge_hal_fifo_t);
108 			break;
109 		case XGE_HAL_CHANNEL_TYPE_RING:
110 			xge_assert(post_qid + 1 >= XGE_HAL_MIN_RING_NUM &&
111 				post_qid + 1 <= XGE_HAL_MAX_RING_NUM);
112 			size = sizeof(xge_hal_ring_t);
113 			break;
114 		default :
115 			xge_assert(size);
116 			break;
117 
118 	}
119 
120 
121 	/* allocate FIFO channel */
122 	channel = (xge_hal_channel_t *) xge_os_malloc(hldev->pdev, size);
123 	if (channel == NULL) {
124 		return NULL;
125 	}
126 	xge_os_memzero(channel, size);
127 
128 	channel->pdev		= hldev->pdev;
129 	channel->regh0		= hldev->regh0;
130 	channel->regh1		= hldev->regh1;
131 	channel->type		= type;
132 	channel->devh		= devh;
133 	channel->post_qid	= post_qid;
134 	channel->compl_qid	= 0;
135 
136 	return channel;
137 }
138 
139 void __hal_channel_free(xge_hal_channel_t *channel)
140 {
141 	int size = 0;
142 
143 	xge_assert(channel->pdev);
144 
145 	switch(channel->type) {
146 		case XGE_HAL_CHANNEL_TYPE_FIFO:
147 			size = sizeof(xge_hal_fifo_t);
148 			break;
149 		case XGE_HAL_CHANNEL_TYPE_RING:
150 			size = sizeof(xge_hal_ring_t);
151 			break;
152 		case XGE_HAL_CHANNEL_TYPE_SEND_QUEUE:
153 		case XGE_HAL_CHANNEL_TYPE_RECEIVE_QUEUE:
154 		case XGE_HAL_CHANNEL_TYPE_COMPLETION_QUEUE:
155 		case XGE_HAL_CHANNEL_TYPE_UP_MESSAGE_QUEUE:
156 		case XGE_HAL_CHANNEL_TYPE_DOWN_MESSAGE_QUEUE:
157 			xge_assert(size);
158 			break;
159 		default:
160 			break;
161 	}
162 
163 	xge_os_free(channel->pdev, channel, size);
164 }
165 
166 xge_hal_status_e
167 __hal_channel_initialize (xge_hal_channel_h channelh,
168 		xge_hal_channel_attr_t *attr, void **reserve_arr,
169 		int reserve_initial, int reserve_max, int reserve_threshold)
170 {
171 	xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
172 	xge_hal_device_t *hldev;
173 
174 	hldev = (xge_hal_device_t *)channel->devh;
175 
176 	channel->dtr_term = attr->dtr_term;
177 	channel->dtr_init = attr->dtr_init;
178 	channel->callback = attr->callback;
179 	channel->userdata = attr->userdata;
180 	channel->flags = attr->flags;
181 	channel->per_dtr_space = attr->per_dtr_space;
182 
183 	channel->reserve_arr = reserve_arr;
184 	channel->reserve_initial = reserve_initial;
185 	channel->reserve_max = reserve_max;
186 	channel->reserve_length = channel->reserve_initial;
187 	channel->reserve_threshold = reserve_threshold;
188 	channel->reserve_top = 0;
189 	channel->saved_arr = (void **) xge_os_malloc(hldev->pdev,
190 					   sizeof(void*)*channel->reserve_max);
191 	if (channel->saved_arr == NULL) {
192 		return XGE_HAL_ERR_OUT_OF_MEMORY;
193 	}
194 	xge_os_memzero(channel->saved_arr, sizeof(void*)*channel->reserve_max);
195 	channel->free_arr = channel->saved_arr;
196 	channel->free_length = channel->reserve_initial;
197 	channel->work_arr = (void **) xge_os_malloc(hldev->pdev,
198 				  sizeof(void*)*channel->reserve_max);
199 	if (channel->work_arr == NULL) {
200 		return XGE_HAL_ERR_OUT_OF_MEMORY;
201 	}
202 	xge_os_memzero(channel->work_arr,
203                        sizeof(void*)*channel->reserve_max);
204 	channel->post_index = 0;
205 	channel->compl_index = 0;
206 	channel->length = channel->reserve_initial;
207 
208 	channel->orig_arr = (void **) xge_os_malloc(hldev->pdev,
209 						sizeof(void*)*channel->reserve_max);
210 	if (channel->orig_arr == NULL)
211 		return XGE_HAL_ERR_OUT_OF_MEMORY;
212 
213 	xge_os_memzero(channel->orig_arr, sizeof(void*)*channel->reserve_max);
214 
215 #if defined(XGE_HAL_RX_MULTI_FREE_IRQ) || defined(XGE_HAL_TX_MULTI_FREE_IRQ)
216 	xge_os_spin_lock_init_irq(&channel->free_lock, hldev->irqh);
217 #elif defined(XGE_HAL_RX_MULTI_FREE) || defined(XGE_HAL_TX_MULTI_FREE)
218 	xge_os_spin_lock_init(&channel->free_lock, hldev->pdev);
219 #endif
220 
221 	return XGE_HAL_OK;
222 }
223 
224 void __hal_channel_terminate(xge_hal_channel_h channelh)
225 {
226 	xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
227 	xge_hal_device_t *hldev;
228 
229 	hldev = (xge_hal_device_t *)channel->devh;
230 
231 	xge_assert(channel->pdev);
232 	/* undo changes made at channel_initialize() */
233 	if (channel->work_arr) {
234 		xge_os_free(channel->pdev, channel->work_arr,
235 		          sizeof(void*)*channel->reserve_max);
236 		channel->work_arr = NULL;
237 	}
238 
239 	if (channel->saved_arr) {
240 		xge_os_free(channel->pdev, channel->saved_arr,
241 		          sizeof(void*)*channel->reserve_max);
242 		channel->saved_arr = NULL;
243 	}
244 
245 	if (channel->orig_arr) {
246 		xge_os_free(channel->pdev, channel->orig_arr,
247 		          sizeof(void*)*channel->reserve_max);
248 		channel->orig_arr = NULL;
249 	}
250 
251 #if defined(XGE_HAL_RX_MULTI_FREE_IRQ) || defined(XGE_HAL_TX_MULTI_FREE_IRQ)
252 	xge_os_spin_lock_destroy_irq(&channel->free_lock, hldev->irqh);
253 #elif defined(XGE_HAL_RX_MULTI_FREE) || defined(XGE_HAL_TX_MULTI_FREE)
254 	xge_os_spin_lock_destroy(&channel->free_lock, hldev->pdev);
255 #endif
256 }
257 
258 /**
259  * xge_hal_channel_open - Open communication channel.
260  * @devh: HAL device, pointer to xge_hal_device_t structure.
261  * @attr: Contains attributes required to open
262  *        the channel.
263  * @channelh:  The channel handle. On success (XGE_HAL_OK) HAL fills
264  * this "out" parameter with a valid channel handle.
265  * @reopen: See  xge_hal_channel_reopen_e{}.
266  *
267  * Open communication channel with the device.
268  *
269  * HAL uses (persistent) channel configuration to allocate both channel
270  * and Xframe Tx and Rx descriptors.
271  * Notes:
272  *     1) The channel config data is fed into HAL prior to
273  *        xge_hal_channel_open().
274  *
275  *     2) The corresponding hardware queues must be already configured and
276  *        enabled.
277  *
278  *     3) Either down or up queue may be omitted, in which case the channel
279  *        is treated as _unidirectional_.
280  *
281  *     4) Post and completion queue may be the same, in which case the channel
282  *        is said to have "in-band completions".
283  *
284  * Note that free_channels list is not protected. i.e. caller must provide
285  * safe context.
286  *
287  * Returns: XGE_HAL_OK  - success.
288  * XGE_HAL_ERR_CHANNEL_NOT_FOUND - Unable to locate the channel.
289  * XGE_HAL_ERR_OUT_OF_MEMORY - Memory allocation failed.
290  *
291  * See also: xge_hal_channel_attr_t{}.
292  * Usage: See ex_open{}.
293  */
294 xge_hal_status_e
295 xge_hal_channel_open(xge_hal_device_h devh,
296 		     xge_hal_channel_attr_t *attr,
297 		     xge_hal_channel_h *channelh,
298 		     xge_hal_channel_reopen_e reopen)
299 {
300 	xge_list_t *item;
301 	int i;
302 	xge_hal_status_e status = XGE_HAL_OK;
303 	xge_hal_channel_t *channel = NULL;
304 	xge_hal_device_t *device = (xge_hal_device_t *)devh;
305 
306 	xge_assert(device);
307 	xge_assert(attr);
308 
309 	*channelh = NULL;
310 
311 	/* find channel */
312 		xge_list_for_each(item, &device->free_channels) {
313 			xge_hal_channel_t *tmp;
314 
315 			tmp = xge_container_of(item, xge_hal_channel_t, item);
316 			if (tmp->type == attr->type &&
317 			tmp->post_qid == attr->post_qid &&
318 			tmp->compl_qid == attr->compl_qid) {
319 				channel = tmp;
320 				break;
321 			}
322 		}
323 
324 		if (channel == NULL) {
325 			return XGE_HAL_ERR_CHANNEL_NOT_FOUND;
326 		}
327 
328 	xge_assert((channel->type == XGE_HAL_CHANNEL_TYPE_FIFO) ||
329 		(channel->type == XGE_HAL_CHANNEL_TYPE_RING));
330 
331 	if (reopen == XGE_HAL_CHANNEL_OC_NORMAL) {
332 		/* allocate memory, initialize pointers, etc */
333 		switch(channel->type) {
334 			case XGE_HAL_CHANNEL_TYPE_FIFO:
335 				status = __hal_fifo_open(channel, attr);
336 				break;
337 			case XGE_HAL_CHANNEL_TYPE_RING:
338 				status = __hal_ring_open(channel, attr);
339 				break;
340 			case XGE_HAL_CHANNEL_TYPE_SEND_QUEUE:
341 			case XGE_HAL_CHANNEL_TYPE_RECEIVE_QUEUE:
342 			case XGE_HAL_CHANNEL_TYPE_COMPLETION_QUEUE:
343 			case XGE_HAL_CHANNEL_TYPE_UP_MESSAGE_QUEUE:
344 			case XGE_HAL_CHANNEL_TYPE_DOWN_MESSAGE_QUEUE:
345 				status = XGE_HAL_FAIL;
346 				break;
347 			default:
348 				break;
349 		}
350 
351 		if (status == XGE_HAL_OK) {
352 			for (i = 0; i < channel->reserve_initial; i++) {
353 				channel->orig_arr[i] =
354 					channel->reserve_arr[i];
355 			}
356 		}
357 		else
358 			return status;
359 	} else {
360 	        xge_assert(reopen == XGE_HAL_CHANNEL_RESET_ONLY);
361 
362 		for (i = 0; i < channel->reserve_initial; i++) {
363 			channel->reserve_arr[i] = channel->orig_arr[i];
364 			channel->free_arr[i] = NULL;
365 		}
366 		channel->free_length = channel->reserve_initial;
367 		channel->reserve_length = channel->reserve_initial;
368 		channel->reserve_top = 0;
369 		channel->post_index = 0;
370 		channel->compl_index = 0;
371                 if (channel->type == XGE_HAL_CHANNEL_TYPE_RING) {
372 			status = __hal_ring_initial_replenish(channel,
373 							      reopen);
374                         if (status != XGE_HAL_OK)
375                                 return status;
376 		}
377 	}
378 
379 	/* move channel to the open state list */
380 
381 	switch(channel->type) {
382 		case XGE_HAL_CHANNEL_TYPE_FIFO:
383 			xge_list_remove(&channel->item);
384 			xge_list_insert(&channel->item, &device->fifo_channels);
385 			break;
386 		case XGE_HAL_CHANNEL_TYPE_RING:
387 			xge_list_remove(&channel->item);
388 			xge_list_insert(&channel->item, &device->ring_channels);
389 			break;
390 		case XGE_HAL_CHANNEL_TYPE_SEND_QUEUE:
391 		case XGE_HAL_CHANNEL_TYPE_RECEIVE_QUEUE:
392 		case XGE_HAL_CHANNEL_TYPE_COMPLETION_QUEUE:
393 		case XGE_HAL_CHANNEL_TYPE_UP_MESSAGE_QUEUE:
394 		case XGE_HAL_CHANNEL_TYPE_DOWN_MESSAGE_QUEUE:
395 			xge_assert(channel->type == XGE_HAL_CHANNEL_TYPE_FIFO ||
396 				   channel->type == XGE_HAL_CHANNEL_TYPE_RING);
397 			break;
398 		default:
399 			break;
400 	}
401 	channel->is_open = 1;
402 	channel->terminating = 0;
403 	/*
404 	 * The magic check the argument validity, has to be
405 	 * removed before 03/01/2005.
406 	 */
407 	channel->magic = XGE_HAL_MAGIC;
408 
409 	*channelh = channel;
410 
411 	return XGE_HAL_OK;
412 }
413 
414 /**
415  * xge_hal_channel_abort - Abort the channel.
416  * @channelh: Channel handle.
417  * @reopen: See  xge_hal_channel_reopen_e{}.
418  *
419  * Terminate (via xge_hal_channel_dtr_term_f{}) all channel descriptors.
420  * Currently used internally only by HAL, as part of its
421  * xge_hal_channel_close() and xge_hal_channel_open() in case
422  * of fatal error.
423  *
424  * See also: xge_hal_channel_dtr_term_f{}.
425  */
426 void xge_hal_channel_abort(xge_hal_channel_h channelh,
427                            xge_hal_channel_reopen_e reopen)
428 {
429 	xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
430 	xge_hal_dtr_h dtr;
431 #ifdef XGE_OS_MEMORY_CHECK
432 	int check_cnt = 0;
433 #endif
434 	int free_length_sav;
435 	int reserve_top_sav;
436 
437 	if (channel->dtr_term == NULL) {
438 		return;
439 	}
440 
441 	free_length_sav = channel->free_length;
442 	while (__hal_channel_dtr_next_freelist(channelh, &dtr) == XGE_HAL_OK) {
443 #ifdef XGE_OS_MEMORY_CHECK
444 #ifdef XGE_DEBUG_ASSERT
445 		if (channel->type == XGE_HAL_CHANNEL_TYPE_FIFO) {
446 		    xge_assert(!__hal_fifo_txdl_priv(dtr)->allocated);
447 		} else {
448 		    if (channel->type == XGE_HAL_CHANNEL_TYPE_RING) {
449 			    xge_assert(!__hal_ring_rxd_priv((xge_hal_ring_t * ) channelh, dtr)->allocated);
450 		    }
451 		}
452 #endif
453 		check_cnt++;
454 #endif
455 		channel->dtr_term(channel, dtr, XGE_HAL_DTR_STATE_FREED,
456 				  channel->userdata, reopen);
457 	}
458 	channel->free_length = free_length_sav;
459 
460 	while (__hal_channel_dtr_next_not_completed(channelh, &dtr) ==
461 	       XGE_HAL_OK) {
462 #ifdef XGE_OS_MEMORY_CHECK
463 #ifdef XGE_DEBUG_ASSERT
464 		if (channel->type == XGE_HAL_CHANNEL_TYPE_FIFO) {
465 			xge_assert(__hal_fifo_txdl_priv(dtr)->allocated);
466 		} else {
467 		    if (channel->type == XGE_HAL_CHANNEL_TYPE_RING) {
468 			xge_assert(__hal_ring_rxd_priv((xge_hal_ring_t * ) channelh, dtr)
469 				   ->allocated);
470 		    }
471 		}
472 #endif
473 		check_cnt++;
474 #endif
475 		channel->dtr_term(channel, dtr, XGE_HAL_DTR_STATE_POSTED,
476 				  channel->userdata, reopen);
477 
478 	}
479 
480 	reserve_top_sav = channel->reserve_top;
481 	while (__hal_channel_dtr_next_reservelist(channelh, &dtr) ==
482 							XGE_HAL_OK) {
483 #ifdef XGE_OS_MEMORY_CHECK
484 #ifdef XGE_DEBUG_ASSERT
485 		if (channel->type == XGE_HAL_CHANNEL_TYPE_FIFO) {
486 		    xge_assert(!__hal_fifo_txdl_priv(dtr)->allocated);
487 		} else {
488 		    if (channel->type == XGE_HAL_CHANNEL_TYPE_RING) {
489 			xge_assert(!__hal_ring_rxd_priv((xge_hal_ring_t * ) channelh, dtr)->allocated);
490 		    }
491 		}
492 #endif
493 		check_cnt++;
494 #endif
495 		channel->dtr_term(channel, dtr, XGE_HAL_DTR_STATE_AVAIL,
496 				  channel->userdata, reopen);
497 	}
498 	channel->reserve_top = reserve_top_sav;
499 
500 	xge_assert(channel->reserve_length ==
501                 (channel->free_length + channel->reserve_top));
502 
503 #ifdef XGE_OS_MEMORY_CHECK
504 	xge_assert(check_cnt == channel->reserve_initial);
505 #endif
506 
507 }
508 
509 /**
510  * xge_hal_channel_close - Close communication channel.
511  * @channelh: The channel handle.
512  * @reopen: See  xge_hal_channel_reopen_e{}.
513  *
514  * Will close previously opened channel and deallocate associated resources.
515  * Channel must be opened otherwise assert will be generated.
516  * Note that free_channels list is not protected. i.e. caller must provide
517  * safe context.
518  */
519 void xge_hal_channel_close(xge_hal_channel_h channelh,
520                            xge_hal_channel_reopen_e reopen)
521 {
522 	xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
523 	xge_hal_device_t *hldev;
524 	xge_list_t *item;
525 	xge_assert(channel);
526 	xge_assert(channel->type < XGE_HAL_CHANNEL_TYPE_MAX);
527 
528 	hldev = (xge_hal_device_t *)channel->devh;
529 	channel->is_open = 0;
530 	channel->magic = XGE_HAL_DEAD;
531 
532 		/* sanity check: make sure channel is not in free list */
533 		xge_list_for_each(item, &hldev->free_channels) {
534 			xge_hal_channel_t *tmp;
535 
536 			tmp = xge_container_of(item, xge_hal_channel_t, item);
537 			xge_assert(!tmp->is_open);
538 			if (channel == tmp) {
539 				return;
540 			}
541 		}
542 
543 	xge_hal_channel_abort(channel, reopen);
544 
545 	xge_assert((channel->type == XGE_HAL_CHANNEL_TYPE_FIFO) ||
546 		   (channel->type == XGE_HAL_CHANNEL_TYPE_RING));
547 
548 	if (reopen == XGE_HAL_CHANNEL_OC_NORMAL) {
549 		/* de-allocate */
550 		switch(channel->type) {
551 			case XGE_HAL_CHANNEL_TYPE_FIFO:
552 				__hal_fifo_close(channelh);
553 				break;
554 			case XGE_HAL_CHANNEL_TYPE_RING:
555 				__hal_ring_close(channelh);
556 				break;
557 			case XGE_HAL_CHANNEL_TYPE_SEND_QUEUE:
558 			case XGE_HAL_CHANNEL_TYPE_RECEIVE_QUEUE:
559 			case XGE_HAL_CHANNEL_TYPE_COMPLETION_QUEUE:
560 			case XGE_HAL_CHANNEL_TYPE_UP_MESSAGE_QUEUE:
561 			case XGE_HAL_CHANNEL_TYPE_DOWN_MESSAGE_QUEUE:
562 				xge_assert(channel->type == XGE_HAL_CHANNEL_TYPE_FIFO ||
563 				    channel->type == XGE_HAL_CHANNEL_TYPE_RING);
564 				break;
565 			default:
566 				break;
567 		}
568 	}
569 
570 	/* move channel back to free state list */
571 	xge_list_remove(&channel->item);
572 		xge_list_insert(&channel->item, &hldev->free_channels);
573 
574 		if (xge_list_is_empty(&hldev->fifo_channels) &&
575 			xge_list_is_empty(&hldev->ring_channels)) {
576 			/* clear msix_idx in case of following HW reset */
577 			hldev->reset_needed_after_close = 1;
578 		}
579 }
580