1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  *
21  * Copyright (c) 2002-2006 Neterion, Inc.
22  */
23 
24 /*
25  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
26  * Use is subject to license terms.
27  */
28 
29 #ifndef XGE_HAL_CHANNEL_H
30 #define XGE_HAL_CHANNEL_H
31 
32 #include "xge-os-pal.h"
33 #include "xge-list.h"
34 #include "xgehal-types.h"
35 #include "xgehal-stats.h"
36 
37 __EXTERN_BEGIN_DECLS
38 
39 /**
40  * enum xge_hal_channel_type_e - Enumerated channel types.
41  * @XGE_HAL_CHANNEL_TYPE_FIFO: fifo.
42  * @XGE_HAL_CHANNEL_TYPE_RING: ring.
43  * @XGE_HAL_CHANNEL_TYPE_SEND_QUEUE: Send Queue
44  * @XGE_HAL_CHANNEL_TYPE_RECEIVE_QUEUE: Receive Queue
45  * @XGE_HAL_CHANNEL_TYPE_COMPLETION_QUEUE: Receive queue completion queue
46  * @XGE_HAL_CHANNEL_TYPE_UP_MESSAGE_QUEUE: Up message queue
47  * @XGE_HAL_CHANNEL_TYPE_DOWN_MESSAGE_QUEUE: Down message queue
48  * @XGE_HAL_CHANNEL_TYPE_MAX: Maximum number of HAL-supported
49  * (and recognized) channel types. Currently: two.
50  *
51  * Enumerated channel types. Currently there are only two link-layer
52  * channels - Xframe fifo and Xframe ring. In the future the list will grow.
53  */
54 typedef enum xge_hal_channel_type_e {
55 	XGE_HAL_CHANNEL_TYPE_FIFO,
56 	XGE_HAL_CHANNEL_TYPE_RING,
57 	XGE_HAL_CHANNEL_TYPE_SEND_QUEUE,
58 	XGE_HAL_CHANNEL_TYPE_RECEIVE_QUEUE,
59 	XGE_HAL_CHANNEL_TYPE_COMPLETION_QUEUE,
60 	XGE_HAL_CHANNEL_TYPE_UP_MESSAGE_QUEUE,
61 	XGE_HAL_CHANNEL_TYPE_DOWN_MESSAGE_QUEUE,
62 	XGE_HAL_CHANNEL_TYPE_MAX
63 } xge_hal_channel_type_e;
64 
65 /**
66  * enum xge_hal_channel_flag_e - Channel flags.
67  * @XGE_HAL_CHANNEL_FLAG_NONE: zero (nil) flag.
68  * @XGE_HAL_CHANNEL_FLAG_USE_TX_LOCK: use lock when posting transmit
69  * descriptor.
70  * @XGE_HAL_CHANNEL_FLAG_FREE_RXD: to-be-defined.
71  *
72  * Channel opening flags. Reserved for future usage.
73  */
74 typedef enum xge_hal_channel_flag_e {
75 	XGE_HAL_CHANNEL_FLAG_NONE		= 0x0,
76 	XGE_HAL_CHANNEL_FLAG_USE_TX_LOCK	= 0x1,
77 	XGE_HAL_CHANNEL_FLAG_FREE_RXD	        = 0x2,
78 	XGE_HAL_CHANNEL_FLAG_USE_RX_POLLING     = 0x4
79 } xge_hal_channel_flag_e;
80 
81 /**
82  * enum xge_hal_dtr_state_e - Descriptor (DTR) state.
83  * @XGE_HAL_DTR_STATE_NONE: Invalid state.
84  * @XGE_HAL_DTR_STATE_AVAIL: Descriptor is available for reservation
85  * (via xge_hal_fifo_dtr_reserve(), xge_hal_ring_dtr_reserve(), etc.).
86  * @XGE_HAL_DTR_STATE_POSTED: Descriptor is posted for processing by the
87  * device.
88  * @XGE_HAL_DTR_STATE_FREED: Descriptor is free and can be reused for
89  * filling-in and posting later.
90  *
91  * Xframe/HAL descriptor states. For more on descriptor states and transitions
92  * please refer to ch_intern{}.
93  *
94  * See also: xge_hal_channel_dtr_term_f{}.
95  */
96 typedef enum xge_hal_dtr_state_e {
97 	XGE_HAL_DTR_STATE_NONE		= 0,
98 	XGE_HAL_DTR_STATE_AVAIL		= 1,
99 	XGE_HAL_DTR_STATE_POSTED	= 2,
100 	XGE_HAL_DTR_STATE_FREED		= 3
101 } xge_hal_dtr_state_e;
102 
103 /**
104  * enum xge_hal_channel_reopen_e - Channel open, close, or reopen option.
105  * @XGE_HAL_CHANNEL_RESET_ONLY: Do not (de)allocate channel; used with
106  * xge_hal_channel_open(), xge_hal_channel_close().
107  * @XGE_HAL_CHANNEL_OC_NORMAL: Do (de)allocate channel; used with
108  * xge_hal_channel_open(), xge_hal_channel_close().
109  *
110  * Enumerates options used with channel open and close operations.
111  * The @XGE_HAL_CHANNEL_RESET_ONLY can be used when resetting the device;
112  * in this case there is actually no need to free and then again malloc
113  * the memory (including DMA-able memory) used for channel operation.
114  */
115 typedef enum xge_hal_channel_reopen_e {
116 	XGE_HAL_CHANNEL_RESET_ONLY	= 1,
117 	XGE_HAL_CHANNEL_OC_NORMAL	= 2
118 } xge_hal_channel_reopen_e;
119 
120 /**
121  * function xge_hal_channel_callback_f - Channel callback.
122  * @channelh: Channel "containing" 1 or more completed descriptors.
123  * @dtrh: First completed descriptor.
124  * @t_code: Transfer code, as per Xframe User Guide.
125  *          Returned by HAL.
126  * @host_control: Opaque 64bit data stored by ULD inside the Xframe
127  *            descriptor prior to posting the latter on the channel
128  *            via xge_hal_fifo_dtr_post() or xge_hal_ring_dtr_post().
129  *            The @host_control is returned as is to the ULD with each
130  *            completed descriptor.
131  * @userdata: Opaque per-channel data specified at channel open
132  *            time, via xge_hal_channel_open().
133  *
134  * Channel completion callback (type declaration). A single per-channel
135  * callback is specified at channel open time, via
136  * xge_hal_channel_open().
137  * Typically gets called as part of the processing of the Interrupt
138  * Service Routine.
139  *
140  * Channel callback gets called by HAL if, and only if, there is at least
141  * one new completion on a given ring or fifo channel. Upon processing the
142  * first @dtrh ULD is _supposed_ to continue consuming completions
143  * using�one of the following HAL APIs:
144  *    - xge_hal_fifo_dtr_next_completed()
145  *      or
146  *    - xge_hal_ring_dtr_next_completed().
147  *
148  * Note that failure to process new completions in a timely fashion
149  * leads to XGE_HAL_INF_OUT_OF_DESCRIPTORS condition.
150  *
151  * Non-zero @t_code means failure to process (transmit or receive, depending
152  * on the channel type) the descriptor.
153  *
154  * In the "transmit" case the failure could happen, for instance, when the
155  * link is down, in which case Xframe completes the descriptor because it
156  * is not able to send the data out.
157  *
158  * For details please refer to Xframe User Guide.
159  *
160  * See also: xge_hal_fifo_dtr_next_completed(),
161  * xge_hal_ring_dtr_next_completed(), xge_hal_channel_dtr_term_f{}.
162  */
163 typedef xge_hal_status_e (*xge_hal_channel_callback_f)
164 				(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
165 				 u8 t_code, void *userdata);
166 
167 /**
168  * function xge_hal_channel_dtr_init_f - Initialize descriptor callback.
169  * @channelh: Channel "containing" the @dtrh descriptor.
170  * @dtrh: Descriptor.
171  * @index: Index of the descriptor in the channel's set of descriptors.
172  * @userdata: Per-channel user data (a.k.a. context) specified at
173  * channel open time, via xge_hal_channel_open().
174  * @reopen: See  xge_hal_channel_reopen_e{}.
175  *
176  * Initialize descriptor callback. Unless NULL is specified in the
177  * xge_hal_channel_attr_t{} structure passed to xge_hal_channel_open()),
178  * HAL invokes the callback as part of the xge_hal_channel_open()
179  * implementation.
180  * For the ring type of channel the ULD is expected to fill in this descriptor
181  * with buffer(s) and control information.
182  * For the fifo type of channel the ULD could use the callback to
183  * pre-set DMA mappings and/or alignment buffers.
184  *
185  * See also: xge_hal_channel_attr_t{}, xge_hal_channel_dtr_term_f{}.
186  */
187 typedef xge_hal_status_e (*xge_hal_channel_dtr_init_f)
188 				(xge_hal_channel_h channelh,
189 				 xge_hal_dtr_h dtrh,
190 				 int index,
191 				 void *userdata,
192 				 xge_hal_channel_reopen_e reopen);
193 
194 /**
195  * function xge_hal_channel_dtr_term_f - Terminate descriptor callback.
196  * @channelh: Channel "containing" the @dtrh descriptor.
197  * @dtrh: First completed descriptor.
198  * @state: One of the xge_hal_dtr_state_e{} enumerated states.
199  * @userdata: Per-channel user data (a.k.a. context) specified at
200  * channel open time, via xge_hal_channel_open().
201  * @reopen: See  xge_hal_channel_reopen_e{}.
202  *
203  * Terminate descriptor callback. Unless NULL is specified in the
204  * xge_hal_channel_attr_t{} structure passed to xge_hal_channel_open()),
205  * HAL invokes the callback as part of closing the corresponding
206  * channel, prior to de-allocating the channel and associated data
207  * structures (including descriptors).
208  * ULD should utilize the callback to (for instance) unmap
209  * and free DMA data buffers associated with the posted (state =
210  * XGE_HAL_DTR_STATE_POSTED) descriptors,
211  * as well as other relevant cleanup functions.
212  *
213  * See also: xge_hal_channel_attr_t{}, xge_hal_channel_dtr_init_f{}.
214  */
215 typedef void (*xge_hal_channel_dtr_term_f) (xge_hal_channel_h channelh,
216 					    xge_hal_dtr_h dtrh,
217 					    xge_hal_dtr_state_e	state,
218 					    void *userdata,
219 					    xge_hal_channel_reopen_e reopen);
220 
221 
222 /**
223  * struct xge_hal_channel_attr_t - Channel open "template".
224  * @type: xge_hal_channel_type_e channel type.
225  * @vp_id: Virtual path id
226  * @post_qid: Queue ID to post descriptors. For the link layer this
227  *            number should be in the 0..7 range.
228  * @compl_qid: Completion queue ID. Must be set to zero for the link layer.
229  * @callback: Channel completion callback. HAL invokes the callback when there
230  *            are new completions on that channel. In many implementations
231  *            the @callback executes in the hw interrupt context.
232  * @dtr_init: Channel's descriptor-initialize callback.
233  *            See xge_hal_channel_dtr_init_f{}.
234  *            If not NULL, HAL invokes the callback when opening
235  *            the channel via xge_hal_channel_open().
236  * @dtr_term: Channel's descriptor-terminate callback. If not NULL,
237  *          HAL invokes the callback when closing the corresponding channel.
238  *          See also xge_hal_channel_dtr_term_f{}.
239  * @userdata: User-defined "context" of _that_ channel. Passed back to the
240  *            user as one of the @callback, @dtr_init, and @dtr_term arguments.
241  * @per_dtr_space: If specified (i.e., greater than zero): extra space
242  *              reserved by HAL per each transmit or receive (depending on the
243  *              channel type) descriptor. Can be used to store,
244  *              and retrieve on completion, information specific
245  *              to the upper-layer.
246  * @flags: xge_hal_channel_flag_e enumerated flags.
247  *
248  * Channel open "template". User fills the structure with channel
249  * attributes and passes it to xge_hal_channel_open().
250  * Usage: See ex_open{}.
251  */
252 typedef struct xge_hal_channel_attr_t {
253 	xge_hal_channel_type_e		type;
254 	int				post_qid;
255 	int				compl_qid;
256 	xge_hal_channel_callback_f	callback;
257 	xge_hal_channel_dtr_init_f	dtr_init;
258 	xge_hal_channel_dtr_term_f	dtr_term;
259 	void				*userdata;
260 	int				per_dtr_space;
261 	xge_hal_channel_flag_e		flags;
262 } xge_hal_channel_attr_t;
263 
264 /*
265  * xge_hal_channel_t
266  * ---------- complete/free section ---------------
267  * @item: List item; used to maintain a list of open channels.
268  * @callback: Channel completion callback. See
269  * xge_hal_channel_callback_f.
270  * @compl_index: Completion index. At any point in time points on the
271  *               position in the channel, which will contain next
272  *               to-be-completed descriptor.
273  * @length: Channel length. Currently allocated number of descriptors.
274  *          The channel length "grows" when more descriptors get allocated.
275  *          See _hal_mempool_grow.
276  * @free_arr: Free array. Contains completed descriptors that were freed
277  *            (i.e., handed over back to HAL) by ULD.
278  *            See xge_hal_fifo_dtr_free(), xge_hal_ring_dtr_free().
279  * @free_lock: Lock to protect @free_arr.
280  * ----------- reserve/post section ---------------
281  * @post_index: Post index. At any point in time points on the
282  *              position in the channel, which'll contain next to-be-posted
283  *              descriptor.
284  * @post_lock: Lock to serialize multiple concurrent "posters" of descriptors
285  *             on the given channel.
286  * @reserve_arr: Reserve array. Contains descriptors that can be reserved
287  *               by ULD for the subsequent send or receive operation.
288  *               See xge_hal_fifo_dtr_reserve(),
289  *               xge_hal_ring_dtr_reserve().
290  * @reserve_length: Length of the @reserve_arr. The length dynamically
291  *                  changes: it decrements each time descriptor is reserved.
292  * @reserve_lock: Lock to serialize multiple concurrent threads accessing
293  *                @reserve_arr.
294  * @reserve_threshold: Reserve threshold. Minimal number of free descriptors
295  *                     that ought to be preserved in the channel at all times.
296  *                     Note that @reserve_threshold >= 0 &&
297  *                     @reserve_threshold < @reserve_max.
298  * ------------ common section --------------------
299  * @devh: Device handle. HAL device object that contains _this_ channel.
300  * @dmah: Channel's DMA address. Used to synchronize (to/from device)
301  *        descriptors.
302  * @regh0: Base address of the device memory space handle. Copied from HAL device
303  *         at channel open time.
304  * @regh1: Base address of the device memory space handle. Copied from HAL device
305  *         at channel open time.
306  * @userdata: Per-channel opaque (void*) user-defined context, which may be
307  *            upper-layer driver object, ULP connection, etc.
308  *            Once channel is open, @userdata is passed back to user via
309  *            xge_hal_channel_callback_f.
310  * @work_arr: Work array. Contains descriptors posted to the channel.
311  *            Note that at any point in time @work_arr contains 3 types of
312  *            descriptors:
313  *            1) posted but not yet consumed by Xframe device;
314  *            2) consumed but not yet completed;
315  *            3) completed but not yet freed
316  *            (via xge_hal_fifo_dtr_free() or xge_hal_ring_dtr_free())
317  * @saved_arr: Array used internally to optimize channel full-duplex
318  *             operation.
319  * @stats: Channel statistcis. Includes HAL internal counters, including
320  *         for instance, number of times out-of-descriptors
321  *         (see XGE_HAL_INF_OUT_OF_DESCRIPTORS) condition happened.
322  * ------------- "slow" section  ------------------
323  * @type: Channel type. See xge_hal_channel_type_e{}.
324  * @vp_id: Virtual path id
325  * @post_qid: Identifies Xframe queue used for posting descriptors.
326  * @compl_qid: Identifies Xframe completion queue.
327  * @flags: Channel flags. See xge_hal_channel_flag_e{}.
328  * @reserve_initial: Initial number of descriptors allocated at channel open
329  *                   time (see xge_hal_channel_open()). The number of
330  *                   channel descriptors can grow at runtime
331  *                   up to @reserve_max value.
332  * @reserve_max: Maximum number of channel descriptors. See @reserve_initial.
333  * @is_open: True, if channel is open; false - otherwise.
334  * @per_dtr_space: Per-descriptor space (in bytes) that channel user can utilize
335  *                 to store per-operation control information.
336  * HAL channel object. HAL devices (see xge_hal_device_t{}) contains
337  * zero or more channels. HAL channel contains zero or more descriptors. The
338  * latter are used by ULD(s) to manage the device and/or send and receive data
339  * to remote peer(s) via the channel.
340  *
341  * See also: xge_hal_channel_type_e{}, xge_hal_channel_flag_e,
342  * xge_hal_channel_callback_f{}
343  */
344 typedef struct {
345 	/* complete/free section */
346 	xge_list_t			item;
347 	xge_hal_channel_callback_f	callback;
348 	void				**free_arr;
349 	int				length;
350 	int				free_length;
351 #if defined(XGE_HAL_RX_MULTI_FREE_IRQ) || defined(XGE_HAL_TX_MULTI_FREE_IRQ) || \
352     defined(XGE_HAL_RX_MULTI_FREE) || defined(XGE_HAL_TX_MULTI_FREE)
353 	spinlock_t			free_lock;
354 #endif
355 	int				compl_index;
356 	unsigned int			usage_cnt;
357 	unsigned int			poll_bytes;
358 
359 	/* reserve/post data path section */
360 	int				terminating;
361 #ifdef __XGE_WIN__
362 	int				__xge_os_attr_cacheline_aligned
363 					post_index;
364 #else
365 	int				post_index
366 					__xge_os_attr_cacheline_aligned;
367 #endif
368 	spinlock_t			reserve_lock;
369 	spinlock_t			post_lock;
370 
371 	void				**reserve_arr;
372 	int				reserve_length;
373 	int				reserve_threshold;
374 	int				reserve_top;
375 	int                             unused1;
376 
377 	/* common section */
378 	xge_hal_device_h		devh;
379 	pci_dev_h                       pdev;
380 	pci_reg_h			regh0;
381 	pci_reg_h			regh1;
382 	void				*userdata;
383 	void				**work_arr;
384 	void				**saved_arr;
385 	void				**orig_arr;
386 	xge_hal_stats_channel_info_t	stats;
387 
388 	/* slow section */
389 	xge_hal_channel_type_e		type;
390 	int				post_qid;
391 	int				compl_qid;
392 	xge_hal_channel_flag_e		flags;
393 	int				reserve_initial;
394 	int				reserve_max;
395 	int				is_open;
396 	int				per_dtr_space;
397 	xge_hal_channel_dtr_term_f	dtr_term;
398 	xge_hal_channel_dtr_init_f	dtr_init;
399 	/* MSI stuff */
400 	u32				msi_msg;
401 	u8				rti;
402 	u8				tti;
403 	u16                             unused2;
404 	/* MSI-X stuff */
405 	u64				msix_address;
406 	u32				msix_data;
407 	int				msix_idx;
408 	volatile int			in_interrupt;
409         unsigned int			magic;
410 #ifdef __XGE_WIN__
411 } __xge_os_attr_cacheline_aligned xge_hal_channel_t ;
412 #else
413 } xge_hal_channel_t __xge_os_attr_cacheline_aligned;
414 #endif
415 
416 /* ========================== CHANNEL PRIVATE API ========================= */
417 
418 xge_hal_status_e
419 __hal_channel_initialize(xge_hal_channel_h channelh,
420 		xge_hal_channel_attr_t *attr, void **reserve_arr,
421 		int reserve_initial, int reserve_max, int reserve_threshold);
422 
423 void __hal_channel_terminate(xge_hal_channel_h channelh);
424 
425 xge_hal_channel_t*
426 __hal_channel_allocate(xge_hal_device_h devh, int post_qid,
427 		xge_hal_channel_type_e	type);
428 
429 void __hal_channel_free(xge_hal_channel_t *channel);
430 
431 #if defined(XGE_DEBUG_FP) && (XGE_DEBUG_FP & XGE_DEBUG_FP_CHANNEL)
432 #define __HAL_STATIC_CHANNEL
433 #define __HAL_INLINE_CHANNEL
434 
435 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
436 __hal_channel_dtr_alloc(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh);
437 
438 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
439 __hal_channel_dtr_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh);
440 
441 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
442 __hal_channel_dtr_try_complete(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh);
443 
444 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
445 __hal_channel_dtr_complete(xge_hal_channel_h channelh);
446 
447 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
448 __hal_channel_dtr_free(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh);
449 
450 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
451 __hal_channel_dtr_dealloc(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh);
452 
453 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
454 __hal_channel_dtr_restore(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
455 			  int offset);
456 
457 /* ========================== CHANNEL PUBLIC API ========================= */
458 
459 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
460 xge_hal_channel_dtr_count(xge_hal_channel_h channelh);
461 
462 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void*
463 xge_hal_channel_userdata(xge_hal_channel_h channelh);
464 
465 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
466 xge_hal_channel_id(xge_hal_channel_h channelh);
467 
468 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
469 xge_hal_check_alignment(dma_addr_t dma_pointer, int size, int alignment,
470 		int copy_size);
471 
472 #else /* XGE_FASTPATH_EXTERN */
473 #define __HAL_STATIC_CHANNEL static
474 #define __HAL_INLINE_CHANNEL inline
475 #include "xgehal-channel-fp.c"
476 #endif /* XGE_FASTPATH_INLINE */
477 
478 xge_hal_status_e
479 xge_hal_channel_open(xge_hal_device_h hldev, xge_hal_channel_attr_t *attr,
480 		     xge_hal_channel_h *channel,
481 		     xge_hal_channel_reopen_e reopen);
482 
483 void xge_hal_channel_close(xge_hal_channel_h channelh,
484                            xge_hal_channel_reopen_e reopen);
485 
486 void xge_hal_channel_abort(xge_hal_channel_h channelh,
487                            xge_hal_channel_reopen_e reopen);
488 
489 __EXTERN_END_DECLS
490 
491 #endif /* XGE_HAL_CHANNEL_H */
492