xref: /illumos-gate/usr/src/uts/common/io/chxge/sge.h (revision 2d6eb4a5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * This file is part of the Chelsio T1 Ethernet driver.
29  *
30  * Copyright (C) 2003-2005 Chelsio Communications.  All rights reserved.
31  */
32 
33 #ifndef _CHELSIO_SGE_H
34 #define	_CHELSIO_SGE_H
35 
36 #ifdef __cplusplus
37 extern "C" {
38 #endif
39 
40 #include <sys/types.h>
41 
42 #include "osdep.h"
43 
44 #define	MBLK_MAX 8
45 
46 #define	spin_lock mutex_enter
47 #define	spin_unlock mutex_exit
48 #define	atomic_sub(a, b) atomic_add_32(b, -(a))
49 #define	atomic_add(a, b) atomic_add_32(b, (a))
50 #define	atomic_read(a) (a)
51 #define	atomic_set(a, b) (*(a) = b)
52 #define	spinlock_t kmutex_t
53 #define	dma_addr_t uint64_t
54 #define	wmb() membar_producer()
55 #define	doorbell_pio(sge, cmd) sge_ring_doorbell(sge, cmd)
56 #define	skb_reserve(skb, offset) (skb->b_rptr += offset)
57 #define	__skb_pull(skb, len) (skb->b_rptr += len)
58 #define	skb_put(skb, len) ((skb)->b_wptr  = (skb)->b_rptr + (len))
59 #define	skb_pull(skb, len) (skb->b_rptr += len)
60 #define	unlikely(a) (a)
61 #define	likely(a) (a)
62 #define	SKB_DATA_ALIGN(X) (((X) + (sizeof (long)-1)) & ~(sizeof (long)-1))
63 #define	t1_is_T1B(adap) adapter_matches_type(adap, CHBT_TERM_T1, TERM_T1B)
64 #define	t1_is_T1C(adap) adapter_matches_type(adap, CHBT_TERM_T1, TERM_T1C)
65 
66 #define	SGE_SM_BUF_SZ(sa)	(sa->ch_sm_buf_sz)
67 #define	SGE_BG_BUF_SZ(sa)	(sa->ch_bg_buf_sz)
68 
69 #define	SGE_CMDQ_N		2
70 #define	SGE_FREELQ_N		2
71 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
72 #define	SGE_CMDQ0_E_N		4096
73 #define	SGE_CMDQ1_E_N		128
74 #define	SGE_FREELQ0_E_N		2048
75 #define	SGE_FREELQ1_E_N		1024
76 #define	SGE_RESPQ_E_N		7168    /* |CMDQ0| + |FREELQ0| + |FREELQ1| */
77 #else
78 #define	SGE_CMDQ0_E_N		2048
79 #define	SGE_CMDQ1_E_N		128
80 #define	SGE_FREELQ0_E_N		4096
81 #define	SGE_FREELQ1_E_N		1024
82 #define	SGE_RESPQ_E_N		7168    /* |CMDQ0| + |FREELQ0| + |FREELQ1| */
83 #endif  /* CONFIG_CHELSIO_T1_OFFLOAD */
84 #define	SGE_BATCH_THRESH	16
85 #define	SGE_INTR_BUCKETSIZE	100
86 #define	SGE_INTR_MAXBUCKETS	11
87 #define	SGE_INTRTIMER0		1
88 #define	SGE_INTRTIMER1		30
89 #define	SGE_INTRTIMER_NRES	10000
90 #define	SGE_RX_COPY_THRESHOLD	256
91 #define	SGE_RX_OFFSET		2
92 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
93 #define	SGE_RX_SM_BUF_SIZE(sa)	1536
94 #else
95 #define	SGE_RX_SM_BUF_SIZE(sa)	(sa->ch_sm_buf_sz)
96 #endif
97 
98 /*
99  * CPL5 Defines
100  */
101 #define	FLITSTOBYTES    8
102 
103 #define	CPL_FORMAT_0_SIZE 8
104 #define	CPL_FORMAT_1_SIZE 16
105 #define	CPL_FORMAT_2_SIZE 24
106 #define	CPL_FORMAT_3_SIZE 32
107 #define	CPL_FORMAT_4_SIZE 40
108 #define	CPL_FORMAT_5_SIZE 48
109 
110 #define	TID_MASK 0xffffff
111 
112 #define	SZ_CPL_RX_PKT CPL_FORMAT_0_SIZE
113 
114 #if BYTE_ORDER == BIG_ENDIAN
115 
116 typedef struct {
117 	u32 AddrLow;
118 	u32 GenerationBit: 1;
119 	u32 BufferLength: 31;
120 	u32 RespQueueSelector: 4;
121 	u32 ResponseTokens: 12;
122 	u32 CmdId: 8;
123 	u32 Reserved: 3;
124 	u32 TokenValid: 1;
125 	u32 Eop: 1;
126 	u32 Sop: 1;
127 	u32 DataValid: 1;
128 	u32 GenerationBit2: 1;
129 	u32 AddrHigh;
130 } CmdQueueEntry;
131 
132 
133 #elif BYTE_ORDER == LITTLE_ENDIAN
134 
135 
136 typedef struct {
137 	u32 BufferLength: 31;
138 	u32 GenerationBit: 1;
139 	u32 AddrLow;
140 	u32 AddrHigh;
141 	u32 GenerationBit2: 1;
142 	u32 DataValid: 1;
143 	u32 Sop: 1;
144 	u32 Eop: 1;
145 	u32 TokenValid: 1;
146 	u32 Reserved: 3;
147 	u32 CmdId: 8;
148 	u32 ResponseTokens: 12;
149 	u32 RespQueueSelector: 4;
150 } CmdQueueEntry;
151 
152 #endif
153 
154 
155 typedef CmdQueueEntry cmdQ_e;
156 
157 #if BYTE_ORDER == BIG_ENDIAN
158 
159 typedef struct {
160 	u32 Qsleeping: 4;
161 	u32 Cmdq1CreditReturn: 5;
162 	u32 Cmdq1DmaComplete: 5;
163 	u32 Cmdq0CreditReturn: 5;
164 	u32 Cmdq0DmaComplete: 5;
165 	u32 FreelistQid: 2;
166 	u32 CreditValid: 1;
167 	u32 DataValid: 1;
168 	u32 Offload: 1;
169 	u32 Eop: 1;
170 	u32 Sop: 1;
171 	u32 GenerationBit: 1;
172 	u32 BufferLength;
173 } ResponseQueueEntry;
174 
175 
176 #elif BYTE_ORDER == LITTLE_ENDIAN
177 
178 
179 typedef struct {
180 	u32 BufferLength;
181 	u32 GenerationBit: 1;
182 	u32 Sop: 1;
183 	u32 Eop: 1;
184 	u32 Offload: 1;
185 	u32 DataValid: 1;
186 	u32 CreditValid: 1;
187 	u32 FreelistQid: 2;
188 	u32 Cmdq0DmaComplete: 5;
189 	u32 Cmdq0CreditReturn: 5;
190 	u32 Cmdq1DmaComplete: 5;
191 	u32 Cmdq1CreditReturn: 5;
192 	u32 Qsleeping: 4;
193 } ResponseQueueEntry;
194 
195 #endif
196 
197 typedef ResponseQueueEntry respQ_e;
198 
199 #if BYTE_ORDER == BIG_ENDIAN
200 
201 
202 typedef struct {
203 	u32 AddrLow;
204 	u32 GenerationBit: 1;
205 	u32 BufferLength: 31;
206 	u32 Reserved: 31;
207 	u32 GenerationBit2: 1;
208 	u32 AddrHigh;
209 } FLQueueEntry;
210 
211 
212 #elif BYTE_ORDER == LITTLE_ENDIAN
213 
214 
215 typedef struct {
216 	u32 BufferLength: 31;
217 	u32 GenerationBit: 1;
218 	u32 AddrLow;
219 	u32 AddrHigh;
220 	u32 GenerationBit2: 1;
221 	u32 Reserved: 31;
222 } FLQueueEntry;
223 
224 
225 #endif
226 
227 typedef FLQueueEntry freelQ_e;
228 
229 /*
230  * Command QUEUE meta entry format.
231  */
232 typedef struct cmdQ_ce {
233 	void *ce_mp;		/* head mblk of pkt */
234 	free_dh_t *ce_dh;	/* ddi dma handle */
235 	uint_t ce_flg;		/* flag 0 - NIC descriptor; 1 - TOE */
236 	uint_t ce_len;		/* length of mblk component */
237 	uint64_t ce_pa;		/* physical address */
238 } cmdQ_ce_t;
239 
240 /*
241  * command queue control structure
242  */
243 typedef struct cmdQ {
244 	u32 cq_credits;		/* # available descriptors for Xmit */
245 	u32 cq_asleep;		/* HW DMA Fetch status */
246 	u32 cq_pio_pidx;	/* Variable updated on Doorbell */
247 	u32 cq_entries_n;	/* # entries for Xmit */
248 	u32 cq_pidx;		/* producer index (SW) */
249 	u32 cq_complete;		/* Shadow consumer index (HW) */
250 	u32 cq_cidx;		/* consumer index (HW) */
251 	u32 cq_genbit;		/* current generation (=valid) bit */
252 	cmdQ_e *cq_entries;
253 	cmdQ_ce_t *cq_centries;
254 	spinlock_t cq_qlock;
255 	uint64_t cq_pa;		/* may not be needed */
256 	ulong_t cq_dh;
257 	ulong_t cq_ah;		/* may not be needed */
258 } cmdQ_t;
259 
260 /*
261  * free list queue control structure
262  */
263 typedef struct freelQ {
264 	u32 fq_id;	/* 0 queue 0, 1 queue 1 */
265 	u32 fq_credits;	/* # available RX buffer descriptors */
266 	u32 fq_entries_n;	/* # RX buffer descriptors */
267 	u32 fq_pidx;	    /* producer index (SW) */
268 	u32 fq_cidx;	    /* consumer index (HW) */
269 	u32 fq_genbit;	  /* current generation (=valid) bit */
270 	u32 fq_rx_buffer_size;  /* size buffer on this freelist */
271 	freelQ_e *fq_entries;   /* HW freelist descriptor Q */
272 	struct freelQ_ce *fq_centries;  /* SW freelist conext descriptor Q */
273 	uint64_t fq_pa;	 /* may not be needed */
274 	ulong_t fq_dh;
275 	ulong_t fq_ah;
276 	u32 fq_pause_on_thresh;
277 	u32 fq_pause_off_thresh;
278 } freelQ_t;
279 
280 /*
281  * response queue control structure
282  */
283 typedef struct respQ {
284 	u32 rq_credits;	 /* # avail response Q entries */
285 	u32 rq_credits_pend;    /* # not yet returned entries */
286 	u32 rq_credits_thresh;  /* return threshold */
287 	u32 rq_entries_n;	/* # response Q descriptors */
288 	u32 rq_pidx;	    /* producer index (HW) */
289 	u32 rq_cidx;	    /* consumer index (SW) */
290 	u32 rq_genbit;	  /* current generation(=valid) bit */
291 	respQ_e *rq_entries;    /* HW response Q */
292 	uint64_t rq_pa;	 /* may not be needed */
293 	ulong_t rq_dh;
294 	ulong_t rq_ah;
295 } reapQ_t;
296 
297 struct sge_intr_counts {
298 	uint32_t respQ_empty;		/* # times respQ empty */
299 	uint32_t respQ_overflow;	/* # respQ overflow (fatal) */
300 	uint32_t freelistQ_empty;	/* # times freelist empty */
301 	uint32_t pkt_too_big;		/* packet too large (fatal) */
302 	uint32_t pkt_mismatch;
303 	uint32_t cmdQ_full[2];		/* not HW intr, host cmdQ[] full */
304 	uint32_t tx_reclaims[2];
305 	uint32_t tx_msg_pullups;	/* # of tx pkt coelescing events */
306 	uint32_t tx_hdr_pullups;	/* # of tx hdr coelescing events */
307 	uint32_t tx_tcp_ip_frag;	/* # of ip fragmentes for tcp data */
308 	uint32_t tx_udp_ip_frag;	/* # of ip fragmentes for udp data */
309 	uint32_t tx_soft_cksums;	/* # of Software checksums done. */
310 	uint32_t tx_need_cpl_space;	/* # of allocs for cpl header */
311 	uint32_t tx_multi_mblks;	/* # of Multi mblk packets */
312 	uint32_t tx_no_dvma1;		/* # of dvma mapping failures */
313 	uint32_t tx_no_dvma2;		/* # of dvma mapping failures */
314 	uint32_t tx_no_dma1;		/* # of dma mapping failures */
315 	uint32_t tx_no_dma2;		/* # of dma mapping failures */
316 	uint32_t rx_cmdq0;		/* # of Qsleeping CMDQ0's */
317 	uint32_t rx_cmdq1;		/* # of Qsleeping CMDQ1's */
318 	uint32_t rx_flq0;		/* # of Qsleeping FL0's */
319 	uint32_t rx_flq1;		/* # of Qsleeping FL1's */
320 	uint32_t rx_flq0_sz;		/* size of freelist-0 buffers */
321 	uint32_t rx_flq1_sz;		/* size of freelist-1 buffers */
322 	uint32_t rx_pkt_drops;		/* # intentionally dropped packets */
323 	uint32_t rx_pkt_copied;		/* # times packets copied by sge */
324 	uint32_t rx_pause_on;		/* # of system pause on's required. */
325 	uint32_t rx_pause_off;		/* # of system pauses off's required. */
326 	uint32_t rx_pause_ms;		/* micro seconds while paused */
327 	uint32_t rx_pause_spike;	/* maximum time paused */
328 	uint32_t rx_fl_credits;		/* Current free list credit usage. */
329 	uint32_t rx_flbuf_fails;	/* # of freelist buf alloc fails. */
330 	uint32_t rx_flbuf_allocs;	/* # of freelist buf allocs. */
331 	uint32_t rx_badEopSop;		/* # of times bad Eop/Sop received */
332 	uint32_t rx_flq0_cnt;	/* # of times free list Q 0 entry used */
333 	uint32_t rx_flq1_cnt;	/* # of times free list Q 1 entry used */
334 	uint32_t arp_sent;		/* # times arp packet sent */
335 #ifdef SUN_KSTATS
336 	uint32_t tx_doorbells;
337 	uint32_t intr_doorbells;
338 	uint32_t intr1_doorbells;
339 	uint32_t sleep_cnt;
340 	uint32_t pe_allocb_cnt;
341 	uint32_t tx_descs[MBLK_MAX];
342 #endif
343 };
344 
345 #ifdef SUN_KSTATS
346 typedef struct sge_intr_counts *p_ch_stats_t;
347 
348 /*
349  * Driver maintained kernel statistics.
350  */
351 typedef struct _ch_kstat_t {
352 	/*
353 	 * Link Input/Output stats
354 	 */
355 	kstat_named_t respQ_empty;	/* # times respQ empty */
356 	kstat_named_t respQ_overflow;	/* # respQ overflow (fatal) */
357 	kstat_named_t freelistQ_empty;	/* # times freelist empty */
358 	kstat_named_t pkt_too_big;	/* packet too large (fatal) */
359 	kstat_named_t pkt_mismatch;
360 	kstat_named_t cmdQ_full[2];	/* not HW intr, host cmdQ[] full */
361 	kstat_named_t tx_reclaims[2];	/* # of tx reclaims called */
362 	kstat_named_t tx_msg_pullups;	/* # of tx pkt coelescing events */
363 	kstat_named_t tx_hdr_pullups;	/* # of tx hdr coelescing events */
364 	kstat_named_t tx_tcp_ip_frag;	/* # of ip fragmentes for tcp data */
365 	kstat_named_t tx_udp_ip_frag;	/* # of ip fragmentes for udp data */
366 	kstat_named_t tx_soft_cksums;	/* # of Software checksums done. */
367 	kstat_named_t tx_need_cpl_space;	/* # of allocs for cpl header */
368 	kstat_named_t tx_multi_mblks;	/* # of multi fragment packets */
369 	kstat_named_t tx_no_dvma1;	/* # of dvma mapping failures */
370 	kstat_named_t tx_no_dvma2;	/* # of dvma mapping failures */
371 	kstat_named_t tx_no_dma1;	/* # of dma mapping failures */
372 	kstat_named_t tx_no_dma2;	/* # of dma mapping failures */
373 	kstat_named_t rx_cmdq0;		/* # times Qsleeping cmdq0 */
374 	kstat_named_t rx_cmdq1;		/* # times Qsleeping cmdq1 */
375 	kstat_named_t rx_flq0;		/* # times Qsleeping flq0 */
376 	kstat_named_t rx_flq0_sz;	/* size of freelist-0 buffers */
377 	kstat_named_t rx_flq1;		/* # times Qsleeping flq1 */
378 	kstat_named_t rx_flq1_sz;	/* size of freelist-1 buffers */
379 	kstat_named_t rx_pkt_drops;	/* # times packets dropped by sge */
380 	kstat_named_t rx_pkt_copied;	/* # intentionally copied packets */
381 	kstat_named_t rx_pause_on;	/* # of system pause on's required. */
382 	kstat_named_t rx_pause_off;	/* # of system pauses off's required. */
383 	kstat_named_t rx_pause_ms;	/* micro seconds while paused. */
384 	kstat_named_t rx_pause_spike;	/* maximum time paused. */
385 	kstat_named_t rx_fl_credits;	/* Current free list credit usage. */
386 	kstat_named_t rx_flbuf_fails;	/* # of freelist buf alloc fails. */
387 	kstat_named_t rx_flbuf_allocs;	/* # of freelist buf allocs. */
388 	kstat_named_t rx_badEopSop;	/* # of times bad Eop/Sop received */
389 	kstat_named_t rx_flq0_cnt; /* # of times free list Q 0 entry used */
390 	kstat_named_t rx_flq1_cnt; /* # of times free list Q 1 entry used */
391 	kstat_named_t arp_sent;		/* # times arp packet sent */
392 
393 	kstat_named_t tx_doorbells;
394 	kstat_named_t intr_doorbells;
395 	kstat_named_t intr1_doorbells;
396 	kstat_named_t sleep_cnt;
397 	kstat_named_t pe_allocb_cnt;
398 	kstat_named_t tx_descs[MBLK_MAX];
399 } ch_kstat_t;
400 typedef ch_kstat_t *p_ch_kstat_t;
401 #endif
402 
403 typedef struct _pesge {
404 	peobj *obj;			/* adapter backpointer */
405 	struct freelQ freelQ[2];	/* freelist Q(s) */
406 	struct respQ respQ;		/* response Q instatiation */
407 	uint32_t rx_pkt_pad;		/* RX padding for T2 packets (hw) */
408 	uint32_t rx_offset;		/* RX padding for T1 packets (sw) */
409 	uint32_t jumbo_fl;		/* jumbo freelist Q index */
410 	uint32_t intrtimer[SGE_INTR_MAXBUCKETS];	/* timer values */
411 	uint32_t currIndex;		/* current index into intrtimer[] */
412 	uint32_t intrtimer_nres;	/* no resource interrupt timer value */
413 	uint32_t sge_control;		/* shadow content of sge control reg */
414 	struct sge_intr_counts intr_cnt;
415 #ifdef SUN_KSTATS
416 	p_kstat_t ksp;
417 #endif
418 	ch_cyclic_t espi_wa_cyclic;
419 	uint32_t ptimeout;
420 	void *pskb;
421 	struct cmdQ cmdQ[2];	    /* command Q(s) */
422 	int do_udp_csum;
423 	int do_tcp_csum;
424 } _pesge;
425 
426 /*
427  * ce_flg flag values
428  */
429 #define	DH_DMA  1
430 #define	DH_DVMA 2
431 #define	DH_TOE  3
432 #define	DH_ARP  8
433 
434 typedef struct freelQ_ce {
435 	void *fe_mp;		/* head mblk of pkt */
436 	ulong_t fe_dh;		/* ddi dma handle */
437 	uint_t  fe_len;		/* length of mblk component */
438 	uint64_t fe_pa;		/* physical address */
439 } freelQ_ce_t;
440 
441 pesge *t1_sge_create(ch_t *, struct sge_params *);
442 
443 extern int  t1_sge_destroy(pesge* sge);
444 extern int  sge_data_out(pesge*, int,  mblk_t *, cmdQ_ce_t *, int, uint32_t);
445 extern int  sge_data_in(pesge *);
446 extern int  sge_start(pesge*);
447 extern int  sge_stop(pesge *);
448 extern int t1_sge_configure(pesge *sge, struct sge_params *p);
449 
450 extern int  t1_sge_intr_error_handler(pesge*);
451 extern int  t1_sge_intr_enable(pesge*);
452 extern int  t1_sge_intr_disable(pesge*);
453 extern int  t1_sge_intr_clear(pesge*);
454 extern u32  t1_sge_get_ptimeout(ch_t *);
455 extern void t1_sge_set_ptimeout(ch_t *, u32);
456 
457 extern struct sge_intr_counts *sge_get_stat(pesge *);
458 extern void sge_add_fake_arp(pesge *, void *);
459 
460 /*
461  * Default SGE settings
462  */
463 #define	SGE_CMDQ0_CNT	(512)
464 #define	SGE_FLQ0_CNT	(512)
465 #define	SGE_RESPQ_CNT	(1024)
466 
467 /*
468  * the structures below were taken from cpl5_cmd.h. It turns out that there
469  * is a number of   #includes    that causes build problems. For now, we're
470  * putting a private copy here. When the sge code is made common, then this
471  * problem will need to be resolved.
472  */
473 
474 typedef uint8_t  __u8;
475 typedef uint32_t __u32;
476 typedef uint16_t __u16;
477 
478 union opcode_tid {
479     __u32 opcode_tid;
480     __u8 opcode;
481 };
482 
483 /*
484  * We want this header's alignment to be no more stringent than 2-byte aligned.
485  * All fields are u8 or u16 except for the length.  However that field is not
486  * used so we break it into 2 16-bit parts to easily meet our alignment needs.
487  */
488 struct cpl_tx_pkt {
489     __u8 opcode;
490 #if BYTE_ORDER == BIG_ENDIAN
491     __u8 rsvd:1;
492     __u8 vlan_valid:1;
493     __u8 l4_csum_dis:1;
494     __u8 ip_csum_dis:1;
495     __u8 iff:4;
496 #else
497     __u8 iff:4;
498     __u8 ip_csum_dis:1;
499     __u8 l4_csum_dis:1;
500     __u8 vlan_valid:1;
501     __u8 rsvd:1;
502 #endif
503     __u16 vlan;
504     __u16 len_hi;
505     __u16 len_lo;
506 };
507 
508 #define	CPL_TX_PKT 0xb2
509 #define	SZ_CPL_TX_PKT CPL_FORMAT_0_SIZE
510 
511 struct cpl_rx_data {
512     union opcode_tid ot;
513     __u32 len;
514     __u32 seq;
515     __u16 urg;
516     __u8  rsvd;
517     __u8  status;
518 };
519 
520 struct cpl_rx_pkt {
521     __u8 opcode;
522 #if BYTE_ORDER == LITTLE_ENDIAN
523     __u8 iff:4;
524     __u8 csum_valid:1;
525     __u8 bad_pkt:1;
526     __u8 vlan_valid:1;
527     __u8 rsvd:1;
528 #else
529     __u8 rsvd:1;
530     __u8 vlan_valid:1;
531     __u8 bad_pkt:1;
532     __u8 csum_valid:1;
533     __u8 iff:4;
534 #endif
535     __u16 csum;
536     __u16 vlan;
537     __u16 len;
538 };
539 
540 #ifdef __cplusplus
541 }
542 #endif
543 
544 #endif /* _CHELSIO_SGE_H */
545