1d39a76e7Sxw /*
2d39a76e7Sxw * CDDL HEADER START
3d39a76e7Sxw *
4d39a76e7Sxw * The contents of this file are subject to the terms of the
5d39a76e7Sxw * Common Development and Distribution License (the "License").
6d39a76e7Sxw * You may not use this file except in compliance with the License.
7d39a76e7Sxw *
8d39a76e7Sxw * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9d39a76e7Sxw * or http://www.opensolaris.org/os/licensing.
10d39a76e7Sxw * See the License for the specific language governing permissions
11d39a76e7Sxw * and limitations under the License.
12d39a76e7Sxw *
13d39a76e7Sxw * When distributing Covered Code, include this CDDL HEADER in each
14d39a76e7Sxw * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15d39a76e7Sxw * If applicable, add the following below this CDDL HEADER, with the
16d39a76e7Sxw * fields enclosed by brackets "[]" replaced with your own identifying
17d39a76e7Sxw * information: Portions Copyright [yyyy] [name of copyright owner]
18d39a76e7Sxw *
19d39a76e7Sxw * CDDL HEADER END
20d39a76e7Sxw */
21d39a76e7Sxw
22d39a76e7Sxw /*
23d39a76e7Sxw * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24d39a76e7Sxw * Use is subject to license terms.
25d39a76e7Sxw */
26d39a76e7Sxw
27d39a76e7Sxw /*
28d39a76e7Sxw * This file is part of the Chelsio T1 Ethernet driver.
29d39a76e7Sxw *
30d39a76e7Sxw * Copyright (C) 2003-2005 Chelsio Communications. All rights reserved.
31d39a76e7Sxw */
32d39a76e7Sxw
33d39a76e7Sxw #include <sys/types.h>
34d39a76e7Sxw #include <sys/param.h>
35d39a76e7Sxw #include <sys/cmn_err.h>
36d39a76e7Sxw #include <sys/sunddi.h>
37d39a76e7Sxw #include <sys/kmem.h>
38d39a76e7Sxw #include <sys/cmn_err.h>
39d39a76e7Sxw #include <sys/byteorder.h>
40d39a76e7Sxw #include <sys/atomic.h>
41d39a76e7Sxw #include <sys/stropts.h>
42d39a76e7Sxw #include <sys/stream.h>
43d39a76e7Sxw #include <sys/strsubr.h>
44d39a76e7Sxw #include <sys/dlpi.h>
45d39a76e7Sxw #include <sys/kstat.h>
46d39a76e7Sxw #include <sys/ethernet.h>
47d39a76e7Sxw #include <netinet/in.h>
48d39a76e7Sxw #include <netinet/udp.h>
49d39a76e7Sxw #include <inet/common.h>
50d39a76e7Sxw #include <inet/nd.h>
51d39a76e7Sxw #include <inet/ip.h>
52d39a76e7Sxw #include <inet/tcp.h>
53d39a76e7Sxw #include <netinet/udp.h>
54d39a76e7Sxw #include <sys/gld.h>
55d39a76e7Sxw #include "ostypes.h"
56d39a76e7Sxw #include "common.h"
57d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_1G
58d39a76e7Sxw #include "fpga_defs.h"
59d39a76e7Sxw #endif
60d39a76e7Sxw #include "regs.h"
61d39a76e7Sxw #include "suni1x10gexp_regs.h"
62d39a76e7Sxw #include "sge.h"
63d39a76e7Sxw #include "espi.h"
64d39a76e7Sxw
65d39a76e7Sxw #include "ch.h"
66d39a76e7Sxw
67d39a76e7Sxw extern uint32_t buffers_in_use[];
68d39a76e7Sxw
69d39a76e7Sxw uint32_t sge_cmdq0_cnt = SGE_CMDQ0_E_N;
70d39a76e7Sxw uint32_t sge_cmdq1_cnt = SGE_CMDQ1_E_N;
71d39a76e7Sxw uint32_t sge_flq0_cnt = SGE_FREELQ0_E_N;
72d39a76e7Sxw uint32_t sge_flq1_cnt = SGE_FREELQ1_E_N;
73d39a76e7Sxw uint32_t sge_respq_cnt = SGE_RESPQ_E_N;
74d39a76e7Sxw
75d39a76e7Sxw uint32_t sge_cmdq0_cnt_orig = SGE_CMDQ0_E_N;
76d39a76e7Sxw uint32_t sge_cmdq1_cnt_orig = SGE_CMDQ1_E_N;
77d39a76e7Sxw uint32_t sge_flq0_cnt_orig = SGE_FREELQ0_E_N;
78d39a76e7Sxw uint32_t sge_flq1_cnt_orig = SGE_FREELQ1_E_N;
79d39a76e7Sxw uint32_t sge_respq_cnt_orig = SGE_RESPQ_E_N;
80d39a76e7Sxw
81d39a76e7Sxw #ifdef HOST_PAUSE
82d39a76e7Sxw uint32_t do_host_pause = 1;
83d39a76e7Sxw uint32_t flq_pause_window = 64;
84d39a76e7Sxw #endif
85d39a76e7Sxw
86d39a76e7Sxw static uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb,
87d39a76e7Sxw ulong_t *dh);
88d39a76e7Sxw void pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t);
89d39a76e7Sxw
90d39a76e7Sxw static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q,
91d39a76e7Sxw unsigned int len, unsigned int offload);
92d39a76e7Sxw #ifdef HOST_PAUSE
93d39a76e7Sxw static void t1_sge_check_pause(pesge *sge, struct freelQ *Q);
94d39a76e7Sxw #endif
95d39a76e7Sxw static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q);
96d39a76e7Sxw static void freelQs_empty(pesge *sge);
97d39a76e7Sxw static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend);
98d39a76e7Sxw static int alloc_rx_resources(pesge *sge, struct sge_params *p);
99d39a76e7Sxw static int alloc_tx_resources(pesge *sge, struct sge_params *p);
100d39a76e7Sxw static inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size,
101d39a76e7Sxw int base_reg_lo, int base_reg_hi, int size_reg);
102d39a76e7Sxw static void configure_sge(pesge *sge, struct sge_params *p);
103d39a76e7Sxw static void free_freelQ_buffers(pesge *sge, struct freelQ *Q);
104d39a76e7Sxw static void free_rx_resources(pesge *sge);
105d39a76e7Sxw static void free_tx_resources(pesge *sge);
106d39a76e7Sxw static inline unsigned int jumbo_payload_capacity(pesge *sge);
107d39a76e7Sxw #ifdef SUN_KSTATS
108d39a76e7Sxw static int sge_kstat_setup(pesge *);
109d39a76e7Sxw static void sge_kstat_remove(pesge *);
110d39a76e7Sxw static int sge_kstat_update(p_kstat_t, int);
111d39a76e7Sxw #endif
112d39a76e7Sxw static uint16_t calc_ocsum(mblk_t *, int);
113d39a76e7Sxw
114d39a76e7Sxw /*
115d39a76e7Sxw * Local routines.
116d39a76e7Sxw */
117d39a76e7Sxw static inline void sge_ring_doorbell(pesge *sge, u32 control_reg);
118d39a76e7Sxw
119d39a76e7Sxw static inline void
sge_ring_doorbell(pesge * sge,u32 control_reg)120d39a76e7Sxw sge_ring_doorbell(pesge *sge, u32 control_reg)
121d39a76e7Sxw {
122d39a76e7Sxw membar_producer();
123d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_DOORBELL, control_reg);
124d39a76e7Sxw }
125d39a76e7Sxw
126d39a76e7Sxw /*
127d39a76e7Sxw * DESC:
128d39a76e7Sxw *
129d39a76e7Sxw * NOTES: Must have at least 1 command queue and 1 freelist queue.
130d39a76e7Sxw *
131d39a76e7Sxw */
132d39a76e7Sxw pesge *
t1_sge_create(ch_t * sa,struct sge_params * p)133d39a76e7Sxw t1_sge_create(ch_t *sa, struct sge_params *p)
134d39a76e7Sxw {
135d39a76e7Sxw pesge *sge;
136d39a76e7Sxw
137d39a76e7Sxw sge = t1_os_malloc_wait_zero(sizeof (pesge));
138d39a76e7Sxw
139d39a76e7Sxw if (sge == NULL)
140d39a76e7Sxw goto error_no_mem;
141d39a76e7Sxw
142d39a76e7Sxw memset(sge, 0, sizeof (*sge));
143d39a76e7Sxw
144d39a76e7Sxw /*
145d39a76e7Sxw * PR2928 & PR3309
146d39a76e7Sxw * set default timeout value - 20 msec
147d39a76e7Sxw * we set the initial value to 2 which gurantees at least one tick.
148d39a76e7Sxw */
149d39a76e7Sxw if (is_T2(sa))
150d39a76e7Sxw sge->ptimeout = 1;
151d39a76e7Sxw
152d39a76e7Sxw sge->obj = sa;
153d39a76e7Sxw #ifdef SUN_KSTATS
154d39a76e7Sxw if (sge_kstat_setup(sge) != 0)
155d39a76e7Sxw goto t1_sge_create_fail1;
156d39a76e7Sxw #endif
157d39a76e7Sxw p->cmdQ_size[0] = sge_cmdq0_cnt;
158d39a76e7Sxw p->cmdQ_size[1] = sge_cmdq1_cnt;
159d39a76e7Sxw
160d39a76e7Sxw /* note that jumbo frame index is inverted for T2 */
161d39a76e7Sxw if (is_T2(sa)) {
162d39a76e7Sxw p->freelQ_size[1] = sge_flq0_cnt;
163d39a76e7Sxw p->freelQ_size[0] = sge_flq1_cnt;
164d39a76e7Sxw } else {
165d39a76e7Sxw p->freelQ_size[0] = sge_flq0_cnt;
166d39a76e7Sxw p->freelQ_size[1] = sge_flq1_cnt;
167d39a76e7Sxw }
168d39a76e7Sxw
169d39a76e7Sxw #if CH_DEBUG
170d39a76e7Sxw /* DEBUG only */
171d39a76e7Sxw cmn_err(CE_NOTE, "sge: %p\n", sge);
172d39a76e7Sxw cmn_err(CE_NOTE, "&sge->cmdQ[0]: %p\n", &sge->cmdQ[0]);
173d39a76e7Sxw cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]);
174d39a76e7Sxw cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]);
175d39a76e7Sxw cmn_err(CE_NOTE, "&sge->respQ: %p\n", &sge->respQ);
176d39a76e7Sxw cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt);
177d39a76e7Sxw #endif
178d39a76e7Sxw #ifdef SUN_KSTATS
179d39a76e7Sxw goto error_no_mem;
180d39a76e7Sxw
181d39a76e7Sxw t1_sge_create_fail1:
182d39a76e7Sxw t1_os_free(sge, sizeof (pesge));
183d39a76e7Sxw sge = NULL;
184d39a76e7Sxw #endif
185d39a76e7Sxw error_no_mem:
186d39a76e7Sxw return (sge);
187d39a76e7Sxw }
188d39a76e7Sxw
189d39a76e7Sxw int
t1_sge_destroy(pesge * sge)190d39a76e7Sxw t1_sge_destroy(pesge* sge)
191d39a76e7Sxw {
192d39a76e7Sxw if (sge != NULL) {
193d39a76e7Sxw free_tx_resources(sge);
194d39a76e7Sxw free_rx_resources(sge);
195d39a76e7Sxw
196d39a76e7Sxw /* PR2928 & PR3309 */
197d39a76e7Sxw if ((is_T2(sge->obj)) && (sge->pskb))
198d39a76e7Sxw pe_free_fake_arp(sge->pskb);
199d39a76e7Sxw #ifdef SUN_KSTATS
200d39a76e7Sxw sge_kstat_remove(sge);
201d39a76e7Sxw #endif
202d39a76e7Sxw t1_os_free(sge, sizeof (pesge));
203d39a76e7Sxw }
204d39a76e7Sxw return (0);
205d39a76e7Sxw }
206d39a76e7Sxw
207d39a76e7Sxw /*
208d39a76e7Sxw * PR2928 & PR3309
209d39a76e7Sxw * call out event from timeout
210d39a76e7Sxw *
211d39a76e7Sxw * there is a potential race between the timeout and the close.
212d39a76e7Sxw * unless we protect the timeout, the close could occur at the
213d39a76e7Sxw * same time. Then if the timeout service routine was slow or
214d39a76e7Sxw * interrupted, the sge_stop() could complete with a timeoutID
215d39a76e7Sxw * that has expired, thus letting another timeout occur. If the
216d39a76e7Sxw * service routine was delayed still further, a detach could occur.
217d39a76e7Sxw * the second time could then end up accessing memory that has been
218d39a76e7Sxw * released back to the system. Bad things could then occur. We
219d39a76e7Sxw * set a flag in sge_stop() to tell the service routine not to
220d39a76e7Sxw * issue further timeouts. sge_stop() will block until a timeout
221d39a76e7Sxw * has occured. If the command Q is full then we shouldn't put out
222d39a76e7Sxw * an arp.
223d39a76e7Sxw */
224d39a76e7Sxw
225d39a76e7Sxw void
t1_espi_workaround(ch_t * adapter)226d39a76e7Sxw t1_espi_workaround(ch_t *adapter)
227d39a76e7Sxw {
228d39a76e7Sxw pesge *sge = adapter->sge;
229d39a76e7Sxw ch_t *chp = (ch_t *)sge->obj;
230d39a76e7Sxw int rv = 1;
231d39a76e7Sxw
232d39a76e7Sxw if ((chp->ch_state == PERUNNING) &&
233*01af90edSToomas Soome atomic_read(sge->cmdQ[0].cq_asleep)) {
234d39a76e7Sxw u32 seop;
235d39a76e7Sxw seop = t1_espi_get_mon(adapter, 0x930, 0);
236d39a76e7Sxw if ((seop & 0xfff0fff) == 0xfff) {
237d39a76e7Sxw /* after first arp */
238f172c8abSToomas Soome if (sge->pskb) {
239d39a76e7Sxw rv = pe_start(adapter, (mblk_t *)sge->pskb,
240d39a76e7Sxw CH_ARP);
241d39a76e7Sxw if (!rv)
242d39a76e7Sxw sge->intr_cnt.arp_sent++;
243f172c8abSToomas Soome }
244d39a76e7Sxw }
245d39a76e7Sxw }
246d39a76e7Sxw #ifdef HOST_PAUSE
247d39a76e7Sxw /*
248d39a76e7Sxw * If we are already in sge_data_in, then we can skip calling
249d39a76e7Sxw * t1_sge_check_pause() this clock cycle. lockstat showed that
250d39a76e7Sxw * we were blocking on the mutex ~ 2% of the time.
251d39a76e7Sxw */
252d39a76e7Sxw if (mutex_tryenter(&adapter->ch_intr)) {
253d39a76e7Sxw t1_sge_check_pause(sge, &sge->freelQ[0]);
254d39a76e7Sxw t1_sge_check_pause(sge, &sge->freelQ[1]);
255d39a76e7Sxw mutex_exit(&adapter->ch_intr);
256d39a76e7Sxw }
257d39a76e7Sxw #endif
258d39a76e7Sxw }
259d39a76e7Sxw
260d39a76e7Sxw int
sge_start(pesge * sge)261d39a76e7Sxw sge_start(pesge *sge)
262d39a76e7Sxw {
263d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control);
264d39a76e7Sxw /* PR2928 & PR3309, also need to avoid Pause deadlock */
265d39a76e7Sxw ch_init_cyclic(sge->obj, &sge->espi_wa_cyclic,
266d39a76e7Sxw (void (*)(void *))t1_espi_workaround, sge->obj);
267d39a76e7Sxw ch_start_cyclic(&sge->espi_wa_cyclic, sge->ptimeout);
268d39a76e7Sxw return (0);
269d39a76e7Sxw }
270d39a76e7Sxw
271d39a76e7Sxw /*
272d39a76e7Sxw * Disables SGE queues.
273d39a76e7Sxw */
274d39a76e7Sxw int
sge_stop(pesge * sge)275d39a76e7Sxw sge_stop(pesge *sge)
276d39a76e7Sxw {
277d39a76e7Sxw uint32_t status;
278d39a76e7Sxw int loops;
279d39a76e7Sxw
280d39a76e7Sxw DBGASSERT(sge);
281d39a76e7Sxw
282d39a76e7Sxw /* PR2928 & PR3309, also need to avoid Pause deadlock */
283d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_CONTROL, 0x0);
284d39a76e7Sxw
285d39a76e7Sxw /* wait until there's no more outstanding interrupts pending */
286d39a76e7Sxw loops = 0;
287d39a76e7Sxw do {
288d39a76e7Sxw status = t1_read_reg_4(sge->obj, A_SG_INT_CAUSE);
289d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, status);
290d39a76e7Sxw drv_usecwait(125);
291d39a76e7Sxw loops++;
292d39a76e7Sxw } while (status && (loops < 1000));
293d39a76e7Sxw
294d39a76e7Sxw ch_stop_cyclic(&sge->espi_wa_cyclic);
295d39a76e7Sxw
296d39a76e7Sxw return (0);
297d39a76e7Sxw }
298d39a76e7Sxw
299d39a76e7Sxw uint32_t sge_cmdq_send_fail;
300d39a76e7Sxw
301d39a76e7Sxw int
sge_data_out(pesge * sge,int qid,mblk_t * m0,cmdQ_ce_t * cmp,int count,uint32_t flg)302d39a76e7Sxw sge_data_out(pesge* sge, int qid, mblk_t *m0,
303f172c8abSToomas Soome cmdQ_ce_t *cmp, int count, uint32_t flg)
304d39a76e7Sxw {
305d39a76e7Sxw struct cmdQ *Q = &sge->cmdQ[qid];
306d39a76e7Sxw ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh;
307d39a76e7Sxw spinlock_t *qlock = &Q->cq_qlock;
308d39a76e7Sxw cmdQ_e *e;
309d39a76e7Sxw cmdQ_e *q = Q->cq_entries;
310d39a76e7Sxw uint32_t credits;
311d39a76e7Sxw uint32_t pidx;
312d39a76e7Sxw uint32_t genbit;
313d39a76e7Sxw uint32_t entries_n = Q->cq_entries_n;
314d39a76e7Sxw cmdQ_ce_t *ce;
315d39a76e7Sxw cmdQ_ce_t *cq = Q->cq_centries;
316d39a76e7Sxw dma_addr_t mapping;
317d39a76e7Sxw uint32_t j = 0;
318d39a76e7Sxw uint32_t offset;
319d39a76e7Sxw #if defined(TX_CKSUM_FIX)
320d39a76e7Sxw uint16_t csum;
321d39a76e7Sxw uint16_t *csum_loc;
322d39a76e7Sxw #endif
323d39a76e7Sxw #ifdef TX_THREAD_RECLAIM
324d39a76e7Sxw uint32_t reclaim_cnt;
325d39a76e7Sxw #endif
326d39a76e7Sxw
327d39a76e7Sxw /*
328d39a76e7Sxw * We must exit if we don't have enough free command queue entries
329d39a76e7Sxw * available.
330d39a76e7Sxw */
331d39a76e7Sxw
332d39a76e7Sxw spin_lock(qlock);
333d39a76e7Sxw
334d39a76e7Sxw #if defined(TX_CKSUM_FIX)
335d39a76e7Sxw /*
336d39a76e7Sxw * This checksum fix will address a fragmented datagram
337d39a76e7Sxw * checksum error. Which will lead to the next packet after
338d39a76e7Sxw * the last packet with the More fragment bit set having its
339d39a76e7Sxw * checksum corrupted. When the packet reaches this point
340d39a76e7Sxw * the 'flg' variable indicates whether a checksum is needed
341d39a76e7Sxw * or not. The algorithm is as follows, if the current packet
342d39a76e7Sxw * is a More fragment set the count of packets to be checksummed
343d39a76e7Sxw * after it to 3. If it't not and the count of is more than 0
344d39a76e7Sxw * then calculate the checksum in software, if a hardware checksum
345d39a76e7Sxw * was requested. Then decrment the count. Same algorithm applies
346d39a76e7Sxw * to TCP.
347d39a76e7Sxw */
348d39a76e7Sxw if (flg & CH_UDP_MF) {
349d39a76e7Sxw sge->do_udp_csum = 3;
350d39a76e7Sxw } else if ((flg & CH_UDP) && (sge->do_udp_csum != 0)) {
351d39a76e7Sxw if ((flg & CH_NO_HWCKSUM) == 0) {
352d39a76e7Sxw /*
353d39a76e7Sxw * Calc Checksum here.
354d39a76e7Sxw */
355d39a76e7Sxw csum = calc_ocsum(m0,
356d39a76e7Sxw sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
357d39a76e7Sxw csum_loc = (uint16_t *)(m0->b_rptr +
358d39a76e7Sxw sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
359d39a76e7Sxw csum_loc += (((*(char *)csum_loc) & 0x0f) << 1);
360d39a76e7Sxw
361d39a76e7Sxw sge->intr_cnt.tx_soft_cksums++;
362d39a76e7Sxw ((struct udphdr *)(csum_loc))->uh_sum = csum;
363d39a76e7Sxw ((struct cpl_tx_pkt *)m0->b_rptr)->l4_csum_dis = 1;
364d39a76e7Sxw }
365d39a76e7Sxw sge->do_udp_csum--;
366d39a76e7Sxw } else if (flg & CH_TCP_MF) {
367d39a76e7Sxw sge->do_tcp_csum = 3;
368d39a76e7Sxw } else if (sge->do_tcp_csum != 0) {
369d39a76e7Sxw if ((flg & CH_NO_HWCKSUM) == 0) {
370d39a76e7Sxw sge->intr_cnt.tx_soft_cksums++;
371d39a76e7Sxw /*
372d39a76e7Sxw * Calc Checksum here.
373d39a76e7Sxw */
374d39a76e7Sxw }
375d39a76e7Sxw sge->do_tcp_csum--;
376d39a76e7Sxw }
377d39a76e7Sxw #endif /* TX_CKSUM_FIX */
378d39a76e7Sxw #ifdef TX_THREAD_RECLAIM
379d39a76e7Sxw reclaim_cnt = Q->cq_complete;
380d39a76e7Sxw if (reclaim_cnt > SGE_BATCH_THRESH) {
381d39a76e7Sxw sge->intr_cnt.tx_reclaims[qid]++;
382d39a76e7Sxw free_cmdQ_buffers(sge, Q, reclaim_cnt);
383d39a76e7Sxw Q->cq_complete = 0;
384d39a76e7Sxw }
385d39a76e7Sxw #endif
386d39a76e7Sxw genbit = Q->cq_genbit;
387d39a76e7Sxw pidx = Q->cq_pidx;
388d39a76e7Sxw credits = Q->cq_credits;
389d39a76e7Sxw
390d39a76e7Sxw if ((credits - 1) < count) {
391d39a76e7Sxw spin_unlock(qlock);
392d39a76e7Sxw sge->intr_cnt.cmdQ_full[qid]++;
393d39a76e7Sxw return (1);
394d39a76e7Sxw }
395d39a76e7Sxw
396d39a76e7Sxw atomic_sub(count, &Q->cq_credits);
397d39a76e7Sxw Q->cq_pidx += count;
398d39a76e7Sxw if (Q->cq_pidx >= entries_n) {
399d39a76e7Sxw Q->cq_pidx -= entries_n;
400d39a76e7Sxw Q->cq_genbit ^= 1;
401d39a76e7Sxw }
402d39a76e7Sxw
403d39a76e7Sxw spin_unlock(qlock);
404d39a76e7Sxw
405d39a76e7Sxw #ifdef SUN_KSTATS
406d39a76e7Sxw if (count > MBLK_MAX)
407d39a76e7Sxw sge->intr_cnt.tx_descs[MBLK_MAX - 1]++;
408d39a76e7Sxw else
409d39a76e7Sxw sge->intr_cnt.tx_descs[count]++;
410d39a76e7Sxw #endif
411d39a76e7Sxw
412d39a76e7Sxw ce = &cq[pidx];
413d39a76e7Sxw *ce = *cmp;
414d39a76e7Sxw mapping = cmp->ce_pa;
415d39a76e7Sxw j++;
416d39a76e7Sxw
417d39a76e7Sxw e = &q[pidx];
418d39a76e7Sxw
419d39a76e7Sxw offset = (caddr_t)e - (caddr_t)q;
420d39a76e7Sxw
421d39a76e7Sxw e->Sop = 1;
422d39a76e7Sxw e->DataValid = 1;
423d39a76e7Sxw e->BufferLength = cmp->ce_len;
424d39a76e7Sxw e->AddrHigh = ((u64)mapping >> 32);
425d39a76e7Sxw e->AddrLow = ((u64)mapping & 0xffffffff);
426d39a76e7Sxw
427d39a76e7Sxw --count;
428d39a76e7Sxw if (count > 0) {
429d39a76e7Sxw unsigned int i;
430d39a76e7Sxw
431d39a76e7Sxw e->Eop = 0;
432d39a76e7Sxw wmb();
433d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit;
434d39a76e7Sxw
435d39a76e7Sxw for (i = 0; i < count; i++) {
436d39a76e7Sxw
437d39a76e7Sxw ce++;
438d39a76e7Sxw e++;
439d39a76e7Sxw cmp++;
440d39a76e7Sxw if (++pidx == entries_n) {
441d39a76e7Sxw pidx = 0;
442d39a76e7Sxw genbit ^= 1;
443d39a76e7Sxw /* sync from offset to end of cmdQ */
444d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(offset),
445d39a76e7Sxw j*sizeof (*e), DDI_DMA_SYNC_FORDEV);
446d39a76e7Sxw offset = j = 0;
447d39a76e7Sxw ce = cq;
448d39a76e7Sxw e = q;
449d39a76e7Sxw }
450d39a76e7Sxw
451d39a76e7Sxw *ce = *cmp;
452d39a76e7Sxw mapping = cmp->ce_pa;
453d39a76e7Sxw j++;
454d39a76e7Sxw e->Sop = 0;
455d39a76e7Sxw e->DataValid = 1;
456d39a76e7Sxw e->BufferLength = cmp->ce_len;
457d39a76e7Sxw e->AddrHigh = ((u64)mapping >> 32);
458d39a76e7Sxw e->AddrLow = ((u64)mapping & 0xffffffff);
459d39a76e7Sxw
460d39a76e7Sxw if (i < (count - 1)) {
461d39a76e7Sxw e->Eop = 0;
462d39a76e7Sxw wmb();
463d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit;
464d39a76e7Sxw }
465d39a76e7Sxw }
466d39a76e7Sxw }
467d39a76e7Sxw
468d39a76e7Sxw ce->ce_mp = m0;
469d39a76e7Sxw
470d39a76e7Sxw e->Eop = 1;
471d39a76e7Sxw wmb();
472d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit;
473d39a76e7Sxw
474d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(offset), j*sizeof (*e),
475d39a76e7Sxw DDI_DMA_SYNC_FORDEV);
476d39a76e7Sxw
477d39a76e7Sxw /*
478d39a76e7Sxw * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
479d39a76e7Sxw * the doorbell if the Q is asleep. There is a natural race, where
480d39a76e7Sxw * the hardware is going to sleep just after we checked, however,
481d39a76e7Sxw * then the interrupt handler will detect the outstanding TX packet
482d39a76e7Sxw * and ring the doorbell for us.
483d39a76e7Sxw */
484d39a76e7Sxw if (qid) {
485d39a76e7Sxw doorbell_pio(sge, F_CMDQ1_ENABLE);
486d39a76e7Sxw } else {
487d39a76e7Sxw if (atomic_read(Q->cq_asleep)) {
488d39a76e7Sxw atomic_set(&Q->cq_asleep, 0);
489d39a76e7Sxw /* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */
490d39a76e7Sxw atomic_set(&Q->cq_pio_pidx, Q->cq_pidx);
491d39a76e7Sxw }
492d39a76e7Sxw }
493d39a76e7Sxw doorbell_pio(sge, F_CMDQ0_ENABLE);
494d39a76e7Sxw
495d39a76e7Sxw return (0);
496d39a76e7Sxw }
497d39a76e7Sxw
498d39a76e7Sxw #define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
499d39a76e7Sxw
500d39a76e7Sxw /*
501d39a76e7Sxw * Disable SGE error interrupts.
502d39a76e7Sxw */
503d39a76e7Sxw int
t1_sge_intr_disable(pesge * sge)504d39a76e7Sxw t1_sge_intr_disable(pesge* sge)
505d39a76e7Sxw {
506d39a76e7Sxw u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
507d39a76e7Sxw
508d39a76e7Sxw t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK);
509d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, 0);
510d39a76e7Sxw return (0);
511d39a76e7Sxw }
512d39a76e7Sxw
513d39a76e7Sxw #define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
514d39a76e7Sxw F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
515d39a76e7Sxw
516d39a76e7Sxw /*
517d39a76e7Sxw * Enable SGE error interrupts.
518d39a76e7Sxw */
519d39a76e7Sxw int
t1_sge_intr_enable(pesge * sge)520d39a76e7Sxw t1_sge_intr_enable(pesge* sge)
521d39a76e7Sxw {
522d39a76e7Sxw u32 en = SGE_INT_ENABLE;
523d39a76e7Sxw u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
524d39a76e7Sxw
525d39a76e7Sxw t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK);
526d39a76e7Sxw
527d39a76e7Sxw if (sge->obj->ch_flags & TSO_CAPABLE)
528d39a76e7Sxw en &= ~F_PACKET_TOO_BIG;
529d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, en);
530d39a76e7Sxw return (0);
531d39a76e7Sxw }
532d39a76e7Sxw
533d39a76e7Sxw /*
534d39a76e7Sxw * Clear SGE error interrupts.
535d39a76e7Sxw */
536d39a76e7Sxw int
t1_sge_intr_clear(pesge * sge)537d39a76e7Sxw t1_sge_intr_clear(pesge* sge)
538d39a76e7Sxw {
539d39a76e7Sxw t1_write_reg_4(sge->obj, A_PL_CAUSE, SGE_PL_INTR_MASK);
540d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, 0xffffffff);
541d39a76e7Sxw return (0);
542d39a76e7Sxw }
543d39a76e7Sxw
544d39a76e7Sxw #define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
545d39a76e7Sxw
546d39a76e7Sxw int
t1_sge_intr_error_handler(pesge * sge)547d39a76e7Sxw t1_sge_intr_error_handler(pesge *sge)
548d39a76e7Sxw {
549d39a76e7Sxw peobj *obj = sge->obj;
550d39a76e7Sxw u32 cause = t1_read_reg_4(obj, A_SG_INT_CAUSE);
551d39a76e7Sxw
552d39a76e7Sxw if (cause & F_RESPQ_EXHAUSTED)
553d39a76e7Sxw sge->intr_cnt.respQ_empty++;
554d39a76e7Sxw if (cause & F_RESPQ_OVERFLOW) {
555d39a76e7Sxw sge->intr_cnt.respQ_overflow++;
556d39a76e7Sxw cmn_err(CE_WARN, "%s: SGE response queue overflow\n",
557d39a76e7Sxw obj->ch_name);
558d39a76e7Sxw }
559d39a76e7Sxw if (cause & F_FL_EXHAUSTED) {
560d39a76e7Sxw sge->intr_cnt.freelistQ_empty++;
561d39a76e7Sxw freelQs_empty(sge);
562d39a76e7Sxw }
563d39a76e7Sxw if (cause & F_PACKET_TOO_BIG) {
564d39a76e7Sxw sge->intr_cnt.pkt_too_big++;
565d39a76e7Sxw cmn_err(CE_WARN, "%s: SGE max packet size exceeded\n",
566d39a76e7Sxw obj->ch_name);
567d39a76e7Sxw }
568d39a76e7Sxw if (cause & F_PACKET_MISMATCH) {
569d39a76e7Sxw sge->intr_cnt.pkt_mismatch++;
570d39a76e7Sxw cmn_err(CE_WARN, "%s: SGE packet mismatch\n",
571d39a76e7Sxw obj->ch_name);
572d39a76e7Sxw }
573d39a76e7Sxw if (cause & SGE_INT_FATAL)
574d39a76e7Sxw t1_fatal_err(obj);
575d39a76e7Sxw
576d39a76e7Sxw t1_write_reg_4(obj, A_SG_INT_CAUSE, cause);
577d39a76e7Sxw return (0);
578d39a76e7Sxw }
579d39a76e7Sxw
580d39a76e7Sxw /*
581d39a76e7Sxw *
582d39a76e7Sxw * PARAM: sge - SGE instance pointer.
583d39a76e7Sxw */
584d39a76e7Sxw int
sge_data_in(pesge * sge)585d39a76e7Sxw sge_data_in(pesge *sge)
586d39a76e7Sxw {
587d39a76e7Sxw peobj *adapter = sge->obj;
588d39a76e7Sxw struct respQ *Q = &sge->respQ;
589d39a76e7Sxw respQ_e *e; /* response queue entry */
590d39a76e7Sxw respQ_e *q = Q->rq_entries; /* base response queue */
591d39a76e7Sxw uint32_t cidx = Q->rq_cidx;
592d39a76e7Sxw uint32_t genbit = Q->rq_genbit;
593d39a76e7Sxw uint32_t entries_n = Q->rq_entries_n;
594d39a76e7Sxw uint32_t credits = Q->rq_credits;
595d39a76e7Sxw uint32_t credits_thresh = Q->rq_credits_thresh;
596d39a76e7Sxw uint32_t ret = 0;
597d39a76e7Sxw #ifndef TX_THREAD_RECLAIM
598d39a76e7Sxw uint32_t credits_pend[2] = {0, 0};
599d39a76e7Sxw #endif
600d39a76e7Sxw uint32_t flags = 0;
601d39a76e7Sxw uint32_t flagt;
602d39a76e7Sxw ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh;
603d39a76e7Sxw
604d39a76e7Sxw t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA);
605d39a76e7Sxw
606d39a76e7Sxw /*
607d39a76e7Sxw * Catch the case where an interrupt arrives
608d39a76e7Sxw * early.
609d39a76e7Sxw */
610d39a76e7Sxw if ((q == NULL) || (dh == NULL)) {
611d39a76e7Sxw goto check_slow_ints;
612d39a76e7Sxw }
613d39a76e7Sxw
614d39a76e7Sxw /* initial response queue entry */
615d39a76e7Sxw e = &q[cidx];
616d39a76e7Sxw
617d39a76e7Sxw /* pull physical memory of response queue entry into cache */
618d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
619d39a76e7Sxw sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
620d39a76e7Sxw
621d39a76e7Sxw while (e->GenerationBit == genbit) {
622d39a76e7Sxw if (--credits < credits_thresh) {
623d39a76e7Sxw uint32_t n = entries_n - credits - 1;
624d39a76e7Sxw t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n);
625d39a76e7Sxw credits += n;
626d39a76e7Sxw }
627d39a76e7Sxw if (likely(e->DataValid)) {
628d39a76e7Sxw (void) t1_sge_rx(sge, &sge->freelQ[e->FreelistQid],
629d39a76e7Sxw e->BufferLength, e->Offload);
630d39a76e7Sxw if ((e->Sop != 1) || (e->Eop != 1)) {
631d39a76e7Sxw sge->intr_cnt.rx_badEopSop++;
632d39a76e7Sxw cmn_err(CE_WARN, "bad Sop %d or Eop %d: %d",
633d39a76e7Sxw e->Sop, e->Eop, e->BufferLength);
634d39a76e7Sxw }
635d39a76e7Sxw }
636d39a76e7Sxw flagt = e->Qsleeping;
637d39a76e7Sxw flags |= flagt;
638d39a76e7Sxw if (flagt & F_CMDQ0_ENABLE)
639d39a76e7Sxw sge->intr_cnt.rx_cmdq0++;
640d39a76e7Sxw if (flagt & F_CMDQ1_ENABLE)
641d39a76e7Sxw sge->intr_cnt.rx_cmdq1++;
642d39a76e7Sxw if (flagt & F_FL0_ENABLE)
643d39a76e7Sxw sge->intr_cnt.rx_flq0++;
644d39a76e7Sxw if (flagt & F_FL1_ENABLE)
645d39a76e7Sxw sge->intr_cnt.rx_flq1++;
646d39a76e7Sxw #ifdef TX_THREAD_RECLAIM
647d39a76e7Sxw spin_lock(&sge->cmdQ[0].cq_qlock);
648d39a76e7Sxw sge->cmdQ[0].cq_complete += e->Cmdq0CreditReturn;
649d39a76e7Sxw spin_unlock(&sge->cmdQ[0].cq_qlock);
650d39a76e7Sxw spin_lock(&sge->cmdQ[1].cq_qlock);
651d39a76e7Sxw sge->cmdQ[1].cq_complete += e->Cmdq1CreditReturn;
652d39a76e7Sxw if ((adapter->ch_blked) &&
653d39a76e7Sxw (sge->cmdQ[0].cq_complete +
654d39a76e7Sxw sge->cmdQ[1].cq_complete) > 16) {
655d39a76e7Sxw adapter->ch_blked = 0;
656d39a76e7Sxw ch_gld_ok(adapter);
657d39a76e7Sxw }
658d39a76e7Sxw spin_unlock(&sge->cmdQ[1].cq_qlock);
659d39a76e7Sxw #else
660d39a76e7Sxw credits_pend[0] += e->Cmdq0CreditReturn;
661d39a76e7Sxw credits_pend[1] += e->Cmdq1CreditReturn;
662d39a76e7Sxw #ifdef CONFIG_SMP
663d39a76e7Sxw if (unlikely(credits_pend[0] > SGE_BATCH_THRESH)) {
664d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
665d39a76e7Sxw credits_pend[0] = 0;
666d39a76e7Sxw }
667d39a76e7Sxw if (unlikely(credits_pend[1] > SGE_BATCH_THRESH)) {
668d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
669d39a76e7Sxw credits_pend[1] = 0;
670d39a76e7Sxw }
671d39a76e7Sxw #endif
672d39a76e7Sxw #endif
673d39a76e7Sxw #ifdef HOST_PAUSE
674d39a76e7Sxw t1_sge_check_pause(sge, &sge->freelQ[e->FreelistQid]);
675d39a76e7Sxw #endif
676d39a76e7Sxw e++;
677d39a76e7Sxw if (unlikely(++cidx == entries_n)) {
678d39a76e7Sxw cidx = 0;
679d39a76e7Sxw genbit ^= 1;
680d39a76e7Sxw e = q;
681d39a76e7Sxw }
682d39a76e7Sxw
683d39a76e7Sxw /* pull physical memory of response queue entry into cache */
684d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
685d39a76e7Sxw sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
686d39a76e7Sxw
687d39a76e7Sxw ret = 1;
688d39a76e7Sxw }
689d39a76e7Sxw
690d39a76e7Sxw #ifndef TX_THREAD_RECLAIM
691d39a76e7Sxw if (credits_pend[0])
692d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
693d39a76e7Sxw if (credits_pend[1])
694d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
695d39a76e7Sxw #endif
696d39a76e7Sxw if (flags & F_CMDQ0_ENABLE) {
697d39a76e7Sxw struct cmdQ *cmdQ = &sge->cmdQ[0];
698d39a76e7Sxw atomic_set(&cmdQ->cq_asleep, 1);
699d39a76e7Sxw if (atomic_read(cmdQ->cq_pio_pidx) != cmdQ->cq_pidx) {
700d39a76e7Sxw doorbell_pio(sge, F_CMDQ0_ENABLE);
701d39a76e7Sxw atomic_set(&cmdQ->cq_pio_pidx, cmdQ->cq_pidx);
702d39a76e7Sxw }
703d39a76e7Sxw }
704d39a76e7Sxw
705d39a76e7Sxw /* the SGE told us one of the free lists is empty */
706d39a76e7Sxw if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE)))
707d39a76e7Sxw freelQs_empty(sge);
708d39a76e7Sxw
709d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD
710d39a76e7Sxw if (adapter->ch_tx_overflow_mutex)
711d39a76e7Sxw mutex_enter(adapter->ch_tx_overflow_mutex);
712d39a76e7Sxw if (adapter->ch_blked &&
713d39a76e7Sxw (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) &&
714d39a76e7Sxw (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) {
715d39a76e7Sxw adapter->ch_blked = 0;
716d39a76e7Sxw if (adapter->ch_tx_overflow_cv)
717d39a76e7Sxw cv_broadcast(adapter->ch_tx_overflow_cv);
718d39a76e7Sxw ch_gld_ok(adapter);
719d39a76e7Sxw }
720d39a76e7Sxw if (adapter->ch_tx_overflow_mutex)
721d39a76e7Sxw mutex_exit(adapter->ch_tx_overflow_mutex);
722d39a76e7Sxw #else
723d39a76e7Sxw #ifndef TX_THREAD_RECLAIM
724d39a76e7Sxw if (adapter->ch_blked &&
725d39a76e7Sxw (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) &&
726d39a76e7Sxw (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) {
727d39a76e7Sxw adapter->ch_blked = 0;
728d39a76e7Sxw ch_gld_ok(adapter);
729d39a76e7Sxw }
730d39a76e7Sxw #endif
731d39a76e7Sxw #endif /* CONFIG_CHELSIO_T1_OFFLOAD */
732d39a76e7Sxw
733d39a76e7Sxw Q->rq_genbit = genbit;
734d39a76e7Sxw Q->rq_cidx = cidx;
735d39a76e7Sxw Q->rq_credits = credits;
736d39a76e7Sxw
737d39a76e7Sxw t1_write_reg_4(adapter, A_SG_SLEEPING, cidx);
738d39a76e7Sxw
739d39a76e7Sxw check_slow_ints:
740d39a76e7Sxw /* handle non-data interrupts */
741d39a76e7Sxw if (unlikely(!ret))
742d39a76e7Sxw ret = t1_slow_intr_handler(adapter);
743d39a76e7Sxw
744d39a76e7Sxw return (ret);
745d39a76e7Sxw }
746d39a76e7Sxw
747d39a76e7Sxw /*
748d39a76e7Sxw * allocate a mblk with DMA mapped mblk.
749d39a76e7Sxw * When checksum offload is enabled, we start the DMA at a 2 byte offset so
750d39a76e7Sxw * the IP header will be aligned. We do this for sparc only.
751d39a76e7Sxw */
752d39a76e7Sxw static uint64_t
os_freelist_buffer_alloc(ch_t * sa,int sz,mblk_t ** mb,ulong_t * dh)753d39a76e7Sxw os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh)
754d39a76e7Sxw {
755d39a76e7Sxw ch_esb_t *ch_get_small_rbuf(ch_t *sa);
756d39a76e7Sxw ch_esb_t *ch_get_big_rbuf(ch_t *sa);
757d39a76e7Sxw ch_esb_t *rbp;
758d39a76e7Sxw uint32_t rxoff = sa->sge->rx_offset;
759d39a76e7Sxw
760d39a76e7Sxw if (sz == SGE_SM_BUF_SZ(sa)) {
761d39a76e7Sxw /* get pre-mapped buffer */
762d39a76e7Sxw if ((rbp = ch_get_small_rbuf(sa)) == NULL) {
763d39a76e7Sxw sa->norcvbuf++;
764d39a76e7Sxw return ((uint64_t)0);
765d39a76e7Sxw }
766d39a76e7Sxw
767d39a76e7Sxw *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
768d39a76e7Sxw SGE_SM_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
769d39a76e7Sxw if (*mb == NULL) {
770d39a76e7Sxw mutex_enter(&sa->ch_small_esbl);
771d39a76e7Sxw rbp->cs_next = sa->ch_small_esb_free;
772d39a76e7Sxw sa->ch_small_esb_free = rbp;
773d39a76e7Sxw mutex_exit(&sa->ch_small_esbl);
774d39a76e7Sxw return ((uint64_t)0);
775d39a76e7Sxw }
776d39a76e7Sxw *dh = rbp->cs_dh;
777d39a76e7Sxw
778d39a76e7Sxw return (rbp->cs_pa + rxoff);
779d39a76e7Sxw } else {
780d39a76e7Sxw /* get pre-mapped buffer */
781d39a76e7Sxw if ((rbp = ch_get_big_rbuf(sa)) == NULL) {
782d39a76e7Sxw sa->norcvbuf++;
783d39a76e7Sxw return ((uint64_t)0);
784d39a76e7Sxw }
785d39a76e7Sxw
786d39a76e7Sxw *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
787d39a76e7Sxw SGE_BG_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
788d39a76e7Sxw if (*mb == NULL) {
789d39a76e7Sxw mutex_enter(&sa->ch_big_esbl);
790d39a76e7Sxw rbp->cs_next = sa->ch_big_esb_free;
791d39a76e7Sxw sa->ch_big_esb_free = rbp;
792d39a76e7Sxw mutex_exit(&sa->ch_big_esbl);
793d39a76e7Sxw return ((uint64_t)0);
794d39a76e7Sxw }
795d39a76e7Sxw *dh = rbp->cs_dh;
796d39a76e7Sxw
797d39a76e7Sxw return (rbp->cs_pa + rxoff);
798d39a76e7Sxw }
799d39a76e7Sxw }
800d39a76e7Sxw
801d39a76e7Sxw static inline unsigned int
t1_sge_rx(pesge * sge,struct freelQ * Q,unsigned int len,unsigned int offload)802d39a76e7Sxw t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload)
803d39a76e7Sxw {
804d39a76e7Sxw mblk_t *skb;
805d39a76e7Sxw peobj *adapter = sge->obj;
806d39a76e7Sxw struct freelQ_ce *cq = Q->fq_centries;
807d39a76e7Sxw struct freelQ_ce *ce = &cq[Q->fq_cidx];
808d39a76e7Sxw ddi_dma_handle_t dh = (ddi_dma_handle_t)ce->fe_dh;
809d39a76e7Sxw uint32_t cidx = Q->fq_cidx;
810d39a76e7Sxw uint32_t entries_n = Q->fq_entries_n;
811d39a76e7Sxw uint32_t sz = Q->fq_rx_buffer_size;
812d39a76e7Sxw uint32_t useit = 1;
813d39a76e7Sxw uint32_t rxoff = sge->rx_offset;
814d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD
815d39a76e7Sxw uint32_t rv;
816d39a76e7Sxw #endif
817d39a76e7Sxw
818d39a76e7Sxw if (Q->fq_id)
819d39a76e7Sxw sge->intr_cnt.rx_flq1_cnt++;
820d39a76e7Sxw else
821d39a76e7Sxw sge->intr_cnt.rx_flq0_cnt++;
822d39a76e7Sxw /*
823d39a76e7Sxw * If pkt size falls below threshold, then we'll copy data to
824d39a76e7Sxw * an blk and reuse mblk.
825d39a76e7Sxw *
826d39a76e7Sxw * NOTE that rxoff is 2 for T1 adapters. We align the the start
827d39a76e7Sxw * of the DMA buffer begin at rxoff offset for T1 cards instead of
828d39a76e7Sxw * at the beginning of the buffer, thus the length of the received
829d39a76e7Sxw * data does not include this offset. We therefore always add
830d39a76e7Sxw * SGE_RX_OFFSET to the allocb size so we have space to provide the
831d39a76e7Sxw * offset for the copied data.
832d39a76e7Sxw */
833d39a76e7Sxw #ifdef HOST_PAUSE
834d39a76e7Sxw /*
835d39a76e7Sxw * If we have Host pause compiled in, then we look at the
836d39a76e7Sxw * free list, if the pause is on and we're not in offload
837d39a76e7Sxw * mode then we drop packets, this is designed to avoid
838d39a76e7Sxw * overwhelming the machine. If the machine is powerfull enough
839d39a76e7Sxw * this will not happen. The 'rx_pkt_drops' will show when
840d39a76e7Sxw * packets are being dropped and how much.
841d39a76e7Sxw */
842d39a76e7Sxw if ((offload == 0) && adapter->pause_on) {
843d39a76e7Sxw freelQ_e *e;
844d39a76e7Sxw /* Ditch the packet and reuse original buffer */
845d39a76e7Sxw e = &Q->fq_entries[cidx];
846d39a76e7Sxw e->GenerationBit ^= 1;
847d39a76e7Sxw e->GenerationBit2 ^= 1;
848d39a76e7Sxw sge->intr_cnt.rx_pkt_drops++;
849d39a76e7Sxw goto rx_entry_consumed;
850d39a76e7Sxw } else if (((adapter->pause_on ||
851d39a76e7Sxw (len <= SGE_RX_COPY_THRESHOLD)) &&
852d39a76e7Sxw (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI))))
853d39a76e7Sxw #else
854d39a76e7Sxw if ((len <= SGE_RX_COPY_THRESHOLD) &&
855d39a76e7Sxw (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI)))
856d39a76e7Sxw #endif
857d39a76e7Sxw {
858d39a76e7Sxw freelQ_e *e;
859d39a76e7Sxw char *src = (char *)((mblk_t *)ce->fe_mp)->b_rptr;
860d39a76e7Sxw
861d39a76e7Sxw /*
862d39a76e7Sxw * pull physical memory of pkt data into cache
863d39a76e7Sxw * Note that len does not include offset for T1.
864d39a76e7Sxw */
865d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(rxoff), len,
866d39a76e7Sxw DDI_DMA_SYNC_FORKERNEL);
867d39a76e7Sxw
868d39a76e7Sxw if (offload == 0) {
869d39a76e7Sxw /*
870d39a76e7Sxw * create 2 byte offset so IP header aligned on
871d39a76e7Sxw * 4 byte boundry
872d39a76e7Sxw */
873d39a76e7Sxw skb_reserve(skb, SGE_RX_OFFSET);
874d39a76e7Sxw /*
875d39a76e7Sxw * if hardware inserted 2 byte offset then need to
876d39a76e7Sxw * start copying with extra offset
877d39a76e7Sxw */
878d39a76e7Sxw src += sge->rx_pkt_pad;
879d39a76e7Sxw }
880d39a76e7Sxw memcpy(skb->b_rptr, src, len);
881d39a76e7Sxw useit = 0; /* mblk copy, don't inc esballoc in use cnt */
882d39a76e7Sxw
883d39a76e7Sxw /* so we can reuse original buffer */
884d39a76e7Sxw e = &Q->fq_entries[cidx];
885d39a76e7Sxw e->GenerationBit ^= 1;
886d39a76e7Sxw e->GenerationBit2 ^= 1;
887d39a76e7Sxw sge->intr_cnt.rx_pkt_copied++;
888d39a76e7Sxw } else {
889d39a76e7Sxw /* consume buffer off the ring */
890d39a76e7Sxw skb = ce->fe_mp;
891d39a76e7Sxw ce->fe_mp = NULL;
892d39a76e7Sxw
893d39a76e7Sxw /*
894d39a76e7Sxw * if not offload (tunneled pkt), & hardward padded, then
895d39a76e7Sxw * adjust start of pkt to point to start of data i.e.
896d39a76e7Sxw * skip pad (2 bytes).
897d39a76e7Sxw */
898d39a76e7Sxw if (!offload && sge->rx_pkt_pad)
899d39a76e7Sxw __skb_pull(skb, SGE_RX_OFFSET);
900d39a76e7Sxw
901d39a76e7Sxw /*
902d39a76e7Sxw * pull physical memory of pkt data into cache
903d39a76e7Sxw * Note that len does not include offset for T1.
904d39a76e7Sxw */
905d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(rxoff), len,
906d39a76e7Sxw DDI_DMA_SYNC_FORKERNEL);
907d39a76e7Sxw }
908d39a76e7Sxw
909d39a76e7Sxw /* set length of data in skb */
910d39a76e7Sxw skb_put(skb, len);
911d39a76e7Sxw
912d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD
913d39a76e7Sxw if (likely(offload)) {
914d39a76e7Sxw if (likely(toe_running(adapter))) {
915d39a76e7Sxw /* sends pkt upstream to toe layer */
916d39a76e7Sxw if (useit) {
917f172c8abSToomas Soome uint_t index;
918f172c8abSToomas Soome if (sz == SGE_SM_BUF_SZ(adapter))
919f172c8abSToomas Soome index = adapter->ch_sm_index;
920f172c8abSToomas Soome else
921f172c8abSToomas Soome index = adapter->ch_big_index;
922f172c8abSToomas Soome atomic_add(1, &buffers_in_use[index]);
923d39a76e7Sxw }
924d39a76e7Sxw if (adapter->toe_rcv)
925d39a76e7Sxw adapter->toe_rcv(adapter->ch_toeinst, skb);
926d39a76e7Sxw else
927d39a76e7Sxw freemsg(skb);
928d39a76e7Sxw } else {
929d39a76e7Sxw cmn_err(CE_WARN,
930d39a76e7Sxw "%s: unexpected offloaded packet, cmd %u\n",
931d39a76e7Sxw adapter->ch_name, *skb->b_rptr);
932d39a76e7Sxw
933d39a76e7Sxw /* discard packet */
934d39a76e7Sxw freemsg(skb);
935d39a76e7Sxw }
936d39a76e7Sxw }
937d39a76e7Sxw #else
938d39a76e7Sxw if (unlikely(offload)) {
939d39a76e7Sxw cmn_err(CE_WARN,
940d39a76e7Sxw "%s: unexpected offloaded packet, cmd %u\n",
941d39a76e7Sxw adapter->ch_name, *skb->b_rptr);
942d39a76e7Sxw
943d39a76e7Sxw /* discard paket */
944d39a76e7Sxw freemsg(skb);
945d39a76e7Sxw }
946d39a76e7Sxw #endif
947d39a76e7Sxw else {
948d39a76e7Sxw struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->b_rptr;
949d39a76e7Sxw int flg = 0;
950d39a76e7Sxw uint32_t cksum;
951d39a76e7Sxw
952d39a76e7Sxw /* adjust beginning of data to skip CPL header */
953d39a76e7Sxw skb_pull(skb, SZ_CPL_RX_PKT);
954d39a76e7Sxw
955d39a76e7Sxw /* extract checksum from CPL header here */
956d39a76e7Sxw
957d39a76e7Sxw /*
958d39a76e7Sxw * bump count of mlbks in used by protocol stack(s)
959d39a76e7Sxw */
960d39a76e7Sxw if (useit) {
961d39a76e7Sxw if (sz == SGE_SM_BUF_SZ(adapter)) {
962d39a76e7Sxw atomic_add(1,
963d39a76e7Sxw &buffers_in_use[adapter->ch_sm_index]);
964d39a76e7Sxw } else {
965d39a76e7Sxw atomic_add(1,
966d39a76e7Sxw &buffers_in_use[adapter->ch_big_index]);
967d39a76e7Sxw }
968d39a76e7Sxw }
969d39a76e7Sxw
970d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD
971d39a76e7Sxw /*
972d39a76e7Sxw * let the TOE layer have a crack at the packet first.
973d39a76e7Sxw */
974d39a76e7Sxw if (adapter->toe_tunnel) {
975d39a76e7Sxw rv = adapter->toe_tunnel(adapter->ch_toeinst, skb);
976d39a76e7Sxw /*
977d39a76e7Sxw * The TOE may have consumed the packet.
978d39a76e7Sxw */
979d39a76e7Sxw if (rv)
980d39a76e7Sxw goto rx_entry_consumed;
981d39a76e7Sxw }
982d39a76e7Sxw #endif /* CONFIG_CHELSIO_T1_OFFLOAD */
983d39a76e7Sxw
984d39a76e7Sxw cksum = p->csum;
985d39a76e7Sxw
986d39a76e7Sxw /*
987d39a76e7Sxw * NOTE: 14+9 = size of MAC + offset to IP protocol field
988d39a76e7Sxw */
989d39a76e7Sxw if (adapter->ch_config.cksum_enabled &&
990d39a76e7Sxw (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) ==
991d39a76e7Sxw ETHERTYPE_IP) &&
992d39a76e7Sxw ((skb->b_rptr[14+9] == IPPROTO_TCP) ||
993d39a76e7Sxw (skb->b_rptr[14+9] == IPPROTO_UDP))) {
994d39a76e7Sxw flg = 1;
995d39a76e7Sxw }
996d39a76e7Sxw
997d39a76e7Sxw ch_send_up(adapter, skb, cksum, flg);
998d39a76e7Sxw }
999d39a76e7Sxw
1000d39a76e7Sxw rx_entry_consumed:
1001d39a76e7Sxw
1002d39a76e7Sxw if (++cidx == entries_n)
1003d39a76e7Sxw cidx = 0;
1004d39a76e7Sxw
1005d39a76e7Sxw Q->fq_cidx = cidx;
1006d39a76e7Sxw
1007d39a76e7Sxw if (unlikely(--Q->fq_credits < (entries_n>>2)))
1008d39a76e7Sxw /* allocate new buffers on the free list */
1009d39a76e7Sxw alloc_freelQ_buffers(sge, Q);
1010d39a76e7Sxw return (1);
1011d39a76e7Sxw }
1012d39a76e7Sxw
1013d39a76e7Sxw #ifdef HOST_PAUSE
1014d39a76e7Sxw static void
t1_sge_check_pause(pesge * sge,struct freelQ * Q)1015d39a76e7Sxw t1_sge_check_pause(pesge *sge, struct freelQ *Q)
1016d39a76e7Sxw {
1017d39a76e7Sxw peobj *adapter = sge->obj;
1018d39a76e7Sxw
1019d39a76e7Sxw /*
1020d39a76e7Sxw * If the number of available credits shrinks below
1021d39a76e7Sxw * the Pause on threshold then enable the pause and
1022d39a76e7Sxw * try and allocate more buffers.
1023d39a76e7Sxw * On the next pass, if there's more credits returned
1024d39a76e7Sxw * then check that you've went above the pause
1025d39a76e7Sxw * threshold and then disable the pause.
1026d39a76e7Sxw */
1027d39a76e7Sxw if (Q->fq_credits < Q->fq_pause_on_thresh) {
1028d39a76e7Sxw if (do_host_pause) {
1029d39a76e7Sxw sge->intr_cnt.rx_pause_on++;
1030d39a76e7Sxw adapter->txxg_cfg1 |=
1031d39a76e7Sxw SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1032d39a76e7Sxw (void) t1_tpi_write(adapter,
1033d39a76e7Sxw SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1034d39a76e7Sxw adapter->txxg_cfg1);
1035d39a76e7Sxw adapter->pause_on = 1;
1036d39a76e7Sxw adapter->pause_time = gethrtime();
1037d39a76e7Sxw }
1038d39a76e7Sxw alloc_freelQ_buffers(sge, Q);
1039d39a76e7Sxw } else if ((adapter->pause_on) &&
1040d39a76e7Sxw (Q->fq_credits > Q->fq_pause_off_thresh)) {
1041d39a76e7Sxw hrtime_t time;
1042d39a76e7Sxw sge->intr_cnt.rx_pause_off++;
1043d39a76e7Sxw adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1044d39a76e7Sxw (void) t1_tpi_write(adapter,
1045d39a76e7Sxw SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1046d39a76e7Sxw adapter->txxg_cfg1);
1047d39a76e7Sxw adapter->pause_on = 0;
1048d39a76e7Sxw time = (gethrtime() - adapter->pause_time)/1000;
1049d39a76e7Sxw sge->intr_cnt.rx_pause_ms += time;
1050d39a76e7Sxw if (time > sge->intr_cnt.rx_pause_spike)
1051d39a76e7Sxw sge->intr_cnt.rx_pause_spike = (uint32_t)time;
1052d39a76e7Sxw }
1053d39a76e7Sxw sge->intr_cnt.rx_fl_credits = Q->fq_credits;
1054d39a76e7Sxw }
1055d39a76e7Sxw #endif /* HOST_PAUSE */
1056d39a76e7Sxw
1057d39a76e7Sxw static void
alloc_freelQ_buffers(pesge * sge,struct freelQ * Q)1058d39a76e7Sxw alloc_freelQ_buffers(pesge *sge, struct freelQ *Q)
1059d39a76e7Sxw {
1060d39a76e7Sxw uint32_t pidx = Q->fq_pidx;
1061d39a76e7Sxw struct freelQ_ce *ce = &Q->fq_centries[pidx];
1062d39a76e7Sxw freelQ_e *fq = Q->fq_entries; /* base of freelist Q */
1063d39a76e7Sxw freelQ_e *e = &Q->fq_entries[pidx];
1064d39a76e7Sxw uint32_t sz = Q->fq_rx_buffer_size;
1065d39a76e7Sxw uint32_t rxoff = sge->rx_offset;
1066d39a76e7Sxw uint32_t credits = Q->fq_credits;
1067d39a76e7Sxw uint32_t entries_n = Q->fq_entries_n;
1068d39a76e7Sxw uint32_t genbit = Q->fq_genbit;
1069d39a76e7Sxw ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh;
1070d39a76e7Sxw ulong_t dh;
1071d39a76e7Sxw uint64_t mapping;
1072d39a76e7Sxw off_t offset = (off_t)((caddr_t)e - (caddr_t)fq);
1073d39a76e7Sxw size_t len = 0;
1074d39a76e7Sxw
1075d39a76e7Sxw while (credits < entries_n) {
1076d39a76e7Sxw if (e->GenerationBit != genbit) {
1077d39a76e7Sxw mblk_t *skb;
1078d39a76e7Sxw
1079d39a76e7Sxw mapping = os_freelist_buffer_alloc(sge->obj, sz,
1080d39a76e7Sxw &skb, &dh);
1081d39a76e7Sxw if (mapping == 0) {
1082d39a76e7Sxw sge->intr_cnt.rx_flbuf_fails++;
1083d39a76e7Sxw break;
1084d39a76e7Sxw }
1085d39a76e7Sxw sge->intr_cnt.rx_flbuf_allocs++;
1086d39a76e7Sxw
1087d39a76e7Sxw ce->fe_mp = skb;
1088d39a76e7Sxw ce->fe_dh = dh;
1089d39a76e7Sxw
1090d39a76e7Sxw /*
1091d39a76e7Sxw * Note that for T1, we've started the beginning of
1092d39a76e7Sxw * of the buffer by an offset of 2 bytes. We thus
1093d39a76e7Sxw * decrement the length to account for this.
1094d39a76e7Sxw */
1095d39a76e7Sxw e->AddrLow = (u32)mapping;
1096d39a76e7Sxw e->AddrHigh = (u64)mapping >> 32;
1097d39a76e7Sxw e->BufferLength = sz - rxoff;
1098d39a76e7Sxw wmb();
1099d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit;
1100d39a76e7Sxw }
1101d39a76e7Sxw
1102d39a76e7Sxw len += sizeof (*e);
1103d39a76e7Sxw
1104d39a76e7Sxw ce++;
1105d39a76e7Sxw e++;
1106d39a76e7Sxw credits++;
1107d39a76e7Sxw if (++pidx == entries_n) {
1108d39a76e7Sxw /*
1109d39a76e7Sxw * sync freelist entries to physical memory up to
1110d39a76e7Sxw * end of the table.
1111d39a76e7Sxw */
1112d39a76e7Sxw (void) ddi_dma_sync(th, offset, len,
1113d39a76e7Sxw DDI_DMA_SYNC_FORDEV);
1114d39a76e7Sxw offset = 0;
1115d39a76e7Sxw len = 0;
1116d39a76e7Sxw
1117d39a76e7Sxw pidx = 0;
1118d39a76e7Sxw genbit ^= 1;
1119d39a76e7Sxw ce = Q->fq_centries;
1120d39a76e7Sxw e = Q->fq_entries;
1121d39a76e7Sxw }
1122d39a76e7Sxw }
1123d39a76e7Sxw
1124d39a76e7Sxw /* sync freelist entries that have been modified. */
1125d39a76e7Sxw if (len)
1126d39a76e7Sxw (void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV);
1127d39a76e7Sxw
1128d39a76e7Sxw Q->fq_genbit = genbit;
1129d39a76e7Sxw Q->fq_pidx = pidx;
1130d39a76e7Sxw Q->fq_credits = credits;
1131d39a76e7Sxw }
1132d39a76e7Sxw
1133d39a76e7Sxw static void
freelQs_empty(pesge * sge)1134d39a76e7Sxw freelQs_empty(pesge *sge)
1135d39a76e7Sxw {
1136d39a76e7Sxw u32 irq_reg = t1_read_reg_4(sge->obj, A_SG_INT_ENABLE);
1137d39a76e7Sxw u32 irqholdoff_reg;
1138d39a76e7Sxw
1139d39a76e7Sxw alloc_freelQ_buffers(sge, &sge->freelQ[0]);
1140d39a76e7Sxw alloc_freelQ_buffers(sge, &sge->freelQ[1]);
1141d39a76e7Sxw
1142d39a76e7Sxw if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) &&
1143d39a76e7Sxw (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) {
1144d39a76e7Sxw irq_reg |= F_FL_EXHAUSTED;
1145d39a76e7Sxw irqholdoff_reg = sge->intrtimer[sge->currIndex];
1146d39a76e7Sxw } else {
1147d39a76e7Sxw /* Clear the F_FL_EXHAUSTED interrupts for now */
1148d39a76e7Sxw irq_reg &= ~F_FL_EXHAUSTED;
1149d39a76e7Sxw irqholdoff_reg = sge->intrtimer_nres;
1150d39a76e7Sxw }
1151d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg);
1152d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, irq_reg);
1153d39a76e7Sxw
1154d39a76e7Sxw /* We reenable the Qs to force an Freelist GTS interrupt later */
1155d39a76e7Sxw doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE);
1156d39a76e7Sxw }
1157d39a76e7Sxw
1158d39a76e7Sxw /*
1159d39a76e7Sxw * Frees 'credits_pend' TX buffers and returns the credits to Q->credits.
1160d39a76e7Sxw * Free xmit buffers
1161d39a76e7Sxw */
1162d39a76e7Sxw static void
free_cmdQ_buffers(pesge * sge,struct cmdQ * Q,unsigned int credits_pend)1163d39a76e7Sxw free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend)
1164d39a76e7Sxw {
1165d39a76e7Sxw mblk_t *skb;
1166d39a76e7Sxw struct cmdQ_ce *ce;
1167d39a76e7Sxw struct cmdQ_ce *cq = Q->cq_centries;
1168d39a76e7Sxw uint32_t entries_n = Q->cq_entries_n;
1169d39a76e7Sxw uint32_t cidx = Q->cq_cidx;
1170d39a76e7Sxw uint32_t i = credits_pend;
1171d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1172d39a76e7Sxw ch_t *chp = sge->obj;
1173d39a76e7Sxw #endif
1174d39a76e7Sxw ce = &cq[cidx];
1175d39a76e7Sxw
1176d39a76e7Sxw while (i--) {
1177d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1178d39a76e7Sxw /* if flag set, then toe buffer */
1179d39a76e7Sxw switch (ce->ce_flg & 0x7) {
1180d39a76e7Sxw case DH_DMA:
1181d39a76e7Sxw if (ce->ce_dh) {
1182d39a76e7Sxw ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1183d39a76e7Sxw ce->ce_dh = NULL; /* may not be needed */
1184d39a76e7Sxw }
1185d39a76e7Sxw skb = ce->ce_mp;
118611abda1eSToomas Soome if (skb && ((ce->ce_flg & CH_ARP) == 0)) {
1187d39a76e7Sxw freemsg(skb);
1188d39a76e7Sxw }
1189d39a76e7Sxw ce->ce_mp = NULL;
1190d39a76e7Sxw break;
1191d39a76e7Sxw
1192d39a76e7Sxw #if defined(__sparc)
1193d39a76e7Sxw case DH_DVMA:
1194d39a76e7Sxw if (ce->ce_dh) {
1195d39a76e7Sxw ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1196d39a76e7Sxw ce->ce_dh = NULL; /* may not be needed */
1197d39a76e7Sxw }
1198d39a76e7Sxw skb = ce->ce_mp;
119911abda1eSToomas Soome if (skb && ((ce->ce_flg & CH_ARP) == 0)) {
1200d39a76e7Sxw freemsg(skb);
1201d39a76e7Sxw }
1202d39a76e7Sxw ce->ce_mp = NULL;
1203d39a76e7Sxw break;
1204d39a76e7Sxw #endif /* __sparc */
1205d39a76e7Sxw
1206d39a76e7Sxw case DH_TOE:
1207d39a76e7Sxw chp->toe_free(chp->ch_toeinst, (tbuf_t *)(ce->ce_mp));
1208d39a76e7Sxw ce->ce_mp = NULL;
1209d39a76e7Sxw break;
1210d39a76e7Sxw }
1211d39a76e7Sxw #else /* CONFIG_CHELSIO_T1_OFFLOAD */
1212d39a76e7Sxw if (ce->ce_dh) {
1213d39a76e7Sxw if ((ce->ce_flg & 7) == DH_DMA) {
1214d39a76e7Sxw ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1215d39a76e7Sxw }
1216d39a76e7Sxw #if defined(__sparc)
1217d39a76e7Sxw else {
1218d39a76e7Sxw ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1219d39a76e7Sxw }
1220d39a76e7Sxw #endif /* __sparc */
1221d39a76e7Sxw ce->ce_dh = NULL; /* may not be needed */
1222d39a76e7Sxw }
1223d39a76e7Sxw
1224d39a76e7Sxw skb = ce->ce_mp;
122511abda1eSToomas Soome if (skb && ((ce->ce_flg & CH_ARP) == 0)) {
1226d39a76e7Sxw freemsg(skb);
1227d39a76e7Sxw }
1228d39a76e7Sxw ce->ce_mp = NULL;
1229d39a76e7Sxw #endif /* !CONFIG_CHELSIO_T1_OFFLOAD */
1230d39a76e7Sxw
1231d39a76e7Sxw ce++;
1232d39a76e7Sxw if (++cidx == entries_n) {
1233d39a76e7Sxw cidx = 0;
1234d39a76e7Sxw ce = cq;
1235d39a76e7Sxw }
1236d39a76e7Sxw }
1237d39a76e7Sxw
1238d39a76e7Sxw Q->cq_cidx = cidx;
1239d39a76e7Sxw atomic_add(credits_pend, &Q->cq_credits);
1240d39a76e7Sxw }
1241d39a76e7Sxw
1242d39a76e7Sxw struct sge_intr_counts *
sge_get_stat(pesge * sge)1243d39a76e7Sxw sge_get_stat(pesge *sge)
1244d39a76e7Sxw {
1245d39a76e7Sxw return (&sge->intr_cnt);
1246d39a76e7Sxw }
1247d39a76e7Sxw
1248d39a76e7Sxw /*
1249d39a76e7Sxw * Allocates both RX and TX resources and configures the SGE. However,
1250d39a76e7Sxw * the hardware is not enabled yet.
1251d39a76e7Sxw *
1252d39a76e7Sxw * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic.
1253d39a76e7Sxw * jumbo_fl is set to the index of the freelist containing the jumbo buffers.
1254d39a76e7Sxw */
1255d39a76e7Sxw int
t1_sge_configure(pesge * sge,struct sge_params * p)1256d39a76e7Sxw t1_sge_configure(pesge *sge, struct sge_params *p)
1257d39a76e7Sxw {
1258d39a76e7Sxw sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET;
1259d39a76e7Sxw sge->jumbo_fl = t1_is_T1B(sge->obj) ? 1 : 0;
1260d39a76e7Sxw /* if we're a T2 card, then we have hardware offset support */
1261d39a76e7Sxw sge->rx_offset = t1_is_T1B(sge->obj) ? SGE_RX_OFFSET: 0;
1262d39a76e7Sxw
1263d39a76e7Sxw if (alloc_rx_resources(sge, p))
1264d39a76e7Sxw return (-ENOMEM);
1265d39a76e7Sxw if (alloc_tx_resources(sge, p)) {
1266d39a76e7Sxw free_rx_resources(sge);
1267d39a76e7Sxw return (-ENOMEM);
1268d39a76e7Sxw }
1269d39a76e7Sxw configure_sge(sge, p);
1270d39a76e7Sxw
1271d39a76e7Sxw /*
1272d39a76e7Sxw * Now that we have sized the free lists calculate the payload
1273d39a76e7Sxw * capacity of the large buffers. Other parts of the driver use
1274d39a76e7Sxw * this to set the max offload coalescing size so that RX packets
1275d39a76e7Sxw * do not overflow our large buffers.
1276d39a76e7Sxw */
1277d39a76e7Sxw p->large_buf_capacity = jumbo_payload_capacity(sge);
1278d39a76e7Sxw return (0);
1279d39a76e7Sxw }
1280d39a76e7Sxw
1281d39a76e7Sxw /*
1282d39a76e7Sxw * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
1283d39a76e7Sxw * response Q.
1284d39a76e7Sxw */
1285d39a76e7Sxw static int
alloc_rx_resources(pesge * sge,struct sge_params * p)1286d39a76e7Sxw alloc_rx_resources(pesge *sge, struct sge_params *p)
1287d39a76e7Sxw {
1288d39a76e7Sxw unsigned int size, i;
1289d39a76e7Sxw
1290d39a76e7Sxw for (i = 0; i < SGE_FREELQ_N; i++) {
1291d39a76e7Sxw struct freelQ *Q = &sge->freelQ[i];
1292d39a76e7Sxw
1293d39a76e7Sxw Q->fq_id = i;
1294d39a76e7Sxw Q->fq_genbit = 1;
1295d39a76e7Sxw Q->fq_entries_n = p->freelQ_size[i];
1296d39a76e7Sxw #ifdef HOST_PAUSE
1297d39a76e7Sxw Q->fq_pause_on_thresh = flq_pause_window;
1298d39a76e7Sxw Q->fq_pause_off_thresh = Q->fq_entries_n >> 1;
1299d39a76e7Sxw #endif
1300d39a76e7Sxw size = sizeof (freelQ_e) * Q->fq_entries_n;
1301d39a76e7Sxw
1302d39a76e7Sxw Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1303d39a76e7Sxw size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT);
1304d39a76e7Sxw
1305d39a76e7Sxw
1306d39a76e7Sxw if (!Q->fq_entries)
1307d39a76e7Sxw goto err_no_mem;
1308d39a76e7Sxw memset(Q->fq_entries, 0, size);
1309d39a76e7Sxw size = sizeof (struct freelQ_ce) * Q->fq_entries_n;
1310d39a76e7Sxw Q->fq_centries = t1_os_malloc_wait_zero(size);
1311d39a76e7Sxw if (!Q->fq_centries)
1312d39a76e7Sxw goto err_no_mem;
1313d39a76e7Sxw memset(Q->fq_centries, 0, size);
1314d39a76e7Sxw }
1315d39a76e7Sxw
1316d39a76e7Sxw /*
1317d39a76e7Sxw * Calculate the buffer sizes for the two free lists. FL0 accommodates
1318d39a76e7Sxw * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
1319d39a76e7Sxw * including all the sk_buff overhead.
1320d39a76e7Sxw * For T1C FL0 and FL1 are reversed.
1321d39a76e7Sxw */
1322d39a76e7Sxw #ifdef NOTYET
1323d39a76e7Sxw sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE +
1324d39a76e7Sxw sizeof (struct cpl_rx_data) +
1325d39a76e7Sxw SGE_RX_OFFSET - sge->rx_pkt_pad;
1326d39a76e7Sxw #else
1327d39a76e7Sxw sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size =
1328d39a76e7Sxw sge->obj->ch_sm_buf_sz;
1329d39a76e7Sxw if (is_T2(sge->obj))
1330d39a76e7Sxw sge->intr_cnt.rx_flq1_sz = sge->obj->ch_sm_buf_sz;
1331d39a76e7Sxw else
1332d39a76e7Sxw sge->intr_cnt.rx_flq0_sz = sge->obj->ch_sm_buf_sz;
1333d39a76e7Sxw #endif
1334d39a76e7Sxw #ifdef NOTYET
1335d39a76e7Sxw sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) -
1336d39a76e7Sxw SKB_DATA_ALIGN(sizeof (struct skb_shared_info));
1337d39a76e7Sxw #else
1338d39a76e7Sxw sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz;
1339d39a76e7Sxw if (is_T2(sge->obj))
1340d39a76e7Sxw sge->intr_cnt.rx_flq0_sz = sge->obj->ch_bg_buf_sz;
1341d39a76e7Sxw else
1342d39a76e7Sxw sge->intr_cnt.rx_flq1_sz = sge->obj->ch_bg_buf_sz;
1343d39a76e7Sxw #endif
1344d39a76e7Sxw
1345d39a76e7Sxw sge->respQ.rq_genbit = 1;
1346d39a76e7Sxw sge->respQ.rq_entries_n = sge_respq_cnt;
1347d39a76e7Sxw sge->respQ.rq_credits = sge_respq_cnt;
1348d39a76e7Sxw sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2);
1349d39a76e7Sxw size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1350d39a76e7Sxw
1351d39a76e7Sxw sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1352d39a76e7Sxw size, &(sge->respQ.rq_pa), &(sge->respQ.rq_dh),
1353d39a76e7Sxw &(sge->respQ.rq_ah), 0);
1354d39a76e7Sxw
1355d39a76e7Sxw if (!sge->respQ.rq_entries)
1356d39a76e7Sxw goto err_no_mem;
1357d39a76e7Sxw memset(sge->respQ.rq_entries, 0, size);
1358d39a76e7Sxw return (0);
1359d39a76e7Sxw
1360d39a76e7Sxw err_no_mem:
1361d39a76e7Sxw free_rx_resources(sge);
1362d39a76e7Sxw return (1);
1363d39a76e7Sxw }
1364d39a76e7Sxw
1365d39a76e7Sxw /*
1366d39a76e7Sxw * Allocates basic TX resources, consisting of memory mapped command Qs.
1367d39a76e7Sxw */
1368d39a76e7Sxw static int
alloc_tx_resources(pesge * sge,struct sge_params * p)1369d39a76e7Sxw alloc_tx_resources(pesge *sge, struct sge_params *p)
1370d39a76e7Sxw {
1371d39a76e7Sxw unsigned int size, i;
1372d39a76e7Sxw
1373d39a76e7Sxw for (i = 0; i < SGE_CMDQ_N; i++) {
1374d39a76e7Sxw struct cmdQ *Q = &sge->cmdQ[i];
1375d39a76e7Sxw
1376d39a76e7Sxw Q->cq_genbit = 1;
1377d39a76e7Sxw Q->cq_entries_n = p->cmdQ_size[i];
1378d39a76e7Sxw atomic_set(&Q->cq_credits, Q->cq_entries_n);
1379d39a76e7Sxw atomic_set(&Q->cq_asleep, 1);
1380d39a76e7Sxw
1381d39a76e7Sxw mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER,
1382d39a76e7Sxw sge->obj->ch_icookp);
1383d39a76e7Sxw
1384d39a76e7Sxw size = sizeof (cmdQ_e) * Q->cq_entries_n;
1385d39a76e7Sxw Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1386d39a76e7Sxw size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT);
1387d39a76e7Sxw
1388d39a76e7Sxw if (!Q->cq_entries)
1389d39a76e7Sxw goto err_no_mem;
1390d39a76e7Sxw memset(Q->cq_entries, 0, size);
1391d39a76e7Sxw size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1392d39a76e7Sxw Q->cq_centries = t1_os_malloc_wait_zero(size);
1393d39a76e7Sxw if (!Q->cq_centries)
1394d39a76e7Sxw goto err_no_mem;
1395d39a76e7Sxw memset(Q->cq_centries, 0, size);
1396d39a76e7Sxw
1397d39a76e7Sxw /* allocate pre-mapped dma headers */
1398d39a76e7Sxw pe_dma_handle_init(sge->obj, Q->cq_entries_n);
1399d39a76e7Sxw }
1400d39a76e7Sxw
1401d39a76e7Sxw return (0);
1402d39a76e7Sxw
1403d39a76e7Sxw err_no_mem:
1404d39a76e7Sxw free_tx_resources(sge);
1405d39a76e7Sxw return (1);
1406d39a76e7Sxw }
1407d39a76e7Sxw
1408d39a76e7Sxw /*
1409d39a76e7Sxw * Sets the interrupt latency timer when the adaptive Rx coalescing
1410d39a76e7Sxw * is turned off. Do nothing when it is turned on again.
1411d39a76e7Sxw *
1412d39a76e7Sxw * This routine relies on the fact that the caller has already set
1413d39a76e7Sxw * the adaptive policy in adapter->sge_params before calling it.
1414d39a76e7Sxw */
1415d39a76e7Sxw int
t1_sge_set_coalesce_params(pesge * sge,struct sge_params * p)1416d39a76e7Sxw t1_sge_set_coalesce_params(pesge *sge, struct sge_params *p)
1417d39a76e7Sxw {
1418d39a76e7Sxw if (!p->coalesce_enable) {
1419d39a76e7Sxw u32 newTimer = p->rx_coalesce_usecs *
1420d39a76e7Sxw (board_info(sge->obj)->clock_core / 1000000);
1421d39a76e7Sxw
1422d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INTRTIMER, newTimer);
1423d39a76e7Sxw }
1424d39a76e7Sxw return (0);
1425d39a76e7Sxw }
1426d39a76e7Sxw
1427d39a76e7Sxw /*
1428d39a76e7Sxw * Programs the various SGE registers. However, the engine is not yet enabled,
1429d39a76e7Sxw * but sge->sge_control is setup and ready to go.
1430d39a76e7Sxw */
1431d39a76e7Sxw static void
configure_sge(pesge * sge,struct sge_params * p)1432d39a76e7Sxw configure_sge(pesge *sge, struct sge_params *p)
1433d39a76e7Sxw {
1434d39a76e7Sxw ch_t *ap = sge->obj;
1435d39a76e7Sxw int i;
1436d39a76e7Sxw
1437d39a76e7Sxw t1_write_reg_4(ap, A_SG_CONTROL, 0);
1438d39a76e7Sxw
1439d39a76e7Sxw setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n,
1440d39a76e7Sxw A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
1441d39a76e7Sxw setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n,
1442d39a76e7Sxw A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
1443d39a76e7Sxw setup_ring_params(ap, sge->freelQ[0].fq_pa,
1444d39a76e7Sxw sge->freelQ[0].fq_entries_n, A_SG_FL0BASELWR,
1445d39a76e7Sxw A_SG_FL0BASEUPR, A_SG_FL0SIZE);
1446d39a76e7Sxw setup_ring_params(ap, sge->freelQ[1].fq_pa,
1447d39a76e7Sxw sge->freelQ[1].fq_entries_n, A_SG_FL1BASELWR,
1448d39a76e7Sxw A_SG_FL1BASEUPR, A_SG_FL1SIZE);
1449d39a76e7Sxw
1450d39a76e7Sxw /* The threshold comparison uses <. */
1451d39a76e7Sxw t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) -
1452d39a76e7Sxw SZ_CPL_RX_PKT - sge->rx_pkt_pad - sge->rx_offset + 1);
1453d39a76e7Sxw setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n,
1454d39a76e7Sxw A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
1455d39a76e7Sxw t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n);
1456d39a76e7Sxw sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
1457d39a76e7Sxw F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
1458d39a76e7Sxw V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
1459d39a76e7Sxw #if 1
1460d39a76e7Sxw /*
1461d39a76e7Sxw * if the the following bit is not set, then we'll get an
1462d39a76e7Sxw * interrupt everytime command Q 0 goes empty. Since we're
1463d39a76e7Sxw * always ringing the doorbell, we can turn it on.
1464d39a76e7Sxw */
1465d39a76e7Sxw F_DISABLE_CMDQ0_GTS |
1466d39a76e7Sxw #endif
1467d39a76e7Sxw V_RX_PKT_OFFSET(sge->rx_pkt_pad);
1468d39a76e7Sxw
1469d39a76e7Sxw #if BYTE_ORDER == BIG_ENDIAN
1470d39a76e7Sxw sge->sge_control |= F_ENABLE_BIG_ENDIAN;
1471d39a76e7Sxw #endif
1472d39a76e7Sxw
1473d39a76e7Sxw /*
1474d39a76e7Sxw * Initialize the SGE Interrupt Timer arrray:
1475d39a76e7Sxw * intrtimer[0] = (SGE_INTRTIMER0) usec
1476d39a76e7Sxw * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec
1477d39a76e7Sxw * intrtimer[10] = (SGE_INTRTIMER1) usec
1478d39a76e7Sxw *
1479d39a76e7Sxw */
1480d39a76e7Sxw sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000;
1481d39a76e7Sxw for (i = 1; i < SGE_INTR_MAXBUCKETS - 1; ++i) {
1482d39a76e7Sxw sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i);
1483d39a76e7Sxw sge->intrtimer[i] *= sge->intrtimer[0];
1484d39a76e7Sxw }
1485d39a76e7Sxw sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] =
1486d39a76e7Sxw sge->intrtimer[0] * SGE_INTRTIMER1;
1487d39a76e7Sxw /* Initialize resource timer */
1488d39a76e7Sxw sge->intrtimer_nres = (uint32_t)(sge->intrtimer[0] *
1489d39a76e7Sxw SGE_INTRTIMER_NRES);
1490d39a76e7Sxw /* Finally finish initialization of intrtimer[0] */
1491d39a76e7Sxw sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0);
1492d39a76e7Sxw /* Initialize for a throughput oriented workload */
1493d39a76e7Sxw sge->currIndex = SGE_INTR_MAXBUCKETS - 1;
1494d39a76e7Sxw
1495d39a76e7Sxw if (p->coalesce_enable)
1496d39a76e7Sxw t1_write_reg_4(ap, A_SG_INTRTIMER,
1497d39a76e7Sxw sge->intrtimer[sge->currIndex]);
1498d39a76e7Sxw else
1499d39a76e7Sxw (void) t1_sge_set_coalesce_params(sge, p);
1500d39a76e7Sxw }
1501d39a76e7Sxw
1502d39a76e7Sxw static inline void
setup_ring_params(ch_t * adapter,u64 addr,u32 size,int base_reg_lo,int base_reg_hi,int size_reg)1503d39a76e7Sxw setup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo,
1504d39a76e7Sxw int base_reg_hi, int size_reg)
1505d39a76e7Sxw {
1506d39a76e7Sxw t1_write_reg_4(adapter, base_reg_lo, (u32)addr);
1507d39a76e7Sxw t1_write_reg_4(adapter, base_reg_hi, addr >> 32);
1508d39a76e7Sxw t1_write_reg_4(adapter, size_reg, size);
1509d39a76e7Sxw }
1510d39a76e7Sxw
1511d39a76e7Sxw /*
1512d39a76e7Sxw * Frees RX resources.
1513d39a76e7Sxw */
1514d39a76e7Sxw static void
free_rx_resources(pesge * sge)1515d39a76e7Sxw free_rx_resources(pesge *sge)
1516d39a76e7Sxw {
1517d39a76e7Sxw unsigned int size, i;
1518d39a76e7Sxw
1519d39a76e7Sxw if (sge->respQ.rq_entries) {
1520d39a76e7Sxw size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1521d39a76e7Sxw
1522d39a76e7Sxw pe_os_free_contig(sge->obj, size, sge->respQ.rq_entries,
1523d39a76e7Sxw sge->respQ.rq_pa, sge->respQ.rq_dh, sge->respQ.rq_ah);
1524d39a76e7Sxw }
1525d39a76e7Sxw
1526d39a76e7Sxw for (i = 0; i < SGE_FREELQ_N; i++) {
1527d39a76e7Sxw struct freelQ *Q = &sge->freelQ[i];
1528d39a76e7Sxw
1529d39a76e7Sxw if (Q->fq_centries) {
1530d39a76e7Sxw free_freelQ_buffers(sge, Q);
1531d39a76e7Sxw
1532d39a76e7Sxw t1_os_free(Q->fq_centries,
1533d39a76e7Sxw Q->fq_entries_n * sizeof (freelQ_ce_t));
1534d39a76e7Sxw }
1535d39a76e7Sxw if (Q->fq_entries) {
1536d39a76e7Sxw size = sizeof (freelQ_e) * Q->fq_entries_n;
1537d39a76e7Sxw
1538d39a76e7Sxw /* free the freelist queue */
1539d39a76e7Sxw pe_os_free_contig(sge->obj, size, Q->fq_entries,
1540d39a76e7Sxw Q->fq_pa, Q->fq_dh, Q->fq_ah);
1541d39a76e7Sxw
1542d39a76e7Sxw }
1543d39a76e7Sxw }
1544d39a76e7Sxw }
1545d39a76e7Sxw
1546d39a76e7Sxw /*
1547d39a76e7Sxw * Frees all RX buffers on the freelist Q. The caller must make sure that
1548d39a76e7Sxw * the SGE is turned off before calling this function.
1549d39a76e7Sxw */
1550d39a76e7Sxw static void
free_freelQ_buffers(pesge * sge,struct freelQ * Q)1551d39a76e7Sxw free_freelQ_buffers(pesge *sge, struct freelQ *Q)
1552d39a76e7Sxw {
1553d39a76e7Sxw struct freelQ_ce *ce;
1554d39a76e7Sxw struct freelQ_ce *cq = Q->fq_centries;
1555d39a76e7Sxw uint32_t credits = Q->fq_credits;
1556d39a76e7Sxw uint32_t entries_n = Q->fq_entries_n;
1557d39a76e7Sxw uint32_t cidx = Q->fq_cidx;
1558d39a76e7Sxw uint32_t i = Q->fq_id;
1559d39a76e7Sxw
1560d39a76e7Sxw ce = &cq[cidx];
1561d39a76e7Sxw
1562d39a76e7Sxw credits = entries_n;
1563d39a76e7Sxw while (credits--) {
1564d39a76e7Sxw mblk_t *mp;
1565d39a76e7Sxw if ((mp = ce->fe_mp) != NULL) {
1566d39a76e7Sxw /* bump in-use count of receive buffers */
1567d39a76e7Sxw if (i != sge->jumbo_fl) {
1568d39a76e7Sxw atomic_add(1,
1569d39a76e7Sxw &buffers_in_use[sge->obj->ch_sm_index]);
1570d39a76e7Sxw } else {
1571d39a76e7Sxw atomic_add(1,
1572d39a76e7Sxw &buffers_in_use[sge->obj->ch_big_index]);
1573d39a76e7Sxw }
1574d39a76e7Sxw
1575d39a76e7Sxw /*
1576d39a76e7Sxw * note. freeb() callback of esb-alloced mblk will
1577d39a76e7Sxw * cause receive buffer to be put back on sa free list.
1578d39a76e7Sxw */
1579d39a76e7Sxw freeb(mp);
1580d39a76e7Sxw ce->fe_mp = NULL;
1581d39a76e7Sxw }
1582d39a76e7Sxw
1583d39a76e7Sxw ce++;
1584d39a76e7Sxw if (++cidx == entries_n) {
1585d39a76e7Sxw cidx = 0;
1586d39a76e7Sxw ce = cq;
1587d39a76e7Sxw }
1588d39a76e7Sxw }
1589d39a76e7Sxw
1590d39a76e7Sxw Q->fq_cidx = cidx;
1591d39a76e7Sxw Q->fq_credits = credits;
1592d39a76e7Sxw }
1593d39a76e7Sxw
1594d39a76e7Sxw /*
1595d39a76e7Sxw * Free TX resources.
1596d39a76e7Sxw *
1597d39a76e7Sxw * Assumes that SGE is stopped and all interrupts are disabled.
1598d39a76e7Sxw */
1599d39a76e7Sxw static void
free_tx_resources(pesge * sge)1600d39a76e7Sxw free_tx_resources(pesge *sge)
1601d39a76e7Sxw {
1602d39a76e7Sxw unsigned int size;
1603d39a76e7Sxw uint32_t i;
1604d39a76e7Sxw
1605d39a76e7Sxw for (i = 0; i < SGE_CMDQ_N; i++) {
1606d39a76e7Sxw struct cmdQ *Q = &sge->cmdQ[i];
1607d39a76e7Sxw
1608d39a76e7Sxw if (Q->cq_centries) {
1609d39a76e7Sxw unsigned int pending = Q->cq_entries_n -
1610d39a76e7Sxw atomic_read(Q->cq_credits);
1611d39a76e7Sxw
1612d39a76e7Sxw mutex_destroy(&Q->cq_qlock);
1613d39a76e7Sxw
1614d39a76e7Sxw if (pending)
1615d39a76e7Sxw free_cmdQ_buffers(sge, Q, pending);
1616d39a76e7Sxw
1617d39a76e7Sxw size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1618d39a76e7Sxw t1_os_free(Q->cq_centries, size);
1619d39a76e7Sxw }
1620d39a76e7Sxw
1621d39a76e7Sxw if (Q->cq_entries) {
1622d39a76e7Sxw size = sizeof (cmdQ_e) * Q->cq_entries_n;
1623d39a76e7Sxw pe_os_free_contig(sge->obj, size, Q->cq_entries,
1624d39a76e7Sxw Q->cq_pa, Q->cq_dh, Q->cq_ah);
1625d39a76e7Sxw }
1626d39a76e7Sxw }
1627d39a76e7Sxw }
1628d39a76e7Sxw
1629d39a76e7Sxw /*
1630d39a76e7Sxw * Return the payload capacity of the jumbo free-list buffers.
1631d39a76e7Sxw */
jumbo_payload_capacity(pesge * sge)1632d39a76e7Sxw static inline unsigned int jumbo_payload_capacity(pesge *sge)
1633d39a76e7Sxw {
1634d39a76e7Sxw return (sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size -
1635d39a76e7Sxw sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset);
1636d39a76e7Sxw }
1637d39a76e7Sxw
1638d39a76e7Sxw /* PR2928 & PR3309 */
1639d39a76e7Sxw void
t1_sge_set_ptimeout(adapter_t * adapter,u32 val)1640d39a76e7Sxw t1_sge_set_ptimeout(adapter_t *adapter, u32 val)
1641d39a76e7Sxw {
1642d39a76e7Sxw pesge *sge = adapter->sge;
1643d39a76e7Sxw
1644d39a76e7Sxw if (is_T2(adapter))
1645d39a76e7Sxw sge->ptimeout = max(val, 1);
1646d39a76e7Sxw }
1647d39a76e7Sxw
1648d39a76e7Sxw /* PR2928 & PR3309 */
1649d39a76e7Sxw u32
t1_sge_get_ptimeout(adapter_t * adapter)1650d39a76e7Sxw t1_sge_get_ptimeout(adapter_t *adapter)
1651d39a76e7Sxw {
1652d39a76e7Sxw pesge *sge = adapter->sge;
1653d39a76e7Sxw
1654d39a76e7Sxw return (is_T2(adapter) ? sge->ptimeout : 0);
1655d39a76e7Sxw }
1656d39a76e7Sxw
1657d39a76e7Sxw void
sge_add_fake_arp(pesge * sge,void * bp)1658d39a76e7Sxw sge_add_fake_arp(pesge *sge, void *bp)
1659d39a76e7Sxw {
1660d39a76e7Sxw sge->pskb = bp;
1661d39a76e7Sxw }
1662d39a76e7Sxw
1663d39a76e7Sxw #ifdef SUN_KSTATS
1664d39a76e7Sxw static int
sge_kstat_setup(pesge * sge)1665d39a76e7Sxw sge_kstat_setup(pesge *sge)
1666d39a76e7Sxw {
1667d39a76e7Sxw int status;
1668d39a76e7Sxw p_kstat_t ksp;
1669d39a76e7Sxw size_t ch_kstat_sz;
1670d39a76e7Sxw p_ch_kstat_t chkp;
1671d39a76e7Sxw char kstat_name[32];
1672d39a76e7Sxw int instance;
1673d39a76e7Sxw int i;
1674d39a76e7Sxw
1675d39a76e7Sxw status = -1;
1676d39a76e7Sxw ch_kstat_sz = sizeof (ch_kstat_t);
1677d39a76e7Sxw instance = ddi_get_instance(sge->obj->ch_dip);
1678d39a76e7Sxw if ((ksp = kstat_create(CHNAME "_debug", instance,
1679d39a76e7Sxw NULL, "net_debug", KSTAT_TYPE_NAMED,
1680d39a76e7Sxw ch_kstat_sz / sizeof (kstat_named_t), 0)) == NULL)
1681d39a76e7Sxw goto sge_kstat_setup_exit;
1682d39a76e7Sxw chkp = (p_ch_kstat_t)ksp->ks_data;
1683d39a76e7Sxw kstat_named_init(&chkp->respQ_empty, "respQ_empty",
1684d39a76e7Sxw KSTAT_DATA_UINT32);
1685d39a76e7Sxw kstat_named_init(&chkp->respQ_overflow, "respQ_overflow",
1686d39a76e7Sxw KSTAT_DATA_UINT32);
1687d39a76e7Sxw kstat_named_init(&chkp->freelistQ_empty, "freelistQ_empty",
1688d39a76e7Sxw KSTAT_DATA_UINT32);
1689d39a76e7Sxw kstat_named_init(&chkp->pkt_too_big, "pkt_too_big",
1690d39a76e7Sxw KSTAT_DATA_UINT32);
1691d39a76e7Sxw kstat_named_init(&chkp->pkt_mismatch, "pkt_mismatch",
1692d39a76e7Sxw KSTAT_DATA_UINT32);
1693d39a76e7Sxw kstat_named_init(&chkp->cmdQ_full[0], "cmdQ_full[0]",
1694d39a76e7Sxw KSTAT_DATA_UINT32);
1695d39a76e7Sxw kstat_named_init(&chkp->cmdQ_full[1], "cmdQ_full[1]",
1696d39a76e7Sxw KSTAT_DATA_UINT32);
1697d39a76e7Sxw kstat_named_init(&chkp->tx_reclaims[0], "tx_reclaims[0]",
1698d39a76e7Sxw KSTAT_DATA_UINT32);
1699d39a76e7Sxw kstat_named_init(&chkp->tx_reclaims[1], "tx_reclaims[1]",
1700d39a76e7Sxw KSTAT_DATA_UINT32);
1701d39a76e7Sxw kstat_named_init(&chkp->tx_msg_pullups, "tx_msg_pullups",
1702d39a76e7Sxw KSTAT_DATA_UINT32);
1703d39a76e7Sxw kstat_named_init(&chkp->tx_hdr_pullups, "tx_hdr_pullups",
1704d39a76e7Sxw KSTAT_DATA_UINT32);
1705d39a76e7Sxw kstat_named_init(&chkp->tx_tcp_ip_frag, "tx_tcp_ip_frag",
1706d39a76e7Sxw KSTAT_DATA_UINT32);
1707d39a76e7Sxw kstat_named_init(&chkp->tx_udp_ip_frag, "tx_udp_ip_frag",
1708d39a76e7Sxw KSTAT_DATA_UINT32);
1709d39a76e7Sxw kstat_named_init(&chkp->tx_soft_cksums, "tx_soft_cksums",
1710d39a76e7Sxw KSTAT_DATA_UINT32);
1711d39a76e7Sxw kstat_named_init(&chkp->tx_need_cpl_space, "tx_need_cpl_space",
1712d39a76e7Sxw KSTAT_DATA_UINT32);
1713d39a76e7Sxw kstat_named_init(&chkp->tx_multi_mblks, "tx_multi_mblks",
1714d39a76e7Sxw KSTAT_DATA_UINT32);
1715d39a76e7Sxw kstat_named_init(&chkp->tx_no_dvma1, "tx_num_multi_dvma_fails",
1716d39a76e7Sxw KSTAT_DATA_UINT32);
1717d39a76e7Sxw kstat_named_init(&chkp->tx_no_dvma2, "tx_num_single_dvma_fails",
1718d39a76e7Sxw KSTAT_DATA_UINT32);
1719d39a76e7Sxw kstat_named_init(&chkp->tx_no_dma1, "tx_num_multi_dma_fails",
1720d39a76e7Sxw KSTAT_DATA_UINT32);
1721d39a76e7Sxw kstat_named_init(&chkp->tx_no_dma2, "tx_num_single_dma_fails",
1722d39a76e7Sxw KSTAT_DATA_UINT32);
1723d39a76e7Sxw kstat_named_init(&chkp->rx_cmdq0, "rx_cmdq0",
1724d39a76e7Sxw KSTAT_DATA_UINT32);
1725d39a76e7Sxw kstat_named_init(&chkp->rx_cmdq1, "rx_cmdq1",
1726d39a76e7Sxw KSTAT_DATA_UINT32);
1727d39a76e7Sxw kstat_named_init(&chkp->rx_flq0, "rx_flq0",
1728d39a76e7Sxw KSTAT_DATA_UINT32);
1729d39a76e7Sxw kstat_named_init(&chkp->rx_flq1, "rx_flq1",
1730d39a76e7Sxw KSTAT_DATA_UINT32);
1731d39a76e7Sxw kstat_named_init(&chkp->rx_flq0_sz, "rx_flq0_buffer_sz",
1732d39a76e7Sxw KSTAT_DATA_UINT32);
1733d39a76e7Sxw kstat_named_init(&chkp->rx_flq1_sz, "rx_flq1_buffer_sz",
1734d39a76e7Sxw KSTAT_DATA_UINT32);
1735d39a76e7Sxw kstat_named_init(&chkp->rx_pkt_drops, "rx_pkt_drops",
1736d39a76e7Sxw KSTAT_DATA_UINT32);
1737d39a76e7Sxw kstat_named_init(&chkp->rx_pkt_copied, "rx_pkt_copied",
1738d39a76e7Sxw KSTAT_DATA_UINT32);
1739d39a76e7Sxw kstat_named_init(&chkp->rx_pause_on, "rx_pause_on",
1740d39a76e7Sxw KSTAT_DATA_UINT32);
1741d39a76e7Sxw kstat_named_init(&chkp->rx_pause_off, "rx_pause_off",
1742d39a76e7Sxw KSTAT_DATA_UINT32);
1743d39a76e7Sxw kstat_named_init(&chkp->rx_pause_ms, "rx_pause_ms",
1744d39a76e7Sxw KSTAT_DATA_UINT32);
1745d39a76e7Sxw kstat_named_init(&chkp->rx_pause_spike, "rx_pause_spike",
1746d39a76e7Sxw KSTAT_DATA_UINT32);
1747d39a76e7Sxw kstat_named_init(&chkp->rx_fl_credits, "rx_fl_credits",
1748d39a76e7Sxw KSTAT_DATA_UINT32);
1749d39a76e7Sxw kstat_named_init(&chkp->rx_flbuf_fails, "rx_flbuf_fails",
1750d39a76e7Sxw KSTAT_DATA_UINT32);
1751d39a76e7Sxw kstat_named_init(&chkp->rx_flbuf_allocs, "rx_flbuf_allocs",
1752d39a76e7Sxw KSTAT_DATA_UINT32);
1753d39a76e7Sxw kstat_named_init(&chkp->rx_badEopSop, "rx_badEopSop",
1754d39a76e7Sxw KSTAT_DATA_UINT32);
1755d39a76e7Sxw kstat_named_init(&chkp->rx_flq0_cnt, "rx_flq0_cnt",
1756d39a76e7Sxw KSTAT_DATA_UINT32);
1757d39a76e7Sxw kstat_named_init(&chkp->rx_flq1_cnt, "rx_flq1_cnt",
1758d39a76e7Sxw KSTAT_DATA_UINT32);
1759d39a76e7Sxw kstat_named_init(&chkp->arp_sent, "arp_sent",
1760d39a76e7Sxw KSTAT_DATA_UINT32);
1761d39a76e7Sxw kstat_named_init(&chkp->tx_doorbells, "tx_doorbells",
1762d39a76e7Sxw KSTAT_DATA_UINT32);
1763d39a76e7Sxw kstat_named_init(&chkp->intr_doorbells, "intr_doorbells",
1764d39a76e7Sxw KSTAT_DATA_UINT32);
1765d39a76e7Sxw kstat_named_init(&chkp->intr1_doorbells, "intr1_doorbells",
1766d39a76e7Sxw KSTAT_DATA_UINT32);
1767d39a76e7Sxw kstat_named_init(&chkp->sleep_cnt, "sleep_cnt",
1768d39a76e7Sxw KSTAT_DATA_UINT32);
1769d39a76e7Sxw kstat_named_init(&chkp->pe_allocb_cnt, "pe_allocb_cnt",
1770d39a76e7Sxw KSTAT_DATA_UINT32);
1771d39a76e7Sxw for (i = 0; i < MBLK_MAX; i++) {
1772d39a76e7Sxw (void) sprintf(kstat_name, "tx_descs[%02d]", i);
1773d39a76e7Sxw kstat_named_init(&chkp->tx_descs[i],
1774d39a76e7Sxw kstat_name, KSTAT_DATA_UINT32);
1775d39a76e7Sxw }
1776d39a76e7Sxw ksp->ks_update = sge_kstat_update;
1777d39a76e7Sxw ksp->ks_private = (void *)sge;
1778d39a76e7Sxw sge->ksp = ksp;
1779d39a76e7Sxw kstat_install(ksp);
1780d39a76e7Sxw status = 0;
1781d39a76e7Sxw
1782d39a76e7Sxw sge_kstat_setup_exit:
1783d39a76e7Sxw return (status);
1784d39a76e7Sxw }
1785d39a76e7Sxw
1786d39a76e7Sxw static void
sge_kstat_remove(pesge * sge)1787d39a76e7Sxw sge_kstat_remove(pesge *sge)
1788d39a76e7Sxw {
1789d39a76e7Sxw if (sge->ksp)
1790d39a76e7Sxw kstat_delete(sge->ksp);
1791d39a76e7Sxw }
1792d39a76e7Sxw
1793d39a76e7Sxw static int
sge_kstat_update(p_kstat_t ksp,int rw)1794d39a76e7Sxw sge_kstat_update(p_kstat_t ksp, int rw)
1795d39a76e7Sxw {
1796d39a76e7Sxw pesge *sge;
1797d39a76e7Sxw p_ch_stats_t statsp;
1798d39a76e7Sxw p_ch_kstat_t chkp;
1799d39a76e7Sxw int i;
1800d39a76e7Sxw
1801d39a76e7Sxw sge = (pesge *)ksp->ks_private;
1802d39a76e7Sxw statsp = (p_ch_stats_t)&sge->intr_cnt;
1803d39a76e7Sxw chkp = (p_ch_kstat_t)ksp->ks_data;
1804d39a76e7Sxw if (rw == KSTAT_WRITE) {
1805d39a76e7Sxw statsp->respQ_empty = chkp->respQ_empty.value.ui32;
1806d39a76e7Sxw statsp->respQ_overflow = chkp->respQ_overflow.value.ui32;
1807d39a76e7Sxw statsp->freelistQ_empty = chkp->freelistQ_empty.value.ui32;
1808d39a76e7Sxw statsp->pkt_too_big = chkp->pkt_too_big.value.ui32;
1809d39a76e7Sxw statsp->pkt_mismatch = chkp->pkt_mismatch.value.ui32;
1810d39a76e7Sxw statsp->cmdQ_full[0] = chkp->cmdQ_full[0].value.ui32;
1811d39a76e7Sxw statsp->cmdQ_full[1] = chkp->cmdQ_full[1].value.ui32;
1812d39a76e7Sxw statsp->tx_reclaims[0] = chkp->tx_reclaims[0].value.ui32;
1813d39a76e7Sxw statsp->tx_reclaims[1] = chkp->tx_reclaims[1].value.ui32;
1814d39a76e7Sxw statsp->tx_msg_pullups = chkp->tx_msg_pullups.value.ui32;
1815d39a76e7Sxw statsp->tx_hdr_pullups = chkp->tx_hdr_pullups.value.ui32;
1816d39a76e7Sxw statsp->tx_tcp_ip_frag = chkp->tx_tcp_ip_frag.value.ui32;
1817d39a76e7Sxw statsp->tx_udp_ip_frag = chkp->tx_udp_ip_frag.value.ui32;
1818d39a76e7Sxw statsp->tx_soft_cksums = chkp->tx_soft_cksums.value.ui32;
1819d39a76e7Sxw statsp->tx_need_cpl_space
1820d39a76e7Sxw = chkp->tx_need_cpl_space.value.ui32;
1821d39a76e7Sxw statsp->tx_multi_mblks = chkp->tx_multi_mblks.value.ui32;
1822d39a76e7Sxw statsp->tx_no_dvma1 = chkp->tx_no_dvma1.value.ui32;
1823d39a76e7Sxw statsp->tx_no_dvma2 = chkp->tx_no_dvma2.value.ui32;
1824d39a76e7Sxw statsp->tx_no_dma1 = chkp->tx_no_dma1.value.ui32;
1825d39a76e7Sxw statsp->tx_no_dma2 = chkp->tx_no_dma2.value.ui32;
1826d39a76e7Sxw statsp->rx_cmdq0 = chkp->rx_cmdq0.value.ui32;
1827d39a76e7Sxw statsp->rx_cmdq1 = chkp->rx_cmdq1.value.ui32;
1828d39a76e7Sxw statsp->rx_flq0 = chkp->rx_flq0.value.ui32;
1829d39a76e7Sxw statsp->rx_flq1 = chkp->rx_flq1.value.ui32;
1830d39a76e7Sxw statsp->rx_flq0_sz = chkp->rx_flq0_sz.value.ui32;
1831d39a76e7Sxw statsp->rx_flq1_sz = chkp->rx_flq1_sz.value.ui32;
1832d39a76e7Sxw statsp->rx_pkt_drops = chkp->rx_pkt_drops.value.ui32;
1833d39a76e7Sxw statsp->rx_pkt_copied = chkp->rx_pkt_copied.value.ui32;
1834d39a76e7Sxw statsp->rx_pause_on = chkp->rx_pause_on.value.ui32;
1835d39a76e7Sxw statsp->rx_pause_off = chkp->rx_pause_off.value.ui32;
1836d39a76e7Sxw statsp->rx_pause_ms = chkp->rx_pause_ms.value.ui32;
1837d39a76e7Sxw statsp->rx_pause_spike = chkp->rx_pause_spike.value.ui32;
1838d39a76e7Sxw statsp->rx_fl_credits = chkp->rx_fl_credits.value.ui32;
1839d39a76e7Sxw statsp->rx_flbuf_fails = chkp->rx_flbuf_fails.value.ui32;
1840d39a76e7Sxw statsp->rx_flbuf_allocs = chkp->rx_flbuf_allocs.value.ui32;
1841d39a76e7Sxw statsp->rx_badEopSop = chkp->rx_badEopSop.value.ui32;
1842d39a76e7Sxw statsp->rx_flq0_cnt = chkp->rx_flq0_cnt.value.ui32;
1843d39a76e7Sxw statsp->rx_flq1_cnt = chkp->rx_flq1_cnt.value.ui32;
1844d39a76e7Sxw statsp->arp_sent = chkp->arp_sent.value.ui32;
1845d39a76e7Sxw statsp->tx_doorbells = chkp->tx_doorbells.value.ui32;
1846d39a76e7Sxw statsp->intr_doorbells = chkp->intr_doorbells.value.ui32;
1847d39a76e7Sxw statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32;
1848d39a76e7Sxw statsp->sleep_cnt = chkp->sleep_cnt.value.ui32;
1849d39a76e7Sxw statsp->pe_allocb_cnt = chkp->pe_allocb_cnt.value.ui32;
1850d39a76e7Sxw for (i = 0; i < MBLK_MAX; i++) {
1851d39a76e7Sxw statsp->tx_descs[i] = chkp->tx_descs[i].value.ui32;
1852d39a76e7Sxw }
1853d39a76e7Sxw } else {
1854d39a76e7Sxw chkp->respQ_empty.value.ui32 = statsp->respQ_empty;
1855d39a76e7Sxw chkp->respQ_overflow.value.ui32 = statsp->respQ_overflow;
1856d39a76e7Sxw chkp->freelistQ_empty.value.ui32
1857d39a76e7Sxw = statsp->freelistQ_empty;
1858d39a76e7Sxw chkp->pkt_too_big.value.ui32 = statsp->pkt_too_big;
1859d39a76e7Sxw chkp->pkt_mismatch.value.ui32 = statsp->pkt_mismatch;
1860d39a76e7Sxw chkp->cmdQ_full[0].value.ui32 = statsp->cmdQ_full[0];
1861d39a76e7Sxw chkp->cmdQ_full[1].value.ui32 = statsp->cmdQ_full[1];
1862d39a76e7Sxw chkp->tx_reclaims[0].value.ui32 = statsp->tx_reclaims[0];
1863d39a76e7Sxw chkp->tx_reclaims[1].value.ui32 = statsp->tx_reclaims[1];
1864d39a76e7Sxw chkp->tx_msg_pullups.value.ui32 = statsp->tx_msg_pullups;
1865d39a76e7Sxw chkp->tx_hdr_pullups.value.ui32 = statsp->tx_hdr_pullups;
1866d39a76e7Sxw chkp->tx_tcp_ip_frag.value.ui32 = statsp->tx_tcp_ip_frag;
1867d39a76e7Sxw chkp->tx_udp_ip_frag.value.ui32 = statsp->tx_udp_ip_frag;
1868d39a76e7Sxw chkp->tx_soft_cksums.value.ui32 = statsp->tx_soft_cksums;
1869d39a76e7Sxw chkp->tx_need_cpl_space.value.ui32
1870d39a76e7Sxw = statsp->tx_need_cpl_space;
1871d39a76e7Sxw chkp->tx_multi_mblks.value.ui32 = statsp->tx_multi_mblks;
1872d39a76e7Sxw chkp->tx_no_dvma1.value.ui32 = statsp->tx_no_dvma1;
1873d39a76e7Sxw chkp->tx_no_dvma2.value.ui32 = statsp->tx_no_dvma2;
1874d39a76e7Sxw chkp->tx_no_dma1.value.ui32 = statsp->tx_no_dma1;
1875d39a76e7Sxw chkp->tx_no_dma2.value.ui32 = statsp->tx_no_dma2;
1876d39a76e7Sxw chkp->rx_cmdq0.value.ui32 = statsp->rx_cmdq0;
1877d39a76e7Sxw chkp->rx_cmdq1.value.ui32 = statsp->rx_cmdq1;
1878d39a76e7Sxw chkp->rx_flq0.value.ui32 = statsp->rx_flq0;
1879d39a76e7Sxw chkp->rx_flq1.value.ui32 = statsp->rx_flq1;
1880d39a76e7Sxw chkp->rx_flq0_sz.value.ui32 = statsp->rx_flq0_sz;
1881d39a76e7Sxw chkp->rx_flq1_sz.value.ui32 = statsp->rx_flq1_sz;
1882d39a76e7Sxw chkp->rx_pkt_drops.value.ui32 = statsp->rx_pkt_drops;
1883d39a76e7Sxw chkp->rx_pkt_copied.value.ui32 = statsp->rx_pkt_copied;
1884d39a76e7Sxw chkp->rx_pause_on.value.ui32 = statsp->rx_pause_on;
1885d39a76e7Sxw chkp->rx_pause_off.value.ui32 = statsp->rx_pause_off;
1886d39a76e7Sxw chkp->rx_pause_ms.value.ui32 = statsp->rx_pause_ms;
1887d39a76e7Sxw chkp->rx_pause_spike.value.ui32 = statsp->rx_pause_spike;
1888d39a76e7Sxw chkp->rx_fl_credits.value.ui32 = statsp->rx_fl_credits;
1889d39a76e7Sxw chkp->rx_flbuf_fails.value.ui32
1890d39a76e7Sxw = statsp->rx_flbuf_fails;
1891d39a76e7Sxw chkp->rx_flbuf_allocs.value.ui32
1892d39a76e7Sxw = statsp->rx_flbuf_allocs;
1893d39a76e7Sxw chkp->rx_badEopSop.value.ui32 = statsp->rx_badEopSop;
1894d39a76e7Sxw chkp->rx_flq0_cnt.value.ui32 = statsp->rx_flq0_cnt;
1895d39a76e7Sxw chkp->rx_flq1_cnt.value.ui32 = statsp->rx_flq1_cnt;
1896d39a76e7Sxw chkp->arp_sent.value.ui32 = statsp->arp_sent;
1897d39a76e7Sxw chkp->tx_doorbells.value.ui32 = statsp->tx_doorbells;
1898d39a76e7Sxw chkp->intr_doorbells.value.ui32 = statsp->intr_doorbells;
1899d39a76e7Sxw chkp->intr1_doorbells.value.ui32
1900d39a76e7Sxw = statsp->intr1_doorbells;
1901d39a76e7Sxw chkp->sleep_cnt.value.ui32 = statsp->sleep_cnt;
1902d39a76e7Sxw chkp->pe_allocb_cnt.value.ui32 = statsp->pe_allocb_cnt;
1903d39a76e7Sxw for (i = 0; i < MBLK_MAX; i++) {
1904d39a76e7Sxw chkp->tx_descs[i].value.ui32 = statsp->tx_descs[i];
1905d39a76e7Sxw }
1906d39a76e7Sxw }
1907d39a76e7Sxw return (0);
1908d39a76e7Sxw }
1909d39a76e7Sxw #endif
1910d39a76e7Sxw
1911d39a76e7Sxw static uint16_t
calc_ocsum(mblk_t * mp,int offset)1912d39a76e7Sxw calc_ocsum(mblk_t *mp, int offset)
1913d39a76e7Sxw {
1914d39a76e7Sxw uint8_t *addrp;
1915d39a76e7Sxw uint32_t src;
1916d39a76e7Sxw uint32_t dst;
1917d39a76e7Sxw
1918d39a76e7Sxw ipha_t *ihdr = (ipha_t *)(mp->b_rptr + offset);
1919d39a76e7Sxw uint32_t sum;
1920d39a76e7Sxw int iplen = IPH_HDR_LENGTH(ihdr);
1921d39a76e7Sxw struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen);
1922d39a76e7Sxw uchar_t *byte;
1923d39a76e7Sxw int len;
1924d39a76e7Sxw
1925d39a76e7Sxw addrp = (uint8_t *)&ihdr->ipha_src;
1926d39a76e7Sxw src = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1927d39a76e7Sxw ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1928d39a76e7Sxw
1929d39a76e7Sxw addrp = (uint8_t *)&ihdr->ipha_dst;
1930d39a76e7Sxw dst = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1931d39a76e7Sxw ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1932d39a76e7Sxw
1933d39a76e7Sxw sum = (uint16_t)(src >> 16) +
1934d39a76e7Sxw (uint16_t)(src) +
1935d39a76e7Sxw (uint16_t)(dst >> 16) +
1936d39a76e7Sxw (uint16_t)(dst) + (udpp->uh_ulen + htons(IPPROTO_UDP));
1937d39a76e7Sxw
1938d39a76e7Sxw sum = (uint16_t)(sum >> 16) + (uint16_t)(sum);
1939d39a76e7Sxw
1940d39a76e7Sxw if (sum > 0xffff)
1941d39a76e7Sxw sum -= 0xffff;
1942d39a76e7Sxw
1943d39a76e7Sxw udpp->uh_sum = 0;
1944d39a76e7Sxw byte = mp->b_rptr + offset + iplen;
1945d39a76e7Sxw do {
1946d39a76e7Sxw len = (mp->b_wptr - byte);
1947d39a76e7Sxw sum = bcksum(byte, len, sum);
1948d39a76e7Sxw if (sum > 0xffff)
1949d39a76e7Sxw sum -= 0xffff;
1950d39a76e7Sxw mp = mp->b_cont;
1951d39a76e7Sxw if (mp)
1952d39a76e7Sxw byte = mp->b_rptr;
1953d39a76e7Sxw } while (mp);
1954d39a76e7Sxw
1955d39a76e7Sxw sum = ~sum & 0xffff;
1956d39a76e7Sxw
1957d39a76e7Sxw return (sum);
1958d39a76e7Sxw }
1959