1d39a76e7Sxw /* 2d39a76e7Sxw * CDDL HEADER START 3d39a76e7Sxw * 4d39a76e7Sxw * The contents of this file are subject to the terms of the 5d39a76e7Sxw * Common Development and Distribution License (the "License"). 6d39a76e7Sxw * You may not use this file except in compliance with the License. 7d39a76e7Sxw * 8d39a76e7Sxw * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9d39a76e7Sxw * or http://www.opensolaris.org/os/licensing. 10d39a76e7Sxw * See the License for the specific language governing permissions 11d39a76e7Sxw * and limitations under the License. 12d39a76e7Sxw * 13d39a76e7Sxw * When distributing Covered Code, include this CDDL HEADER in each 14d39a76e7Sxw * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15d39a76e7Sxw * If applicable, add the following below this CDDL HEADER, with the 16d39a76e7Sxw * fields enclosed by brackets "[]" replaced with your own identifying 17d39a76e7Sxw * information: Portions Copyright [yyyy] [name of copyright owner] 18d39a76e7Sxw * 19d39a76e7Sxw * CDDL HEADER END 20d39a76e7Sxw */ 21d39a76e7Sxw 22d39a76e7Sxw /* 23d39a76e7Sxw * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24d39a76e7Sxw * Use is subject to license terms. 25d39a76e7Sxw */ 26d39a76e7Sxw 27d39a76e7Sxw /* 28d39a76e7Sxw * This file is part of the Chelsio T1 Ethernet driver. 29d39a76e7Sxw * 30d39a76e7Sxw * Copyright (C) 2003-2005 Chelsio Communications. All rights reserved. 31d39a76e7Sxw */ 32d39a76e7Sxw 33d39a76e7Sxw #include <sys/types.h> 34d39a76e7Sxw #include <sys/param.h> 35d39a76e7Sxw #include <sys/cmn_err.h> 36d39a76e7Sxw #include <sys/sunddi.h> 37d39a76e7Sxw #include <sys/kmem.h> 38d39a76e7Sxw #include <sys/cmn_err.h> 39d39a76e7Sxw #include <sys/byteorder.h> 40d39a76e7Sxw #include <sys/atomic.h> 41d39a76e7Sxw #include <sys/stropts.h> 42d39a76e7Sxw #include <sys/stream.h> 43d39a76e7Sxw #include <sys/strsubr.h> 44d39a76e7Sxw #include <sys/dlpi.h> 45d39a76e7Sxw #include <sys/kstat.h> 46d39a76e7Sxw #include <sys/ethernet.h> 47d39a76e7Sxw #include <netinet/in.h> 48d39a76e7Sxw #include <netinet/udp.h> 49d39a76e7Sxw #include <inet/common.h> 50d39a76e7Sxw #include <inet/nd.h> 51d39a76e7Sxw #include <inet/ip.h> 52d39a76e7Sxw #include <inet/tcp.h> 53d39a76e7Sxw #include <netinet/udp.h> 54d39a76e7Sxw #include <sys/gld.h> 55d39a76e7Sxw #include "ostypes.h" 56d39a76e7Sxw #include "common.h" 57d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_1G 58d39a76e7Sxw #include "fpga_defs.h" 59d39a76e7Sxw #endif 60d39a76e7Sxw #include "regs.h" 61d39a76e7Sxw #include "suni1x10gexp_regs.h" 62d39a76e7Sxw #include "sge.h" 63d39a76e7Sxw #include "espi.h" 64d39a76e7Sxw 65d39a76e7Sxw #include "ch.h" 66d39a76e7Sxw 67d39a76e7Sxw extern uint32_t buffers_in_use[]; 68d39a76e7Sxw 69d39a76e7Sxw uint32_t sge_cmdq0_cnt = SGE_CMDQ0_E_N; 70d39a76e7Sxw uint32_t sge_cmdq1_cnt = SGE_CMDQ1_E_N; 71d39a76e7Sxw uint32_t sge_flq0_cnt = SGE_FREELQ0_E_N; 72d39a76e7Sxw uint32_t sge_flq1_cnt = SGE_FREELQ1_E_N; 73d39a76e7Sxw uint32_t sge_respq_cnt = SGE_RESPQ_E_N; 74d39a76e7Sxw 75d39a76e7Sxw uint32_t sge_cmdq0_cnt_orig = SGE_CMDQ0_E_N; 76d39a76e7Sxw uint32_t sge_cmdq1_cnt_orig = SGE_CMDQ1_E_N; 77d39a76e7Sxw uint32_t sge_flq0_cnt_orig = SGE_FREELQ0_E_N; 78d39a76e7Sxw uint32_t sge_flq1_cnt_orig = SGE_FREELQ1_E_N; 79d39a76e7Sxw uint32_t sge_respq_cnt_orig = SGE_RESPQ_E_N; 80d39a76e7Sxw 81d39a76e7Sxw #ifdef HOST_PAUSE 82d39a76e7Sxw uint32_t do_host_pause = 1; 83d39a76e7Sxw uint32_t flq_pause_window = 64; 84d39a76e7Sxw #endif 85d39a76e7Sxw 86d39a76e7Sxw static uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, 87d39a76e7Sxw ulong_t *dh); 88d39a76e7Sxw void pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t); 89d39a76e7Sxw 90d39a76e7Sxw static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q, 91d39a76e7Sxw unsigned int len, unsigned int offload); 92d39a76e7Sxw #ifdef HOST_PAUSE 93d39a76e7Sxw static void t1_sge_check_pause(pesge *sge, struct freelQ *Q); 94d39a76e7Sxw #endif 95d39a76e7Sxw static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q); 96d39a76e7Sxw static void freelQs_empty(pesge *sge); 97d39a76e7Sxw static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend); 98d39a76e7Sxw static int alloc_rx_resources(pesge *sge, struct sge_params *p); 99d39a76e7Sxw static int alloc_tx_resources(pesge *sge, struct sge_params *p); 100d39a76e7Sxw static inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size, 101d39a76e7Sxw int base_reg_lo, int base_reg_hi, int size_reg); 102d39a76e7Sxw static void configure_sge(pesge *sge, struct sge_params *p); 103d39a76e7Sxw static void free_freelQ_buffers(pesge *sge, struct freelQ *Q); 104d39a76e7Sxw static void free_rx_resources(pesge *sge); 105d39a76e7Sxw static void free_tx_resources(pesge *sge); 106d39a76e7Sxw static inline unsigned int jumbo_payload_capacity(pesge *sge); 107d39a76e7Sxw #ifdef SUN_KSTATS 108d39a76e7Sxw static int sge_kstat_setup(pesge *); 109d39a76e7Sxw static void sge_kstat_remove(pesge *); 110d39a76e7Sxw static int sge_kstat_update(p_kstat_t, int); 111d39a76e7Sxw #endif 112d39a76e7Sxw static uint16_t calc_ocsum(mblk_t *, int); 113d39a76e7Sxw 114d39a76e7Sxw /* 115d39a76e7Sxw * Local routines. 116d39a76e7Sxw */ 117d39a76e7Sxw static inline void sge_ring_doorbell(pesge *sge, u32 control_reg); 118d39a76e7Sxw 119d39a76e7Sxw static inline void 120d39a76e7Sxw sge_ring_doorbell(pesge *sge, u32 control_reg) 121d39a76e7Sxw { 122d39a76e7Sxw membar_producer(); 123d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_DOORBELL, control_reg); 124d39a76e7Sxw } 125d39a76e7Sxw 126d39a76e7Sxw /* 127d39a76e7Sxw * DESC: 128d39a76e7Sxw * 129d39a76e7Sxw * NOTES: Must have at least 1 command queue and 1 freelist queue. 130d39a76e7Sxw * 131d39a76e7Sxw */ 132d39a76e7Sxw pesge * 133d39a76e7Sxw t1_sge_create(ch_t *sa, struct sge_params *p) 134d39a76e7Sxw { 135d39a76e7Sxw pesge *sge; 136d39a76e7Sxw 137d39a76e7Sxw sge = t1_os_malloc_wait_zero(sizeof (pesge)); 138d39a76e7Sxw 139d39a76e7Sxw if (sge == NULL) 140d39a76e7Sxw goto error_no_mem; 141d39a76e7Sxw 142d39a76e7Sxw memset(sge, 0, sizeof (*sge)); 143d39a76e7Sxw 144d39a76e7Sxw /* 145d39a76e7Sxw * PR2928 & PR3309 146d39a76e7Sxw * set default timeout value - 20 msec 147d39a76e7Sxw * we set the initial value to 2 which gurantees at least one tick. 148d39a76e7Sxw */ 149d39a76e7Sxw if (is_T2(sa)) 150d39a76e7Sxw sge->ptimeout = 1; 151d39a76e7Sxw 152d39a76e7Sxw sge->obj = sa; 153d39a76e7Sxw #ifdef SUN_KSTATS 154d39a76e7Sxw if (sge_kstat_setup(sge) != 0) 155d39a76e7Sxw goto t1_sge_create_fail1; 156d39a76e7Sxw #endif 157d39a76e7Sxw p->cmdQ_size[0] = sge_cmdq0_cnt; 158d39a76e7Sxw p->cmdQ_size[1] = sge_cmdq1_cnt; 159d39a76e7Sxw 160d39a76e7Sxw /* note that jumbo frame index is inverted for T2 */ 161d39a76e7Sxw if (is_T2(sa)) { 162d39a76e7Sxw p->freelQ_size[1] = sge_flq0_cnt; 163d39a76e7Sxw p->freelQ_size[0] = sge_flq1_cnt; 164d39a76e7Sxw } else { 165d39a76e7Sxw p->freelQ_size[0] = sge_flq0_cnt; 166d39a76e7Sxw p->freelQ_size[1] = sge_flq1_cnt; 167d39a76e7Sxw } 168d39a76e7Sxw 169d39a76e7Sxw #if CH_DEBUG 170d39a76e7Sxw /* DEBUG only */ 171d39a76e7Sxw cmn_err(CE_NOTE, "sge: %p\n", sge); 172d39a76e7Sxw cmn_err(CE_NOTE, "&sge->cmdQ[0]: %p\n", &sge->cmdQ[0]); 173d39a76e7Sxw cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]); 174d39a76e7Sxw cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]); 175d39a76e7Sxw cmn_err(CE_NOTE, "&sge->respQ: %p\n", &sge->respQ); 176d39a76e7Sxw cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt); 177d39a76e7Sxw #endif 178d39a76e7Sxw #ifdef SUN_KSTATS 179d39a76e7Sxw goto error_no_mem; 180d39a76e7Sxw 181d39a76e7Sxw t1_sge_create_fail1: 182d39a76e7Sxw t1_os_free(sge, sizeof (pesge)); 183d39a76e7Sxw sge = NULL; 184d39a76e7Sxw #endif 185d39a76e7Sxw error_no_mem: 186d39a76e7Sxw return (sge); 187d39a76e7Sxw } 188d39a76e7Sxw 189d39a76e7Sxw int 190d39a76e7Sxw t1_sge_destroy(pesge* sge) 191d39a76e7Sxw { 192d39a76e7Sxw if (sge != NULL) { 193d39a76e7Sxw free_tx_resources(sge); 194d39a76e7Sxw free_rx_resources(sge); 195d39a76e7Sxw 196d39a76e7Sxw /* PR2928 & PR3309 */ 197d39a76e7Sxw if ((is_T2(sge->obj)) && (sge->pskb)) 198d39a76e7Sxw pe_free_fake_arp(sge->pskb); 199d39a76e7Sxw #ifdef SUN_KSTATS 200d39a76e7Sxw sge_kstat_remove(sge); 201d39a76e7Sxw #endif 202d39a76e7Sxw t1_os_free(sge, sizeof (pesge)); 203d39a76e7Sxw } 204d39a76e7Sxw return (0); 205d39a76e7Sxw } 206d39a76e7Sxw 207d39a76e7Sxw /* 208d39a76e7Sxw * PR2928 & PR3309 209d39a76e7Sxw * call out event from timeout 210d39a76e7Sxw * 211d39a76e7Sxw * there is a potential race between the timeout and the close. 212d39a76e7Sxw * unless we protect the timeout, the close could occur at the 213d39a76e7Sxw * same time. Then if the timeout service routine was slow or 214d39a76e7Sxw * interrupted, the sge_stop() could complete with a timeoutID 215d39a76e7Sxw * that has expired, thus letting another timeout occur. If the 216d39a76e7Sxw * service routine was delayed still further, a detach could occur. 217d39a76e7Sxw * the second time could then end up accessing memory that has been 218d39a76e7Sxw * released back to the system. Bad things could then occur. We 219d39a76e7Sxw * set a flag in sge_stop() to tell the service routine not to 220d39a76e7Sxw * issue further timeouts. sge_stop() will block until a timeout 221d39a76e7Sxw * has occured. If the command Q is full then we shouldn't put out 222d39a76e7Sxw * an arp. 223d39a76e7Sxw */ 224d39a76e7Sxw 225d39a76e7Sxw void 226d39a76e7Sxw t1_espi_workaround(ch_t *adapter) 227d39a76e7Sxw { 228d39a76e7Sxw pesge *sge = adapter->sge; 229d39a76e7Sxw ch_t *chp = (ch_t *)sge->obj; 230d39a76e7Sxw int rv = 1; 231d39a76e7Sxw 232d39a76e7Sxw if ((chp->ch_state == PERUNNING) && 233d39a76e7Sxw atomic_read(&sge->cmdQ[0].cq_asleep)) { 234d39a76e7Sxw u32 seop; 235d39a76e7Sxw seop = t1_espi_get_mon(adapter, 0x930, 0); 236d39a76e7Sxw if ((seop & 0xfff0fff) == 0xfff) { 237d39a76e7Sxw /* after first arp */ 238*f172c8abSToomas Soome if (sge->pskb) { 239d39a76e7Sxw rv = pe_start(adapter, (mblk_t *)sge->pskb, 240d39a76e7Sxw CH_ARP); 241d39a76e7Sxw if (!rv) 242d39a76e7Sxw sge->intr_cnt.arp_sent++; 243*f172c8abSToomas Soome } 244d39a76e7Sxw } 245d39a76e7Sxw } 246d39a76e7Sxw #ifdef HOST_PAUSE 247d39a76e7Sxw /* 248d39a76e7Sxw * If we are already in sge_data_in, then we can skip calling 249d39a76e7Sxw * t1_sge_check_pause() this clock cycle. lockstat showed that 250d39a76e7Sxw * we were blocking on the mutex ~ 2% of the time. 251d39a76e7Sxw */ 252d39a76e7Sxw if (mutex_tryenter(&adapter->ch_intr)) { 253d39a76e7Sxw t1_sge_check_pause(sge, &sge->freelQ[0]); 254d39a76e7Sxw t1_sge_check_pause(sge, &sge->freelQ[1]); 255d39a76e7Sxw mutex_exit(&adapter->ch_intr); 256d39a76e7Sxw } 257d39a76e7Sxw #endif 258d39a76e7Sxw } 259d39a76e7Sxw 260d39a76e7Sxw int 261d39a76e7Sxw sge_start(pesge *sge) 262d39a76e7Sxw { 263d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control); 264d39a76e7Sxw /* PR2928 & PR3309, also need to avoid Pause deadlock */ 265d39a76e7Sxw ch_init_cyclic(sge->obj, &sge->espi_wa_cyclic, 266d39a76e7Sxw (void (*)(void *))t1_espi_workaround, sge->obj); 267d39a76e7Sxw ch_start_cyclic(&sge->espi_wa_cyclic, sge->ptimeout); 268d39a76e7Sxw return (0); 269d39a76e7Sxw } 270d39a76e7Sxw 271d39a76e7Sxw /* 272d39a76e7Sxw * Disables SGE queues. 273d39a76e7Sxw */ 274d39a76e7Sxw int 275d39a76e7Sxw sge_stop(pesge *sge) 276d39a76e7Sxw { 277d39a76e7Sxw uint32_t status; 278d39a76e7Sxw int loops; 279d39a76e7Sxw 280d39a76e7Sxw DBGASSERT(sge); 281d39a76e7Sxw 282d39a76e7Sxw /* PR2928 & PR3309, also need to avoid Pause deadlock */ 283d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_CONTROL, 0x0); 284d39a76e7Sxw 285d39a76e7Sxw /* wait until there's no more outstanding interrupts pending */ 286d39a76e7Sxw loops = 0; 287d39a76e7Sxw do { 288d39a76e7Sxw status = t1_read_reg_4(sge->obj, A_SG_INT_CAUSE); 289d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, status); 290d39a76e7Sxw drv_usecwait(125); 291d39a76e7Sxw loops++; 292d39a76e7Sxw } while (status && (loops < 1000)); 293d39a76e7Sxw 294d39a76e7Sxw ch_stop_cyclic(&sge->espi_wa_cyclic); 295d39a76e7Sxw 296d39a76e7Sxw return (0); 297d39a76e7Sxw } 298d39a76e7Sxw 299d39a76e7Sxw uint32_t sge_cmdq_send_fail; 300d39a76e7Sxw 301d39a76e7Sxw int 302d39a76e7Sxw sge_data_out(pesge* sge, int qid, mblk_t *m0, 303*f172c8abSToomas Soome cmdQ_ce_t *cmp, int count, uint32_t flg) 304d39a76e7Sxw { 305d39a76e7Sxw struct cmdQ *Q = &sge->cmdQ[qid]; 306d39a76e7Sxw ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh; 307d39a76e7Sxw spinlock_t *qlock = &Q->cq_qlock; 308d39a76e7Sxw cmdQ_e *e; 309d39a76e7Sxw cmdQ_e *q = Q->cq_entries; 310d39a76e7Sxw uint32_t credits; 311d39a76e7Sxw uint32_t pidx; 312d39a76e7Sxw uint32_t genbit; 313d39a76e7Sxw uint32_t entries_n = Q->cq_entries_n; 314d39a76e7Sxw cmdQ_ce_t *ce; 315d39a76e7Sxw cmdQ_ce_t *cq = Q->cq_centries; 316d39a76e7Sxw dma_addr_t mapping; 317d39a76e7Sxw uint32_t j = 0; 318d39a76e7Sxw uint32_t offset; 319d39a76e7Sxw #if defined(TX_CKSUM_FIX) 320d39a76e7Sxw uint16_t csum; 321d39a76e7Sxw uint16_t *csum_loc; 322d39a76e7Sxw #endif 323d39a76e7Sxw #ifdef TX_THREAD_RECLAIM 324d39a76e7Sxw uint32_t reclaim_cnt; 325d39a76e7Sxw #endif 326d39a76e7Sxw 327d39a76e7Sxw /* 328d39a76e7Sxw * We must exit if we don't have enough free command queue entries 329d39a76e7Sxw * available. 330d39a76e7Sxw */ 331d39a76e7Sxw 332d39a76e7Sxw spin_lock(qlock); 333d39a76e7Sxw 334d39a76e7Sxw #if defined(TX_CKSUM_FIX) 335d39a76e7Sxw /* 336d39a76e7Sxw * This checksum fix will address a fragmented datagram 337d39a76e7Sxw * checksum error. Which will lead to the next packet after 338d39a76e7Sxw * the last packet with the More fragment bit set having its 339d39a76e7Sxw * checksum corrupted. When the packet reaches this point 340d39a76e7Sxw * the 'flg' variable indicates whether a checksum is needed 341d39a76e7Sxw * or not. The algorithm is as follows, if the current packet 342d39a76e7Sxw * is a More fragment set the count of packets to be checksummed 343d39a76e7Sxw * after it to 3. If it't not and the count of is more than 0 344d39a76e7Sxw * then calculate the checksum in software, if a hardware checksum 345d39a76e7Sxw * was requested. Then decrment the count. Same algorithm applies 346d39a76e7Sxw * to TCP. 347d39a76e7Sxw */ 348d39a76e7Sxw if (flg & CH_UDP_MF) { 349d39a76e7Sxw sge->do_udp_csum = 3; 350d39a76e7Sxw } else if ((flg & CH_UDP) && (sge->do_udp_csum != 0)) { 351d39a76e7Sxw if ((flg & CH_NO_HWCKSUM) == 0) { 352d39a76e7Sxw /* 353d39a76e7Sxw * Calc Checksum here. 354d39a76e7Sxw */ 355d39a76e7Sxw csum = calc_ocsum(m0, 356d39a76e7Sxw sizeof (struct ether_header) + CPL_FORMAT_0_SIZE); 357d39a76e7Sxw csum_loc = (uint16_t *)(m0->b_rptr + 358d39a76e7Sxw sizeof (struct ether_header) + CPL_FORMAT_0_SIZE); 359d39a76e7Sxw csum_loc += (((*(char *)csum_loc) & 0x0f) << 1); 360d39a76e7Sxw 361d39a76e7Sxw sge->intr_cnt.tx_soft_cksums++; 362d39a76e7Sxw ((struct udphdr *)(csum_loc))->uh_sum = csum; 363d39a76e7Sxw ((struct cpl_tx_pkt *)m0->b_rptr)->l4_csum_dis = 1; 364d39a76e7Sxw } 365d39a76e7Sxw sge->do_udp_csum--; 366d39a76e7Sxw } else if (flg & CH_TCP_MF) { 367d39a76e7Sxw sge->do_tcp_csum = 3; 368d39a76e7Sxw } else if (sge->do_tcp_csum != 0) { 369d39a76e7Sxw if ((flg & CH_NO_HWCKSUM) == 0) { 370d39a76e7Sxw sge->intr_cnt.tx_soft_cksums++; 371d39a76e7Sxw /* 372d39a76e7Sxw * Calc Checksum here. 373d39a76e7Sxw */ 374d39a76e7Sxw } 375d39a76e7Sxw sge->do_tcp_csum--; 376d39a76e7Sxw } 377d39a76e7Sxw #endif /* TX_CKSUM_FIX */ 378d39a76e7Sxw #ifdef TX_THREAD_RECLAIM 379d39a76e7Sxw reclaim_cnt = Q->cq_complete; 380d39a76e7Sxw if (reclaim_cnt > SGE_BATCH_THRESH) { 381d39a76e7Sxw sge->intr_cnt.tx_reclaims[qid]++; 382d39a76e7Sxw free_cmdQ_buffers(sge, Q, reclaim_cnt); 383d39a76e7Sxw Q->cq_complete = 0; 384d39a76e7Sxw } 385d39a76e7Sxw #endif 386d39a76e7Sxw genbit = Q->cq_genbit; 387d39a76e7Sxw pidx = Q->cq_pidx; 388d39a76e7Sxw credits = Q->cq_credits; 389d39a76e7Sxw 390d39a76e7Sxw if ((credits - 1) < count) { 391d39a76e7Sxw spin_unlock(qlock); 392d39a76e7Sxw sge->intr_cnt.cmdQ_full[qid]++; 393d39a76e7Sxw return (1); 394d39a76e7Sxw } 395d39a76e7Sxw 396d39a76e7Sxw atomic_sub(count, &Q->cq_credits); 397d39a76e7Sxw Q->cq_pidx += count; 398d39a76e7Sxw if (Q->cq_pidx >= entries_n) { 399d39a76e7Sxw Q->cq_pidx -= entries_n; 400d39a76e7Sxw Q->cq_genbit ^= 1; 401d39a76e7Sxw } 402d39a76e7Sxw 403d39a76e7Sxw spin_unlock(qlock); 404d39a76e7Sxw 405d39a76e7Sxw #ifdef SUN_KSTATS 406d39a76e7Sxw if (count > MBLK_MAX) 407d39a76e7Sxw sge->intr_cnt.tx_descs[MBLK_MAX - 1]++; 408d39a76e7Sxw else 409d39a76e7Sxw sge->intr_cnt.tx_descs[count]++; 410d39a76e7Sxw #endif 411d39a76e7Sxw 412d39a76e7Sxw ce = &cq[pidx]; 413d39a76e7Sxw *ce = *cmp; 414d39a76e7Sxw mapping = cmp->ce_pa; 415d39a76e7Sxw j++; 416d39a76e7Sxw 417d39a76e7Sxw e = &q[pidx]; 418d39a76e7Sxw 419d39a76e7Sxw offset = (caddr_t)e - (caddr_t)q; 420d39a76e7Sxw 421d39a76e7Sxw e->Sop = 1; 422d39a76e7Sxw e->DataValid = 1; 423d39a76e7Sxw e->BufferLength = cmp->ce_len; 424d39a76e7Sxw e->AddrHigh = ((u64)mapping >> 32); 425d39a76e7Sxw e->AddrLow = ((u64)mapping & 0xffffffff); 426d39a76e7Sxw 427d39a76e7Sxw --count; 428d39a76e7Sxw if (count > 0) { 429d39a76e7Sxw unsigned int i; 430d39a76e7Sxw 431d39a76e7Sxw e->Eop = 0; 432d39a76e7Sxw wmb(); 433d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit; 434d39a76e7Sxw 435d39a76e7Sxw for (i = 0; i < count; i++) { 436d39a76e7Sxw 437d39a76e7Sxw ce++; 438d39a76e7Sxw e++; 439d39a76e7Sxw cmp++; 440d39a76e7Sxw if (++pidx == entries_n) { 441d39a76e7Sxw pidx = 0; 442d39a76e7Sxw genbit ^= 1; 443d39a76e7Sxw /* sync from offset to end of cmdQ */ 444d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(offset), 445d39a76e7Sxw j*sizeof (*e), DDI_DMA_SYNC_FORDEV); 446d39a76e7Sxw offset = j = 0; 447d39a76e7Sxw ce = cq; 448d39a76e7Sxw e = q; 449d39a76e7Sxw } 450d39a76e7Sxw 451d39a76e7Sxw *ce = *cmp; 452d39a76e7Sxw mapping = cmp->ce_pa; 453d39a76e7Sxw j++; 454d39a76e7Sxw e->Sop = 0; 455d39a76e7Sxw e->DataValid = 1; 456d39a76e7Sxw e->BufferLength = cmp->ce_len; 457d39a76e7Sxw e->AddrHigh = ((u64)mapping >> 32); 458d39a76e7Sxw e->AddrLow = ((u64)mapping & 0xffffffff); 459d39a76e7Sxw 460d39a76e7Sxw if (i < (count - 1)) { 461d39a76e7Sxw e->Eop = 0; 462d39a76e7Sxw wmb(); 463d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit; 464d39a76e7Sxw } 465d39a76e7Sxw } 466d39a76e7Sxw } 467d39a76e7Sxw 468d39a76e7Sxw ce->ce_mp = m0; 469d39a76e7Sxw 470d39a76e7Sxw e->Eop = 1; 471d39a76e7Sxw wmb(); 472d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit; 473d39a76e7Sxw 474d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(offset), j*sizeof (*e), 475d39a76e7Sxw DDI_DMA_SYNC_FORDEV); 476d39a76e7Sxw 477d39a76e7Sxw /* 478d39a76e7Sxw * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring 479d39a76e7Sxw * the doorbell if the Q is asleep. There is a natural race, where 480d39a76e7Sxw * the hardware is going to sleep just after we checked, however, 481d39a76e7Sxw * then the interrupt handler will detect the outstanding TX packet 482d39a76e7Sxw * and ring the doorbell for us. 483d39a76e7Sxw */ 484d39a76e7Sxw if (qid) { 485d39a76e7Sxw doorbell_pio(sge, F_CMDQ1_ENABLE); 486d39a76e7Sxw } else { 487d39a76e7Sxw if (atomic_read(Q->cq_asleep)) { 488d39a76e7Sxw atomic_set(&Q->cq_asleep, 0); 489d39a76e7Sxw /* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */ 490d39a76e7Sxw atomic_set(&Q->cq_pio_pidx, Q->cq_pidx); 491d39a76e7Sxw } 492d39a76e7Sxw } 493d39a76e7Sxw doorbell_pio(sge, F_CMDQ0_ENABLE); 494d39a76e7Sxw 495d39a76e7Sxw return (0); 496d39a76e7Sxw } 497d39a76e7Sxw 498d39a76e7Sxw #define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA) 499d39a76e7Sxw 500d39a76e7Sxw /* 501d39a76e7Sxw * Disable SGE error interrupts. 502d39a76e7Sxw */ 503d39a76e7Sxw int 504d39a76e7Sxw t1_sge_intr_disable(pesge* sge) 505d39a76e7Sxw { 506d39a76e7Sxw u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE); 507d39a76e7Sxw 508d39a76e7Sxw t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK); 509d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, 0); 510d39a76e7Sxw return (0); 511d39a76e7Sxw } 512d39a76e7Sxw 513d39a76e7Sxw #define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \ 514d39a76e7Sxw F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 515d39a76e7Sxw 516d39a76e7Sxw /* 517d39a76e7Sxw * Enable SGE error interrupts. 518d39a76e7Sxw */ 519d39a76e7Sxw int 520d39a76e7Sxw t1_sge_intr_enable(pesge* sge) 521d39a76e7Sxw { 522d39a76e7Sxw u32 en = SGE_INT_ENABLE; 523d39a76e7Sxw u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE); 524d39a76e7Sxw 525d39a76e7Sxw t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK); 526d39a76e7Sxw 527d39a76e7Sxw if (sge->obj->ch_flags & TSO_CAPABLE) 528d39a76e7Sxw en &= ~F_PACKET_TOO_BIG; 529d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, en); 530d39a76e7Sxw return (0); 531d39a76e7Sxw } 532d39a76e7Sxw 533d39a76e7Sxw /* 534d39a76e7Sxw * Clear SGE error interrupts. 535d39a76e7Sxw */ 536d39a76e7Sxw int 537d39a76e7Sxw t1_sge_intr_clear(pesge* sge) 538d39a76e7Sxw { 539d39a76e7Sxw t1_write_reg_4(sge->obj, A_PL_CAUSE, SGE_PL_INTR_MASK); 540d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, 0xffffffff); 541d39a76e7Sxw return (0); 542d39a76e7Sxw } 543d39a76e7Sxw 544d39a76e7Sxw #define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 545d39a76e7Sxw 546d39a76e7Sxw int 547d39a76e7Sxw t1_sge_intr_error_handler(pesge *sge) 548d39a76e7Sxw { 549d39a76e7Sxw peobj *obj = sge->obj; 550d39a76e7Sxw u32 cause = t1_read_reg_4(obj, A_SG_INT_CAUSE); 551d39a76e7Sxw 552d39a76e7Sxw if (cause & F_RESPQ_EXHAUSTED) 553d39a76e7Sxw sge->intr_cnt.respQ_empty++; 554d39a76e7Sxw if (cause & F_RESPQ_OVERFLOW) { 555d39a76e7Sxw sge->intr_cnt.respQ_overflow++; 556d39a76e7Sxw cmn_err(CE_WARN, "%s: SGE response queue overflow\n", 557d39a76e7Sxw obj->ch_name); 558d39a76e7Sxw } 559d39a76e7Sxw if (cause & F_FL_EXHAUSTED) { 560d39a76e7Sxw sge->intr_cnt.freelistQ_empty++; 561d39a76e7Sxw freelQs_empty(sge); 562d39a76e7Sxw } 563d39a76e7Sxw if (cause & F_PACKET_TOO_BIG) { 564d39a76e7Sxw sge->intr_cnt.pkt_too_big++; 565d39a76e7Sxw cmn_err(CE_WARN, "%s: SGE max packet size exceeded\n", 566d39a76e7Sxw obj->ch_name); 567d39a76e7Sxw } 568d39a76e7Sxw if (cause & F_PACKET_MISMATCH) { 569d39a76e7Sxw sge->intr_cnt.pkt_mismatch++; 570d39a76e7Sxw cmn_err(CE_WARN, "%s: SGE packet mismatch\n", 571d39a76e7Sxw obj->ch_name); 572d39a76e7Sxw } 573d39a76e7Sxw if (cause & SGE_INT_FATAL) 574d39a76e7Sxw t1_fatal_err(obj); 575d39a76e7Sxw 576d39a76e7Sxw t1_write_reg_4(obj, A_SG_INT_CAUSE, cause); 577d39a76e7Sxw return (0); 578d39a76e7Sxw } 579d39a76e7Sxw 580d39a76e7Sxw /* 581d39a76e7Sxw * 582d39a76e7Sxw * PARAM: sge - SGE instance pointer. 583d39a76e7Sxw */ 584d39a76e7Sxw int 585d39a76e7Sxw sge_data_in(pesge *sge) 586d39a76e7Sxw { 587d39a76e7Sxw peobj *adapter = sge->obj; 588d39a76e7Sxw struct respQ *Q = &sge->respQ; 589d39a76e7Sxw respQ_e *e; /* response queue entry */ 590d39a76e7Sxw respQ_e *q = Q->rq_entries; /* base response queue */ 591d39a76e7Sxw uint32_t cidx = Q->rq_cidx; 592d39a76e7Sxw uint32_t genbit = Q->rq_genbit; 593d39a76e7Sxw uint32_t entries_n = Q->rq_entries_n; 594d39a76e7Sxw uint32_t credits = Q->rq_credits; 595d39a76e7Sxw uint32_t credits_thresh = Q->rq_credits_thresh; 596d39a76e7Sxw uint32_t ret = 0; 597d39a76e7Sxw #ifndef TX_THREAD_RECLAIM 598d39a76e7Sxw uint32_t credits_pend[2] = {0, 0}; 599d39a76e7Sxw #endif 600d39a76e7Sxw uint32_t flags = 0; 601d39a76e7Sxw uint32_t flagt; 602d39a76e7Sxw ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh; 603d39a76e7Sxw 604d39a76e7Sxw t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA); 605d39a76e7Sxw 606d39a76e7Sxw /* 607d39a76e7Sxw * Catch the case where an interrupt arrives 608d39a76e7Sxw * early. 609d39a76e7Sxw */ 610d39a76e7Sxw if ((q == NULL) || (dh == NULL)) { 611d39a76e7Sxw goto check_slow_ints; 612d39a76e7Sxw } 613d39a76e7Sxw 614d39a76e7Sxw /* initial response queue entry */ 615d39a76e7Sxw e = &q[cidx]; 616d39a76e7Sxw 617d39a76e7Sxw /* pull physical memory of response queue entry into cache */ 618d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q), 619d39a76e7Sxw sizeof (*e), DDI_DMA_SYNC_FORKERNEL); 620d39a76e7Sxw 621d39a76e7Sxw while (e->GenerationBit == genbit) { 622d39a76e7Sxw if (--credits < credits_thresh) { 623d39a76e7Sxw uint32_t n = entries_n - credits - 1; 624d39a76e7Sxw t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n); 625d39a76e7Sxw credits += n; 626d39a76e7Sxw } 627d39a76e7Sxw if (likely(e->DataValid)) { 628d39a76e7Sxw (void) t1_sge_rx(sge, &sge->freelQ[e->FreelistQid], 629d39a76e7Sxw e->BufferLength, e->Offload); 630d39a76e7Sxw if ((e->Sop != 1) || (e->Eop != 1)) { 631d39a76e7Sxw sge->intr_cnt.rx_badEopSop++; 632d39a76e7Sxw cmn_err(CE_WARN, "bad Sop %d or Eop %d: %d", 633d39a76e7Sxw e->Sop, e->Eop, e->BufferLength); 634d39a76e7Sxw } 635d39a76e7Sxw } 636d39a76e7Sxw flagt = e->Qsleeping; 637d39a76e7Sxw flags |= flagt; 638d39a76e7Sxw if (flagt & F_CMDQ0_ENABLE) 639d39a76e7Sxw sge->intr_cnt.rx_cmdq0++; 640d39a76e7Sxw if (flagt & F_CMDQ1_ENABLE) 641d39a76e7Sxw sge->intr_cnt.rx_cmdq1++; 642d39a76e7Sxw if (flagt & F_FL0_ENABLE) 643d39a76e7Sxw sge->intr_cnt.rx_flq0++; 644d39a76e7Sxw if (flagt & F_FL1_ENABLE) 645d39a76e7Sxw sge->intr_cnt.rx_flq1++; 646d39a76e7Sxw #ifdef TX_THREAD_RECLAIM 647d39a76e7Sxw spin_lock(&sge->cmdQ[0].cq_qlock); 648d39a76e7Sxw sge->cmdQ[0].cq_complete += e->Cmdq0CreditReturn; 649d39a76e7Sxw spin_unlock(&sge->cmdQ[0].cq_qlock); 650d39a76e7Sxw spin_lock(&sge->cmdQ[1].cq_qlock); 651d39a76e7Sxw sge->cmdQ[1].cq_complete += e->Cmdq1CreditReturn; 652d39a76e7Sxw if ((adapter->ch_blked) && 653d39a76e7Sxw (sge->cmdQ[0].cq_complete + 654d39a76e7Sxw sge->cmdQ[1].cq_complete) > 16) { 655d39a76e7Sxw adapter->ch_blked = 0; 656d39a76e7Sxw ch_gld_ok(adapter); 657d39a76e7Sxw } 658d39a76e7Sxw spin_unlock(&sge->cmdQ[1].cq_qlock); 659d39a76e7Sxw #else 660d39a76e7Sxw credits_pend[0] += e->Cmdq0CreditReturn; 661d39a76e7Sxw credits_pend[1] += e->Cmdq1CreditReturn; 662d39a76e7Sxw #ifdef CONFIG_SMP 663d39a76e7Sxw if (unlikely(credits_pend[0] > SGE_BATCH_THRESH)) { 664d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]); 665d39a76e7Sxw credits_pend[0] = 0; 666d39a76e7Sxw } 667d39a76e7Sxw if (unlikely(credits_pend[1] > SGE_BATCH_THRESH)) { 668d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]); 669d39a76e7Sxw credits_pend[1] = 0; 670d39a76e7Sxw } 671d39a76e7Sxw #endif 672d39a76e7Sxw #endif 673d39a76e7Sxw #ifdef HOST_PAUSE 674d39a76e7Sxw t1_sge_check_pause(sge, &sge->freelQ[e->FreelistQid]); 675d39a76e7Sxw #endif 676d39a76e7Sxw e++; 677d39a76e7Sxw if (unlikely(++cidx == entries_n)) { 678d39a76e7Sxw cidx = 0; 679d39a76e7Sxw genbit ^= 1; 680d39a76e7Sxw e = q; 681d39a76e7Sxw } 682d39a76e7Sxw 683d39a76e7Sxw /* pull physical memory of response queue entry into cache */ 684d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q), 685d39a76e7Sxw sizeof (*e), DDI_DMA_SYNC_FORKERNEL); 686d39a76e7Sxw 687d39a76e7Sxw ret = 1; 688d39a76e7Sxw } 689d39a76e7Sxw 690d39a76e7Sxw #ifndef TX_THREAD_RECLAIM 691d39a76e7Sxw if (credits_pend[0]) 692d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]); 693d39a76e7Sxw if (credits_pend[1]) 694d39a76e7Sxw free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]); 695d39a76e7Sxw #endif 696d39a76e7Sxw if (flags & F_CMDQ0_ENABLE) { 697d39a76e7Sxw struct cmdQ *cmdQ = &sge->cmdQ[0]; 698d39a76e7Sxw atomic_set(&cmdQ->cq_asleep, 1); 699d39a76e7Sxw if (atomic_read(cmdQ->cq_pio_pidx) != cmdQ->cq_pidx) { 700d39a76e7Sxw doorbell_pio(sge, F_CMDQ0_ENABLE); 701d39a76e7Sxw atomic_set(&cmdQ->cq_pio_pidx, cmdQ->cq_pidx); 702d39a76e7Sxw } 703d39a76e7Sxw } 704d39a76e7Sxw 705d39a76e7Sxw /* the SGE told us one of the free lists is empty */ 706d39a76e7Sxw if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE))) 707d39a76e7Sxw freelQs_empty(sge); 708d39a76e7Sxw 709d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD 710d39a76e7Sxw if (adapter->ch_tx_overflow_mutex) 711d39a76e7Sxw mutex_enter(adapter->ch_tx_overflow_mutex); 712d39a76e7Sxw if (adapter->ch_blked && 713d39a76e7Sxw (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) && 714d39a76e7Sxw (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) { 715d39a76e7Sxw adapter->ch_blked = 0; 716d39a76e7Sxw if (adapter->ch_tx_overflow_cv) 717d39a76e7Sxw cv_broadcast(adapter->ch_tx_overflow_cv); 718d39a76e7Sxw ch_gld_ok(adapter); 719d39a76e7Sxw } 720d39a76e7Sxw if (adapter->ch_tx_overflow_mutex) 721d39a76e7Sxw mutex_exit(adapter->ch_tx_overflow_mutex); 722d39a76e7Sxw #else 723d39a76e7Sxw #ifndef TX_THREAD_RECLAIM 724d39a76e7Sxw if (adapter->ch_blked && 725d39a76e7Sxw (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) && 726d39a76e7Sxw (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) { 727d39a76e7Sxw adapter->ch_blked = 0; 728d39a76e7Sxw ch_gld_ok(adapter); 729d39a76e7Sxw } 730d39a76e7Sxw #endif 731d39a76e7Sxw #endif /* CONFIG_CHELSIO_T1_OFFLOAD */ 732d39a76e7Sxw 733d39a76e7Sxw Q->rq_genbit = genbit; 734d39a76e7Sxw Q->rq_cidx = cidx; 735d39a76e7Sxw Q->rq_credits = credits; 736d39a76e7Sxw 737d39a76e7Sxw t1_write_reg_4(adapter, A_SG_SLEEPING, cidx); 738d39a76e7Sxw 739d39a76e7Sxw check_slow_ints: 740d39a76e7Sxw /* handle non-data interrupts */ 741d39a76e7Sxw if (unlikely(!ret)) 742d39a76e7Sxw ret = t1_slow_intr_handler(adapter); 743d39a76e7Sxw 744d39a76e7Sxw return (ret); 745d39a76e7Sxw } 746d39a76e7Sxw 747d39a76e7Sxw /* 748d39a76e7Sxw * allocate a mblk with DMA mapped mblk. 749d39a76e7Sxw * When checksum offload is enabled, we start the DMA at a 2 byte offset so 750d39a76e7Sxw * the IP header will be aligned. We do this for sparc only. 751d39a76e7Sxw */ 752d39a76e7Sxw static uint64_t 753d39a76e7Sxw os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh) 754d39a76e7Sxw { 755d39a76e7Sxw ch_esb_t *ch_get_small_rbuf(ch_t *sa); 756d39a76e7Sxw ch_esb_t *ch_get_big_rbuf(ch_t *sa); 757d39a76e7Sxw ch_esb_t *rbp; 758d39a76e7Sxw uint32_t rxoff = sa->sge->rx_offset; 759d39a76e7Sxw 760d39a76e7Sxw if (sz == SGE_SM_BUF_SZ(sa)) { 761d39a76e7Sxw /* get pre-mapped buffer */ 762d39a76e7Sxw if ((rbp = ch_get_small_rbuf(sa)) == NULL) { 763d39a76e7Sxw sa->norcvbuf++; 764d39a76e7Sxw return ((uint64_t)0); 765d39a76e7Sxw } 766d39a76e7Sxw 767d39a76e7Sxw *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff, 768d39a76e7Sxw SGE_SM_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn); 769d39a76e7Sxw if (*mb == NULL) { 770d39a76e7Sxw mutex_enter(&sa->ch_small_esbl); 771d39a76e7Sxw rbp->cs_next = sa->ch_small_esb_free; 772d39a76e7Sxw sa->ch_small_esb_free = rbp; 773d39a76e7Sxw mutex_exit(&sa->ch_small_esbl); 774d39a76e7Sxw return ((uint64_t)0); 775d39a76e7Sxw } 776d39a76e7Sxw *dh = rbp->cs_dh; 777d39a76e7Sxw 778d39a76e7Sxw return (rbp->cs_pa + rxoff); 779d39a76e7Sxw } else { 780d39a76e7Sxw /* get pre-mapped buffer */ 781d39a76e7Sxw if ((rbp = ch_get_big_rbuf(sa)) == NULL) { 782d39a76e7Sxw sa->norcvbuf++; 783d39a76e7Sxw return ((uint64_t)0); 784d39a76e7Sxw } 785d39a76e7Sxw 786d39a76e7Sxw *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff, 787d39a76e7Sxw SGE_BG_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn); 788d39a76e7Sxw if (*mb == NULL) { 789d39a76e7Sxw mutex_enter(&sa->ch_big_esbl); 790d39a76e7Sxw rbp->cs_next = sa->ch_big_esb_free; 791d39a76e7Sxw sa->ch_big_esb_free = rbp; 792d39a76e7Sxw mutex_exit(&sa->ch_big_esbl); 793d39a76e7Sxw return ((uint64_t)0); 794d39a76e7Sxw } 795d39a76e7Sxw *dh = rbp->cs_dh; 796d39a76e7Sxw 797d39a76e7Sxw return (rbp->cs_pa + rxoff); 798d39a76e7Sxw } 799d39a76e7Sxw } 800d39a76e7Sxw 801d39a76e7Sxw static inline unsigned int 802d39a76e7Sxw t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload) 803d39a76e7Sxw { 804d39a76e7Sxw mblk_t *skb; 805d39a76e7Sxw peobj *adapter = sge->obj; 806d39a76e7Sxw struct freelQ_ce *cq = Q->fq_centries; 807d39a76e7Sxw struct freelQ_ce *ce = &cq[Q->fq_cidx]; 808d39a76e7Sxw ddi_dma_handle_t dh = (ddi_dma_handle_t)ce->fe_dh; 809d39a76e7Sxw uint32_t cidx = Q->fq_cidx; 810d39a76e7Sxw uint32_t entries_n = Q->fq_entries_n; 811d39a76e7Sxw uint32_t sz = Q->fq_rx_buffer_size; 812d39a76e7Sxw uint32_t useit = 1; 813d39a76e7Sxw uint32_t rxoff = sge->rx_offset; 814d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD 815d39a76e7Sxw uint32_t rv; 816d39a76e7Sxw #endif 817d39a76e7Sxw 818d39a76e7Sxw if (Q->fq_id) 819d39a76e7Sxw sge->intr_cnt.rx_flq1_cnt++; 820d39a76e7Sxw else 821d39a76e7Sxw sge->intr_cnt.rx_flq0_cnt++; 822d39a76e7Sxw /* 823d39a76e7Sxw * If pkt size falls below threshold, then we'll copy data to 824d39a76e7Sxw * an blk and reuse mblk. 825d39a76e7Sxw * 826d39a76e7Sxw * NOTE that rxoff is 2 for T1 adapters. We align the the start 827d39a76e7Sxw * of the DMA buffer begin at rxoff offset for T1 cards instead of 828d39a76e7Sxw * at the beginning of the buffer, thus the length of the received 829d39a76e7Sxw * data does not include this offset. We therefore always add 830d39a76e7Sxw * SGE_RX_OFFSET to the allocb size so we have space to provide the 831d39a76e7Sxw * offset for the copied data. 832d39a76e7Sxw */ 833d39a76e7Sxw #ifdef HOST_PAUSE 834d39a76e7Sxw /* 835d39a76e7Sxw * If we have Host pause compiled in, then we look at the 836d39a76e7Sxw * free list, if the pause is on and we're not in offload 837d39a76e7Sxw * mode then we drop packets, this is designed to avoid 838d39a76e7Sxw * overwhelming the machine. If the machine is powerfull enough 839d39a76e7Sxw * this will not happen. The 'rx_pkt_drops' will show when 840d39a76e7Sxw * packets are being dropped and how much. 841d39a76e7Sxw */ 842d39a76e7Sxw if ((offload == 0) && adapter->pause_on) { 843d39a76e7Sxw freelQ_e *e; 844d39a76e7Sxw /* Ditch the packet and reuse original buffer */ 845d39a76e7Sxw e = &Q->fq_entries[cidx]; 846d39a76e7Sxw e->GenerationBit ^= 1; 847d39a76e7Sxw e->GenerationBit2 ^= 1; 848d39a76e7Sxw sge->intr_cnt.rx_pkt_drops++; 849d39a76e7Sxw goto rx_entry_consumed; 850d39a76e7Sxw } else if (((adapter->pause_on || 851d39a76e7Sxw (len <= SGE_RX_COPY_THRESHOLD)) && 852d39a76e7Sxw (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI)))) 853d39a76e7Sxw #else 854d39a76e7Sxw if ((len <= SGE_RX_COPY_THRESHOLD) && 855d39a76e7Sxw (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI))) 856d39a76e7Sxw #endif 857d39a76e7Sxw { 858d39a76e7Sxw freelQ_e *e; 859d39a76e7Sxw char *src = (char *)((mblk_t *)ce->fe_mp)->b_rptr; 860d39a76e7Sxw 861d39a76e7Sxw /* 862d39a76e7Sxw * pull physical memory of pkt data into cache 863d39a76e7Sxw * Note that len does not include offset for T1. 864d39a76e7Sxw */ 865d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(rxoff), len, 866d39a76e7Sxw DDI_DMA_SYNC_FORKERNEL); 867d39a76e7Sxw 868d39a76e7Sxw if (offload == 0) { 869d39a76e7Sxw /* 870d39a76e7Sxw * create 2 byte offset so IP header aligned on 871d39a76e7Sxw * 4 byte boundry 872d39a76e7Sxw */ 873d39a76e7Sxw skb_reserve(skb, SGE_RX_OFFSET); 874d39a76e7Sxw /* 875d39a76e7Sxw * if hardware inserted 2 byte offset then need to 876d39a76e7Sxw * start copying with extra offset 877d39a76e7Sxw */ 878d39a76e7Sxw src += sge->rx_pkt_pad; 879d39a76e7Sxw } 880d39a76e7Sxw memcpy(skb->b_rptr, src, len); 881d39a76e7Sxw useit = 0; /* mblk copy, don't inc esballoc in use cnt */ 882d39a76e7Sxw 883d39a76e7Sxw /* so we can reuse original buffer */ 884d39a76e7Sxw e = &Q->fq_entries[cidx]; 885d39a76e7Sxw e->GenerationBit ^= 1; 886d39a76e7Sxw e->GenerationBit2 ^= 1; 887d39a76e7Sxw sge->intr_cnt.rx_pkt_copied++; 888d39a76e7Sxw } else { 889d39a76e7Sxw /* consume buffer off the ring */ 890d39a76e7Sxw skb = ce->fe_mp; 891d39a76e7Sxw ce->fe_mp = NULL; 892d39a76e7Sxw 893d39a76e7Sxw /* 894d39a76e7Sxw * if not offload (tunneled pkt), & hardward padded, then 895d39a76e7Sxw * adjust start of pkt to point to start of data i.e. 896d39a76e7Sxw * skip pad (2 bytes). 897d39a76e7Sxw */ 898d39a76e7Sxw if (!offload && sge->rx_pkt_pad) 899d39a76e7Sxw __skb_pull(skb, SGE_RX_OFFSET); 900d39a76e7Sxw 901d39a76e7Sxw /* 902d39a76e7Sxw * pull physical memory of pkt data into cache 903d39a76e7Sxw * Note that len does not include offset for T1. 904d39a76e7Sxw */ 905d39a76e7Sxw (void) ddi_dma_sync(dh, (off_t)(rxoff), len, 906d39a76e7Sxw DDI_DMA_SYNC_FORKERNEL); 907d39a76e7Sxw } 908d39a76e7Sxw 909d39a76e7Sxw /* set length of data in skb */ 910d39a76e7Sxw skb_put(skb, len); 911d39a76e7Sxw 912d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD 913d39a76e7Sxw if (likely(offload)) { 914d39a76e7Sxw if (likely(toe_running(adapter))) { 915d39a76e7Sxw /* sends pkt upstream to toe layer */ 916d39a76e7Sxw if (useit) { 917*f172c8abSToomas Soome uint_t index; 918*f172c8abSToomas Soome if (sz == SGE_SM_BUF_SZ(adapter)) 919*f172c8abSToomas Soome index = adapter->ch_sm_index; 920*f172c8abSToomas Soome else 921*f172c8abSToomas Soome index = adapter->ch_big_index; 922*f172c8abSToomas Soome atomic_add(1, &buffers_in_use[index]); 923d39a76e7Sxw } 924d39a76e7Sxw if (adapter->toe_rcv) 925d39a76e7Sxw adapter->toe_rcv(adapter->ch_toeinst, skb); 926d39a76e7Sxw else 927d39a76e7Sxw freemsg(skb); 928d39a76e7Sxw } else { 929d39a76e7Sxw cmn_err(CE_WARN, 930d39a76e7Sxw "%s: unexpected offloaded packet, cmd %u\n", 931d39a76e7Sxw adapter->ch_name, *skb->b_rptr); 932d39a76e7Sxw 933d39a76e7Sxw /* discard packet */ 934d39a76e7Sxw freemsg(skb); 935d39a76e7Sxw } 936d39a76e7Sxw } 937d39a76e7Sxw #else 938d39a76e7Sxw if (unlikely(offload)) { 939d39a76e7Sxw cmn_err(CE_WARN, 940d39a76e7Sxw "%s: unexpected offloaded packet, cmd %u\n", 941d39a76e7Sxw adapter->ch_name, *skb->b_rptr); 942d39a76e7Sxw 943d39a76e7Sxw /* discard paket */ 944d39a76e7Sxw freemsg(skb); 945d39a76e7Sxw } 946d39a76e7Sxw #endif 947d39a76e7Sxw else { 948d39a76e7Sxw struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->b_rptr; 949d39a76e7Sxw int flg = 0; 950d39a76e7Sxw uint32_t cksum; 951d39a76e7Sxw 952d39a76e7Sxw /* adjust beginning of data to skip CPL header */ 953d39a76e7Sxw skb_pull(skb, SZ_CPL_RX_PKT); 954d39a76e7Sxw 955d39a76e7Sxw /* extract checksum from CPL header here */ 956d39a76e7Sxw 957d39a76e7Sxw /* 958d39a76e7Sxw * bump count of mlbks in used by protocol stack(s) 959d39a76e7Sxw */ 960d39a76e7Sxw if (useit) { 961d39a76e7Sxw if (sz == SGE_SM_BUF_SZ(adapter)) { 962d39a76e7Sxw atomic_add(1, 963d39a76e7Sxw &buffers_in_use[adapter->ch_sm_index]); 964d39a76e7Sxw } else { 965d39a76e7Sxw atomic_add(1, 966d39a76e7Sxw &buffers_in_use[adapter->ch_big_index]); 967d39a76e7Sxw } 968d39a76e7Sxw } 969d39a76e7Sxw 970d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD 971d39a76e7Sxw /* 972d39a76e7Sxw * let the TOE layer have a crack at the packet first. 973d39a76e7Sxw */ 974d39a76e7Sxw if (adapter->toe_tunnel) { 975d39a76e7Sxw rv = adapter->toe_tunnel(adapter->ch_toeinst, skb); 976d39a76e7Sxw /* 977d39a76e7Sxw * The TOE may have consumed the packet. 978d39a76e7Sxw */ 979d39a76e7Sxw if (rv) 980d39a76e7Sxw goto rx_entry_consumed; 981d39a76e7Sxw } 982d39a76e7Sxw #endif /* CONFIG_CHELSIO_T1_OFFLOAD */ 983d39a76e7Sxw 984d39a76e7Sxw cksum = p->csum; 985d39a76e7Sxw 986d39a76e7Sxw /* 987d39a76e7Sxw * NOTE: 14+9 = size of MAC + offset to IP protocol field 988d39a76e7Sxw */ 989d39a76e7Sxw if (adapter->ch_config.cksum_enabled && 990d39a76e7Sxw (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) == 991d39a76e7Sxw ETHERTYPE_IP) && 992d39a76e7Sxw ((skb->b_rptr[14+9] == IPPROTO_TCP) || 993d39a76e7Sxw (skb->b_rptr[14+9] == IPPROTO_UDP))) { 994d39a76e7Sxw flg = 1; 995d39a76e7Sxw } 996d39a76e7Sxw 997d39a76e7Sxw ch_send_up(adapter, skb, cksum, flg); 998d39a76e7Sxw } 999d39a76e7Sxw 1000d39a76e7Sxw rx_entry_consumed: 1001d39a76e7Sxw 1002d39a76e7Sxw if (++cidx == entries_n) 1003d39a76e7Sxw cidx = 0; 1004d39a76e7Sxw 1005d39a76e7Sxw Q->fq_cidx = cidx; 1006d39a76e7Sxw 1007d39a76e7Sxw if (unlikely(--Q->fq_credits < (entries_n>>2))) 1008d39a76e7Sxw /* allocate new buffers on the free list */ 1009d39a76e7Sxw alloc_freelQ_buffers(sge, Q); 1010d39a76e7Sxw return (1); 1011d39a76e7Sxw } 1012d39a76e7Sxw 1013d39a76e7Sxw #ifdef HOST_PAUSE 1014d39a76e7Sxw static void 1015d39a76e7Sxw t1_sge_check_pause(pesge *sge, struct freelQ *Q) 1016d39a76e7Sxw { 1017d39a76e7Sxw peobj *adapter = sge->obj; 1018d39a76e7Sxw 1019d39a76e7Sxw /* 1020d39a76e7Sxw * If the number of available credits shrinks below 1021d39a76e7Sxw * the Pause on threshold then enable the pause and 1022d39a76e7Sxw * try and allocate more buffers. 1023d39a76e7Sxw * On the next pass, if there's more credits returned 1024d39a76e7Sxw * then check that you've went above the pause 1025d39a76e7Sxw * threshold and then disable the pause. 1026d39a76e7Sxw */ 1027d39a76e7Sxw if (Q->fq_credits < Q->fq_pause_on_thresh) { 1028d39a76e7Sxw if (do_host_pause) { 1029d39a76e7Sxw sge->intr_cnt.rx_pause_on++; 1030d39a76e7Sxw adapter->txxg_cfg1 |= 1031d39a76e7Sxw SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE; 1032d39a76e7Sxw (void) t1_tpi_write(adapter, 1033d39a76e7Sxw SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2, 1034d39a76e7Sxw adapter->txxg_cfg1); 1035d39a76e7Sxw adapter->pause_on = 1; 1036d39a76e7Sxw adapter->pause_time = gethrtime(); 1037d39a76e7Sxw } 1038d39a76e7Sxw alloc_freelQ_buffers(sge, Q); 1039d39a76e7Sxw } else if ((adapter->pause_on) && 1040d39a76e7Sxw (Q->fq_credits > Q->fq_pause_off_thresh)) { 1041d39a76e7Sxw hrtime_t time; 1042d39a76e7Sxw sge->intr_cnt.rx_pause_off++; 1043d39a76e7Sxw adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE; 1044d39a76e7Sxw (void) t1_tpi_write(adapter, 1045d39a76e7Sxw SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2, 1046d39a76e7Sxw adapter->txxg_cfg1); 1047d39a76e7Sxw adapter->pause_on = 0; 1048d39a76e7Sxw time = (gethrtime() - adapter->pause_time)/1000; 1049d39a76e7Sxw sge->intr_cnt.rx_pause_ms += time; 1050d39a76e7Sxw if (time > sge->intr_cnt.rx_pause_spike) 1051d39a76e7Sxw sge->intr_cnt.rx_pause_spike = (uint32_t)time; 1052d39a76e7Sxw } 1053d39a76e7Sxw sge->intr_cnt.rx_fl_credits = Q->fq_credits; 1054d39a76e7Sxw } 1055d39a76e7Sxw #endif /* HOST_PAUSE */ 1056d39a76e7Sxw 1057d39a76e7Sxw static void 1058d39a76e7Sxw alloc_freelQ_buffers(pesge *sge, struct freelQ *Q) 1059d39a76e7Sxw { 1060d39a76e7Sxw uint32_t pidx = Q->fq_pidx; 1061d39a76e7Sxw struct freelQ_ce *ce = &Q->fq_centries[pidx]; 1062d39a76e7Sxw freelQ_e *fq = Q->fq_entries; /* base of freelist Q */ 1063d39a76e7Sxw freelQ_e *e = &Q->fq_entries[pidx]; 1064d39a76e7Sxw uint32_t sz = Q->fq_rx_buffer_size; 1065d39a76e7Sxw uint32_t rxoff = sge->rx_offset; 1066d39a76e7Sxw uint32_t credits = Q->fq_credits; 1067d39a76e7Sxw uint32_t entries_n = Q->fq_entries_n; 1068d39a76e7Sxw uint32_t genbit = Q->fq_genbit; 1069d39a76e7Sxw ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh; 1070d39a76e7Sxw ulong_t dh; 1071d39a76e7Sxw uint64_t mapping; 1072d39a76e7Sxw off_t offset = (off_t)((caddr_t)e - (caddr_t)fq); 1073d39a76e7Sxw size_t len = 0; 1074d39a76e7Sxw 1075d39a76e7Sxw while (credits < entries_n) { 1076d39a76e7Sxw if (e->GenerationBit != genbit) { 1077d39a76e7Sxw mblk_t *skb; 1078d39a76e7Sxw 1079d39a76e7Sxw mapping = os_freelist_buffer_alloc(sge->obj, sz, 1080d39a76e7Sxw &skb, &dh); 1081d39a76e7Sxw if (mapping == 0) { 1082d39a76e7Sxw sge->intr_cnt.rx_flbuf_fails++; 1083d39a76e7Sxw break; 1084d39a76e7Sxw } 1085d39a76e7Sxw sge->intr_cnt.rx_flbuf_allocs++; 1086d39a76e7Sxw 1087d39a76e7Sxw ce->fe_mp = skb; 1088d39a76e7Sxw ce->fe_dh = dh; 1089d39a76e7Sxw 1090d39a76e7Sxw /* 1091d39a76e7Sxw * Note that for T1, we've started the beginning of 1092d39a76e7Sxw * of the buffer by an offset of 2 bytes. We thus 1093d39a76e7Sxw * decrement the length to account for this. 1094d39a76e7Sxw */ 1095d39a76e7Sxw e->AddrLow = (u32)mapping; 1096d39a76e7Sxw e->AddrHigh = (u64)mapping >> 32; 1097d39a76e7Sxw e->BufferLength = sz - rxoff; 1098d39a76e7Sxw wmb(); 1099d39a76e7Sxw e->GenerationBit = e->GenerationBit2 = genbit; 1100d39a76e7Sxw } 1101d39a76e7Sxw 1102d39a76e7Sxw len += sizeof (*e); 1103d39a76e7Sxw 1104d39a76e7Sxw ce++; 1105d39a76e7Sxw e++; 1106d39a76e7Sxw credits++; 1107d39a76e7Sxw if (++pidx == entries_n) { 1108d39a76e7Sxw /* 1109d39a76e7Sxw * sync freelist entries to physical memory up to 1110d39a76e7Sxw * end of the table. 1111d39a76e7Sxw */ 1112d39a76e7Sxw (void) ddi_dma_sync(th, offset, len, 1113d39a76e7Sxw DDI_DMA_SYNC_FORDEV); 1114d39a76e7Sxw offset = 0; 1115d39a76e7Sxw len = 0; 1116d39a76e7Sxw 1117d39a76e7Sxw pidx = 0; 1118d39a76e7Sxw genbit ^= 1; 1119d39a76e7Sxw ce = Q->fq_centries; 1120d39a76e7Sxw e = Q->fq_entries; 1121d39a76e7Sxw } 1122d39a76e7Sxw } 1123d39a76e7Sxw 1124d39a76e7Sxw /* sync freelist entries that have been modified. */ 1125d39a76e7Sxw if (len) 1126d39a76e7Sxw (void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV); 1127d39a76e7Sxw 1128d39a76e7Sxw Q->fq_genbit = genbit; 1129d39a76e7Sxw Q->fq_pidx = pidx; 1130d39a76e7Sxw Q->fq_credits = credits; 1131d39a76e7Sxw } 1132d39a76e7Sxw 1133d39a76e7Sxw static void 1134d39a76e7Sxw freelQs_empty(pesge *sge) 1135d39a76e7Sxw { 1136d39a76e7Sxw u32 irq_reg = t1_read_reg_4(sge->obj, A_SG_INT_ENABLE); 1137d39a76e7Sxw u32 irqholdoff_reg; 1138d39a76e7Sxw 1139d39a76e7Sxw alloc_freelQ_buffers(sge, &sge->freelQ[0]); 1140d39a76e7Sxw alloc_freelQ_buffers(sge, &sge->freelQ[1]); 1141d39a76e7Sxw 1142d39a76e7Sxw if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) && 1143d39a76e7Sxw (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) { 1144d39a76e7Sxw irq_reg |= F_FL_EXHAUSTED; 1145d39a76e7Sxw irqholdoff_reg = sge->intrtimer[sge->currIndex]; 1146d39a76e7Sxw } else { 1147d39a76e7Sxw /* Clear the F_FL_EXHAUSTED interrupts for now */ 1148d39a76e7Sxw irq_reg &= ~F_FL_EXHAUSTED; 1149d39a76e7Sxw irqholdoff_reg = sge->intrtimer_nres; 1150d39a76e7Sxw } 1151d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg); 1152d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, irq_reg); 1153d39a76e7Sxw 1154d39a76e7Sxw /* We reenable the Qs to force an Freelist GTS interrupt later */ 1155d39a76e7Sxw doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE); 1156d39a76e7Sxw } 1157d39a76e7Sxw 1158d39a76e7Sxw /* 1159d39a76e7Sxw * Frees 'credits_pend' TX buffers and returns the credits to Q->credits. 1160d39a76e7Sxw * Free xmit buffers 1161d39a76e7Sxw */ 1162d39a76e7Sxw static void 1163d39a76e7Sxw free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend) 1164d39a76e7Sxw { 1165d39a76e7Sxw mblk_t *skb; 1166d39a76e7Sxw struct cmdQ_ce *ce; 1167d39a76e7Sxw struct cmdQ_ce *cq = Q->cq_centries; 1168d39a76e7Sxw uint32_t entries_n = Q->cq_entries_n; 1169d39a76e7Sxw uint32_t cidx = Q->cq_cidx; 1170d39a76e7Sxw uint32_t i = credits_pend; 1171d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD 1172d39a76e7Sxw ch_t *chp = sge->obj; 1173d39a76e7Sxw #endif 1174d39a76e7Sxw ce = &cq[cidx]; 1175d39a76e7Sxw 1176d39a76e7Sxw while (i--) { 1177d39a76e7Sxw #ifdef CONFIG_CHELSIO_T1_OFFLOAD 1178d39a76e7Sxw /* if flag set, then toe buffer */ 1179d39a76e7Sxw switch (ce->ce_flg & 0x7) { 1180d39a76e7Sxw case DH_DMA: 1181d39a76e7Sxw if (ce->ce_dh) { 1182d39a76e7Sxw ch_unbind_dma_handle(sge->obj, ce->ce_dh); 1183d39a76e7Sxw ce->ce_dh = NULL; /* may not be needed */ 1184d39a76e7Sxw } 1185d39a76e7Sxw skb = ce->ce_mp; 1186d39a76e7Sxw if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1187d39a76e7Sxw freemsg(skb); 1188d39a76e7Sxw } 1189d39a76e7Sxw ce->ce_mp = NULL; 1190d39a76e7Sxw break; 1191d39a76e7Sxw 1192d39a76e7Sxw #if defined(__sparc) 1193d39a76e7Sxw case DH_DVMA: 1194d39a76e7Sxw if (ce->ce_dh) { 1195d39a76e7Sxw ch_unbind_dvma_handle(sge->obj, ce->ce_dh); 1196d39a76e7Sxw ce->ce_dh = NULL; /* may not be needed */ 1197d39a76e7Sxw } 1198d39a76e7Sxw skb = ce->ce_mp; 1199d39a76e7Sxw if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1200d39a76e7Sxw freemsg(skb); 1201d39a76e7Sxw } 1202d39a76e7Sxw ce->ce_mp = NULL; 1203d39a76e7Sxw break; 1204d39a76e7Sxw #endif /* __sparc */ 1205d39a76e7Sxw 1206d39a76e7Sxw case DH_TOE: 1207d39a76e7Sxw chp->toe_free(chp->ch_toeinst, (tbuf_t *)(ce->ce_mp)); 1208d39a76e7Sxw ce->ce_mp = NULL; 1209d39a76e7Sxw break; 1210d39a76e7Sxw } 1211d39a76e7Sxw #else /* CONFIG_CHELSIO_T1_OFFLOAD */ 1212d39a76e7Sxw if (ce->ce_dh) { 1213d39a76e7Sxw if ((ce->ce_flg & 7) == DH_DMA) { 1214d39a76e7Sxw ch_unbind_dma_handle(sge->obj, ce->ce_dh); 1215d39a76e7Sxw } 1216d39a76e7Sxw #if defined(__sparc) 1217d39a76e7Sxw else { 1218d39a76e7Sxw ch_unbind_dvma_handle(sge->obj, ce->ce_dh); 1219d39a76e7Sxw } 1220d39a76e7Sxw #endif /* __sparc */ 1221d39a76e7Sxw ce->ce_dh = NULL; /* may not be needed */ 1222d39a76e7Sxw } 1223d39a76e7Sxw 1224d39a76e7Sxw skb = ce->ce_mp; 1225d39a76e7Sxw if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1226d39a76e7Sxw freemsg(skb); 1227d39a76e7Sxw } 1228d39a76e7Sxw ce->ce_mp = NULL; 1229d39a76e7Sxw #endif /* !CONFIG_CHELSIO_T1_OFFLOAD */ 1230d39a76e7Sxw 1231d39a76e7Sxw ce++; 1232d39a76e7Sxw if (++cidx == entries_n) { 1233d39a76e7Sxw cidx = 0; 1234d39a76e7Sxw ce = cq; 1235d39a76e7Sxw } 1236d39a76e7Sxw } 1237d39a76e7Sxw 1238d39a76e7Sxw Q->cq_cidx = cidx; 1239d39a76e7Sxw atomic_add(credits_pend, &Q->cq_credits); 1240d39a76e7Sxw } 1241d39a76e7Sxw 1242d39a76e7Sxw struct sge_intr_counts * 1243d39a76e7Sxw sge_get_stat(pesge *sge) 1244d39a76e7Sxw { 1245d39a76e7Sxw return (&sge->intr_cnt); 1246d39a76e7Sxw } 1247d39a76e7Sxw 1248d39a76e7Sxw /* 1249d39a76e7Sxw * Allocates both RX and TX resources and configures the SGE. However, 1250d39a76e7Sxw * the hardware is not enabled yet. 1251d39a76e7Sxw * 1252d39a76e7Sxw * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic. 1253d39a76e7Sxw * jumbo_fl is set to the index of the freelist containing the jumbo buffers. 1254d39a76e7Sxw */ 1255d39a76e7Sxw int 1256d39a76e7Sxw t1_sge_configure(pesge *sge, struct sge_params *p) 1257d39a76e7Sxw { 1258d39a76e7Sxw sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET; 1259d39a76e7Sxw sge->jumbo_fl = t1_is_T1B(sge->obj) ? 1 : 0; 1260d39a76e7Sxw /* if we're a T2 card, then we have hardware offset support */ 1261d39a76e7Sxw sge->rx_offset = t1_is_T1B(sge->obj) ? SGE_RX_OFFSET: 0; 1262d39a76e7Sxw 1263d39a76e7Sxw if (alloc_rx_resources(sge, p)) 1264d39a76e7Sxw return (-ENOMEM); 1265d39a76e7Sxw if (alloc_tx_resources(sge, p)) { 1266d39a76e7Sxw free_rx_resources(sge); 1267d39a76e7Sxw return (-ENOMEM); 1268d39a76e7Sxw } 1269d39a76e7Sxw configure_sge(sge, p); 1270d39a76e7Sxw 1271d39a76e7Sxw /* 1272d39a76e7Sxw * Now that we have sized the free lists calculate the payload 1273d39a76e7Sxw * capacity of the large buffers. Other parts of the driver use 1274d39a76e7Sxw * this to set the max offload coalescing size so that RX packets 1275d39a76e7Sxw * do not overflow our large buffers. 1276d39a76e7Sxw */ 1277d39a76e7Sxw p->large_buf_capacity = jumbo_payload_capacity(sge); 1278d39a76e7Sxw return (0); 1279d39a76e7Sxw } 1280d39a76e7Sxw 1281d39a76e7Sxw /* 1282d39a76e7Sxw * Allocates basic RX resources, consisting of memory mapped freelist Qs and a 1283d39a76e7Sxw * response Q. 1284d39a76e7Sxw */ 1285d39a76e7Sxw static int 1286d39a76e7Sxw alloc_rx_resources(pesge *sge, struct sge_params *p) 1287d39a76e7Sxw { 1288d39a76e7Sxw unsigned int size, i; 1289d39a76e7Sxw 1290d39a76e7Sxw for (i = 0; i < SGE_FREELQ_N; i++) { 1291d39a76e7Sxw struct freelQ *Q = &sge->freelQ[i]; 1292d39a76e7Sxw 1293d39a76e7Sxw Q->fq_id = i; 1294d39a76e7Sxw Q->fq_genbit = 1; 1295d39a76e7Sxw Q->fq_entries_n = p->freelQ_size[i]; 1296d39a76e7Sxw #ifdef HOST_PAUSE 1297d39a76e7Sxw Q->fq_pause_on_thresh = flq_pause_window; 1298d39a76e7Sxw Q->fq_pause_off_thresh = Q->fq_entries_n >> 1; 1299d39a76e7Sxw #endif 1300d39a76e7Sxw size = sizeof (freelQ_e) * Q->fq_entries_n; 1301d39a76e7Sxw 1302d39a76e7Sxw Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1303d39a76e7Sxw size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT); 1304d39a76e7Sxw 1305d39a76e7Sxw 1306d39a76e7Sxw if (!Q->fq_entries) 1307d39a76e7Sxw goto err_no_mem; 1308d39a76e7Sxw memset(Q->fq_entries, 0, size); 1309d39a76e7Sxw size = sizeof (struct freelQ_ce) * Q->fq_entries_n; 1310d39a76e7Sxw Q->fq_centries = t1_os_malloc_wait_zero(size); 1311d39a76e7Sxw if (!Q->fq_centries) 1312d39a76e7Sxw goto err_no_mem; 1313d39a76e7Sxw memset(Q->fq_centries, 0, size); 1314d39a76e7Sxw } 1315d39a76e7Sxw 1316d39a76e7Sxw /* 1317d39a76e7Sxw * Calculate the buffer sizes for the two free lists. FL0 accommodates 1318d39a76e7Sxw * regular sized Ethernet frames, FL1 is sized not to exceed 16K, 1319d39a76e7Sxw * including all the sk_buff overhead. 1320d39a76e7Sxw * For T1C FL0 and FL1 are reversed. 1321d39a76e7Sxw */ 1322d39a76e7Sxw #ifdef NOTYET 1323d39a76e7Sxw sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE + 1324d39a76e7Sxw sizeof (struct cpl_rx_data) + 1325d39a76e7Sxw SGE_RX_OFFSET - sge->rx_pkt_pad; 1326d39a76e7Sxw #else 1327d39a76e7Sxw sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = 1328d39a76e7Sxw sge->obj->ch_sm_buf_sz; 1329d39a76e7Sxw if (is_T2(sge->obj)) 1330d39a76e7Sxw sge->intr_cnt.rx_flq1_sz = sge->obj->ch_sm_buf_sz; 1331d39a76e7Sxw else 1332d39a76e7Sxw sge->intr_cnt.rx_flq0_sz = sge->obj->ch_sm_buf_sz; 1333d39a76e7Sxw #endif 1334d39a76e7Sxw #ifdef NOTYET 1335d39a76e7Sxw sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) - 1336d39a76e7Sxw SKB_DATA_ALIGN(sizeof (struct skb_shared_info)); 1337d39a76e7Sxw #else 1338d39a76e7Sxw sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz; 1339d39a76e7Sxw if (is_T2(sge->obj)) 1340d39a76e7Sxw sge->intr_cnt.rx_flq0_sz = sge->obj->ch_bg_buf_sz; 1341d39a76e7Sxw else 1342d39a76e7Sxw sge->intr_cnt.rx_flq1_sz = sge->obj->ch_bg_buf_sz; 1343d39a76e7Sxw #endif 1344d39a76e7Sxw 1345d39a76e7Sxw sge->respQ.rq_genbit = 1; 1346d39a76e7Sxw sge->respQ.rq_entries_n = sge_respq_cnt; 1347d39a76e7Sxw sge->respQ.rq_credits = sge_respq_cnt; 1348d39a76e7Sxw sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2); 1349d39a76e7Sxw size = sizeof (respQ_e) * sge->respQ.rq_entries_n; 1350d39a76e7Sxw 1351d39a76e7Sxw sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1352d39a76e7Sxw size, &(sge->respQ.rq_pa), &(sge->respQ.rq_dh), 1353d39a76e7Sxw &(sge->respQ.rq_ah), 0); 1354d39a76e7Sxw 1355d39a76e7Sxw if (!sge->respQ.rq_entries) 1356d39a76e7Sxw goto err_no_mem; 1357d39a76e7Sxw memset(sge->respQ.rq_entries, 0, size); 1358d39a76e7Sxw return (0); 1359d39a76e7Sxw 1360d39a76e7Sxw err_no_mem: 1361d39a76e7Sxw free_rx_resources(sge); 1362d39a76e7Sxw return (1); 1363d39a76e7Sxw } 1364d39a76e7Sxw 1365d39a76e7Sxw /* 1366d39a76e7Sxw * Allocates basic TX resources, consisting of memory mapped command Qs. 1367d39a76e7Sxw */ 1368d39a76e7Sxw static int 1369d39a76e7Sxw alloc_tx_resources(pesge *sge, struct sge_params *p) 1370d39a76e7Sxw { 1371d39a76e7Sxw unsigned int size, i; 1372d39a76e7Sxw 1373d39a76e7Sxw for (i = 0; i < SGE_CMDQ_N; i++) { 1374d39a76e7Sxw struct cmdQ *Q = &sge->cmdQ[i]; 1375d39a76e7Sxw 1376d39a76e7Sxw Q->cq_genbit = 1; 1377d39a76e7Sxw Q->cq_entries_n = p->cmdQ_size[i]; 1378d39a76e7Sxw atomic_set(&Q->cq_credits, Q->cq_entries_n); 1379d39a76e7Sxw atomic_set(&Q->cq_asleep, 1); 1380d39a76e7Sxw 1381d39a76e7Sxw mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER, 1382d39a76e7Sxw sge->obj->ch_icookp); 1383d39a76e7Sxw 1384d39a76e7Sxw size = sizeof (cmdQ_e) * Q->cq_entries_n; 1385d39a76e7Sxw Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1386d39a76e7Sxw size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT); 1387d39a76e7Sxw 1388d39a76e7Sxw if (!Q->cq_entries) 1389d39a76e7Sxw goto err_no_mem; 1390d39a76e7Sxw memset(Q->cq_entries, 0, size); 1391d39a76e7Sxw size = sizeof (struct cmdQ_ce) * Q->cq_entries_n; 1392d39a76e7Sxw Q->cq_centries = t1_os_malloc_wait_zero(size); 1393d39a76e7Sxw if (!Q->cq_centries) 1394d39a76e7Sxw goto err_no_mem; 1395d39a76e7Sxw memset(Q->cq_centries, 0, size); 1396d39a76e7Sxw 1397d39a76e7Sxw /* allocate pre-mapped dma headers */ 1398d39a76e7Sxw pe_dma_handle_init(sge->obj, Q->cq_entries_n); 1399d39a76e7Sxw } 1400d39a76e7Sxw 1401d39a76e7Sxw return (0); 1402d39a76e7Sxw 1403d39a76e7Sxw err_no_mem: 1404d39a76e7Sxw free_tx_resources(sge); 1405d39a76e7Sxw return (1); 1406d39a76e7Sxw } 1407d39a76e7Sxw 1408d39a76e7Sxw /* 1409d39a76e7Sxw * Sets the interrupt latency timer when the adaptive Rx coalescing 1410d39a76e7Sxw * is turned off. Do nothing when it is turned on again. 1411d39a76e7Sxw * 1412d39a76e7Sxw * This routine relies on the fact that the caller has already set 1413d39a76e7Sxw * the adaptive policy in adapter->sge_params before calling it. 1414d39a76e7Sxw */ 1415d39a76e7Sxw int 1416d39a76e7Sxw t1_sge_set_coalesce_params(pesge *sge, struct sge_params *p) 1417d39a76e7Sxw { 1418d39a76e7Sxw if (!p->coalesce_enable) { 1419d39a76e7Sxw u32 newTimer = p->rx_coalesce_usecs * 1420d39a76e7Sxw (board_info(sge->obj)->clock_core / 1000000); 1421d39a76e7Sxw 1422d39a76e7Sxw t1_write_reg_4(sge->obj, A_SG_INTRTIMER, newTimer); 1423d39a76e7Sxw } 1424d39a76e7Sxw return (0); 1425d39a76e7Sxw } 1426d39a76e7Sxw 1427d39a76e7Sxw /* 1428d39a76e7Sxw * Programs the various SGE registers. However, the engine is not yet enabled, 1429d39a76e7Sxw * but sge->sge_control is setup and ready to go. 1430d39a76e7Sxw */ 1431d39a76e7Sxw static void 1432d39a76e7Sxw configure_sge(pesge *sge, struct sge_params *p) 1433d39a76e7Sxw { 1434d39a76e7Sxw ch_t *ap = sge->obj; 1435d39a76e7Sxw int i; 1436d39a76e7Sxw 1437d39a76e7Sxw t1_write_reg_4(ap, A_SG_CONTROL, 0); 1438d39a76e7Sxw 1439d39a76e7Sxw setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n, 1440d39a76e7Sxw A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE); 1441d39a76e7Sxw setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n, 1442d39a76e7Sxw A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE); 1443d39a76e7Sxw setup_ring_params(ap, sge->freelQ[0].fq_pa, 1444d39a76e7Sxw sge->freelQ[0].fq_entries_n, A_SG_FL0BASELWR, 1445d39a76e7Sxw A_SG_FL0BASEUPR, A_SG_FL0SIZE); 1446d39a76e7Sxw setup_ring_params(ap, sge->freelQ[1].fq_pa, 1447d39a76e7Sxw sge->freelQ[1].fq_entries_n, A_SG_FL1BASELWR, 1448d39a76e7Sxw A_SG_FL1BASEUPR, A_SG_FL1SIZE); 1449d39a76e7Sxw 1450d39a76e7Sxw /* The threshold comparison uses <. */ 1451d39a76e7Sxw t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) - 1452d39a76e7Sxw SZ_CPL_RX_PKT - sge->rx_pkt_pad - sge->rx_offset + 1); 1453d39a76e7Sxw setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n, 1454d39a76e7Sxw A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE); 1455d39a76e7Sxw t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n); 1456d39a76e7Sxw sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE | 1457d39a76e7Sxw F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE | 1458d39a76e7Sxw V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE | 1459d39a76e7Sxw #if 1 1460d39a76e7Sxw /* 1461d39a76e7Sxw * if the the following bit is not set, then we'll get an 1462d39a76e7Sxw * interrupt everytime command Q 0 goes empty. Since we're 1463d39a76e7Sxw * always ringing the doorbell, we can turn it on. 1464d39a76e7Sxw */ 1465d39a76e7Sxw F_DISABLE_CMDQ0_GTS | 1466d39a76e7Sxw #endif 1467d39a76e7Sxw V_RX_PKT_OFFSET(sge->rx_pkt_pad); 1468d39a76e7Sxw 1469d39a76e7Sxw #if BYTE_ORDER == BIG_ENDIAN 1470d39a76e7Sxw sge->sge_control |= F_ENABLE_BIG_ENDIAN; 1471d39a76e7Sxw #endif 1472d39a76e7Sxw 1473d39a76e7Sxw /* 1474d39a76e7Sxw * Initialize the SGE Interrupt Timer arrray: 1475d39a76e7Sxw * intrtimer[0] = (SGE_INTRTIMER0) usec 1476d39a76e7Sxw * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec 1477d39a76e7Sxw * intrtimer[10] = (SGE_INTRTIMER1) usec 1478d39a76e7Sxw * 1479d39a76e7Sxw */ 1480d39a76e7Sxw sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000; 1481d39a76e7Sxw for (i = 1; i < SGE_INTR_MAXBUCKETS - 1; ++i) { 1482d39a76e7Sxw sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i); 1483d39a76e7Sxw sge->intrtimer[i] *= sge->intrtimer[0]; 1484d39a76e7Sxw } 1485d39a76e7Sxw sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] = 1486d39a76e7Sxw sge->intrtimer[0] * SGE_INTRTIMER1; 1487d39a76e7Sxw /* Initialize resource timer */ 1488d39a76e7Sxw sge->intrtimer_nres = (uint32_t)(sge->intrtimer[0] * 1489d39a76e7Sxw SGE_INTRTIMER_NRES); 1490d39a76e7Sxw /* Finally finish initialization of intrtimer[0] */ 1491d39a76e7Sxw sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0); 1492d39a76e7Sxw /* Initialize for a throughput oriented workload */ 1493d39a76e7Sxw sge->currIndex = SGE_INTR_MAXBUCKETS - 1; 1494d39a76e7Sxw 1495d39a76e7Sxw if (p->coalesce_enable) 1496d39a76e7Sxw t1_write_reg_4(ap, A_SG_INTRTIMER, 1497d39a76e7Sxw sge->intrtimer[sge->currIndex]); 1498d39a76e7Sxw else 1499d39a76e7Sxw (void) t1_sge_set_coalesce_params(sge, p); 1500d39a76e7Sxw } 1501d39a76e7Sxw 1502d39a76e7Sxw static inline void 1503d39a76e7Sxw setup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo, 1504d39a76e7Sxw int base_reg_hi, int size_reg) 1505d39a76e7Sxw { 1506d39a76e7Sxw t1_write_reg_4(adapter, base_reg_lo, (u32)addr); 1507d39a76e7Sxw t1_write_reg_4(adapter, base_reg_hi, addr >> 32); 1508d39a76e7Sxw t1_write_reg_4(adapter, size_reg, size); 1509d39a76e7Sxw } 1510d39a76e7Sxw 1511d39a76e7Sxw /* 1512d39a76e7Sxw * Frees RX resources. 1513d39a76e7Sxw */ 1514d39a76e7Sxw static void 1515d39a76e7Sxw free_rx_resources(pesge *sge) 1516d39a76e7Sxw { 1517d39a76e7Sxw unsigned int size, i; 1518d39a76e7Sxw 1519d39a76e7Sxw if (sge->respQ.rq_entries) { 1520d39a76e7Sxw size = sizeof (respQ_e) * sge->respQ.rq_entries_n; 1521d39a76e7Sxw 1522d39a76e7Sxw pe_os_free_contig(sge->obj, size, sge->respQ.rq_entries, 1523d39a76e7Sxw sge->respQ.rq_pa, sge->respQ.rq_dh, sge->respQ.rq_ah); 1524d39a76e7Sxw } 1525d39a76e7Sxw 1526d39a76e7Sxw for (i = 0; i < SGE_FREELQ_N; i++) { 1527d39a76e7Sxw struct freelQ *Q = &sge->freelQ[i]; 1528d39a76e7Sxw 1529d39a76e7Sxw if (Q->fq_centries) { 1530d39a76e7Sxw free_freelQ_buffers(sge, Q); 1531d39a76e7Sxw 1532d39a76e7Sxw t1_os_free(Q->fq_centries, 1533d39a76e7Sxw Q->fq_entries_n * sizeof (freelQ_ce_t)); 1534d39a76e7Sxw } 1535d39a76e7Sxw if (Q->fq_entries) { 1536d39a76e7Sxw size = sizeof (freelQ_e) * Q->fq_entries_n; 1537d39a76e7Sxw 1538d39a76e7Sxw /* free the freelist queue */ 1539d39a76e7Sxw pe_os_free_contig(sge->obj, size, Q->fq_entries, 1540d39a76e7Sxw Q->fq_pa, Q->fq_dh, Q->fq_ah); 1541d39a76e7Sxw 1542d39a76e7Sxw } 1543d39a76e7Sxw } 1544d39a76e7Sxw } 1545d39a76e7Sxw 1546d39a76e7Sxw /* 1547d39a76e7Sxw * Frees all RX buffers on the freelist Q. The caller must make sure that 1548d39a76e7Sxw * the SGE is turned off before calling this function. 1549d39a76e7Sxw */ 1550d39a76e7Sxw static void 1551d39a76e7Sxw free_freelQ_buffers(pesge *sge, struct freelQ *Q) 1552d39a76e7Sxw { 1553d39a76e7Sxw struct freelQ_ce *ce; 1554d39a76e7Sxw struct freelQ_ce *cq = Q->fq_centries; 1555d39a76e7Sxw uint32_t credits = Q->fq_credits; 1556d39a76e7Sxw uint32_t entries_n = Q->fq_entries_n; 1557d39a76e7Sxw uint32_t cidx = Q->fq_cidx; 1558d39a76e7Sxw uint32_t i = Q->fq_id; 1559d39a76e7Sxw 1560d39a76e7Sxw ce = &cq[cidx]; 1561d39a76e7Sxw 1562d39a76e7Sxw credits = entries_n; 1563d39a76e7Sxw while (credits--) { 1564d39a76e7Sxw mblk_t *mp; 1565d39a76e7Sxw if ((mp = ce->fe_mp) != NULL) { 1566d39a76e7Sxw /* bump in-use count of receive buffers */ 1567d39a76e7Sxw if (i != sge->jumbo_fl) { 1568d39a76e7Sxw atomic_add(1, 1569d39a76e7Sxw &buffers_in_use[sge->obj->ch_sm_index]); 1570d39a76e7Sxw } else { 1571d39a76e7Sxw atomic_add(1, 1572d39a76e7Sxw &buffers_in_use[sge->obj->ch_big_index]); 1573d39a76e7Sxw } 1574d39a76e7Sxw 1575d39a76e7Sxw /* 1576d39a76e7Sxw * note. freeb() callback of esb-alloced mblk will 1577d39a76e7Sxw * cause receive buffer to be put back on sa free list. 1578d39a76e7Sxw */ 1579d39a76e7Sxw freeb(mp); 1580d39a76e7Sxw ce->fe_mp = NULL; 1581d39a76e7Sxw } 1582d39a76e7Sxw 1583d39a76e7Sxw ce++; 1584d39a76e7Sxw if (++cidx == entries_n) { 1585d39a76e7Sxw cidx = 0; 1586d39a76e7Sxw ce = cq; 1587d39a76e7Sxw } 1588d39a76e7Sxw } 1589d39a76e7Sxw 1590d39a76e7Sxw Q->fq_cidx = cidx; 1591d39a76e7Sxw Q->fq_credits = credits; 1592d39a76e7Sxw } 1593d39a76e7Sxw 1594d39a76e7Sxw /* 1595d39a76e7Sxw * Free TX resources. 1596d39a76e7Sxw * 1597d39a76e7Sxw * Assumes that SGE is stopped and all interrupts are disabled. 1598d39a76e7Sxw */ 1599d39a76e7Sxw static void 1600d39a76e7Sxw free_tx_resources(pesge *sge) 1601d39a76e7Sxw { 1602d39a76e7Sxw unsigned int size; 1603d39a76e7Sxw uint32_t i; 1604d39a76e7Sxw 1605d39a76e7Sxw for (i = 0; i < SGE_CMDQ_N; i++) { 1606d39a76e7Sxw struct cmdQ *Q = &sge->cmdQ[i]; 1607d39a76e7Sxw 1608d39a76e7Sxw if (Q->cq_centries) { 1609d39a76e7Sxw unsigned int pending = Q->cq_entries_n - 1610d39a76e7Sxw atomic_read(Q->cq_credits); 1611d39a76e7Sxw 1612d39a76e7Sxw mutex_destroy(&Q->cq_qlock); 1613d39a76e7Sxw 1614d39a76e7Sxw if (pending) 1615d39a76e7Sxw free_cmdQ_buffers(sge, Q, pending); 1616d39a76e7Sxw 1617d39a76e7Sxw size = sizeof (struct cmdQ_ce) * Q->cq_entries_n; 1618d39a76e7Sxw t1_os_free(Q->cq_centries, size); 1619d39a76e7Sxw } 1620d39a76e7Sxw 1621d39a76e7Sxw if (Q->cq_entries) { 1622d39a76e7Sxw size = sizeof (cmdQ_e) * Q->cq_entries_n; 1623d39a76e7Sxw pe_os_free_contig(sge->obj, size, Q->cq_entries, 1624d39a76e7Sxw Q->cq_pa, Q->cq_dh, Q->cq_ah); 1625d39a76e7Sxw } 1626d39a76e7Sxw } 1627d39a76e7Sxw } 1628d39a76e7Sxw 1629d39a76e7Sxw /* 1630d39a76e7Sxw * Return the payload capacity of the jumbo free-list buffers. 1631d39a76e7Sxw */ 1632d39a76e7Sxw static inline unsigned int jumbo_payload_capacity(pesge *sge) 1633d39a76e7Sxw { 1634d39a76e7Sxw return (sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size - 1635d39a76e7Sxw sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset); 1636d39a76e7Sxw } 1637d39a76e7Sxw 1638d39a76e7Sxw /* PR2928 & PR3309 */ 1639d39a76e7Sxw void 1640d39a76e7Sxw t1_sge_set_ptimeout(adapter_t *adapter, u32 val) 1641d39a76e7Sxw { 1642d39a76e7Sxw pesge *sge = adapter->sge; 1643d39a76e7Sxw 1644d39a76e7Sxw if (is_T2(adapter)) 1645d39a76e7Sxw sge->ptimeout = max(val, 1); 1646d39a76e7Sxw } 1647d39a76e7Sxw 1648d39a76e7Sxw /* PR2928 & PR3309 */ 1649d39a76e7Sxw u32 1650d39a76e7Sxw t1_sge_get_ptimeout(adapter_t *adapter) 1651d39a76e7Sxw { 1652d39a76e7Sxw pesge *sge = adapter->sge; 1653d39a76e7Sxw 1654d39a76e7Sxw return (is_T2(adapter) ? sge->ptimeout : 0); 1655d39a76e7Sxw } 1656d39a76e7Sxw 1657d39a76e7Sxw void 1658d39a76e7Sxw sge_add_fake_arp(pesge *sge, void *bp) 1659d39a76e7Sxw { 1660d39a76e7Sxw sge->pskb = bp; 1661d39a76e7Sxw } 1662d39a76e7Sxw 1663d39a76e7Sxw #ifdef SUN_KSTATS 1664d39a76e7Sxw static int 1665d39a76e7Sxw sge_kstat_setup(pesge *sge) 1666d39a76e7Sxw { 1667d39a76e7Sxw int status; 1668d39a76e7Sxw p_kstat_t ksp; 1669d39a76e7Sxw size_t ch_kstat_sz; 1670d39a76e7Sxw p_ch_kstat_t chkp; 1671d39a76e7Sxw char kstat_name[32]; 1672d39a76e7Sxw int instance; 1673d39a76e7Sxw int i; 1674d39a76e7Sxw 1675d39a76e7Sxw status = -1; 1676d39a76e7Sxw ch_kstat_sz = sizeof (ch_kstat_t); 1677d39a76e7Sxw instance = ddi_get_instance(sge->obj->ch_dip); 1678d39a76e7Sxw if ((ksp = kstat_create(CHNAME "_debug", instance, 1679d39a76e7Sxw NULL, "net_debug", KSTAT_TYPE_NAMED, 1680d39a76e7Sxw ch_kstat_sz / sizeof (kstat_named_t), 0)) == NULL) 1681d39a76e7Sxw goto sge_kstat_setup_exit; 1682d39a76e7Sxw chkp = (p_ch_kstat_t)ksp->ks_data; 1683d39a76e7Sxw kstat_named_init(&chkp->respQ_empty, "respQ_empty", 1684d39a76e7Sxw KSTAT_DATA_UINT32); 1685d39a76e7Sxw kstat_named_init(&chkp->respQ_overflow, "respQ_overflow", 1686d39a76e7Sxw KSTAT_DATA_UINT32); 1687d39a76e7Sxw kstat_named_init(&chkp->freelistQ_empty, "freelistQ_empty", 1688d39a76e7Sxw KSTAT_DATA_UINT32); 1689d39a76e7Sxw kstat_named_init(&chkp->pkt_too_big, "pkt_too_big", 1690d39a76e7Sxw KSTAT_DATA_UINT32); 1691d39a76e7Sxw kstat_named_init(&chkp->pkt_mismatch, "pkt_mismatch", 1692d39a76e7Sxw KSTAT_DATA_UINT32); 1693d39a76e7Sxw kstat_named_init(&chkp->cmdQ_full[0], "cmdQ_full[0]", 1694d39a76e7Sxw KSTAT_DATA_UINT32); 1695d39a76e7Sxw kstat_named_init(&chkp->cmdQ_full[1], "cmdQ_full[1]", 1696d39a76e7Sxw KSTAT_DATA_UINT32); 1697d39a76e7Sxw kstat_named_init(&chkp->tx_reclaims[0], "tx_reclaims[0]", 1698d39a76e7Sxw KSTAT_DATA_UINT32); 1699d39a76e7Sxw kstat_named_init(&chkp->tx_reclaims[1], "tx_reclaims[1]", 1700d39a76e7Sxw KSTAT_DATA_UINT32); 1701d39a76e7Sxw kstat_named_init(&chkp->tx_msg_pullups, "tx_msg_pullups", 1702d39a76e7Sxw KSTAT_DATA_UINT32); 1703d39a76e7Sxw kstat_named_init(&chkp->tx_hdr_pullups, "tx_hdr_pullups", 1704d39a76e7Sxw KSTAT_DATA_UINT32); 1705d39a76e7Sxw kstat_named_init(&chkp->tx_tcp_ip_frag, "tx_tcp_ip_frag", 1706d39a76e7Sxw KSTAT_DATA_UINT32); 1707d39a76e7Sxw kstat_named_init(&chkp->tx_udp_ip_frag, "tx_udp_ip_frag", 1708d39a76e7Sxw KSTAT_DATA_UINT32); 1709d39a76e7Sxw kstat_named_init(&chkp->tx_soft_cksums, "tx_soft_cksums", 1710d39a76e7Sxw KSTAT_DATA_UINT32); 1711d39a76e7Sxw kstat_named_init(&chkp->tx_need_cpl_space, "tx_need_cpl_space", 1712d39a76e7Sxw KSTAT_DATA_UINT32); 1713d39a76e7Sxw kstat_named_init(&chkp->tx_multi_mblks, "tx_multi_mblks", 1714d39a76e7Sxw KSTAT_DATA_UINT32); 1715d39a76e7Sxw kstat_named_init(&chkp->tx_no_dvma1, "tx_num_multi_dvma_fails", 1716d39a76e7Sxw KSTAT_DATA_UINT32); 1717d39a76e7Sxw kstat_named_init(&chkp->tx_no_dvma2, "tx_num_single_dvma_fails", 1718d39a76e7Sxw KSTAT_DATA_UINT32); 1719d39a76e7Sxw kstat_named_init(&chkp->tx_no_dma1, "tx_num_multi_dma_fails", 1720d39a76e7Sxw KSTAT_DATA_UINT32); 1721d39a76e7Sxw kstat_named_init(&chkp->tx_no_dma2, "tx_num_single_dma_fails", 1722d39a76e7Sxw KSTAT_DATA_UINT32); 1723d39a76e7Sxw kstat_named_init(&chkp->rx_cmdq0, "rx_cmdq0", 1724d39a76e7Sxw KSTAT_DATA_UINT32); 1725d39a76e7Sxw kstat_named_init(&chkp->rx_cmdq1, "rx_cmdq1", 1726d39a76e7Sxw KSTAT_DATA_UINT32); 1727d39a76e7Sxw kstat_named_init(&chkp->rx_flq0, "rx_flq0", 1728d39a76e7Sxw KSTAT_DATA_UINT32); 1729d39a76e7Sxw kstat_named_init(&chkp->rx_flq1, "rx_flq1", 1730d39a76e7Sxw KSTAT_DATA_UINT32); 1731d39a76e7Sxw kstat_named_init(&chkp->rx_flq0_sz, "rx_flq0_buffer_sz", 1732d39a76e7Sxw KSTAT_DATA_UINT32); 1733d39a76e7Sxw kstat_named_init(&chkp->rx_flq1_sz, "rx_flq1_buffer_sz", 1734d39a76e7Sxw KSTAT_DATA_UINT32); 1735d39a76e7Sxw kstat_named_init(&chkp->rx_pkt_drops, "rx_pkt_drops", 1736d39a76e7Sxw KSTAT_DATA_UINT32); 1737d39a76e7Sxw kstat_named_init(&chkp->rx_pkt_copied, "rx_pkt_copied", 1738d39a76e7Sxw KSTAT_DATA_UINT32); 1739d39a76e7Sxw kstat_named_init(&chkp->rx_pause_on, "rx_pause_on", 1740d39a76e7Sxw KSTAT_DATA_UINT32); 1741d39a76e7Sxw kstat_named_init(&chkp->rx_pause_off, "rx_pause_off", 1742d39a76e7Sxw KSTAT_DATA_UINT32); 1743d39a76e7Sxw kstat_named_init(&chkp->rx_pause_ms, "rx_pause_ms", 1744d39a76e7Sxw KSTAT_DATA_UINT32); 1745d39a76e7Sxw kstat_named_init(&chkp->rx_pause_spike, "rx_pause_spike", 1746d39a76e7Sxw KSTAT_DATA_UINT32); 1747d39a76e7Sxw kstat_named_init(&chkp->rx_fl_credits, "rx_fl_credits", 1748d39a76e7Sxw KSTAT_DATA_UINT32); 1749d39a76e7Sxw kstat_named_init(&chkp->rx_flbuf_fails, "rx_flbuf_fails", 1750d39a76e7Sxw KSTAT_DATA_UINT32); 1751d39a76e7Sxw kstat_named_init(&chkp->rx_flbuf_allocs, "rx_flbuf_allocs", 1752d39a76e7Sxw KSTAT_DATA_UINT32); 1753d39a76e7Sxw kstat_named_init(&chkp->rx_badEopSop, "rx_badEopSop", 1754d39a76e7Sxw KSTAT_DATA_UINT32); 1755d39a76e7Sxw kstat_named_init(&chkp->rx_flq0_cnt, "rx_flq0_cnt", 1756d39a76e7Sxw KSTAT_DATA_UINT32); 1757d39a76e7Sxw kstat_named_init(&chkp->rx_flq1_cnt, "rx_flq1_cnt", 1758d39a76e7Sxw KSTAT_DATA_UINT32); 1759d39a76e7Sxw kstat_named_init(&chkp->arp_sent, "arp_sent", 1760d39a76e7Sxw KSTAT_DATA_UINT32); 1761d39a76e7Sxw kstat_named_init(&chkp->tx_doorbells, "tx_doorbells", 1762d39a76e7Sxw KSTAT_DATA_UINT32); 1763d39a76e7Sxw kstat_named_init(&chkp->intr_doorbells, "intr_doorbells", 1764d39a76e7Sxw KSTAT_DATA_UINT32); 1765d39a76e7Sxw kstat_named_init(&chkp->intr1_doorbells, "intr1_doorbells", 1766d39a76e7Sxw KSTAT_DATA_UINT32); 1767d39a76e7Sxw kstat_named_init(&chkp->sleep_cnt, "sleep_cnt", 1768d39a76e7Sxw KSTAT_DATA_UINT32); 1769d39a76e7Sxw kstat_named_init(&chkp->pe_allocb_cnt, "pe_allocb_cnt", 1770d39a76e7Sxw KSTAT_DATA_UINT32); 1771d39a76e7Sxw for (i = 0; i < MBLK_MAX; i++) { 1772d39a76e7Sxw (void) sprintf(kstat_name, "tx_descs[%02d]", i); 1773d39a76e7Sxw kstat_named_init(&chkp->tx_descs[i], 1774d39a76e7Sxw kstat_name, KSTAT_DATA_UINT32); 1775d39a76e7Sxw } 1776d39a76e7Sxw ksp->ks_update = sge_kstat_update; 1777d39a76e7Sxw ksp->ks_private = (void *)sge; 1778d39a76e7Sxw sge->ksp = ksp; 1779d39a76e7Sxw kstat_install(ksp); 1780d39a76e7Sxw status = 0; 1781d39a76e7Sxw 1782d39a76e7Sxw sge_kstat_setup_exit: 1783d39a76e7Sxw return (status); 1784d39a76e7Sxw } 1785d39a76e7Sxw 1786d39a76e7Sxw static void 1787d39a76e7Sxw sge_kstat_remove(pesge *sge) 1788d39a76e7Sxw { 1789d39a76e7Sxw if (sge->ksp) 1790d39a76e7Sxw kstat_delete(sge->ksp); 1791d39a76e7Sxw } 1792d39a76e7Sxw 1793d39a76e7Sxw static int 1794d39a76e7Sxw sge_kstat_update(p_kstat_t ksp, int rw) 1795d39a76e7Sxw { 1796d39a76e7Sxw pesge *sge; 1797d39a76e7Sxw p_ch_stats_t statsp; 1798d39a76e7Sxw p_ch_kstat_t chkp; 1799d39a76e7Sxw int i; 1800d39a76e7Sxw 1801d39a76e7Sxw sge = (pesge *)ksp->ks_private; 1802d39a76e7Sxw statsp = (p_ch_stats_t)&sge->intr_cnt; 1803d39a76e7Sxw chkp = (p_ch_kstat_t)ksp->ks_data; 1804d39a76e7Sxw if (rw == KSTAT_WRITE) { 1805d39a76e7Sxw statsp->respQ_empty = chkp->respQ_empty.value.ui32; 1806d39a76e7Sxw statsp->respQ_overflow = chkp->respQ_overflow.value.ui32; 1807d39a76e7Sxw statsp->freelistQ_empty = chkp->freelistQ_empty.value.ui32; 1808d39a76e7Sxw statsp->pkt_too_big = chkp->pkt_too_big.value.ui32; 1809d39a76e7Sxw statsp->pkt_mismatch = chkp->pkt_mismatch.value.ui32; 1810d39a76e7Sxw statsp->cmdQ_full[0] = chkp->cmdQ_full[0].value.ui32; 1811d39a76e7Sxw statsp->cmdQ_full[1] = chkp->cmdQ_full[1].value.ui32; 1812d39a76e7Sxw statsp->tx_reclaims[0] = chkp->tx_reclaims[0].value.ui32; 1813d39a76e7Sxw statsp->tx_reclaims[1] = chkp->tx_reclaims[1].value.ui32; 1814d39a76e7Sxw statsp->tx_msg_pullups = chkp->tx_msg_pullups.value.ui32; 1815d39a76e7Sxw statsp->tx_hdr_pullups = chkp->tx_hdr_pullups.value.ui32; 1816d39a76e7Sxw statsp->tx_tcp_ip_frag = chkp->tx_tcp_ip_frag.value.ui32; 1817d39a76e7Sxw statsp->tx_udp_ip_frag = chkp->tx_udp_ip_frag.value.ui32; 1818d39a76e7Sxw statsp->tx_soft_cksums = chkp->tx_soft_cksums.value.ui32; 1819d39a76e7Sxw statsp->tx_need_cpl_space 1820d39a76e7Sxw = chkp->tx_need_cpl_space.value.ui32; 1821d39a76e7Sxw statsp->tx_multi_mblks = chkp->tx_multi_mblks.value.ui32; 1822d39a76e7Sxw statsp->tx_no_dvma1 = chkp->tx_no_dvma1.value.ui32; 1823d39a76e7Sxw statsp->tx_no_dvma2 = chkp->tx_no_dvma2.value.ui32; 1824d39a76e7Sxw statsp->tx_no_dma1 = chkp->tx_no_dma1.value.ui32; 1825d39a76e7Sxw statsp->tx_no_dma2 = chkp->tx_no_dma2.value.ui32; 1826d39a76e7Sxw statsp->rx_cmdq0 = chkp->rx_cmdq0.value.ui32; 1827d39a76e7Sxw statsp->rx_cmdq1 = chkp->rx_cmdq1.value.ui32; 1828d39a76e7Sxw statsp->rx_flq0 = chkp->rx_flq0.value.ui32; 1829d39a76e7Sxw statsp->rx_flq1 = chkp->rx_flq1.value.ui32; 1830d39a76e7Sxw statsp->rx_flq0_sz = chkp->rx_flq0_sz.value.ui32; 1831d39a76e7Sxw statsp->rx_flq1_sz = chkp->rx_flq1_sz.value.ui32; 1832d39a76e7Sxw statsp->rx_pkt_drops = chkp->rx_pkt_drops.value.ui32; 1833d39a76e7Sxw statsp->rx_pkt_copied = chkp->rx_pkt_copied.value.ui32; 1834d39a76e7Sxw statsp->rx_pause_on = chkp->rx_pause_on.value.ui32; 1835d39a76e7Sxw statsp->rx_pause_off = chkp->rx_pause_off.value.ui32; 1836d39a76e7Sxw statsp->rx_pause_ms = chkp->rx_pause_ms.value.ui32; 1837d39a76e7Sxw statsp->rx_pause_spike = chkp->rx_pause_spike.value.ui32; 1838d39a76e7Sxw statsp->rx_fl_credits = chkp->rx_fl_credits.value.ui32; 1839d39a76e7Sxw statsp->rx_flbuf_fails = chkp->rx_flbuf_fails.value.ui32; 1840d39a76e7Sxw statsp->rx_flbuf_allocs = chkp->rx_flbuf_allocs.value.ui32; 1841d39a76e7Sxw statsp->rx_badEopSop = chkp->rx_badEopSop.value.ui32; 1842d39a76e7Sxw statsp->rx_flq0_cnt = chkp->rx_flq0_cnt.value.ui32; 1843d39a76e7Sxw statsp->rx_flq1_cnt = chkp->rx_flq1_cnt.value.ui32; 1844d39a76e7Sxw statsp->arp_sent = chkp->arp_sent.value.ui32; 1845d39a76e7Sxw statsp->tx_doorbells = chkp->tx_doorbells.value.ui32; 1846d39a76e7Sxw statsp->intr_doorbells = chkp->intr_doorbells.value.ui32; 1847d39a76e7Sxw statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32; 1848d39a76e7Sxw statsp->sleep_cnt = chkp->sleep_cnt.value.ui32; 1849d39a76e7Sxw statsp->pe_allocb_cnt = chkp->pe_allocb_cnt.value.ui32; 1850d39a76e7Sxw for (i = 0; i < MBLK_MAX; i++) { 1851d39a76e7Sxw statsp->tx_descs[i] = chkp->tx_descs[i].value.ui32; 1852d39a76e7Sxw } 1853d39a76e7Sxw } else { 1854d39a76e7Sxw chkp->respQ_empty.value.ui32 = statsp->respQ_empty; 1855d39a76e7Sxw chkp->respQ_overflow.value.ui32 = statsp->respQ_overflow; 1856d39a76e7Sxw chkp->freelistQ_empty.value.ui32 1857d39a76e7Sxw = statsp->freelistQ_empty; 1858d39a76e7Sxw chkp->pkt_too_big.value.ui32 = statsp->pkt_too_big; 1859d39a76e7Sxw chkp->pkt_mismatch.value.ui32 = statsp->pkt_mismatch; 1860d39a76e7Sxw chkp->cmdQ_full[0].value.ui32 = statsp->cmdQ_full[0]; 1861d39a76e7Sxw chkp->cmdQ_full[1].value.ui32 = statsp->cmdQ_full[1]; 1862d39a76e7Sxw chkp->tx_reclaims[0].value.ui32 = statsp->tx_reclaims[0]; 1863d39a76e7Sxw chkp->tx_reclaims[1].value.ui32 = statsp->tx_reclaims[1]; 1864d39a76e7Sxw chkp->tx_msg_pullups.value.ui32 = statsp->tx_msg_pullups; 1865d39a76e7Sxw chkp->tx_hdr_pullups.value.ui32 = statsp->tx_hdr_pullups; 1866d39a76e7Sxw chkp->tx_tcp_ip_frag.value.ui32 = statsp->tx_tcp_ip_frag; 1867d39a76e7Sxw chkp->tx_udp_ip_frag.value.ui32 = statsp->tx_udp_ip_frag; 1868d39a76e7Sxw chkp->tx_soft_cksums.value.ui32 = statsp->tx_soft_cksums; 1869d39a76e7Sxw chkp->tx_need_cpl_space.value.ui32 1870d39a76e7Sxw = statsp->tx_need_cpl_space; 1871d39a76e7Sxw chkp->tx_multi_mblks.value.ui32 = statsp->tx_multi_mblks; 1872d39a76e7Sxw chkp->tx_no_dvma1.value.ui32 = statsp->tx_no_dvma1; 1873d39a76e7Sxw chkp->tx_no_dvma2.value.ui32 = statsp->tx_no_dvma2; 1874d39a76e7Sxw chkp->tx_no_dma1.value.ui32 = statsp->tx_no_dma1; 1875d39a76e7Sxw chkp->tx_no_dma2.value.ui32 = statsp->tx_no_dma2; 1876d39a76e7Sxw chkp->rx_cmdq0.value.ui32 = statsp->rx_cmdq0; 1877d39a76e7Sxw chkp->rx_cmdq1.value.ui32 = statsp->rx_cmdq1; 1878d39a76e7Sxw chkp->rx_flq0.value.ui32 = statsp->rx_flq0; 1879d39a76e7Sxw chkp->rx_flq1.value.ui32 = statsp->rx_flq1; 1880d39a76e7Sxw chkp->rx_flq0_sz.value.ui32 = statsp->rx_flq0_sz; 1881d39a76e7Sxw chkp->rx_flq1_sz.value.ui32 = statsp->rx_flq1_sz; 1882d39a76e7Sxw chkp->rx_pkt_drops.value.ui32 = statsp->rx_pkt_drops; 1883d39a76e7Sxw chkp->rx_pkt_copied.value.ui32 = statsp->rx_pkt_copied; 1884d39a76e7Sxw chkp->rx_pause_on.value.ui32 = statsp->rx_pause_on; 1885d39a76e7Sxw chkp->rx_pause_off.value.ui32 = statsp->rx_pause_off; 1886d39a76e7Sxw chkp->rx_pause_ms.value.ui32 = statsp->rx_pause_ms; 1887d39a76e7Sxw chkp->rx_pause_spike.value.ui32 = statsp->rx_pause_spike; 1888d39a76e7Sxw chkp->rx_fl_credits.value.ui32 = statsp->rx_fl_credits; 1889d39a76e7Sxw chkp->rx_flbuf_fails.value.ui32 1890d39a76e7Sxw = statsp->rx_flbuf_fails; 1891d39a76e7Sxw chkp->rx_flbuf_allocs.value.ui32 1892d39a76e7Sxw = statsp->rx_flbuf_allocs; 1893d39a76e7Sxw chkp->rx_badEopSop.value.ui32 = statsp->rx_badEopSop; 1894d39a76e7Sxw chkp->rx_flq0_cnt.value.ui32 = statsp->rx_flq0_cnt; 1895d39a76e7Sxw chkp->rx_flq1_cnt.value.ui32 = statsp->rx_flq1_cnt; 1896d39a76e7Sxw chkp->arp_sent.value.ui32 = statsp->arp_sent; 1897d39a76e7Sxw chkp->tx_doorbells.value.ui32 = statsp->tx_doorbells; 1898d39a76e7Sxw chkp->intr_doorbells.value.ui32 = statsp->intr_doorbells; 1899d39a76e7Sxw chkp->intr1_doorbells.value.ui32 1900d39a76e7Sxw = statsp->intr1_doorbells; 1901d39a76e7Sxw chkp->sleep_cnt.value.ui32 = statsp->sleep_cnt; 1902d39a76e7Sxw chkp->pe_allocb_cnt.value.ui32 = statsp->pe_allocb_cnt; 1903d39a76e7Sxw for (i = 0; i < MBLK_MAX; i++) { 1904d39a76e7Sxw chkp->tx_descs[i].value.ui32 = statsp->tx_descs[i]; 1905d39a76e7Sxw } 1906d39a76e7Sxw } 1907d39a76e7Sxw return (0); 1908d39a76e7Sxw } 1909d39a76e7Sxw #endif 1910d39a76e7Sxw 1911d39a76e7Sxw static uint16_t 1912d39a76e7Sxw calc_ocsum(mblk_t *mp, int offset) 1913d39a76e7Sxw { 1914d39a76e7Sxw uint8_t *addrp; 1915d39a76e7Sxw uint32_t src; 1916d39a76e7Sxw uint32_t dst; 1917d39a76e7Sxw 1918d39a76e7Sxw ipha_t *ihdr = (ipha_t *)(mp->b_rptr + offset); 1919d39a76e7Sxw uint32_t sum; 1920d39a76e7Sxw int iplen = IPH_HDR_LENGTH(ihdr); 1921d39a76e7Sxw struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen); 1922d39a76e7Sxw uchar_t *byte; 1923d39a76e7Sxw int len; 1924d39a76e7Sxw 1925d39a76e7Sxw addrp = (uint8_t *)&ihdr->ipha_src; 1926d39a76e7Sxw src = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) | 1927d39a76e7Sxw ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]); 1928d39a76e7Sxw 1929d39a76e7Sxw addrp = (uint8_t *)&ihdr->ipha_dst; 1930d39a76e7Sxw dst = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) | 1931d39a76e7Sxw ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]); 1932d39a76e7Sxw 1933d39a76e7Sxw sum = (uint16_t)(src >> 16) + 1934d39a76e7Sxw (uint16_t)(src) + 1935d39a76e7Sxw (uint16_t)(dst >> 16) + 1936d39a76e7Sxw (uint16_t)(dst) + (udpp->uh_ulen + htons(IPPROTO_UDP)); 1937d39a76e7Sxw 1938d39a76e7Sxw sum = (uint16_t)(sum >> 16) + (uint16_t)(sum); 1939d39a76e7Sxw 1940d39a76e7Sxw if (sum > 0xffff) 1941d39a76e7Sxw sum -= 0xffff; 1942d39a76e7Sxw 1943d39a76e7Sxw udpp->uh_sum = 0; 1944d39a76e7Sxw byte = mp->b_rptr + offset + iplen; 1945d39a76e7Sxw do { 1946d39a76e7Sxw len = (mp->b_wptr - byte); 1947d39a76e7Sxw sum = bcksum(byte, len, sum); 1948d39a76e7Sxw if (sum > 0xffff) 1949d39a76e7Sxw sum -= 0xffff; 1950d39a76e7Sxw mp = mp->b_cont; 1951d39a76e7Sxw if (mp) 1952d39a76e7Sxw byte = mp->b_rptr; 1953d39a76e7Sxw } while (mp); 1954d39a76e7Sxw 1955d39a76e7Sxw sum = ~sum & 0xffff; 1956d39a76e7Sxw 1957d39a76e7Sxw return (sum); 1958d39a76e7Sxw } 1959