1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include "nge.h"
28
29 #undef NGE_DBG
30 #define NGE_DBG NGE_DBG_RECV
31
32 #define RXD_END 0x20000000
33 #define RXD_ERR 0x40000000
34 #define RXD_OWN 0x80000000
35 #define RXD_CSUM_MSK 0x1C000000
36 #define RXD_BCNT_MSK 0x00003FFF
37
38 #define RXD_CK8G_NO_HSUM 0x0
39 #define RXD_CK8G_TCP_SUM_ERR 0x04000000
40 #define RXD_CK8G_UDP_SUM_ERR 0x08000000
41 #define RXD_CK8G_IP_HSUM_ERR 0x0C000000
42 #define RXD_CK8G_IP_HSUM 0x10000000
43 #define RXD_CK8G_TCP_SUM 0x14000000
44 #define RXD_CK8G_UDP_SUM 0x18000000
45 #define RXD_CK8G_RESV 0x1C000000
46
47 extern ddi_device_acc_attr_t nge_data_accattr;
48
49 /*
50 * Callback code invoked from STREAMs when the recv data buffer is free for
51 * recycling.
52 *
53 * The following table describes function behaviour:
54 *
55 * | mac stopped | mac running
56 * ---------------------------------------------------
57 * buffer delivered | free buffer | recycle buffer
58 * buffer not delivered | do nothing | recycle buffer (*)
59 *
60 * Note (*):
61 * Recycle buffer only if mac state did not change during execution of
62 * function. Otherwise if mac state changed, set buffer delivered & re-enter
63 * function by calling freemsg().
64 */
65
66 void
nge_recv_recycle(caddr_t arg)67 nge_recv_recycle(caddr_t arg)
68 {
69 boolean_t val;
70 boolean_t valid;
71 nge_t *ngep;
72 dma_area_t *bufp;
73 buff_ring_t *brp;
74 nge_sw_statistics_t *sw_stp;
75
76 bufp = (dma_area_t *)arg;
77 ngep = (nge_t *)bufp->private;
78 brp = ngep->buff;
79 sw_stp = &ngep->statistics.sw_statistics;
80
81 /*
82 * Free the buffer directly if the buffer was allocated
83 * previously or mac was stopped.
84 */
85 if (bufp->signature != brp->buf_sign) {
86 if (bufp->rx_delivered == B_TRUE) {
87 nge_free_dma_mem(bufp);
88 kmem_free(bufp, sizeof (dma_area_t));
89 val = nge_atomic_decrease(&brp->rx_hold, 1);
90 ASSERT(val == B_TRUE);
91 }
92 return;
93 }
94
95 /*
96 * recycle the data buffer again and fill them in free ring
97 */
98 bufp->rx_recycle.free_func = nge_recv_recycle;
99 bufp->rx_recycle.free_arg = (caddr_t)bufp;
100
101 bufp->mp = desballoc(DMA_VPTR(*bufp),
102 ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle);
103
104 if (bufp->mp == NULL) {
105 sw_stp->mp_alloc_err++;
106 sw_stp->recy_free++;
107 nge_free_dma_mem(bufp);
108 kmem_free(bufp, sizeof (dma_area_t));
109 val = nge_atomic_decrease(&brp->rx_hold, 1);
110 ASSERT(val == B_TRUE);
111 } else {
112
113 mutex_enter(brp->recycle_lock);
114 if (bufp->signature != brp->buf_sign)
115 valid = B_TRUE;
116 else
117 valid = B_FALSE;
118 bufp->rx_delivered = valid;
119 if (bufp->rx_delivered == B_FALSE) {
120 bufp->next = brp->recycle_list;
121 brp->recycle_list = bufp;
122 }
123 mutex_exit(brp->recycle_lock);
124 if (valid == B_TRUE)
125 /* call nge_rx_recycle again to free it */
126 freemsg(bufp->mp);
127 else {
128 val = nge_atomic_decrease(&brp->rx_hold, 1);
129 ASSERT(val == B_TRUE);
130 }
131 }
132 }
133
134 /*
135 * Checking the rx's BDs (one or more) to receive
136 * one complete packet.
137 * start_index: the start indexer of BDs for one packet.
138 * end_index: the end indexer of BDs for one packet.
139 */
140 static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len);
141 #pragma inline(nge_recv_packet)
142
143 static mblk_t *
nge_recv_packet(nge_t * ngep,uint32_t start_index,size_t len)144 nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len)
145 {
146 uint8_t *rptr;
147 uint32_t minsize;
148 uint32_t maxsize;
149 mblk_t *mp;
150 buff_ring_t *brp;
151 sw_rx_sbd_t *srbdp;
152 dma_area_t *bufp;
153 nge_sw_statistics_t *sw_stp;
154 void *hw_bd_p;
155
156 brp = ngep->buff;
157 minsize = ETHERMIN;
158 maxsize = ngep->max_sdu;
159 sw_stp = &ngep->statistics.sw_statistics;
160 mp = NULL;
161
162 srbdp = &brp->sw_rbds[start_index];
163 DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL);
164 hw_bd_p = DMA_VPTR(srbdp->desc);
165
166 /*
167 * First check the free_list, if it is NULL,
168 * make the recycle_list be free_list.
169 */
170 if (brp->free_list == NULL) {
171 mutex_enter(brp->recycle_lock);
172 brp->free_list = brp->recycle_list;
173 brp->recycle_list = NULL;
174 mutex_exit(brp->recycle_lock);
175 }
176 bufp = brp->free_list;
177 /* If it's not a qualified packet, delete it */
178 if (len > maxsize || len < minsize) {
179 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
180 srbdp->bufp->alength);
181 srbdp->flags = CONTROLER_OWN;
182 return (NULL);
183 }
184
185 /*
186 * If receive packet size is smaller than RX bcopy threshold,
187 * or there is no available buffer in free_list or recycle list,
188 * we use bcopy directly.
189 */
190 if (len <= ngep->param_rxbcopy_threshold || bufp == NULL)
191 brp->rx_bcopy = B_TRUE;
192 else
193 brp->rx_bcopy = B_FALSE;
194
195 if (brp->rx_bcopy) {
196 mp = allocb(len + NGE_HEADROOM, 0);
197 if (mp == NULL) {
198 sw_stp->mp_alloc_err++;
199 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
200 srbdp->bufp->alength);
201 srbdp->flags = CONTROLER_OWN;
202 return (NULL);
203 }
204 rptr = DMA_VPTR(*srbdp->bufp);
205 mp->b_rptr = mp->b_rptr + NGE_HEADROOM;
206 bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len);
207 mp->b_wptr = mp->b_rptr + len;
208 } else {
209 mp = srbdp->bufp->mp;
210 /*
211 * Make sure the packet *contents* 4-byte aligned
212 */
213 mp->b_rptr += NGE_HEADROOM;
214 mp->b_wptr = mp->b_rptr + len;
215 mp->b_next = mp->b_cont = NULL;
216 srbdp->bufp->rx_delivered = B_TRUE;
217 srbdp->bufp = NULL;
218 nge_atomic_increase(&brp->rx_hold, 1);
219
220 /* Fill the buffer from free_list */
221 srbdp->bufp = bufp;
222 brp->free_list = bufp->next;
223 bufp->next = NULL;
224 }
225
226 /* replenish the buffer for hardware descriptor */
227 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
228 srbdp->bufp->alength);
229 srbdp->flags = CONTROLER_OWN;
230 sw_stp->rbytes += len;
231 sw_stp->recv_count++;
232
233 return (mp);
234 }
235
236
237 #define RX_HW_ERR 0x01
238 #define RX_SUM_NO 0x02
239 #define RX_SUM_ERR 0x04
240
241 /*
242 * Statistic the rx's error
243 * and generate a log msg for these.
244 * Note:
245 * RXE, Parity Error, Symbo error, CRC error
246 * have been recored by nvidia's hardware
247 * statistics part (nge_statistics). So it is uncessary to record them by
248 * driver in this place.
249 */
250 static uint32_t
251 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags);
252 #pragma inline(nge_rxsta_handle)
253
254 static uint32_t
nge_rxsta_handle(nge_t * ngep,uint32_t stflag,uint32_t * pflags)255 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags)
256 {
257 uint32_t errors;
258 uint32_t err_flag;
259 nge_sw_statistics_t *sw_stp;
260
261 err_flag = 0;
262 sw_stp = &ngep->statistics.sw_statistics;
263
264 if ((RXD_END & stflag) == 0)
265 return (RX_HW_ERR);
266
267 errors = stflag & RXD_CSUM_MSK;
268 switch (errors) {
269 default:
270 break;
271
272 case RXD_CK8G_TCP_SUM:
273 case RXD_CK8G_UDP_SUM:
274 *pflags |= HCK_IPV4_HDRCKSUM_OK;
275 *pflags |= HCK_FULLCKSUM_OK;
276 break;
277
278 case RXD_CK8G_TCP_SUM_ERR:
279 case RXD_CK8G_UDP_SUM_ERR:
280 sw_stp->tcp_hwsum_err++;
281 *pflags |= HCK_IPV4_HDRCKSUM_OK;
282 break;
283
284 case RXD_CK8G_IP_HSUM:
285 *pflags |= HCK_IPV4_HDRCKSUM_OK;
286 break;
287
288 case RXD_CK8G_NO_HSUM:
289 err_flag |= RX_SUM_NO;
290 break;
291
292 case RXD_CK8G_IP_HSUM_ERR:
293 sw_stp->ip_hwsum_err++;
294 err_flag |= RX_SUM_ERR;
295 break;
296 }
297
298 if ((stflag & RXD_ERR) != 0) {
299
300 err_flag |= RX_HW_ERR;
301 NGE_DEBUG(("Receive desc error, status: 0x%x", stflag));
302 }
303
304 return (err_flag);
305 }
306
307 static mblk_t *
nge_recv_ring(nge_t * ngep)308 nge_recv_ring(nge_t *ngep)
309 {
310 uint32_t stflag;
311 uint32_t flag_err;
312 uint32_t sum_flags;
313 size_t len;
314 uint64_t end_index;
315 uint64_t sync_start;
316 mblk_t *mp;
317 mblk_t **tail;
318 mblk_t *head;
319 recv_ring_t *rrp;
320 buff_ring_t *brp;
321 sw_rx_sbd_t *srbdp;
322 void * hw_bd_p;
323 nge_mode_cntl mode_cntl;
324
325 mp = NULL;
326 head = NULL;
327 tail = &head;
328 rrp = ngep->recv;
329 brp = ngep->buff;
330
331 end_index = sync_start = rrp->prod_index;
332 /* Sync the descriptor for kernel */
333 if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) {
334 (void) ddi_dma_sync(rrp->desc.dma_hdl,
335 sync_start * ngep->desc_attr.rxd_size,
336 ngep->param_recv_max_packet * ngep->desc_attr.rxd_size,
337 DDI_DMA_SYNC_FORKERNEL);
338 } else {
339 (void) ddi_dma_sync(rrp->desc.dma_hdl,
340 sync_start * ngep->desc_attr.rxd_size,
341 0,
342 DDI_DMA_SYNC_FORKERNEL);
343 (void) ddi_dma_sync(rrp->desc.dma_hdl,
344 0,
345 (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) *
346 ngep->desc_attr.rxd_size,
347 DDI_DMA_SYNC_FORKERNEL);
348 }
349
350 /*
351 * Looking through the rx's ring to find the good packets
352 * and try to receive more and more packets in rx's ring
353 */
354 for (;;) {
355 sum_flags = 0;
356 flag_err = 0;
357 end_index = rrp->prod_index;
358 srbdp = &brp->sw_rbds[end_index];
359 hw_bd_p = DMA_VPTR(srbdp->desc);
360 stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len);
361 /*
362 * If there is no packet in receving ring
363 * break the loop
364 */
365 if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags)
366 break;
367
368 ngep->recv_count++;
369 flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags);
370 if ((flag_err & RX_HW_ERR) == 0) {
371 srbdp->flags = NGE_END_PACKET;
372 mp = nge_recv_packet(ngep, end_index, len);
373 } else {
374 /* Hardware error, re-use the buffer */
375 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
376 srbdp->bufp->alength);
377 srbdp->flags = CONTROLER_OWN;
378 }
379 if (mp != NULL) {
380 if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) {
381 mac_hcksum_set(mp, 0, 0, 0, 0, sum_flags);
382 }
383 *tail = mp;
384 tail = &mp->b_next;
385 mp = NULL;
386 }
387 rrp->prod_index = NEXT(end_index, rrp->desc.nslots);
388 if (ngep->recv_count >= ngep->param_recv_max_packet)
389 break;
390 }
391
392 /* Sync the descriptors for device */
393 if (sync_start + ngep->recv_count <= ngep->rx_desc) {
394 (void) ddi_dma_sync(rrp->desc.dma_hdl,
395 sync_start * ngep->desc_attr.rxd_size,
396 ngep->recv_count * ngep->desc_attr.rxd_size,
397 DDI_DMA_SYNC_FORDEV);
398 } else {
399 (void) ddi_dma_sync(rrp->desc.dma_hdl,
400 sync_start * ngep->desc_attr.rxd_size,
401 0,
402 DDI_DMA_SYNC_FORDEV);
403 (void) ddi_dma_sync(rrp->desc.dma_hdl,
404 0,
405 (ngep->recv_count + sync_start - ngep->rx_desc) *
406 ngep->desc_attr.rxd_size,
407 DDI_DMA_SYNC_FORDEV);
408 }
409 mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL);
410 mode_cntl.mode_bits.rxdm = NGE_SET;
411 mode_cntl.mode_bits.tx_rcom_en = NGE_SET;
412 nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val);
413
414 return (head);
415 }
416
417 void
nge_receive(nge_t * ngep)418 nge_receive(nge_t *ngep)
419 {
420 mblk_t *mp;
421 recv_ring_t *rrp;
422 rrp = ngep->recv;
423
424 mp = nge_recv_ring(ngep);
425 mutex_exit(ngep->genlock);
426 if (mp != NULL)
427 mac_rx(ngep->mh, rrp->handle, mp);
428 mutex_enter(ngep->genlock);
429 }
430
431 void
nge_hot_rxd_fill(void * hwd,const ddi_dma_cookie_t * cookie,size_t len)432 nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
433 {
434 uint64_t dmac_addr;
435 hot_rx_bd * hw_bd_p;
436
437 hw_bd_p = (hot_rx_bd *)hwd;
438 dmac_addr = cookie->dmac_laddress + NGE_HEADROOM;
439
440 hw_bd_p->cntl_status.cntl_val = 0;
441
442 hw_bd_p->host_buf_addr_hi = dmac_addr >> 32;
443 hw_bd_p->host_buf_addr_lo = (uint32_t)dmac_addr;
444 hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
445
446 membar_producer();
447 hw_bd_p->cntl_status.control_bits.own = NGE_SET;
448 }
449
450 void
nge_sum_rxd_fill(void * hwd,const ddi_dma_cookie_t * cookie,size_t len)451 nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
452 {
453 sum_rx_bd * hw_bd_p;
454
455 hw_bd_p = hwd;
456
457 hw_bd_p->cntl_status.cntl_val = 0;
458
459 hw_bd_p->host_buf_addr =
460 (uint32_t)(cookie->dmac_address + NGE_HEADROOM);
461 hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
462
463 membar_producer();
464 hw_bd_p->cntl_status.control_bits.own = NGE_SET;
465 }
466
467 uint32_t
nge_hot_rxd_check(const void * hwd,size_t * len)468 nge_hot_rxd_check(const void *hwd, size_t *len)
469 {
470 uint32_t err_flag;
471 const hot_rx_bd * hrbdp;
472
473 hrbdp = hwd;
474 err_flag = hrbdp->cntl_status.cntl_val;
475 *len = err_flag & RXD_BCNT_MSK;
476 return (err_flag);
477 }
478
479 uint32_t
nge_sum_rxd_check(const void * hwd,size_t * len)480 nge_sum_rxd_check(const void *hwd, size_t *len)
481 {
482 uint32_t err_flag;
483 const sum_rx_bd * hrbdp;
484
485 hrbdp = hwd;
486
487 err_flag = hrbdp->cntl_status.cntl_val;
488 *len = err_flag & RXD_BCNT_MSK;
489 return (err_flag);
490 }
491