1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include "nge.h"
28
29#undef	NGE_DBG
30#define	NGE_DBG		NGE_DBG_RECV
31
32#define	RXD_END		0x20000000
33#define	RXD_ERR		0x40000000
34#define	RXD_OWN		0x80000000
35#define	RXD_CSUM_MSK	0x1C000000
36#define	RXD_BCNT_MSK	0x00003FFF
37
38#define	RXD_CK8G_NO_HSUM	0x0
39#define	RXD_CK8G_TCP_SUM_ERR	0x04000000
40#define	RXD_CK8G_UDP_SUM_ERR	0x08000000
41#define	RXD_CK8G_IP_HSUM_ERR	0x0C000000
42#define	RXD_CK8G_IP_HSUM	0x10000000
43#define	RXD_CK8G_TCP_SUM	0x14000000
44#define	RXD_CK8G_UDP_SUM	0x18000000
45#define	RXD_CK8G_RESV		0x1C000000
46
47extern ddi_device_acc_attr_t nge_data_accattr;
48
49/*
50 * Callback code invoked from STREAMs when the recv data buffer is free for
51 * recycling.
52 *
53 * The following table describes function behaviour:
54 *
55 *                      | mac stopped | mac running
56 * ---------------------------------------------------
57 * buffer delivered     | free buffer | recycle buffer
58 * buffer not delivered | do nothing  | recycle buffer (*)
59 *
60 * Note (*):
61 *   Recycle buffer only if mac state did not change during execution of
62 *   function. Otherwise if mac state changed, set buffer delivered & re-enter
63 *   function by calling freemsg().
64 */
65
66void
67nge_recv_recycle(caddr_t arg)
68{
69	boolean_t val;
70	boolean_t valid;
71	nge_t *ngep;
72	dma_area_t *bufp;
73	buff_ring_t *brp;
74	nge_sw_statistics_t *sw_stp;
75
76	bufp = (dma_area_t *)arg;
77	ngep = (nge_t *)bufp->private;
78	brp = ngep->buff;
79	sw_stp = &ngep->statistics.sw_statistics;
80
81	/*
82	 * Free the buffer directly if the buffer was allocated
83	 * previously or mac was stopped.
84	 */
85	if (bufp->signature != brp->buf_sign) {
86		if (bufp->rx_delivered == B_TRUE) {
87			nge_free_dma_mem(bufp);
88			kmem_free(bufp, sizeof (dma_area_t));
89			val = nge_atomic_decrease(&brp->rx_hold, 1);
90			ASSERT(val == B_TRUE);
91		}
92		return;
93	}
94
95	/*
96	 * recycle the data buffer again and fill them in free ring
97	 */
98	bufp->rx_recycle.free_func = nge_recv_recycle;
99	bufp->rx_recycle.free_arg = (caddr_t)bufp;
100
101	bufp->mp = desballoc(DMA_VPTR(*bufp),
102	    ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle);
103
104	if (bufp->mp == NULL) {
105		sw_stp->mp_alloc_err++;
106		sw_stp->recy_free++;
107		nge_free_dma_mem(bufp);
108		kmem_free(bufp, sizeof (dma_area_t));
109		val = nge_atomic_decrease(&brp->rx_hold, 1);
110		ASSERT(val == B_TRUE);
111	} else {
112
113		mutex_enter(brp->recycle_lock);
114		if (bufp->signature != brp->buf_sign)
115			valid = B_TRUE;
116		else
117			valid = B_FALSE;
118		bufp->rx_delivered = valid;
119		if (bufp->rx_delivered == B_FALSE)  {
120			bufp->next = brp->recycle_list;
121			brp->recycle_list = bufp;
122		}
123		mutex_exit(brp->recycle_lock);
124		if (valid == B_TRUE)
125			/* call nge_rx_recycle again to free it */
126			freemsg(bufp->mp);
127		else {
128			val = nge_atomic_decrease(&brp->rx_hold, 1);
129			ASSERT(val == B_TRUE);
130		}
131	}
132}
133
134/*
135 * Checking the rx's BDs (one or more) to receive
136 * one complete packet.
137 * start_index: the start indexer of BDs for one packet.
138 * end_index: the end indexer of BDs for one packet.
139 */
140static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len);
141#pragma	inline(nge_recv_packet)
142
143static mblk_t *
144nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len)
145{
146	uint8_t *rptr;
147	uint32_t minsize;
148	uint32_t maxsize;
149	mblk_t *mp;
150	buff_ring_t *brp;
151	sw_rx_sbd_t *srbdp;
152	dma_area_t *bufp;
153	nge_sw_statistics_t *sw_stp;
154	void *hw_bd_p;
155
156	brp = ngep->buff;
157	minsize = ETHERMIN;
158	maxsize = ngep->max_sdu;
159	sw_stp = &ngep->statistics.sw_statistics;
160	mp = NULL;
161
162	srbdp = &brp->sw_rbds[start_index];
163	DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL);
164	hw_bd_p = DMA_VPTR(srbdp->desc);
165
166	/*
167	 * First check the free_list, if it is NULL,
168	 * make the recycle_list be free_list.
169	 */
170	if (brp->free_list == NULL) {
171		mutex_enter(brp->recycle_lock);
172		brp->free_list = brp->recycle_list;
173		brp->recycle_list = NULL;
174		mutex_exit(brp->recycle_lock);
175	}
176	bufp = brp->free_list;
177	/* If it's not a qualified packet, delete it */
178	if (len > maxsize || len < minsize) {
179		ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
180		    srbdp->bufp->alength);
181		srbdp->flags = CONTROLER_OWN;
182		return (NULL);
183	}
184
185	/*
186	 * If receive packet size is smaller than RX bcopy threshold,
187	 * or there is no available buffer in free_list or recycle list,
188	 * we use bcopy directly.
189	 */
190	if (len <= ngep->param_rxbcopy_threshold || bufp == NULL)
191		brp->rx_bcopy = B_TRUE;
192	else
193		brp->rx_bcopy = B_FALSE;
194
195	if (brp->rx_bcopy) {
196		mp = allocb(len + NGE_HEADROOM, 0);
197		if (mp == NULL) {
198			sw_stp->mp_alloc_err++;
199			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
200			    srbdp->bufp->alength);
201			srbdp->flags = CONTROLER_OWN;
202			return (NULL);
203		}
204		rptr = DMA_VPTR(*srbdp->bufp);
205		mp->b_rptr = mp->b_rptr + NGE_HEADROOM;
206		bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len);
207		mp->b_wptr = mp->b_rptr + len;
208	} else {
209		mp = srbdp->bufp->mp;
210		/*
211		 * Make sure the packet *contents* 4-byte aligned
212		 */
213		mp->b_rptr += NGE_HEADROOM;
214		mp->b_wptr = mp->b_rptr + len;
215		mp->b_next = mp->b_cont = NULL;
216		srbdp->bufp->rx_delivered = B_TRUE;
217		srbdp->bufp = NULL;
218		nge_atomic_increase(&brp->rx_hold, 1);
219
220		/* Fill the buffer from free_list */
221		srbdp->bufp = bufp;
222		brp->free_list = bufp->next;
223		bufp->next = NULL;
224	}
225
226	/* replenish the buffer for hardware descriptor */
227	ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
228	    srbdp->bufp->alength);
229	srbdp->flags = CONTROLER_OWN;
230	sw_stp->rbytes += len;
231	sw_stp->recv_count++;
232
233	return (mp);
234}
235
236
237#define	RX_HW_ERR	0x01
238#define	RX_SUM_NO	0x02
239#define	RX_SUM_ERR	0x04
240
241/*
242 * Statistic the rx's error
243 * and generate a log msg for these.
244 * Note:
245 * RXE, Parity Error, Symbo error, CRC error
246 * have been recored by nvidia's  hardware
247 * statistics part (nge_statistics). So it is uncessary to record them by
248 * driver in this place.
249 */
250static uint32_t
251nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags);
252#pragma	inline(nge_rxsta_handle)
253
254static uint32_t
255nge_rxsta_handle(nge_t *ngep,  uint32_t stflag, uint32_t *pflags)
256{
257	uint32_t errors;
258	uint32_t err_flag;
259	nge_sw_statistics_t *sw_stp;
260
261	err_flag = 0;
262	sw_stp = &ngep->statistics.sw_statistics;
263
264	if ((RXD_END & stflag) == 0)
265		return (RX_HW_ERR);
266
267	errors = stflag & RXD_CSUM_MSK;
268	switch (errors) {
269	default:
270	break;
271
272	case RXD_CK8G_TCP_SUM:
273	case RXD_CK8G_UDP_SUM:
274		*pflags |= HCK_IPV4_HDRCKSUM_OK;
275		*pflags |= HCK_FULLCKSUM_OK;
276		break;
277
278	case RXD_CK8G_TCP_SUM_ERR:
279	case RXD_CK8G_UDP_SUM_ERR:
280		sw_stp->tcp_hwsum_err++;
281		*pflags |= HCK_IPV4_HDRCKSUM_OK;
282		break;
283
284	case RXD_CK8G_IP_HSUM:
285		*pflags |= HCK_IPV4_HDRCKSUM_OK;
286		break;
287
288	case RXD_CK8G_NO_HSUM:
289		err_flag |= RX_SUM_NO;
290		break;
291
292	case RXD_CK8G_IP_HSUM_ERR:
293		sw_stp->ip_hwsum_err++;
294		err_flag |=  RX_SUM_ERR;
295		break;
296	}
297
298	if ((stflag & RXD_ERR) != 0)	{
299
300		err_flag |= RX_HW_ERR;
301		NGE_DEBUG(("Receive desc error, status: 0x%x", stflag));
302	}
303
304	return (err_flag);
305}
306
307static mblk_t *
308nge_recv_ring(nge_t *ngep)
309{
310	uint32_t stflag;
311	uint32_t flag_err;
312	uint32_t sum_flags;
313	size_t len;
314	uint64_t end_index;
315	uint64_t sync_start;
316	mblk_t *mp;
317	mblk_t **tail;
318	mblk_t *head;
319	recv_ring_t *rrp;
320	buff_ring_t *brp;
321	sw_rx_sbd_t *srbdp;
322	void * hw_bd_p;
323	nge_mode_cntl mode_cntl;
324
325	mp = NULL;
326	head = NULL;
327	tail = &head;
328	rrp = ngep->recv;
329	brp = ngep->buff;
330
331	end_index = sync_start = rrp->prod_index;
332	/* Sync the descriptor for kernel */
333	if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) {
334		(void) ddi_dma_sync(rrp->desc.dma_hdl,
335		    sync_start * ngep->desc_attr.rxd_size,
336		    ngep->param_recv_max_packet * ngep->desc_attr.rxd_size,
337		    DDI_DMA_SYNC_FORKERNEL);
338	} else {
339		(void) ddi_dma_sync(rrp->desc.dma_hdl,
340		    sync_start * ngep->desc_attr.rxd_size,
341		    0,
342		    DDI_DMA_SYNC_FORKERNEL);
343		(void) ddi_dma_sync(rrp->desc.dma_hdl,
344		    0,
345		    (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) *
346		    ngep->desc_attr.rxd_size,
347		    DDI_DMA_SYNC_FORKERNEL);
348	}
349
350	/*
351	 * Looking through the rx's ring to find the good packets
352	 * and try to receive more and more packets in rx's ring
353	 */
354	for (;;) {
355		sum_flags = 0;
356		flag_err = 0;
357		end_index = rrp->prod_index;
358		srbdp = &brp->sw_rbds[end_index];
359		hw_bd_p = DMA_VPTR(srbdp->desc);
360		stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len);
361		/*
362		 * If there is no packet in receving ring
363		 * break the loop
364		 */
365		if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags)
366			break;
367
368		ngep->recv_count++;
369		flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags);
370		if ((flag_err & RX_HW_ERR) == 0) {
371			srbdp->flags = NGE_END_PACKET;
372			mp = nge_recv_packet(ngep, end_index, len);
373		} else {
374			/* Hardware error, re-use the buffer */
375			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
376			    srbdp->bufp->alength);
377			srbdp->flags = CONTROLER_OWN;
378		}
379		if (mp != NULL) {
380			if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) {
381				mac_hcksum_set(mp, 0, 0, 0, 0, sum_flags);
382			}
383			*tail = mp;
384			tail = &mp->b_next;
385			mp = NULL;
386		}
387		rrp->prod_index = NEXT(end_index, rrp->desc.nslots);
388		if (ngep->recv_count >= ngep->param_recv_max_packet)
389			break;
390	}
391
392	/* Sync the descriptors for device */
393	if (sync_start + ngep->recv_count <= ngep->rx_desc) {
394		(void) ddi_dma_sync(rrp->desc.dma_hdl,
395		    sync_start * ngep->desc_attr.rxd_size,
396		    ngep->recv_count * ngep->desc_attr.rxd_size,
397		    DDI_DMA_SYNC_FORDEV);
398	} else {
399		(void) ddi_dma_sync(rrp->desc.dma_hdl,
400		    sync_start * ngep->desc_attr.rxd_size,
401		    0,
402		    DDI_DMA_SYNC_FORDEV);
403		(void) ddi_dma_sync(rrp->desc.dma_hdl,
404		    0,
405		    (ngep->recv_count + sync_start - ngep->rx_desc) *
406		    ngep->desc_attr.rxd_size,
407		    DDI_DMA_SYNC_FORDEV);
408	}
409	mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL);
410	mode_cntl.mode_bits.rxdm = NGE_SET;
411	mode_cntl.mode_bits.tx_rcom_en = NGE_SET;
412	nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val);
413
414	return (head);
415}
416
417void
418nge_receive(nge_t *ngep)
419{
420	mblk_t *mp;
421	recv_ring_t *rrp;
422	rrp = ngep->recv;
423
424	mp = nge_recv_ring(ngep);
425	mutex_exit(ngep->genlock);
426	if (mp != NULL)
427		mac_rx(ngep->mh, rrp->handle, mp);
428	mutex_enter(ngep->genlock);
429}
430
431void
432nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
433{
434	uint64_t dmac_addr;
435	hot_rx_bd * hw_bd_p;
436
437	hw_bd_p = (hot_rx_bd *)hwd;
438	dmac_addr = cookie->dmac_laddress + NGE_HEADROOM;
439
440	hw_bd_p->cntl_status.cntl_val = 0;
441
442	hw_bd_p->host_buf_addr_hi = dmac_addr >> 32;
443	hw_bd_p->host_buf_addr_lo = (uint32_t)dmac_addr;
444	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
445
446	membar_producer();
447	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
448}
449
450void
451nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
452{
453	sum_rx_bd * hw_bd_p;
454
455	hw_bd_p = hwd;
456
457	hw_bd_p->cntl_status.cntl_val = 0;
458
459	hw_bd_p->host_buf_addr =
460	    (uint32_t)(cookie->dmac_address + NGE_HEADROOM);
461	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
462
463	membar_producer();
464	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
465}
466
467uint32_t
468nge_hot_rxd_check(const void *hwd, size_t *len)
469{
470	uint32_t err_flag;
471	const hot_rx_bd * hrbdp;
472
473	hrbdp = hwd;
474	err_flag = hrbdp->cntl_status.cntl_val;
475	*len = err_flag & RXD_BCNT_MSK;
476	return (err_flag);
477}
478
479uint32_t
480nge_sum_rxd_check(const void *hwd, size_t *len)
481{
482	uint32_t err_flag;
483	const sum_rx_bd * hrbdp;
484
485	hrbdp = hwd;
486
487	err_flag = hrbdp->cntl_status.cntl_val;
488	*len = err_flag & RXD_BCNT_MSK;
489	return (err_flag);
490}
491