1/*
2 * AMD 10Gb Ethernet driver
3 *
4 * This file is available to you under your choice of the following two
5 * licenses:
6 *
7 * License 1: GPLv2
8 *
9 * Copyright (c) 2014 Advanced Micro Devices, Inc.
10 *
11 * This file is free software; you may copy, redistribute and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation, either version 2 of the License, or (at
14 * your option) any later version.
15 *
16 * This file is distributed in the hope that it will be useful, but
17 * WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
23 *
24 * This file incorporates work covered by the following copyright and
25 * permission notice:
26 *     The Synopsys DWC ETHER XGMAC Software Driver and documentation
27 *     (hereinafter "Software") is an unsupported proprietary work of Synopsys,
28 *     Inc. unless otherwise expressly agreed to in writing between Synopsys
29 *     and you.
30 *
31 *     The Software IS NOT an item of Licensed Software or Licensed Product
32 *     under any End User Software License Agreement or Agreement for Licensed
33 *     Product with Synopsys or any supplement thereto.  Permission is hereby
34 *     granted, free of charge, to any person obtaining a copy of this software
35 *     annotated with this license and the Software, to deal in the Software
36 *     without restriction, including without limitation the rights to use,
37 *     copy, modify, merge, publish, distribute, sublicense, and/or sell copies
38 *     of the Software, and to permit persons to whom the Software is furnished
39 *     to do so, subject to the following conditions:
40 *
41 *     The above copyright notice and this permission notice shall be included
42 *     in all copies or substantial portions of the Software.
43 *
44 *     THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS"
45 *     BASIS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
46 *     TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
47 *     PARTICULAR PURPOSE ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS
48 *     BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
49 *     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
50 *     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
51 *     INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
52 *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
53 *     ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
54 *     THE POSSIBILITY OF SUCH DAMAGE.
55 *
56 *
57 * License 2: Modified BSD
58 *
59 * Copyright (c) 2014 Advanced Micro Devices, Inc.
60 * All rights reserved.
61 *
62 * Redistribution and use in source and binary forms, with or without
63 * modification, are permitted provided that the following conditions are met:
64 *     * Redistributions of source code must retain the above copyright
65 *       notice, this list of conditions and the following disclaimer.
66 *     * Redistributions in binary form must reproduce the above copyright
67 *       notice, this list of conditions and the following disclaimer in the
68 *       documentation and/or other materials provided with the distribution.
69 *     * Neither the name of Advanced Micro Devices, Inc. nor the
70 *       names of its contributors may be used to endorse or promote products
71 *       derived from this software without specific prior written permission.
72 *
73 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
74 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
75 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
76 * ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
77 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
78 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
79 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
80 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
81 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
82 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
83 *
84 * This file incorporates work covered by the following copyright and
85 * permission notice:
86 *     The Synopsys DWC ETHER XGMAC Software Driver and documentation
87 *     (hereinafter "Software") is an unsupported proprietary work of Synopsys,
88 *     Inc. unless otherwise expressly agreed to in writing between Synopsys
89 *     and you.
90 *
91 *     The Software IS NOT an item of Licensed Software or Licensed Product
92 *     under any End User Software License Agreement or Agreement for Licensed
93 *     Product with Synopsys or any supplement thereto.  Permission is hereby
94 *     granted, free of charge, to any person obtaining a copy of this software
95 *     annotated with this license and the Software, to deal in the Software
96 *     without restriction, including without limitation the rights to use,
97 *     copy, modify, merge, publish, distribute, sublicense, and/or sell copies
98 *     of the Software, and to permit persons to whom the Software is furnished
99 *     to do so, subject to the following conditions:
100 *
101 *     The above copyright notice and this permission notice shall be included
102 *     in all copies or substantial portions of the Software.
103 *
104 *     THIS SOFTWARE IS BEING DISTRIBUTED BY SYNOPSYS SOLELY ON AN "AS IS"
105 *     BASIS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
106 *     TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
107 *     PARTICULAR PURPOSE ARE HEREBY DISCLAIMED. IN NO EVENT SHALL SYNOPSYS
108 *     BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
109 *     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
110 *     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
111 *     INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
112 *     CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
113 *     ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
114 *     THE POSSIBILITY OF SUCH DAMAGE.
115 */
116
117#include <sys/cdefs.h>
118__FBSDID("$FreeBSD$");
119
120#include "xgbe.h"
121#include "xgbe-common.h"
122
123static void xgbe_unmap_rdata(struct xgbe_prv_data *, struct xgbe_ring_data *);
124
125static void xgbe_free_ring(struct xgbe_prv_data *pdata,
126			   struct xgbe_ring *ring)
127{
128	struct xgbe_ring_data *rdata;
129	unsigned int i;
130
131	if (!ring)
132		return;
133
134	bus_dmamap_destroy(ring->mbuf_dmat, ring->mbuf_map);
135	bus_dma_tag_destroy(ring->mbuf_dmat);
136
137	ring->mbuf_map = NULL;
138	ring->mbuf_dmat = NULL;
139
140	if (ring->rdata) {
141		for (i = 0; i < ring->rdesc_count; i++) {
142			rdata = XGBE_GET_DESC_DATA(ring, i);
143			xgbe_unmap_rdata(pdata, rdata);
144		}
145
146		free(ring->rdata, M_AXGBE);
147		ring->rdata = NULL;
148	}
149
150	bus_dmamap_unload(ring->rdesc_dmat, ring->rdesc_map);
151	bus_dmamem_free(ring->rdesc_dmat, ring->rdesc, ring->rdesc_map);
152	bus_dma_tag_destroy(ring->rdesc_dmat);
153
154	ring->rdesc_map = NULL;
155	ring->rdesc_dmat = NULL;
156	ring->rdesc = NULL;
157}
158
159static void xgbe_free_ring_resources(struct xgbe_prv_data *pdata)
160{
161	struct xgbe_channel *channel;
162	unsigned int i;
163
164	DBGPR("-->xgbe_free_ring_resources\n");
165
166	channel = pdata->channel;
167	for (i = 0; i < pdata->channel_count; i++, channel++) {
168		xgbe_free_ring(pdata, channel->tx_ring);
169		xgbe_free_ring(pdata, channel->rx_ring);
170	}
171
172	DBGPR("<--xgbe_free_ring_resources\n");
173}
174
175static void xgbe_ring_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg,
176                                int error)
177{
178	if (error)
179		return;
180	*(bus_addr_t *) arg = segs->ds_addr;
181}
182
183static int xgbe_init_ring(struct xgbe_prv_data *pdata,
184			  struct xgbe_ring *ring, unsigned int rdesc_count)
185{
186	bus_size_t len;
187	int err, flags;
188
189	DBGPR("-->xgbe_init_ring\n");
190
191	if (!ring)
192		return 0;
193
194	flags = 0;
195	if (pdata->coherent)
196		flags = BUS_DMA_COHERENT;
197
198	/* Descriptors */
199	ring->rdesc_count = rdesc_count;
200	len = sizeof(struct xgbe_ring_desc) * rdesc_count;
201	err = bus_dma_tag_create(pdata->dmat, 512, 0, BUS_SPACE_MAXADDR,
202	    BUS_SPACE_MAXADDR, NULL, NULL, len, 1, len, flags, NULL, NULL,
203	    &ring->rdesc_dmat);
204	if (err != 0) {
205		printf("Unable to create the DMA tag: %d\n", err);
206		return -err;
207	}
208
209	err = bus_dmamem_alloc(ring->rdesc_dmat, (void **)&ring->rdesc,
210	    BUS_DMA_WAITOK | BUS_DMA_COHERENT, &ring->rdesc_map);
211	if (err != 0) {
212		bus_dma_tag_destroy(ring->rdesc_dmat);
213		printf("Unable to allocate DMA memory: %d\n", err);
214		return -err;
215	}
216	err = bus_dmamap_load(ring->rdesc_dmat, ring->rdesc_map, ring->rdesc,
217	    len, xgbe_ring_dmamap_cb, &ring->rdesc_paddr, 0);
218	if (err != 0) {
219		bus_dmamem_free(ring->rdesc_dmat, ring->rdesc, ring->rdesc_map);
220		bus_dma_tag_destroy(ring->rdesc_dmat);
221		printf("Unable to load DMA memory\n");
222		return -err;
223	}
224
225	/* Descriptor information */
226	ring->rdata = malloc(rdesc_count * sizeof(struct xgbe_ring_data),
227	    M_AXGBE, M_WAITOK | M_ZERO);
228
229	/* Create the space DMA tag for mbufs */
230	err = bus_dma_tag_create(pdata->dmat, 1, 0, BUS_SPACE_MAXADDR,
231	    BUS_SPACE_MAXADDR, NULL, NULL, XGBE_TX_MAX_BUF_SIZE * rdesc_count,
232	    rdesc_count, XGBE_TX_MAX_BUF_SIZE, flags, NULL, NULL,
233	    &ring->mbuf_dmat);
234	if (err != 0)
235		return -err;
236
237	err = bus_dmamap_create(ring->mbuf_dmat, 0, &ring->mbuf_map);
238	if (err != 0)
239		return -err;
240
241	DBGPR("<--xgbe_init_ring\n");
242
243	return 0;
244}
245
246static int xgbe_alloc_ring_resources(struct xgbe_prv_data *pdata)
247{
248	struct xgbe_channel *channel;
249	unsigned int i;
250	int ret;
251
252	DBGPR("-->xgbe_alloc_ring_resources\n");
253
254	channel = pdata->channel;
255	for (i = 0; i < pdata->channel_count; i++, channel++) {
256		ret = xgbe_init_ring(pdata, channel->tx_ring,
257				     pdata->tx_desc_count);
258		if (ret) {
259			printf("error initializing Tx ring\n");
260			goto err_ring;
261		}
262
263		ret = xgbe_init_ring(pdata, channel->rx_ring,
264				     pdata->rx_desc_count);
265		if (ret) {
266			printf("error initializing Rx ring\n");
267			goto err_ring;
268		}
269	}
270
271	DBGPR("<--xgbe_alloc_ring_resources\n");
272
273	return 0;
274
275err_ring:
276	xgbe_free_ring_resources(pdata);
277
278	return ret;
279}
280
281static int xgbe_map_rx_buffer(struct xgbe_prv_data *pdata,
282			      struct xgbe_ring *ring,
283			      struct xgbe_ring_data *rdata)
284{
285	bus_dmamap_t mbuf_map;
286	bus_dma_segment_t segs[2];
287	struct mbuf *m0, *m1;
288	int err, nsegs;
289
290	m0 = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MCLBYTES);
291	if (m0 == NULL)
292		return (-ENOBUFS);
293
294	m1 = m_getjcl(M_NOWAIT, MT_DATA, 0, MCLBYTES);
295	if (m1 == NULL) {
296		m_freem(m0);
297		return (-ENOBUFS);
298	}
299
300	m0->m_next = m1;
301	m0->m_flags |= M_PKTHDR;
302	m0->m_len = MHLEN;
303	m0->m_pkthdr.len = MHLEN + MCLBYTES;
304
305	m1->m_len = MCLBYTES;
306	m1->m_next = NULL;
307	m1->m_pkthdr.len = MCLBYTES;
308
309	err = bus_dmamap_create(ring->mbuf_dmat, 0, &mbuf_map);
310	if (err != 0) {
311		m_freem(m0);
312		return (-err);
313	}
314
315	err = bus_dmamap_load_mbuf_sg(ring->mbuf_dmat, mbuf_map, m0, segs,
316	    &nsegs, BUS_DMA_NOWAIT);
317	if (err != 0) {
318		m_freem(m0);
319		bus_dmamap_destroy(ring->mbuf_dmat, mbuf_map);
320		return (-err);
321	}
322
323	KASSERT(nsegs == 2,
324	    ("xgbe_map_rx_buffer: Unable to handle multiple segments %d",
325	    nsegs));
326
327	rdata->mb = m0;
328	rdata->mbuf_free = 0;
329	rdata->mbuf_dmat = ring->mbuf_dmat;
330	rdata->mbuf_map = mbuf_map;
331	rdata->mbuf_hdr_paddr = segs[0].ds_addr;
332	rdata->mbuf_data_paddr = segs[1].ds_addr;
333
334	return 0;
335}
336
337static void xgbe_wrapper_tx_descriptor_init(struct xgbe_prv_data *pdata)
338{
339	struct xgbe_hw_if *hw_if = &pdata->hw_if;
340	struct xgbe_channel *channel;
341	struct xgbe_ring *ring;
342	struct xgbe_ring_data *rdata;
343	struct xgbe_ring_desc *rdesc;
344	bus_addr_t rdesc_paddr;
345	unsigned int i, j;
346
347	DBGPR("-->xgbe_wrapper_tx_descriptor_init\n");
348
349	channel = pdata->channel;
350	for (i = 0; i < pdata->channel_count; i++, channel++) {
351		ring = channel->tx_ring;
352		if (!ring)
353			break;
354
355		rdesc = ring->rdesc;
356		rdesc_paddr = ring->rdesc_paddr;
357
358		for (j = 0; j < ring->rdesc_count; j++) {
359			rdata = XGBE_GET_DESC_DATA(ring, j);
360
361			rdata->rdesc = rdesc;
362			rdata->rdata_paddr = rdesc_paddr;
363
364			rdesc++;
365			rdesc_paddr += sizeof(struct xgbe_ring_desc);
366		}
367
368		ring->cur = 0;
369		ring->dirty = 0;
370		memset(&ring->tx, 0, sizeof(ring->tx));
371
372		hw_if->tx_desc_init(channel);
373	}
374
375	DBGPR("<--xgbe_wrapper_tx_descriptor_init\n");
376}
377
378static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata)
379{
380	struct xgbe_hw_if *hw_if = &pdata->hw_if;
381	struct xgbe_channel *channel;
382	struct xgbe_ring *ring;
383	struct xgbe_ring_desc *rdesc;
384	struct xgbe_ring_data *rdata;
385	bus_addr_t rdesc_paddr;
386	unsigned int i, j;
387
388	DBGPR("-->xgbe_wrapper_rx_descriptor_init\n");
389
390	channel = pdata->channel;
391	for (i = 0; i < pdata->channel_count; i++, channel++) {
392		ring = channel->rx_ring;
393		if (!ring)
394			break;
395
396		rdesc = ring->rdesc;
397		rdesc_paddr = ring->rdesc_paddr;
398
399		for (j = 0; j < ring->rdesc_count; j++) {
400			rdata = XGBE_GET_DESC_DATA(ring, j);
401
402			rdata->rdesc = rdesc;
403			rdata->rdata_paddr = rdesc_paddr;
404
405			if (xgbe_map_rx_buffer(pdata, ring, rdata))
406				break;
407
408			rdesc++;
409			rdesc_paddr += sizeof(struct xgbe_ring_desc);
410		}
411
412		ring->cur = 0;
413		ring->dirty = 0;
414
415		hw_if->rx_desc_init(channel);
416	}
417}
418
419static void xgbe_unmap_rdata(struct xgbe_prv_data *pdata,
420			     struct xgbe_ring_data *rdata)
421{
422
423	if (rdata->mbuf_map != NULL)
424		bus_dmamap_destroy(rdata->mbuf_dmat, rdata->mbuf_map);
425
426	if (rdata->mbuf_free)
427		m_freem(rdata->mb);
428
429	rdata->mb = NULL;
430	rdata->mbuf_free = 0;
431	rdata->mbuf_hdr_paddr = 0;
432	rdata->mbuf_data_paddr = 0;
433	rdata->mbuf_len = 0;
434
435	memset(&rdata->tx, 0, sizeof(rdata->tx));
436	memset(&rdata->rx, 0, sizeof(rdata->rx));
437}
438
439struct xgbe_map_tx_skb_data {
440	struct xgbe_ring *ring;
441	struct xgbe_packet_data *packet;
442	unsigned int cur_index;
443};
444
445static void xgbe_map_tx_skb_cb(void *callback_arg, bus_dma_segment_t *segs,
446    int nseg, bus_size_t mapsize, int error)
447{
448	struct xgbe_map_tx_skb_data *data;
449	struct xgbe_ring_data *rdata;
450	struct xgbe_ring *ring;
451	int i;
452
453	if (error != 0)
454		return;
455
456	data = callback_arg;
457	ring = data->ring;
458
459	for (i = 0; i < nseg; i++) {
460		rdata = XGBE_GET_DESC_DATA(ring, data->cur_index);
461
462		KASSERT(segs[i].ds_len <= XGBE_TX_MAX_BUF_SIZE,
463		    ("%s: Segment size is too large %ld > %d", __func__,
464		    segs[i].ds_len, XGBE_TX_MAX_BUF_SIZE));
465
466		if (i == 0) {
467			rdata->mbuf_dmat = ring->mbuf_dmat;
468			bus_dmamap_create(ring->mbuf_dmat, 0, &ring->mbuf_map);
469		}
470
471		rdata->mbuf_hdr_paddr = 0;
472		rdata->mbuf_data_paddr = segs[i].ds_addr;
473		rdata->mbuf_len = segs[i].ds_len;
474
475		data->packet->length += rdata->mbuf_len;
476
477		data->cur_index++;
478	}
479}
480
481static int xgbe_map_tx_skb(struct xgbe_channel *channel, struct mbuf *m)
482{
483	struct xgbe_ring *ring = channel->tx_ring;
484	struct xgbe_map_tx_skb_data cbdata;
485	struct xgbe_ring_data *rdata;
486	struct xgbe_packet_data *packet;
487	unsigned int start_index, cur_index;
488	int err;
489
490	DBGPR("-->xgbe_map_tx_skb: cur = %d\n", ring->cur);
491
492	start_index = ring->cur;
493	cur_index = ring->cur;
494
495	packet = &ring->packet_data;
496	packet->rdesc_count = 0;
497	packet->length = 0;
498
499	cbdata.ring = ring;
500	cbdata.packet = packet;
501	cbdata.cur_index = cur_index;
502
503	err = bus_dmamap_load_mbuf(ring->mbuf_dmat, ring->mbuf_map, m,
504	    xgbe_map_tx_skb_cb, &cbdata, BUS_DMA_NOWAIT);
505	if (err != 0) /* TODO: Undo the mapping */
506		return (-err);
507
508	cur_index = cbdata.cur_index;
509
510	/* Save the mbuf address in the last entry. We always have some data
511	 * that has been mapped so rdata is always advanced past the last
512	 * piece of mapped data - use the entry pointed to by cur_index - 1.
513	 */
514	rdata = XGBE_GET_DESC_DATA(ring, cur_index - 1);
515	rdata->mb = m;
516	rdata->mbuf_free = 1;
517
518	/* Save the number of descriptor entries used */
519	packet->rdesc_count = cur_index - start_index;
520
521	DBGPR("<--xgbe_map_tx_skb: count=%u\n", packet->rdesc_count);
522
523	return packet->rdesc_count;
524}
525
526void xgbe_init_function_ptrs_desc(struct xgbe_desc_if *desc_if)
527{
528	DBGPR("-->xgbe_init_function_ptrs_desc\n");
529
530	desc_if->alloc_ring_resources = xgbe_alloc_ring_resources;
531	desc_if->free_ring_resources = xgbe_free_ring_resources;
532	desc_if->map_tx_skb = xgbe_map_tx_skb;
533	desc_if->map_rx_buffer = xgbe_map_rx_buffer;
534	desc_if->unmap_rdata = xgbe_unmap_rdata;
535	desc_if->wrapper_tx_desc_init = xgbe_wrapper_tx_descriptor_init;
536	desc_if->wrapper_rx_desc_init = xgbe_wrapper_rx_descriptor_init;
537
538	DBGPR("<--xgbe_init_function_ptrs_desc\n");
539}
540