1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright 2007-2009 Myricom, Inc.  All rights reserved.
29  * Use is subject to license terms.
30  */
31 
32 /*
33  * Copyright (c) 2014, Joyent, Inc.
34  * Copyright (c) 2016 by Delphix. All rights reserved.
35  */
36 
37 #define	MXGEFW_NDIS
38 #include "myri10ge_var.h"
39 #include "rss_eth_z8e.h"
40 #include "rss_ethp_z8e.h"
41 #include "mcp_gen_header.h"
42 
43 #define	MYRI10GE_MAX_ETHER_MTU 9014
44 #define	MYRI10GE_MAX_GLD_MTU	9000
45 #define	MYRI10GE_MIN_GLD_MTU	1500
46 
47 #define	MYRI10GE_ETH_STOPPED 0
48 #define	MYRI10GE_ETH_STOPPING 1
49 #define	MYRI10GE_ETH_STARTING 2
50 #define	MYRI10GE_ETH_RUNNING 3
51 #define	MYRI10GE_ETH_OPEN_FAILED 4
52 #define	MYRI10GE_ETH_SUSPENDED_RUNNING 5
53 
54 static int myri10ge_small_bytes = 510;
55 static int myri10ge_intr_coal_delay = 125;
56 static int myri10ge_flow_control = 1;
57 #if defined __i386 || defined i386 || defined __i386__ || defined __x86_64__
58 static int myri10ge_nvidia_ecrc_enable = 1;
59 #endif
60 static int myri10ge_mtu_override = 0;
61 static int myri10ge_tx_copylen = 512;
62 static int myri10ge_deassert_wait = 1;
63 static int myri10ge_verbose = 0;
64 static int myri10ge_watchdog_reset = 0;
65 static int myri10ge_use_msix = 1;
66 static int myri10ge_max_slices = -1;
67 static int myri10ge_use_msi = 1;
68 int myri10ge_force_firmware = 0;
69 static boolean_t myri10ge_use_lso = B_TRUE;
70 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT;
71 static int myri10ge_tx_hash = 1;
72 static int myri10ge_lro = 0;
73 static int myri10ge_lro_cnt = 8;
74 int myri10ge_lro_max_aggr = 2;
75 static int myri10ge_lso_copy = 0;
76 static mblk_t *myri10ge_send_wrapper(void *arg, mblk_t *mp);
77 int myri10ge_tx_handles_initial = 128;
78 
79 static	kmutex_t myri10ge_param_lock;
80 static void* myri10ge_db_lastfree;
81 
82 static int myri10ge_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
83 static int myri10ge_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
84 static int myri10ge_quiesce(dev_info_t *dip);
85 
86 DDI_DEFINE_STREAM_OPS(myri10ge_ops, nulldev, nulldev, myri10ge_attach,
87     myri10ge_detach, nodev, NULL, D_MP, NULL, myri10ge_quiesce);
88 
89 
90 static struct modldrv modldrv = {
91 	&mod_driverops,
92 	"Myricom 10G driver (10GbE)",
93 	&myri10ge_ops,
94 };
95 
96 
97 static struct modlinkage modlinkage = {
98 	MODREV_1,
99 	{&modldrv, NULL},
100 };
101 
102 unsigned char myri10ge_broadcastaddr[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
103 
104 static ddi_dma_attr_t myri10ge_misc_dma_attr = {
105 	DMA_ATTR_V0,			/* version number. */
106 	(uint64_t)0,			/* low address */
107 	(uint64_t)0xffffffffffffffffULL, /* high address */
108 	(uint64_t)0x7ffffff,		/* address counter max */
109 	(uint64_t)4096,			/* alignment */
110 	(uint_t)0x7f,			/* burstsizes for 32b and 64b xfers */
111 	(uint32_t)0x1,			/* minimum transfer size */
112 	(uint64_t)0x7fffffff,		/* maximum transfer size */
113 	(uint64_t)0x7fffffff,		/* maximum segment size */
114 	1,				/* scatter/gather list length */
115 	1,				/* granularity */
116 	0				/* attribute flags */
117 };
118 
119 /*
120  * The Myri10GE NIC has the following constraints on receive buffers:
121  * 1) Buffers which cross a 4KB boundary must be aligned to 4KB
122  * 2) Buffers which are not aligned to 4KB must not cross a 4KB boundary
123  */
124 
125 static ddi_dma_attr_t myri10ge_rx_jumbo_dma_attr = {
126 	DMA_ATTR_V0,			/* version number. */
127 	(uint64_t)0,			/* low address */
128 	(uint64_t)0xffffffffffffffffULL, /* high address */
129 	(uint64_t)0x7ffffff,		/* address counter max */
130 	(uint64_t)4096,			/* alignment */
131 	(uint_t)0x7f,			/* burstsizes for 32b and 64b xfers */
132 	(uint32_t)0x1,			/* minimum transfer size */
133 	(uint64_t)0x7fffffff,		/* maximum transfer size */
134 	UINT64_MAX,			/* maximum segment size */
135 	1,				/* scatter/gather list length */
136 	1,				/* granularity */
137 	0				/* attribute flags */
138 };
139 
140 static ddi_dma_attr_t myri10ge_rx_std_dma_attr = {
141 	DMA_ATTR_V0,			/* version number. */
142 	(uint64_t)0,			/* low address */
143 	(uint64_t)0xffffffffffffffffULL, /* high address */
144 	(uint64_t)0x7ffffff,		/* address counter max */
145 #if defined sparc64 || defined __sparcv9
146 	(uint64_t)4096,			/* alignment */
147 #else
148 	(uint64_t)0x80,			/* alignment */
149 #endif
150 	(uint_t)0x7f,			/* burstsizes for 32b and 64b xfers */
151 	(uint32_t)0x1,			/* minimum transfer size */
152 	(uint64_t)0x7fffffff,		/* maximum transfer size */
153 #if defined sparc64 || defined __sparcv9
154 	UINT64_MAX,			/* maximum segment size */
155 #else
156 	(uint64_t)0xfff,		/* maximum segment size */
157 #endif
158 	1,				/* scatter/gather list length */
159 	1,				/* granularity */
160 	0				/* attribute flags */
161 };
162 
163 static ddi_dma_attr_t myri10ge_tx_dma_attr = {
164 	DMA_ATTR_V0,			/* version number. */
165 	(uint64_t)0,			/* low address */
166 	(uint64_t)0xffffffffffffffffULL, /* high address */
167 	(uint64_t)0x7ffffff,		/* address counter max */
168 	(uint64_t)1,			/* alignment */
169 	(uint_t)0x7f,			/* burstsizes for 32b and 64b xfers */
170 	(uint32_t)0x1,			/* minimum transfer size */
171 	(uint64_t)0x7fffffff,		/* maximum transfer size */
172 	UINT64_MAX,			/* maximum segment size */
173 	INT32_MAX,			/* scatter/gather list length */
174 	1,				/* granularity */
175 	0			/* attribute flags */
176 };
177 
178 #if defined sparc64 || defined __sparcv9
179 #define	WC 0
180 #else
181 #define	WC 1
182 #endif
183 
184 struct ddi_device_acc_attr myri10ge_dev_access_attr = {
185 	DDI_DEVICE_ATTR_V0,		/* version */
186 	DDI_NEVERSWAP_ACC,		/* endian flash */
187 #if WC
188 	DDI_MERGING_OK_ACC		/* data order */
189 #else
190 	DDI_STRICTORDER_ACC
191 #endif
192 };
193 
194 static void myri10ge_watchdog(void *arg);
195 
196 #ifdef MYRICOM_PRIV
197 int myri10ge_mtu = MYRI10GE_MAX_ETHER_MTU + MXGEFW_PAD + VLAN_TAGSZ;
198 #define	MYRI10GE_DEFAULT_GLD_MTU	MYRI10GE_MAX_GLD_MTU
199 #else
200 int myri10ge_mtu = ETHERMAX + MXGEFW_PAD + VLAN_TAGSZ;
201 #define	MYRI10GE_DEFAULT_GLD_MTU	MYRI10GE_MIN_GLD_MTU
202 #endif
203 int myri10ge_bigbufs_initial = 1024;
204 int myri10ge_bigbufs_max = 4096;
205 
206 
207 caddr_t
myri10ge_dma_alloc(dev_info_t * dip,size_t len,ddi_dma_attr_t * attr,ddi_device_acc_attr_t * accattr,uint_t alloc_flags,int bind_flags,struct myri10ge_dma_stuff * dma,int warn,int (* wait)(caddr_t))208 myri10ge_dma_alloc(dev_info_t *dip, size_t len,
209     ddi_dma_attr_t *attr, ddi_device_acc_attr_t  *accattr,
210     uint_t alloc_flags, int bind_flags, struct myri10ge_dma_stuff *dma,
211     int warn, int (*wait)(caddr_t))
212 {
213 	caddr_t  kaddr;
214 	size_t real_length;
215 	ddi_dma_cookie_t cookie;
216 	uint_t count;
217 	int err;
218 
219 	err = ddi_dma_alloc_handle(dip, attr, wait,
220 	    NULL, &dma->handle);
221 	if (err != DDI_SUCCESS) {
222 		if (warn)
223 			cmn_err(CE_WARN,
224 			    "myri10ge: ddi_dma_alloc_handle failed\n");
225 		goto abort_with_nothing;
226 	}
227 
228 	err = ddi_dma_mem_alloc(dma->handle, len, accattr, alloc_flags,
229 	    wait, NULL, &kaddr, &real_length,
230 	    &dma->acc_handle);
231 	if (err != DDI_SUCCESS) {
232 		if (warn)
233 			cmn_err(CE_WARN,
234 			    "myri10ge: ddi_dma_mem_alloc failed\n");
235 		goto abort_with_handle;
236 	}
237 
238 	err = ddi_dma_addr_bind_handle(dma->handle, NULL, kaddr, len,
239 	    bind_flags, wait, NULL, &cookie, &count);
240 
241 	if (err != DDI_SUCCESS) {
242 		if (warn)
243 			cmn_err(CE_WARN,
244 			    "myri10ge: ddi_dma_addr_bind_handle failed\n");
245 		goto abort_with_mem;
246 	}
247 
248 	if (count != 1) {
249 		if (warn)
250 			cmn_err(CE_WARN,
251 			    "myri10ge: got too many dma segments ");
252 		goto abort_with_bind;
253 	}
254 	dma->low = htonl(MYRI10GE_LOWPART_TO_U32(cookie.dmac_laddress));
255 	dma->high = htonl(MYRI10GE_HIGHPART_TO_U32(cookie.dmac_laddress));
256 	return (kaddr);
257 
258 abort_with_bind:
259 	(void) ddi_dma_unbind_handle(dma->handle);
260 
261 abort_with_mem:
262 	ddi_dma_mem_free(&dma->acc_handle);
263 
264 abort_with_handle:
265 	ddi_dma_free_handle(&dma->handle);
266 abort_with_nothing:
267 	if (warn) {
268 		cmn_err(CE_WARN, "myri10ge: myri10ge_dma_alloc failed.\n  ");
269 		cmn_err(CE_WARN, "args: dip=%p len=0x%lx ddi_dma_attr=%p\n",
270 		    (void*) dip, len, (void*) attr);
271 		cmn_err(CE_WARN,
272 		    "args: ddi_device_acc_attr=%p  alloc_flags=0x%x\n",
273 		    (void*) accattr, alloc_flags);
274 		cmn_err(CE_WARN, "args: bind_flags=0x%x  dmastuff=%p",
275 		    bind_flags, (void*) dma);
276 	}
277 	return (NULL);
278 
279 }
280 
281 void
myri10ge_dma_free(struct myri10ge_dma_stuff * dma)282 myri10ge_dma_free(struct myri10ge_dma_stuff *dma)
283 {
284 	(void) ddi_dma_unbind_handle(dma->handle);
285 	ddi_dma_mem_free(&dma->acc_handle);
286 	ddi_dma_free_handle(&dma->handle);
287 }
288 
289 static inline void
myri10ge_pio_copy32(void * to,uint32_t * from32,size_t size)290 myri10ge_pio_copy32(void *to, uint32_t *from32, size_t size)
291 {
292 	register volatile uint32_t *to32;
293 	size_t i;
294 
295 	to32 = (volatile uint32_t *) to;
296 	for (i = (size / 4); i; i--) {
297 		*to32 = *from32;
298 		to32++;
299 		from32++;
300 	}
301 }
302 
303 #if defined(_LP64)
304 static inline void
myri10ge_pio_copy64(void * to,uint64_t * from64,size_t size)305 myri10ge_pio_copy64(void *to, uint64_t *from64, size_t size)
306 {
307 	register volatile uint64_t *to64;
308 	size_t i;
309 
310 	to64 = (volatile uint64_t *) to;
311 	for (i = (size / 8); i; i--) {
312 		*to64 = *from64;
313 		to64++;
314 		from64++;
315 	}
316 }
317 #endif
318 
319 /*
320  * This routine copies memory from the host to the NIC.
321  * The "size" argument must always be a multiple of
322  * the size of long (4 or 8 bytes), and to/from must also
323  * be naturally aligned.
324  */
325 static inline void
myri10ge_pio_copy(void * to,void * from,size_t size)326 myri10ge_pio_copy(void *to, void *from, size_t size)
327 {
328 #if !defined(_LP64)
329 	ASSERT((size % 4) == 0);
330 	myri10ge_pio_copy32(to, (uint32_t *)from, size);
331 #else
332 	ASSERT((size % 8) == 0);
333 	myri10ge_pio_copy64(to, (uint64_t *)from, size);
334 #endif
335 }
336 
337 
338 /*
339  * Due to various bugs in Solaris (especially bug 6186772 where the
340  * TCP/UDP checksum is calculated incorrectly on mblk chains with more
341  * than two elements), and the design bug where hardware checksums are
342  * ignored on mblk chains with more than 2 elements, we need to
343  * allocate private pool of physically contiguous receive buffers.
344  */
345 
346 static void
myri10ge_jpool_init(struct myri10ge_slice_state * ss)347 myri10ge_jpool_init(struct myri10ge_slice_state *ss)
348 {
349 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
350 
351 	bzero(jpool, sizeof (*jpool));
352 	mutex_init(&jpool->mtx, NULL, MUTEX_DRIVER,
353 	    ss->mgp->icookie);
354 	jpool->head = NULL;
355 }
356 
357 static void
myri10ge_jpool_fini(struct myri10ge_slice_state * ss)358 myri10ge_jpool_fini(struct myri10ge_slice_state *ss)
359 {
360 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
361 
362 	if (jpool->head != NULL) {
363 		cmn_err(CE_WARN,
364 		    "%s: BUG! myri10ge_jpool_fini called on non-empty pool\n",
365 		    ss->mgp->name);
366 	}
367 	mutex_destroy(&jpool->mtx);
368 }
369 
370 
371 /*
372  * copy an array of mcp_kreq_ether_recv_t's to the mcp.  Copy
373  * at most 32 bytes at a time, so as to avoid involving the software
374  * pio handler in the nic.   We re-write the first segment's low
375  * DMA address to mark it valid only after we write the entire chunk
376  * in a burst
377  */
378 static inline void
myri10ge_submit_8rx(mcp_kreq_ether_recv_t * dst,mcp_kreq_ether_recv_t * src)379 myri10ge_submit_8rx(mcp_kreq_ether_recv_t *dst, mcp_kreq_ether_recv_t *src)
380 {
381 	src->addr_low |= BE_32(1);
382 	myri10ge_pio_copy(dst, src, 4 * sizeof (*src));
383 	mb();
384 	myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src));
385 	mb();
386 	src->addr_low &= ~(BE_32(1));
387 	dst->addr_low = src->addr_low;
388 	mb();
389 }
390 
391 static void
myri10ge_pull_jpool(struct myri10ge_slice_state * ss)392 myri10ge_pull_jpool(struct myri10ge_slice_state *ss)
393 {
394 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
395 	struct myri10ge_jpool_entry *jtail, *j, *jfree;
396 	volatile void *putp;
397 	int i;
398 
399 	/* find tail */
400 	jtail = NULL;
401 	if (jpool->head != NULL) {
402 		j = jpool->head;
403 		while (j->next != NULL)
404 			j = j->next;
405 		jtail = j;
406 	}
407 
408 	/*
409 	 * iterate over all per-CPU caches, and add contents into
410 	 * jpool
411 	 */
412 	for (i = 0; i < MYRI10GE_MAX_CPUS; i++) {
413 		/* take per-CPU free list */
414 		putp = &jpool->cpu[i & MYRI10GE_MAX_CPU_MASK].head;
415 		jfree = atomic_swap_ptr(putp, NULL);
416 		if (jfree == NULL)
417 			continue;
418 
419 		/* append to pool */
420 		if (jtail == NULL) {
421 			jpool->head = jfree;
422 		} else {
423 			jtail->next = jfree;
424 		}
425 		j = jfree;
426 		while (j->next != NULL)
427 			j = j->next;
428 		jtail = j;
429 	}
430 }
431 
432 /*
433  * Transfers buffers from the free pool to the nic
434  * Must be called holding the jpool mutex.
435  */
436 
437 static inline void
myri10ge_restock_jumbos(struct myri10ge_slice_state * ss)438 myri10ge_restock_jumbos(struct myri10ge_slice_state *ss)
439 {
440 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
441 	struct myri10ge_jpool_entry *j;
442 	myri10ge_rx_ring_t *rx;
443 	int i, idx, limit;
444 
445 	rx = &ss->rx_big;
446 	limit = ss->j_rx_cnt + (rx->mask + 1);
447 
448 	for (i = rx->cnt; i != limit; i++) {
449 		idx = i & (rx->mask);
450 		j = jpool->head;
451 		if (j == NULL) {
452 			myri10ge_pull_jpool(ss);
453 			j = jpool->head;
454 			if (j == NULL) {
455 				break;
456 			}
457 		}
458 		jpool->head = j->next;
459 		rx->info[idx].j = j;
460 		rx->shadow[idx].addr_low = j->dma.low;
461 		rx->shadow[idx].addr_high = j->dma.high;
462 		/* copy 4 descriptors (32-bytes) to the mcp at a time */
463 		if ((idx & 7) == 7) {
464 			myri10ge_submit_8rx(&rx->lanai[idx - 7],
465 			    &rx->shadow[idx - 7]);
466 		}
467 	}
468 	rx->cnt = i;
469 }
470 
471 /*
472  * Transfer buffers from the nic to the free pool.
473  * Should be called holding the jpool mutex
474  */
475 
476 static inline void
myri10ge_unstock_jumbos(struct myri10ge_slice_state * ss)477 myri10ge_unstock_jumbos(struct myri10ge_slice_state *ss)
478 {
479 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
480 	struct myri10ge_jpool_entry *j;
481 	myri10ge_rx_ring_t *rx;
482 	int i;
483 
484 	mutex_enter(&jpool->mtx);
485 	rx = &ss->rx_big;
486 
487 	for (i = 0; i < rx->mask + 1; i++) {
488 		j = rx->info[i].j;
489 		rx->info[i].j = NULL;
490 		if (j == NULL)
491 			continue;
492 		j->next = jpool->head;
493 		jpool->head = j;
494 	}
495 	mutex_exit(&jpool->mtx);
496 
497 }
498 
499 
500 /*
501  * Free routine which is called when the mblk allocated via
502  * esballoc() is freed.   Here we return the jumbo buffer
503  * to the free pool, and possibly pass some jumbo buffers
504  * to the nic
505  */
506 
507 static void
myri10ge_jfree_rtn(void * arg)508 myri10ge_jfree_rtn(void *arg)
509 {
510 	struct myri10ge_jpool_entry *j = (struct myri10ge_jpool_entry *)arg;
511 	struct myri10ge_jpool_stuff *jpool;
512 	volatile uintptr_t *putp;
513 	uintptr_t old, new;
514 
515 	jpool = &j->ss->jpool;
516 
517 	/* prepend buffer locklessly to per-CPU freelist */
518 	putp = (void *)&jpool->cpu[CPU->cpu_seqid & MYRI10GE_MAX_CPU_MASK].head;
519 	new = (uintptr_t)j;
520 	do {
521 		old = *putp;
522 		j->next = (void *)old;
523 	} while (atomic_cas_ulong(putp, old, new) != old);
524 }
525 
526 static void
myri10ge_remove_jbuf(struct myri10ge_jpool_entry * j)527 myri10ge_remove_jbuf(struct myri10ge_jpool_entry *j)
528 {
529 	(void) ddi_dma_unbind_handle(j->dma_handle);
530 	ddi_dma_mem_free(&j->acc_handle);
531 	ddi_dma_free_handle(&j->dma_handle);
532 	kmem_free(j, sizeof (*j));
533 }
534 
535 
536 /*
537  * Allocates one physically contiguous descriptor
538  * and add it to the jumbo buffer pool.
539  */
540 
541 static int
myri10ge_add_jbuf(struct myri10ge_slice_state * ss)542 myri10ge_add_jbuf(struct myri10ge_slice_state *ss)
543 {
544 	struct myri10ge_jpool_entry *j;
545 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
546 	ddi_dma_attr_t *rx_dma_attr;
547 	size_t real_length;
548 	ddi_dma_cookie_t cookie;
549 	uint_t count;
550 	int err;
551 
552 	if (myri10ge_mtu < 2048)
553 		rx_dma_attr = &myri10ge_rx_std_dma_attr;
554 	else
555 		rx_dma_attr = &myri10ge_rx_jumbo_dma_attr;
556 
557 again:
558 	j = (struct myri10ge_jpool_entry *)
559 	    kmem_alloc(sizeof (*j), KM_SLEEP);
560 	err = ddi_dma_alloc_handle(ss->mgp->dip, rx_dma_attr,
561 	    DDI_DMA_DONTWAIT, NULL, &j->dma_handle);
562 	if (err != DDI_SUCCESS)
563 		goto abort_with_j;
564 
565 	err = ddi_dma_mem_alloc(j->dma_handle, myri10ge_mtu,
566 	    &myri10ge_dev_access_attr,  DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
567 	    NULL, &j->buf, &real_length, &j->acc_handle);
568 	if (err != DDI_SUCCESS)
569 		goto abort_with_handle;
570 
571 	err = ddi_dma_addr_bind_handle(j->dma_handle, NULL, j->buf,
572 	    real_length, DDI_DMA_READ|DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
573 	    NULL, &cookie, &count);
574 	if (err != DDI_SUCCESS)
575 		goto abort_with_mem;
576 
577 	/*
578 	 * Make certain std MTU buffers do not cross a 4KB boundary:
579 	 *
580 	 * Setting dma_attr_align=4096 will do this, but the system
581 	 * will only allocate 1 RX buffer per 4KB page, rather than 2.
582 	 * Setting dma_attr_granular=4096 *seems* to work around this,
583 	 * but I'm paranoid about future systems no longer honoring
584 	 * this, so fall back to the safe, but memory wasting way if a
585 	 * buffer crosses a 4KB boundary.
586 	 */
587 
588 	if (rx_dma_attr == &myri10ge_rx_std_dma_attr &&
589 	    rx_dma_attr->dma_attr_align != 4096) {
590 		uint32_t start, end;
591 
592 		start = MYRI10GE_LOWPART_TO_U32(cookie.dmac_laddress);
593 		end = start + myri10ge_mtu;
594 		if (((end >> 12) != (start >> 12)) && (start & 4095U)) {
595 			printf("std buffer crossed a 4KB boundary!\n");
596 			myri10ge_remove_jbuf(j);
597 			rx_dma_attr->dma_attr_align = 4096;
598 			rx_dma_attr->dma_attr_seg = UINT64_MAX;
599 			goto again;
600 		}
601 	}
602 
603 	j->dma.low =
604 	    htonl(MYRI10GE_LOWPART_TO_U32(cookie.dmac_laddress));
605 	j->dma.high =
606 	    htonl(MYRI10GE_HIGHPART_TO_U32(cookie.dmac_laddress));
607 	j->ss = ss;
608 
609 
610 	j->free_func.free_func = myri10ge_jfree_rtn;
611 	j->free_func.free_arg = (char *)j;
612 	mutex_enter(&jpool->mtx);
613 	j->next = jpool->head;
614 	jpool->head = j;
615 	jpool->num_alloc++;
616 	mutex_exit(&jpool->mtx);
617 	return (0);
618 
619 abort_with_mem:
620 	ddi_dma_mem_free(&j->acc_handle);
621 
622 abort_with_handle:
623 	ddi_dma_free_handle(&j->dma_handle);
624 
625 abort_with_j:
626 	kmem_free(j, sizeof (*j));
627 
628 	/*
629 	 * If an allocation failed, perhaps it failed because it could
630 	 * not satisfy granularity requirement.  Disable that, and
631 	 * try agin.
632 	 */
633 	if (rx_dma_attr == &myri10ge_rx_std_dma_attr &&
634 	    rx_dma_attr->dma_attr_align != 4096) {
635 			cmn_err(CE_NOTE,
636 			    "!alloc failed, reverting to gran=1\n");
637 			rx_dma_attr->dma_attr_align = 4096;
638 			rx_dma_attr->dma_attr_seg = UINT64_MAX;
639 			goto again;
640 	}
641 	return (err);
642 }
643 
644 static int
myri10ge_jfree_cnt(struct myri10ge_jpool_stuff * jpool)645 myri10ge_jfree_cnt(struct myri10ge_jpool_stuff *jpool)
646 {
647 	int i;
648 	struct myri10ge_jpool_entry *j;
649 
650 	mutex_enter(&jpool->mtx);
651 	j = jpool->head;
652 	i = 0;
653 	while (j != NULL) {
654 		i++;
655 		j = j->next;
656 	}
657 	mutex_exit(&jpool->mtx);
658 	return (i);
659 }
660 
661 static int
myri10ge_add_jbufs(struct myri10ge_slice_state * ss,int num,int total)662 myri10ge_add_jbufs(struct myri10ge_slice_state *ss, int num, int total)
663 {
664 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
665 	int allocated = 0;
666 	int err;
667 	int needed;
668 
669 	/*
670 	 * if total is set, user wants "num" jbufs in the pool,
671 	 * otherwise the user wants to "num" additional jbufs
672 	 * added to the pool
673 	 */
674 	if (total && jpool->num_alloc) {
675 		allocated = myri10ge_jfree_cnt(jpool);
676 		needed = num - allocated;
677 	} else {
678 		needed = num;
679 	}
680 
681 	while (needed > 0) {
682 		needed--;
683 		err = myri10ge_add_jbuf(ss);
684 		if (err == 0) {
685 			allocated++;
686 		}
687 	}
688 	return (allocated);
689 }
690 
691 static void
myri10ge_remove_jbufs(struct myri10ge_slice_state * ss)692 myri10ge_remove_jbufs(struct myri10ge_slice_state *ss)
693 {
694 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
695 	struct myri10ge_jpool_entry *j;
696 
697 	mutex_enter(&jpool->mtx);
698 	myri10ge_pull_jpool(ss);
699 	while (jpool->head != NULL) {
700 		jpool->num_alloc--;
701 		j = jpool->head;
702 		jpool->head = j->next;
703 		myri10ge_remove_jbuf(j);
704 	}
705 	mutex_exit(&jpool->mtx);
706 }
707 
708 static void
myri10ge_carve_up_jbufs_into_small_ring(struct myri10ge_slice_state * ss)709 myri10ge_carve_up_jbufs_into_small_ring(struct myri10ge_slice_state *ss)
710 {
711 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
712 	struct myri10ge_jpool_entry *j = NULL;
713 	caddr_t ptr;
714 	uint32_t dma_low, dma_high;
715 	int idx, len;
716 	unsigned int alloc_size;
717 
718 	dma_low = dma_high = len = 0;
719 	alloc_size = myri10ge_small_bytes + MXGEFW_PAD;
720 	ptr = NULL;
721 	for (idx = 0; idx < ss->rx_small.mask + 1; idx++) {
722 		/* Allocate a jumbo frame and carve it into small frames */
723 		if (len < alloc_size) {
724 			mutex_enter(&jpool->mtx);
725 			/* remove jumbo from freelist */
726 			j = jpool->head;
727 			jpool->head = j->next;
728 			/* place it onto small list */
729 			j->next = ss->small_jpool;
730 			ss->small_jpool = j;
731 			mutex_exit(&jpool->mtx);
732 			len = myri10ge_mtu;
733 			dma_low = ntohl(j->dma.low);
734 			dma_high = ntohl(j->dma.high);
735 			ptr = j->buf;
736 		}
737 		ss->rx_small.info[idx].ptr = ptr;
738 		ss->rx_small.shadow[idx].addr_low = htonl(dma_low);
739 		ss->rx_small.shadow[idx].addr_high = htonl(dma_high);
740 		len -= alloc_size;
741 		ptr += alloc_size;
742 		dma_low += alloc_size;
743 	}
744 }
745 
746 /*
747  * Return the jumbo bufs we carved up for small to the jumbo pool
748  */
749 
750 static void
myri10ge_release_small_jbufs(struct myri10ge_slice_state * ss)751 myri10ge_release_small_jbufs(struct myri10ge_slice_state *ss)
752 {
753 	struct myri10ge_jpool_stuff *jpool = &ss->jpool;
754 	struct myri10ge_jpool_entry *j = NULL;
755 
756 	mutex_enter(&jpool->mtx);
757 	while (ss->small_jpool != NULL) {
758 		j = ss->small_jpool;
759 		ss->small_jpool = j->next;
760 		j->next = jpool->head;
761 		jpool->head = j;
762 	}
763 	mutex_exit(&jpool->mtx);
764 	ss->jbufs_for_smalls = 0;
765 }
766 
767 static int
myri10ge_add_tx_handle(struct myri10ge_slice_state * ss)768 myri10ge_add_tx_handle(struct myri10ge_slice_state *ss)
769 {
770 	myri10ge_tx_ring_t *tx = &ss->tx;
771 	struct myri10ge_priv *mgp = ss->mgp;
772 	struct myri10ge_tx_dma_handle *handle;
773 	int err;
774 
775 	handle = kmem_zalloc(sizeof (*handle), KM_SLEEP);
776 	err = ddi_dma_alloc_handle(mgp->dip,
777 	    &myri10ge_tx_dma_attr,
778 	    DDI_DMA_SLEEP, NULL,
779 	    &handle->h);
780 	if (err) {
781 		static int limit = 0;
782 		if (limit == 0)
783 			cmn_err(CE_WARN, "%s: Falled to alloc tx dma handle\n",
784 			    mgp->name);
785 		limit++;
786 		kmem_free(handle, sizeof (*handle));
787 		return (err);
788 	}
789 	mutex_enter(&tx->handle_lock);
790 	MYRI10GE_SLICE_STAT_INC(tx_handles_alloced);
791 	handle->next = tx->free_tx_handles;
792 	tx->free_tx_handles = handle;
793 	mutex_exit(&tx->handle_lock);
794 	return (DDI_SUCCESS);
795 }
796 
797 static void
myri10ge_remove_tx_handles(struct myri10ge_slice_state * ss)798 myri10ge_remove_tx_handles(struct myri10ge_slice_state *ss)
799 {
800 	myri10ge_tx_ring_t *tx = &ss->tx;
801 	struct myri10ge_tx_dma_handle *handle;
802 	mutex_enter(&tx->handle_lock);
803 
804 	handle = tx->free_tx_handles;
805 	while (handle != NULL) {
806 		tx->free_tx_handles = handle->next;
807 		ddi_dma_free_handle(&handle->h);
808 		kmem_free(handle, sizeof (*handle));
809 		handle = tx->free_tx_handles;
810 		MYRI10GE_SLICE_STAT_DEC(tx_handles_alloced);
811 	}
812 	mutex_exit(&tx->handle_lock);
813 	if (MYRI10GE_SLICE_STAT(tx_handles_alloced) != 0) {
814 		cmn_err(CE_WARN, "%s: %d tx dma handles allocated at close\n",
815 		    ss->mgp->name,
816 		    (int)MYRI10GE_SLICE_STAT(tx_handles_alloced));
817 	}
818 }
819 
820 static void
myri10ge_free_tx_handles(myri10ge_tx_ring_t * tx,struct myri10ge_tx_dma_handle_head * list)821 myri10ge_free_tx_handles(myri10ge_tx_ring_t *tx,
822     struct myri10ge_tx_dma_handle_head *list)
823 {
824 	mutex_enter(&tx->handle_lock);
825 	list->tail->next = tx->free_tx_handles;
826 	tx->free_tx_handles = list->head;
827 	mutex_exit(&tx->handle_lock);
828 }
829 
830 static void
myri10ge_free_tx_handle_slist(myri10ge_tx_ring_t * tx,struct myri10ge_tx_dma_handle * handle)831 myri10ge_free_tx_handle_slist(myri10ge_tx_ring_t *tx,
832     struct myri10ge_tx_dma_handle *handle)
833 {
834 	struct myri10ge_tx_dma_handle_head list;
835 
836 	if (handle == NULL)
837 		return;
838 	list.head = handle;
839 	list.tail = handle;
840 	while (handle != NULL) {
841 		list.tail = handle;
842 		handle = handle->next;
843 	}
844 	myri10ge_free_tx_handles(tx, &list);
845 }
846 
847 static int
myri10ge_alloc_tx_handles(struct myri10ge_slice_state * ss,int count,struct myri10ge_tx_dma_handle ** ret)848 myri10ge_alloc_tx_handles(struct myri10ge_slice_state *ss, int count,
849     struct myri10ge_tx_dma_handle **ret)
850 {
851 	myri10ge_tx_ring_t *tx = &ss->tx;
852 	struct myri10ge_tx_dma_handle *handle;
853 	int err, i;
854 
855 	mutex_enter(&tx->handle_lock);
856 	for (i = 0; i < count; i++) {
857 		handle = tx->free_tx_handles;
858 		while (handle == NULL) {
859 			mutex_exit(&tx->handle_lock);
860 			err = myri10ge_add_tx_handle(ss);
861 			if (err != DDI_SUCCESS) {
862 				goto abort_with_handles;
863 			}
864 			mutex_enter(&tx->handle_lock);
865 			handle = tx->free_tx_handles;
866 		}
867 		tx->free_tx_handles = handle->next;
868 		handle->next = *ret;
869 		*ret = handle;
870 	}
871 	mutex_exit(&tx->handle_lock);
872 	return (DDI_SUCCESS);
873 
874 abort_with_handles:
875 	myri10ge_free_tx_handle_slist(tx, *ret);
876 	return (err);
877 }
878 
879 
880 /*
881  * Frees DMA resources associated with the send ring
882  */
883 static void
myri10ge_unprepare_tx_ring(struct myri10ge_slice_state * ss)884 myri10ge_unprepare_tx_ring(struct myri10ge_slice_state *ss)
885 {
886 	myri10ge_tx_ring_t *tx;
887 	struct myri10ge_tx_dma_handle_head handles;
888 	size_t bytes;
889 	int idx;
890 
891 	tx = &ss->tx;
892 	handles.head = NULL;
893 	handles.tail = NULL;
894 	for (idx = 0; idx < ss->tx.mask + 1; idx++) {
895 		if (tx->info[idx].m) {
896 			(void) ddi_dma_unbind_handle(tx->info[idx].handle->h);
897 			handles.head = tx->info[idx].handle;
898 			if (handles.tail == NULL)
899 				handles.tail = tx->info[idx].handle;
900 			freeb(tx->info[idx].m);
901 			tx->info[idx].m = 0;
902 			tx->info[idx].handle = 0;
903 		}
904 		tx->cp[idx].va = NULL;
905 		myri10ge_dma_free(&tx->cp[idx].dma);
906 	}
907 	bytes = sizeof (*tx->cp) * (tx->mask + 1);
908 	kmem_free(tx->cp, bytes);
909 	tx->cp = NULL;
910 	if (handles.head != NULL)
911 		myri10ge_free_tx_handles(tx, &handles);
912 	myri10ge_remove_tx_handles(ss);
913 }
914 
915 /*
916  * Allocates DMA handles associated with the send ring
917  */
918 static inline int
myri10ge_prepare_tx_ring(struct myri10ge_slice_state * ss)919 myri10ge_prepare_tx_ring(struct myri10ge_slice_state *ss)
920 {
921 	struct myri10ge_tx_dma_handle *handles;
922 	int h;
923 	size_t bytes;
924 
925 	bytes = sizeof (*ss->tx.cp) * (ss->tx.mask + 1);
926 	ss->tx.cp = kmem_zalloc(bytes, KM_SLEEP);
927 	if (ss->tx.cp == NULL) {
928 		cmn_err(CE_WARN,
929 		    "%s: Failed to allocate tx copyblock storage\n",
930 		    ss->mgp->name);
931 		return (DDI_FAILURE);
932 	}
933 
934 
935 	/* allocate the TX copyblocks */
936 	for (h = 0; h < ss->tx.mask + 1; h++) {
937 		ss->tx.cp[h].va = myri10ge_dma_alloc(ss->mgp->dip,
938 		    4096, &myri10ge_rx_jumbo_dma_attr,
939 		    &myri10ge_dev_access_attr, DDI_DMA_STREAMING,
940 		    DDI_DMA_WRITE|DDI_DMA_STREAMING, &ss->tx.cp[h].dma, 1,
941 		    DDI_DMA_DONTWAIT);
942 		if (ss->tx.cp[h].va == NULL) {
943 			cmn_err(CE_WARN, "%s: Failed to allocate tx "
944 			    "copyblock %d\n", ss->mgp->name, h);
945 			goto abort_with_copyblocks;
946 		}
947 	}
948 	/* pre-allocate transmit handles */
949 	handles = NULL;
950 	(void) myri10ge_alloc_tx_handles(ss, myri10ge_tx_handles_initial,
951 	    &handles);
952 	if (handles != NULL)
953 		myri10ge_free_tx_handle_slist(&ss->tx, handles);
954 
955 	return (DDI_SUCCESS);
956 
957 abort_with_copyblocks:
958 	while (h > 0)  {
959 		h--;
960 		myri10ge_dma_free(&ss->tx.cp[h].dma);
961 	}
962 
963 	bytes = sizeof (*ss->tx.cp) * (ss->tx.mask + 1);
964 	kmem_free(ss->tx.cp, bytes);
965 	ss->tx.cp = NULL;
966 	return (DDI_FAILURE);
967 }
968 
969 /*
970  * The eeprom strings on the lanaiX have the format
971  * SN=x\0
972  * MAC=x:x:x:x:x:x\0
973  * PT:ddd mmm xx xx:xx:xx xx\0
974  * PV:ddd mmm xx xx:xx:xx xx\0
975  */
976 static int
myri10ge_read_mac_addr(struct myri10ge_priv * mgp)977 myri10ge_read_mac_addr(struct myri10ge_priv *mgp)
978 {
979 #define	MYRI10GE_NEXT_STRING(p) while (ptr < limit && *ptr++)
980 #define	myri10ge_digit(c) (((c) >= '0' && (c) <= '9') ? ((c) - '0') :	\
981 		(((c) >= 'A' && (c) <= 'F') ? (10 + (c) - 'A') :	\
982 		(((c) >= 'a' && (c) <= 'f') ? (10 + (c) - 'a') : -1)))
983 
984 	char *ptr, *limit;
985 	int i, hv, lv;
986 
987 	ptr = mgp->eeprom_strings;
988 	limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE;
989 
990 	while (*ptr != '\0' && ptr < limit) {
991 		if (memcmp(ptr, "MAC=", 4) == 0) {
992 			ptr += 4;
993 			if (myri10ge_verbose)
994 				printf("%s: mac address = %s\n", mgp->name,
995 				    ptr);
996 			mgp->mac_addr_string = ptr;
997 			for (i = 0; i < 6; i++) {
998 				if ((ptr + 2) > limit)
999 					goto abort;
1000 
1001 				if (*(ptr+1) == ':') {
1002 					hv = 0;
1003 					lv = myri10ge_digit(*ptr); ptr++;
1004 				} else {
1005 					hv = myri10ge_digit(*ptr); ptr++;
1006 					lv = myri10ge_digit(*ptr); ptr++;
1007 				}
1008 				mgp->mac_addr[i] = (hv << 4) | lv;
1009 				ptr++;
1010 			}
1011 		}
1012 		if (memcmp((const void *)ptr, "SN=", 3) == 0) {
1013 			ptr += 3;
1014 			mgp->sn_str = (char *)ptr;
1015 		}
1016 		if (memcmp((const void *)ptr, "PC=", 3) == 0) {
1017 			ptr += 3;
1018 			mgp->pc_str = (char *)ptr;
1019 		}
1020 		MYRI10GE_NEXT_STRING(ptr);
1021 	}
1022 
1023 	return (0);
1024 
1025 abort:
1026 	cmn_err(CE_WARN, "%s: failed to parse eeprom_strings", mgp->name);
1027 	return (ENXIO);
1028 }
1029 
1030 
1031 /*
1032  * Determine the register set containing the PCI resource we
1033  * want to map: the memory-mappable part of the interface. We do
1034  * this by scanning the DDI "reg" property of the interface,
1035  * which is an array of mx_ddi_reg_set structures.
1036  */
1037 static int
myri10ge_reg_set(dev_info_t * dip,int * reg_set,int * span,unsigned long * busno,unsigned long * devno,unsigned long * funcno)1038 myri10ge_reg_set(dev_info_t *dip, int *reg_set, int *span,
1039     unsigned long *busno, unsigned long *devno,
1040     unsigned long *funcno)
1041 {
1042 
1043 #define	REGISTER_NUMBER(ip)	(ip[0] >>  0 & 0xff)
1044 #define	FUNCTION_NUMBER(ip)	(ip[0] >>  8 & 0x07)
1045 #define	DEVICE_NUMBER(ip)	(ip[0] >> 11 & 0x1f)
1046 #define	BUS_NUMBER(ip)		(ip[0] >> 16 & 0xff)
1047 #define	ADDRESS_SPACE(ip)	(ip[0] >> 24 & 0x03)
1048 #define	PCI_ADDR_HIGH(ip)	(ip[1])
1049 #define	PCI_ADDR_LOW(ip)	(ip[2])
1050 #define	PCI_SPAN_HIGH(ip)	(ip[3])
1051 #define	PCI_SPAN_LOW(ip)	(ip[4])
1052 
1053 #define	MX_DDI_REG_SET_32_BIT_MEMORY_SPACE 2
1054 #define	MX_DDI_REG_SET_64_BIT_MEMORY_SPACE 3
1055 
1056 	int *data, i, *rs;
1057 	uint32_t nelementsp;
1058 
1059 #ifdef MYRI10GE_REGSET_VERBOSE
1060 	char *address_space_name[] = { "Configuration Space",
1061 					"I/O Space",
1062 					"32-bit Memory Space",
1063 					"64-bit Memory Space"
1064 	};
1065 #endif
1066 
1067 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1068 	    "reg", &data, &nelementsp) != DDI_SUCCESS) {
1069 		printf("Could not determine register set.\n");
1070 		return (ENXIO);
1071 	}
1072 
1073 #ifdef MYRI10GE_REGSET_VERBOSE
1074 	printf("There are %d register sets.\n", nelementsp / 5);
1075 #endif
1076 	if (!nelementsp) {
1077 		printf("Didn't find any \"reg\" properties.\n");
1078 		ddi_prop_free(data);
1079 		return (ENODEV);
1080 	}
1081 
1082 	/* Scan for the register number. */
1083 	rs = &data[0];
1084 	*busno = BUS_NUMBER(rs);
1085 	*devno = DEVICE_NUMBER(rs);
1086 	*funcno = FUNCTION_NUMBER(rs);
1087 
1088 #ifdef MYRI10GE_REGSET_VERBOSE
1089 	printf("*** Scanning for register number.\n");
1090 #endif
1091 	for (i = 0; i < nelementsp / 5; i++) {
1092 		rs = &data[5 * i];
1093 #ifdef MYRI10GE_REGSET_VERBOSE
1094 		printf("Examining register set %d:\n", i);
1095 		printf("  Register number = %d.\n", REGISTER_NUMBER(rs));
1096 		printf("  Function number = %d.\n", FUNCTION_NUMBER(rs));
1097 		printf("  Device number   = %d.\n", DEVICE_NUMBER(rs));
1098 		printf("  Bus number      = %d.\n", BUS_NUMBER(rs));
1099 		printf("  Address space   = %d (%s ).\n", ADDRESS_SPACE(rs),
1100 		    address_space_name[ADDRESS_SPACE(rs)]);
1101 		printf("  pci address 0x%08x %08x\n", PCI_ADDR_HIGH(rs),
1102 		    PCI_ADDR_LOW(rs));
1103 		printf("  pci span 0x%08x %08x\n", PCI_SPAN_HIGH(rs),
1104 		    PCI_SPAN_LOW(rs));
1105 #endif
1106 		/* We are looking for a memory property. */
1107 
1108 		if (ADDRESS_SPACE(rs) == MX_DDI_REG_SET_64_BIT_MEMORY_SPACE ||
1109 		    ADDRESS_SPACE(rs) == MX_DDI_REG_SET_32_BIT_MEMORY_SPACE) {
1110 			*reg_set = i;
1111 
1112 #ifdef MYRI10GE_REGSET_VERBOSE
1113 			printf("%s uses register set %d.\n",
1114 			    address_space_name[ADDRESS_SPACE(rs)], *reg_set);
1115 #endif
1116 
1117 			*span = (PCI_SPAN_LOW(rs));
1118 #ifdef MYRI10GE_REGSET_VERBOSE
1119 			printf("Board span is 0x%x\n", *span);
1120 #endif
1121 			break;
1122 		}
1123 	}
1124 
1125 	ddi_prop_free(data);
1126 
1127 	/* If no match, fail. */
1128 	if (i >= nelementsp / 5) {
1129 		return (EIO);
1130 	}
1131 
1132 	return (0);
1133 }
1134 
1135 
1136 static int
myri10ge_load_firmware_from_zlib(struct myri10ge_priv * mgp,uint32_t * limit)1137 myri10ge_load_firmware_from_zlib(struct myri10ge_priv *mgp, uint32_t *limit)
1138 {
1139 	void *inflate_buffer;
1140 	int rv, status;
1141 	size_t sram_size = mgp->sram_size - MYRI10GE_EEPROM_STRINGS_SIZE;
1142 	size_t destlen;
1143 	mcp_gen_header_t *hdr;
1144 	unsigned hdr_offset, i;
1145 
1146 
1147 	*limit = 0; /* -Wuninitialized */
1148 	status = 0;
1149 
1150 	inflate_buffer = kmem_zalloc(sram_size, KM_NOSLEEP);
1151 	if (!inflate_buffer) {
1152 		cmn_err(CE_WARN,
1153 		    "%s: Could not allocate buffer to inflate mcp\n",
1154 		    mgp->name);
1155 		return (ENOMEM);
1156 	}
1157 
1158 	destlen = sram_size;
1159 	rv = z_uncompress(inflate_buffer, &destlen, mgp->eth_z8e,
1160 	    mgp->eth_z8e_length);
1161 
1162 	if (rv != Z_OK) {
1163 		cmn_err(CE_WARN, "%s: Could not inflate mcp: %s\n",
1164 		    mgp->name, z_strerror(rv));
1165 		status = ENXIO;
1166 		goto abort;
1167 	}
1168 
1169 	*limit = (uint32_t)destlen;
1170 
1171 	hdr_offset = htonl(*(uint32_t *)(void *)((char *)inflate_buffer +
1172 	    MCP_HEADER_PTR_OFFSET));
1173 	hdr = (void *)((char *)inflate_buffer + hdr_offset);
1174 	if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) {
1175 		cmn_err(CE_WARN, "%s: Bad firmware type: 0x%x\n", mgp->name,
1176 		    ntohl(hdr->mcp_type));
1177 		status = EIO;
1178 		goto abort;
1179 	}
1180 
1181 	/* save firmware version for kstat */
1182 	(void) strncpy(mgp->fw_version, hdr->version, sizeof (mgp->fw_version));
1183 	if (myri10ge_verbose)
1184 		printf("%s: firmware id: %s\n", mgp->name, hdr->version);
1185 
1186 	/* Copy the inflated firmware to NIC SRAM. */
1187 	for (i = 0; i < *limit; i += 256) {
1188 		myri10ge_pio_copy((char *)mgp->sram + MYRI10GE_FW_OFFSET + i,
1189 		    (char *)inflate_buffer + i,
1190 		    min(256U, (unsigned)(*limit - i)));
1191 		mb();
1192 		(void) *(int *)(void *)mgp->sram;
1193 		mb();
1194 	}
1195 
1196 abort:
1197 	kmem_free(inflate_buffer, sram_size);
1198 
1199 	return (status);
1200 
1201 }
1202 
1203 
1204 int
myri10ge_send_cmd(struct myri10ge_priv * mgp,uint32_t cmd,myri10ge_cmd_t * data)1205 myri10ge_send_cmd(struct myri10ge_priv *mgp, uint32_t cmd,
1206     myri10ge_cmd_t *data)
1207 {
1208 	mcp_cmd_t *buf;
1209 	char buf_bytes[sizeof (*buf) + 8];
1210 	volatile mcp_cmd_response_t *response = mgp->cmd;
1211 	volatile char *cmd_addr =
1212 	    (volatile char *)mgp->sram + MXGEFW_ETH_CMD;
1213 	int sleep_total = 0;
1214 
1215 	/* ensure buf is aligned to 8 bytes */
1216 	buf = (mcp_cmd_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
1217 
1218 	buf->data0 = htonl(data->data0);
1219 	buf->data1 = htonl(data->data1);
1220 	buf->data2 = htonl(data->data2);
1221 	buf->cmd = htonl(cmd);
1222 	buf->response_addr.low = mgp->cmd_dma.low;
1223 	buf->response_addr.high = mgp->cmd_dma.high;
1224 	mutex_enter(&mgp->cmd_lock);
1225 	response->result = 0xffffffff;
1226 	mb();
1227 
1228 	myri10ge_pio_copy((void *)cmd_addr, buf, sizeof (*buf));
1229 
1230 	/* wait up to 20ms */
1231 	for (sleep_total = 0; sleep_total < 20; sleep_total++) {
1232 		mb();
1233 		if (response->result != 0xffffffff) {
1234 			if (response->result == 0) {
1235 				data->data0 = ntohl(response->data);
1236 				mutex_exit(&mgp->cmd_lock);
1237 				return (0);
1238 			} else if (ntohl(response->result)
1239 			    == MXGEFW_CMD_UNKNOWN) {
1240 				mutex_exit(&mgp->cmd_lock);
1241 				return (ENOSYS);
1242 			} else if (ntohl(response->result)
1243 			    == MXGEFW_CMD_ERROR_UNALIGNED) {
1244 				mutex_exit(&mgp->cmd_lock);
1245 				return (E2BIG);
1246 			} else {
1247 				cmn_err(CE_WARN,
1248 				    "%s: command %d failed, result = %d\n",
1249 				    mgp->name, cmd, ntohl(response->result));
1250 				mutex_exit(&mgp->cmd_lock);
1251 				return (ENXIO);
1252 			}
1253 		}
1254 		drv_usecwait(1000);
1255 	}
1256 	mutex_exit(&mgp->cmd_lock);
1257 	cmn_err(CE_WARN, "%s: command %d timed out, result = %d\n",
1258 	    mgp->name, cmd, ntohl(response->result));
1259 	return (EAGAIN);
1260 }
1261 
1262 /*
1263  * Enable or disable periodic RDMAs from the host to make certain
1264  * chipsets resend dropped PCIe messages
1265  */
1266 
1267 static void
myri10ge_dummy_rdma(struct myri10ge_priv * mgp,int enable)1268 myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable)
1269 {
1270 	char buf_bytes[72];
1271 	volatile uint32_t *confirm;
1272 	volatile char *submit;
1273 	uint32_t *buf;
1274 	int i;
1275 
1276 	buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
1277 
1278 	/* clear confirmation addr */
1279 	confirm = (volatile uint32_t *)mgp->cmd;
1280 	*confirm = 0;
1281 	mb();
1282 
1283 	/*
1284 	 * send an rdma command to the PCIe engine, and wait for the
1285 	 * response in the confirmation address.  The firmware should
1286 	 *  write a -1 there to indicate it is alive and well
1287 	 */
1288 
1289 	buf[0] = mgp->cmd_dma.high;		/* confirm addr MSW */
1290 	buf[1] = mgp->cmd_dma.low;		/* confirm addr LSW */
1291 	buf[2] = htonl(0xffffffff);		/* confirm data */
1292 	buf[3] = htonl(mgp->cmd_dma.high);	/* dummy addr MSW */
1293 	buf[4] = htonl(mgp->cmd_dma.low);	/* dummy addr LSW */
1294 	buf[5] = htonl(enable);			/* enable? */
1295 
1296 
1297 	submit = (volatile char *)(mgp->sram + MXGEFW_BOOT_DUMMY_RDMA);
1298 
1299 	myri10ge_pio_copy((char *)submit, buf, 64);
1300 	mb();
1301 	drv_usecwait(1000);
1302 	mb();
1303 	i = 0;
1304 	while (*confirm != 0xffffffff && i < 20) {
1305 		drv_usecwait(1000);
1306 		i++;
1307 	}
1308 	if (*confirm != 0xffffffff) {
1309 		cmn_err(CE_WARN, "%s: dummy rdma %s failed (%p = 0x%x)",
1310 		    mgp->name,
1311 		    (enable ? "enable" : "disable"), (void*) confirm, *confirm);
1312 	}
1313 }
1314 
1315 static int
myri10ge_load_firmware(struct myri10ge_priv * mgp)1316 myri10ge_load_firmware(struct myri10ge_priv *mgp)
1317 {
1318 	myri10ge_cmd_t cmd;
1319 	volatile uint32_t *confirm;
1320 	volatile char *submit;
1321 	char buf_bytes[72];
1322 	uint32_t *buf, size;
1323 	int status, i;
1324 
1325 	buf = (uint32_t *)((unsigned long)(buf_bytes + 7) & ~7UL);
1326 
1327 	status = myri10ge_load_firmware_from_zlib(mgp, &size);
1328 	if (status) {
1329 		cmn_err(CE_WARN, "%s: firmware loading failed\n", mgp->name);
1330 		return (status);
1331 	}
1332 
1333 	/* clear confirmation addr */
1334 	confirm = (volatile uint32_t *)mgp->cmd;
1335 	*confirm = 0;
1336 	mb();
1337 
1338 	/*
1339 	 * send a reload command to the bootstrap MCP, and wait for the
1340 	 * response in the confirmation address.  The firmware should
1341 	 * write a -1 there to indicate it is alive and well
1342 	 */
1343 
1344 	buf[0] = mgp->cmd_dma.high;	/* confirm addr MSW */
1345 	buf[1] = mgp->cmd_dma.low;	/* confirm addr LSW */
1346 	buf[2] = htonl(0xffffffff);	/* confirm data */
1347 
1348 	/*
1349 	 * FIX: All newest firmware should un-protect the bottom of
1350 	 * the sram before handoff. However, the very first interfaces
1351 	 * do not. Therefore the handoff copy must skip the first 8 bytes
1352 	 */
1353 	buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */
1354 	buf[4] = htonl(size - 8);	/* length of code */
1355 	buf[5] = htonl(8);		/* where to copy to */
1356 	buf[6] = htonl(0);		/* where to jump to */
1357 
1358 	submit = (volatile char *)(mgp->sram + MXGEFW_BOOT_HANDOFF);
1359 
1360 	myri10ge_pio_copy((char *)submit, buf, 64);
1361 	mb();
1362 	drv_usecwait(1000);
1363 	mb();
1364 	i = 0;
1365 	while (*confirm != 0xffffffff && i < 1000) {
1366 		drv_usecwait(1000);
1367 		i++;
1368 	}
1369 	if (*confirm != 0xffffffff) {
1370 		cmn_err(CE_WARN, "%s: handoff failed (%p = 0x%x)",
1371 		    mgp->name, (void *) confirm, *confirm);
1372 
1373 		return (ENXIO);
1374 	}
1375 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
1376 	if (status != 0) {
1377 		cmn_err(CE_WARN, "%s: failed MXGEFW_CMD_GET_RX_RING_SIZE\n",
1378 		    mgp->name);
1379 		return (ENXIO);
1380 	}
1381 
1382 	mgp->max_intr_slots = 2 * (cmd.data0 / sizeof (mcp_dma_addr_t));
1383 	myri10ge_dummy_rdma(mgp, 1);
1384 	return (0);
1385 }
1386 
1387 static int
myri10ge_m_unicst(void * arg,const uint8_t * addr)1388 myri10ge_m_unicst(void *arg, const uint8_t *addr)
1389 {
1390 	struct myri10ge_priv *mgp = arg;
1391 	myri10ge_cmd_t cmd;
1392 	int status;
1393 
1394 	cmd.data0 = ((addr[0] << 24) | (addr[1] << 16)
1395 	    | (addr[2] << 8) | addr[3]);
1396 
1397 	cmd.data1 = ((addr[4] << 8) | (addr[5]));
1398 
1399 	status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd);
1400 	if (status == 0 && (addr != mgp->mac_addr))
1401 		(void) memcpy(mgp->mac_addr, addr, sizeof (mgp->mac_addr));
1402 
1403 	return (status);
1404 }
1405 
1406 static int
myri10ge_change_pause(struct myri10ge_priv * mgp,int pause)1407 myri10ge_change_pause(struct myri10ge_priv *mgp, int pause)
1408 {
1409 	myri10ge_cmd_t cmd;
1410 	int status;
1411 
1412 	if (pause)
1413 		status = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_FLOW_CONTROL,
1414 		    &cmd);
1415 	else
1416 		status = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_FLOW_CONTROL,
1417 		    &cmd);
1418 
1419 	if (status) {
1420 		cmn_err(CE_WARN, "%s: Failed to set flow control mode\n",
1421 		    mgp->name);
1422 		return (ENXIO);
1423 	}
1424 	mgp->pause = pause;
1425 	return (0);
1426 }
1427 
1428 static void
myri10ge_change_promisc(struct myri10ge_priv * mgp,int promisc)1429 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc)
1430 {
1431 	myri10ge_cmd_t cmd;
1432 	int status;
1433 
1434 	if (promisc)
1435 		status = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_PROMISC, &cmd);
1436 	else
1437 		status = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_PROMISC, &cmd);
1438 
1439 	if (status) {
1440 		cmn_err(CE_WARN, "%s: Failed to set promisc mode\n",
1441 		    mgp->name);
1442 	}
1443 }
1444 
1445 static int
myri10ge_dma_test(struct myri10ge_priv * mgp,int test_type)1446 myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type)
1447 {
1448 	myri10ge_cmd_t cmd;
1449 	int status;
1450 	uint32_t len;
1451 	void *dmabench;
1452 	struct myri10ge_dma_stuff dmabench_dma;
1453 	char *test = " ";
1454 
1455 	/*
1456 	 * Run a small DMA test.
1457 	 * The magic multipliers to the length tell the firmware
1458 	 * tp do DMA read, write, or read+write tests.  The
1459 	 * results are returned in cmd.data0.  The upper 16
1460 	 * bits or the return is the number of transfers completed.
1461 	 * The lower 16 bits is the time in 0.5us ticks that the
1462 	 * transfers took to complete
1463 	 */
1464 
1465 	len = mgp->tx_boundary;
1466 
1467 	dmabench = myri10ge_dma_alloc(mgp->dip, len,
1468 	    &myri10ge_rx_jumbo_dma_attr, &myri10ge_dev_access_attr,
1469 	    DDI_DMA_STREAMING,  DDI_DMA_RDWR|DDI_DMA_STREAMING,
1470 	    &dmabench_dma, 1, DDI_DMA_DONTWAIT);
1471 	mgp->read_dma = mgp->write_dma = mgp->read_write_dma = 0;
1472 	if (dmabench == NULL) {
1473 		cmn_err(CE_WARN, "%s dma benchmark aborted\n", mgp->name);
1474 		return (ENOMEM);
1475 	}
1476 
1477 	cmd.data0 = ntohl(dmabench_dma.low);
1478 	cmd.data1 = ntohl(dmabench_dma.high);
1479 	cmd.data2 = len * 0x10000;
1480 	status = myri10ge_send_cmd(mgp, test_type, &cmd);
1481 	if (status != 0) {
1482 		test = "read";
1483 		goto abort;
1484 	}
1485 	mgp->read_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff);
1486 
1487 	cmd.data0 = ntohl(dmabench_dma.low);
1488 	cmd.data1 = ntohl(dmabench_dma.high);
1489 	cmd.data2 = len * 0x1;
1490 	status = myri10ge_send_cmd(mgp, test_type, &cmd);
1491 	if (status != 0) {
1492 		test = "write";
1493 		goto abort;
1494 	}
1495 	mgp->write_dma = ((cmd.data0>>16) * len * 2) / (cmd.data0 & 0xffff);
1496 
1497 	cmd.data0 = ntohl(dmabench_dma.low);
1498 	cmd.data1 = ntohl(dmabench_dma.high);
1499 	cmd.data2 = len * 0x10001;
1500 	status = myri10ge_send_cmd(mgp, test_type, &cmd);
1501 	if (status != 0) {
1502 		test = "read/write";
1503 		goto abort;
1504 	}
1505 	mgp->read_write_dma = ((cmd.data0>>16) * len * 2 * 2) /
1506 	    (cmd.data0 & 0xffff);
1507 
1508 
1509 abort:
1510 	myri10ge_dma_free(&dmabench_dma);
1511 	if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST)
1512 		cmn_err(CE_WARN, "%s %s dma benchmark failed\n", mgp->name,
1513 		    test);
1514 	return (status);
1515 }
1516 
1517 static int
myri10ge_reset(struct myri10ge_priv * mgp)1518 myri10ge_reset(struct myri10ge_priv *mgp)
1519 {
1520 	myri10ge_cmd_t cmd;
1521 	struct myri10ge_nic_stat *ethstat;
1522 	struct myri10ge_slice_state *ss;
1523 	int i, status;
1524 	size_t bytes;
1525 
1526 	/* send a reset command to the card to see if it is alive */
1527 	(void) memset(&cmd, 0, sizeof (cmd));
1528 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd);
1529 	if (status != 0) {
1530 		cmn_err(CE_WARN, "%s: failed reset\n", mgp->name);
1531 		return (ENXIO);
1532 	}
1533 
1534 	/* Now exchange information about interrupts  */
1535 
1536 	bytes = mgp->max_intr_slots * sizeof (*mgp->ss[0].rx_done.entry);
1537 	cmd.data0 = (uint32_t)bytes;
1538 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd);
1539 
1540 	/*
1541 	 * Even though we already know how many slices are supported
1542 	 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES
1543 	 * has magic side effects, and must be called after a reset.
1544 	 * It must be called prior to calling any RSS related cmds,
1545 	 * including assigning an interrupt queue for anything but
1546 	 * slice 0.  It must also be called *after*
1547 	 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by
1548 	 * the firmware to compute offsets.
1549 	 */
1550 
1551 	if (mgp->num_slices > 1) {
1552 
1553 		/* ask the maximum number of slices it supports */
1554 		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES,
1555 		    &cmd);
1556 		if (status != 0) {
1557 			cmn_err(CE_WARN,
1558 			    "%s: failed to get number of slices\n",
1559 			    mgp->name);
1560 			return (status);
1561 		}
1562 
1563 		/*
1564 		 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior
1565 		 * to setting up the interrupt queue DMA
1566 		 */
1567 
1568 		cmd.data0 = mgp->num_slices;
1569 		cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE |
1570 		    MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES;
1571 		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
1572 		    &cmd);
1573 		if (status != 0) {
1574 			cmn_err(CE_WARN,
1575 			    "%s: failed to set number of slices\n",
1576 			    mgp->name);
1577 			return (status);
1578 		}
1579 	}
1580 	for (i = 0; i < mgp->num_slices; i++) {
1581 		ss = &mgp->ss[i];
1582 		cmd.data0 = ntohl(ss->rx_done.dma.low);
1583 		cmd.data1 = ntohl(ss->rx_done.dma.high);
1584 		cmd.data2 = i;
1585 		status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA,
1586 		    &cmd);
1587 	};
1588 
1589 	status |= myri10ge_send_cmd(mgp,  MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd);
1590 	for (i = 0; i < mgp->num_slices; i++) {
1591 		ss = &mgp->ss[i];
1592 		ss->irq_claim = (volatile unsigned int *)
1593 		    (void *)(mgp->sram + cmd.data0 + 8 * i);
1594 	}
1595 
1596 	if (mgp->ddi_intr_type == DDI_INTR_TYPE_FIXED) {
1597 		status |= myri10ge_send_cmd(mgp,
1598 		    MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, &cmd);
1599 		mgp->irq_deassert = (uint32_t *)(void *)(mgp->sram + cmd.data0);
1600 	}
1601 
1602 	status |= myri10ge_send_cmd(mgp,
1603 	    MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd);
1604 	mgp->intr_coal_delay_ptr = (uint32_t *)(void *)(mgp->sram + cmd.data0);
1605 
1606 	if (status != 0) {
1607 		cmn_err(CE_WARN, "%s: failed set interrupt parameters\n",
1608 		    mgp->name);
1609 		return (status);
1610 	}
1611 
1612 	*mgp->intr_coal_delay_ptr = htonl(mgp->intr_coal_delay);
1613 	(void) myri10ge_dma_test(mgp, MXGEFW_DMA_TEST);
1614 
1615 	/* reset mcp/driver shared state back to 0 */
1616 
1617 	for (i = 0; i < mgp->num_slices; i++) {
1618 		ss = &mgp->ss[i];
1619 		bytes = mgp->max_intr_slots *
1620 		    sizeof (*mgp->ss[0].rx_done.entry);
1621 		(void) memset(ss->rx_done.entry, 0, bytes);
1622 		ss->tx.req = 0;
1623 		ss->tx.done = 0;
1624 		ss->tx.pkt_done = 0;
1625 		ss->rx_big.cnt = 0;
1626 		ss->rx_small.cnt = 0;
1627 		ss->rx_done.idx = 0;
1628 		ss->rx_done.cnt = 0;
1629 		ss->rx_token = 0;
1630 		ss->tx.watchdog_done = 0;
1631 		ss->tx.watchdog_req = 0;
1632 		ss->tx.active = 0;
1633 		ss->tx.activate = 0;
1634 	}
1635 	mgp->watchdog_rx_pause = 0;
1636 	if (mgp->ksp_stat != NULL) {
1637 		ethstat = (struct myri10ge_nic_stat *)mgp->ksp_stat->ks_data;
1638 		ethstat->link_changes.value.ul = 0;
1639 	}
1640 	status = myri10ge_m_unicst(mgp, mgp->mac_addr);
1641 	myri10ge_change_promisc(mgp, 0);
1642 	(void) myri10ge_change_pause(mgp, mgp->pause);
1643 	return (status);
1644 }
1645 
1646 static int
myri10ge_init_toeplitz(struct myri10ge_priv * mgp)1647 myri10ge_init_toeplitz(struct myri10ge_priv *mgp)
1648 {
1649 	myri10ge_cmd_t cmd;
1650 	int i, b, s, t, j;
1651 	int status;
1652 	uint32_t k[8];
1653 	uint32_t tmp;
1654 	uint8_t *key;
1655 
1656 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RSS_KEY_OFFSET,
1657 	    &cmd);
1658 	if (status != 0) {
1659 		cmn_err(CE_WARN, "%s: failed to get rss key\n",
1660 		    mgp->name);
1661 		return (EIO);
1662 	}
1663 	myri10ge_pio_copy32(mgp->rss_key,
1664 	    (uint32_t *)(void*)((char *)mgp->sram + cmd.data0),
1665 	    sizeof (mgp->rss_key));
1666 
1667 	mgp->toeplitz_hash_table = kmem_alloc(sizeof (uint32_t) * 12 * 256,
1668 	    KM_SLEEP);
1669 	key = (uint8_t *)mgp->rss_key;
1670 	t = 0;
1671 	for (b = 0; b < 12; b++) {
1672 		for (s = 0; s < 8; s++) {
1673 			/* Bits: b*8+s, ..., b*8+s+31 */
1674 			k[s] = 0;
1675 			for (j = 0; j < 32; j++) {
1676 				int bit = b*8+s+j;
1677 				bit = 0x1 & (key[bit / 8] >> (7 -(bit & 0x7)));
1678 				k[s] |= bit << (31 - j);
1679 			}
1680 		}
1681 
1682 		for (i = 0; i <= 0xff; i++) {
1683 			tmp = 0;
1684 			if (i & (1 << 7)) { tmp ^= k[0]; }
1685 			if (i & (1 << 6)) { tmp ^= k[1]; }
1686 			if (i & (1 << 5)) { tmp ^= k[2]; }
1687 			if (i & (1 << 4)) { tmp ^= k[3]; }
1688 			if (i & (1 << 3)) { tmp ^= k[4]; }
1689 			if (i & (1 << 2)) { tmp ^= k[5]; }
1690 			if (i & (1 << 1)) { tmp ^= k[6]; }
1691 			if (i & (1 << 0)) { tmp ^= k[7]; }
1692 			mgp->toeplitz_hash_table[t++] = tmp;
1693 		}
1694 	}
1695 	return (0);
1696 }
1697 
1698 static inline struct myri10ge_slice_state *
myri10ge_toeplitz_send_hash(struct myri10ge_priv * mgp,struct ip * ip)1699 myri10ge_toeplitz_send_hash(struct myri10ge_priv *mgp, struct ip *ip)
1700 {
1701 	struct tcphdr *hdr;
1702 	uint32_t saddr, daddr;
1703 	uint32_t hash, slice;
1704 	uint32_t *table = mgp->toeplitz_hash_table;
1705 	uint16_t src, dst;
1706 
1707 	/*
1708 	 * Note hashing order is reversed from how it is done
1709 	 * in the NIC, so as to generate the same hash value
1710 	 * for the connection to try to keep connections CPU local
1711 	 */
1712 
1713 	/* hash on IPv4 src/dst address */
1714 	saddr = ntohl(ip->ip_src.s_addr);
1715 	daddr = ntohl(ip->ip_dst.s_addr);
1716 	hash = table[(256 * 0) + ((daddr >> 24) & 0xff)];
1717 	hash ^= table[(256 * 1) + ((daddr >> 16) & 0xff)];
1718 	hash ^= table[(256 * 2) + ((daddr >> 8) & 0xff)];
1719 	hash ^= table[(256 * 3) + ((daddr) & 0xff)];
1720 	hash ^= table[(256 * 4) + ((saddr >> 24) & 0xff)];
1721 	hash ^= table[(256 * 5) + ((saddr >> 16) & 0xff)];
1722 	hash ^= table[(256 * 6) + ((saddr >> 8) & 0xff)];
1723 	hash ^= table[(256 * 7) + ((saddr) & 0xff)];
1724 	/* hash on TCP port, if required */
1725 	if ((myri10ge_rss_hash & MXGEFW_RSS_HASH_TYPE_TCP_IPV4) &&
1726 	    ip->ip_p == IPPROTO_TCP) {
1727 		hdr = (struct tcphdr *)(void *)
1728 		    (((uint8_t *)ip) +  (ip->ip_hl << 2));
1729 		src = ntohs(hdr->th_sport);
1730 		dst = ntohs(hdr->th_dport);
1731 
1732 		hash ^= table[(256 * 8) + ((dst >> 8) & 0xff)];
1733 		hash ^= table[(256 * 9) + ((dst) & 0xff)];
1734 		hash ^= table[(256 * 10) + ((src >> 8) & 0xff)];
1735 		hash ^= table[(256 * 11) + ((src) & 0xff)];
1736 	}
1737 	slice = (mgp->num_slices - 1) & hash;
1738 	return (&mgp->ss[slice]);
1739 
1740 }
1741 
1742 static inline struct myri10ge_slice_state *
myri10ge_simple_send_hash(struct myri10ge_priv * mgp,struct ip * ip)1743 myri10ge_simple_send_hash(struct myri10ge_priv *mgp, struct ip *ip)
1744 {
1745 	struct tcphdr *hdr;
1746 	uint32_t slice, hash_val;
1747 
1748 
1749 	if (ip->ip_p != IPPROTO_TCP && ip->ip_p != IPPROTO_UDP) {
1750 		return (&mgp->ss[0]);
1751 	}
1752 	hdr = (struct tcphdr *)(void *)(((uint8_t *)ip) +  (ip->ip_hl << 2));
1753 
1754 	/*
1755 	 * Use the second byte of the *destination* address for
1756 	 * MXGEFW_RSS_HASH_TYPE_SRC_PORT, so as to match NIC's hashing
1757 	 */
1758 	hash_val = ntohs(hdr->th_dport) & 0xff;
1759 	if (myri10ge_rss_hash == MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT)
1760 		hash_val += ntohs(hdr->th_sport) & 0xff;
1761 
1762 	slice = (mgp->num_slices - 1) & hash_val;
1763 	return (&mgp->ss[slice]);
1764 }
1765 
1766 static inline struct myri10ge_slice_state *
myri10ge_send_hash(struct myri10ge_priv * mgp,mblk_t * mp)1767 myri10ge_send_hash(struct myri10ge_priv *mgp, mblk_t *mp)
1768 {
1769 	unsigned int slice = 0;
1770 	struct ether_header *eh;
1771 	struct ether_vlan_header *vh;
1772 	struct ip *ip;
1773 	int ehl, ihl;
1774 
1775 	if (mgp->num_slices == 1)
1776 		return (&mgp->ss[0]);
1777 
1778 	if (myri10ge_tx_hash == 0) {
1779 		slice = CPU->cpu_id & (mgp->num_slices - 1);
1780 		return (&mgp->ss[slice]);
1781 	}
1782 
1783 	/*
1784 	 *  ensure it is a TCP or UDP over IPv4 packet, and that the
1785 	 *  headers are in the 1st mblk.  Otherwise, punt
1786 	 */
1787 	ehl = sizeof (*eh);
1788 	ihl = sizeof (*ip);
1789 	if ((MBLKL(mp)) <  (ehl + ihl + 8))
1790 		return (&mgp->ss[0]);
1791 	eh = (struct ether_header *)(void *)mp->b_rptr;
1792 	ip = (struct ip *)(void *)(eh + 1);
1793 	if (eh->ether_type != BE_16(ETHERTYPE_IP)) {
1794 		if (eh->ether_type != BE_16(ETHERTYPE_VLAN))
1795 			return (&mgp->ss[0]);
1796 		vh = (struct ether_vlan_header *)(void *)mp->b_rptr;
1797 		if (vh->ether_type != BE_16(ETHERTYPE_IP))
1798 			return (&mgp->ss[0]);
1799 		ehl += 4;
1800 		ip = (struct ip *)(void *)(vh + 1);
1801 	}
1802 	ihl = ip->ip_hl << 2;
1803 	if (MBLKL(mp) <  (ehl + ihl + 8))
1804 		return (&mgp->ss[0]);
1805 	switch (myri10ge_rss_hash) {
1806 	case MXGEFW_RSS_HASH_TYPE_IPV4:
1807 		/* fallthru */
1808 	case MXGEFW_RSS_HASH_TYPE_TCP_IPV4:
1809 		/* fallthru */
1810 	case (MXGEFW_RSS_HASH_TYPE_IPV4|MXGEFW_RSS_HASH_TYPE_TCP_IPV4):
1811 		return (myri10ge_toeplitz_send_hash(mgp, ip));
1812 	case MXGEFW_RSS_HASH_TYPE_SRC_PORT:
1813 		/* fallthru */
1814 	case MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT:
1815 		return (myri10ge_simple_send_hash(mgp, ip));
1816 	default:
1817 		break;
1818 	}
1819 	return (&mgp->ss[0]);
1820 }
1821 
1822 static int
myri10ge_setup_slice(struct myri10ge_slice_state * ss)1823 myri10ge_setup_slice(struct myri10ge_slice_state *ss)
1824 {
1825 	struct myri10ge_priv *mgp = ss->mgp;
1826 	myri10ge_cmd_t cmd;
1827 	int tx_ring_size, rx_ring_size;
1828 	int tx_ring_entries, rx_ring_entries;
1829 	int slice, status;
1830 	int allocated, idx;
1831 	size_t bytes;
1832 
1833 	slice = ss - mgp->ss;
1834 	cmd.data0 = slice;
1835 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd);
1836 	tx_ring_size = cmd.data0;
1837 	cmd.data0 = slice;
1838 	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd);
1839 	if (status != 0)
1840 		return (status);
1841 	rx_ring_size = cmd.data0;
1842 
1843 	tx_ring_entries = tx_ring_size / sizeof (struct mcp_kreq_ether_send);
1844 	rx_ring_entries = rx_ring_size / sizeof (struct mcp_dma_addr);
1845 	ss->tx.mask = tx_ring_entries - 1;
1846 	ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1;
1847 
1848 	/* get the lanai pointers to the send and receive rings */
1849 
1850 	cmd.data0 = slice;
1851 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, &cmd);
1852 	ss->tx.lanai = (mcp_kreq_ether_send_t *)(void *)(mgp->sram + cmd.data0);
1853 	if (mgp->num_slices > 1) {
1854 		ss->tx.go = (char *)mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice;
1855 		ss->tx.stop = (char *)mgp->sram + MXGEFW_ETH_SEND_STOP +
1856 		    64 * slice;
1857 	} else {
1858 		ss->tx.go = NULL;
1859 		ss->tx.stop = NULL;
1860 	}
1861 
1862 	cmd.data0 = slice;
1863 	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, &cmd);
1864 	ss->rx_small.lanai = (mcp_kreq_ether_recv_t *)
1865 	    (void *)(mgp->sram + cmd.data0);
1866 
1867 	cmd.data0 = slice;
1868 	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd);
1869 	ss->rx_big.lanai = (mcp_kreq_ether_recv_t *)(void *)
1870 	    (mgp->sram + cmd.data0);
1871 
1872 	if (status != 0) {
1873 		cmn_err(CE_WARN,
1874 		    "%s: failed to get ring sizes or locations\n", mgp->name);
1875 		return (status);
1876 	}
1877 
1878 	status = ENOMEM;
1879 	bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
1880 	ss->rx_small.shadow = kmem_zalloc(bytes, KM_SLEEP);
1881 	if (ss->rx_small.shadow == NULL)
1882 		goto abort;
1883 	(void) memset(ss->rx_small.shadow, 0, bytes);
1884 
1885 	bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
1886 	ss->rx_big.shadow = kmem_zalloc(bytes, KM_SLEEP);
1887 	if (ss->rx_big.shadow == NULL)
1888 		goto abort_with_rx_small_shadow;
1889 	(void) memset(ss->rx_big.shadow, 0, bytes);
1890 
1891 	/* allocate the host info rings */
1892 
1893 	bytes = tx_ring_entries * sizeof (*ss->tx.info);
1894 	ss->tx.info = kmem_zalloc(bytes, KM_SLEEP);
1895 	if (ss->tx.info == NULL)
1896 		goto abort_with_rx_big_shadow;
1897 	(void) memset(ss->tx.info, 0, bytes);
1898 
1899 	bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
1900 	ss->rx_small.info = kmem_zalloc(bytes, KM_SLEEP);
1901 	if (ss->rx_small.info == NULL)
1902 		goto abort_with_tx_info;
1903 	(void) memset(ss->rx_small.info, 0, bytes);
1904 
1905 	bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
1906 	ss->rx_big.info = kmem_zalloc(bytes, KM_SLEEP);
1907 	if (ss->rx_big.info == NULL)
1908 		goto abort_with_rx_small_info;
1909 	(void) memset(ss->rx_big.info, 0, bytes);
1910 
1911 	ss->tx.stall = ss->tx.sched = 0;
1912 	ss->tx.stall_early = ss->tx.stall_late = 0;
1913 
1914 	ss->jbufs_for_smalls = 1 + (1 + ss->rx_small.mask) /
1915 	    (myri10ge_mtu / (myri10ge_small_bytes + MXGEFW_PAD));
1916 
1917 	allocated = myri10ge_add_jbufs(ss,
1918 	    myri10ge_bigbufs_initial + ss->jbufs_for_smalls, 1);
1919 	if (allocated < ss->jbufs_for_smalls + myri10ge_bigbufs_initial) {
1920 		cmn_err(CE_WARN,
1921 		    "%s: Could not allocate enough receive buffers (%d/%d)\n",
1922 		    mgp->name, allocated,
1923 		    myri10ge_bigbufs_initial + ss->jbufs_for_smalls);
1924 		goto abort_with_jumbos;
1925 	}
1926 
1927 	myri10ge_carve_up_jbufs_into_small_ring(ss);
1928 	ss->j_rx_cnt = 0;
1929 
1930 	mutex_enter(&ss->jpool.mtx);
1931 	if (allocated < rx_ring_entries)
1932 		ss->jpool.low_water = allocated / 4;
1933 	else
1934 		ss->jpool.low_water = rx_ring_entries / 2;
1935 
1936 	/*
1937 	 * invalidate the big receive ring in case we do not
1938 	 * allocate sufficient jumbos to fill it
1939 	 */
1940 	(void) memset(ss->rx_big.shadow, 1,
1941 	    (ss->rx_big.mask + 1) * sizeof (ss->rx_big.shadow[0]));
1942 	for (idx = 7; idx <= ss->rx_big.mask; idx += 8) {
1943 		myri10ge_submit_8rx(&ss->rx_big.lanai[idx - 7],
1944 		    &ss->rx_big.shadow[idx - 7]);
1945 		mb();
1946 	}
1947 
1948 
1949 	myri10ge_restock_jumbos(ss);
1950 
1951 	for (idx = 7; idx <= ss->rx_small.mask; idx += 8) {
1952 		myri10ge_submit_8rx(&ss->rx_small.lanai[idx - 7],
1953 		    &ss->rx_small.shadow[idx - 7]);
1954 		mb();
1955 	}
1956 	ss->rx_small.cnt = ss->rx_small.mask + 1;
1957 
1958 	mutex_exit(&ss->jpool.mtx);
1959 
1960 	status = myri10ge_prepare_tx_ring(ss);
1961 
1962 	if (status != 0)
1963 		goto abort_with_small_jbufs;
1964 
1965 	cmd.data0 = ntohl(ss->fw_stats_dma.low);
1966 	cmd.data1 = ntohl(ss->fw_stats_dma.high);
1967 	cmd.data2 = sizeof (mcp_irq_data_t);
1968 	cmd.data2 |= (slice << 16);
1969 	bzero(ss->fw_stats, sizeof (*ss->fw_stats));
1970 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd);
1971 	if (status == ENOSYS) {
1972 		cmd.data0 = ntohl(ss->fw_stats_dma.low) +
1973 		    offsetof(mcp_irq_data_t, send_done_count);
1974 		cmd.data1 = ntohl(ss->fw_stats_dma.high);
1975 		status = myri10ge_send_cmd(mgp,
1976 		    MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, &cmd);
1977 	}
1978 	if (status) {
1979 		cmn_err(CE_WARN, "%s: Couldn't set stats DMA\n", mgp->name);
1980 		goto abort_with_tx;
1981 	}
1982 
1983 	return (0);
1984 
1985 abort_with_tx:
1986 	myri10ge_unprepare_tx_ring(ss);
1987 
1988 abort_with_small_jbufs:
1989 	myri10ge_release_small_jbufs(ss);
1990 
1991 abort_with_jumbos:
1992 	if (allocated != 0) {
1993 		mutex_enter(&ss->jpool.mtx);
1994 		ss->jpool.low_water = 0;
1995 		mutex_exit(&ss->jpool.mtx);
1996 		myri10ge_unstock_jumbos(ss);
1997 		myri10ge_remove_jbufs(ss);
1998 	}
1999 
2000 	bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
2001 	kmem_free(ss->rx_big.info, bytes);
2002 
2003 abort_with_rx_small_info:
2004 	bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
2005 	kmem_free(ss->rx_small.info, bytes);
2006 
2007 abort_with_tx_info:
2008 	bytes = tx_ring_entries * sizeof (*ss->tx.info);
2009 	kmem_free(ss->tx.info, bytes);
2010 
2011 abort_with_rx_big_shadow:
2012 	bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
2013 	kmem_free(ss->rx_big.shadow, bytes);
2014 
2015 abort_with_rx_small_shadow:
2016 	bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
2017 	kmem_free(ss->rx_small.shadow, bytes);
2018 abort:
2019 	return (status);
2020 
2021 }
2022 
2023 static void
myri10ge_teardown_slice(struct myri10ge_slice_state * ss)2024 myri10ge_teardown_slice(struct myri10ge_slice_state *ss)
2025 {
2026 	int tx_ring_entries, rx_ring_entries;
2027 	size_t bytes;
2028 
2029 	/* ignore slices that have not been fully setup */
2030 	if (ss->tx.cp == NULL)
2031 		return;
2032 	/* Free the TX copy buffers */
2033 	myri10ge_unprepare_tx_ring(ss);
2034 
2035 	/* stop passing returned buffers to firmware */
2036 
2037 	mutex_enter(&ss->jpool.mtx);
2038 	ss->jpool.low_water = 0;
2039 	mutex_exit(&ss->jpool.mtx);
2040 	myri10ge_release_small_jbufs(ss);
2041 
2042 	/* Release the free jumbo frame pool */
2043 	myri10ge_unstock_jumbos(ss);
2044 	myri10ge_remove_jbufs(ss);
2045 
2046 	rx_ring_entries = ss->rx_big.mask + 1;
2047 	tx_ring_entries = ss->tx.mask + 1;
2048 
2049 	bytes = rx_ring_entries * sizeof (*ss->rx_big.info);
2050 	kmem_free(ss->rx_big.info, bytes);
2051 
2052 	bytes = rx_ring_entries * sizeof (*ss->rx_small.info);
2053 	kmem_free(ss->rx_small.info, bytes);
2054 
2055 	bytes = tx_ring_entries * sizeof (*ss->tx.info);
2056 	kmem_free(ss->tx.info, bytes);
2057 
2058 	bytes = rx_ring_entries * sizeof (*ss->rx_big.shadow);
2059 	kmem_free(ss->rx_big.shadow, bytes);
2060 
2061 	bytes = rx_ring_entries * sizeof (*ss->rx_small.shadow);
2062 	kmem_free(ss->rx_small.shadow, bytes);
2063 
2064 }
2065 static int
myri10ge_start_locked(struct myri10ge_priv * mgp)2066 myri10ge_start_locked(struct myri10ge_priv *mgp)
2067 {
2068 	myri10ge_cmd_t cmd;
2069 	int status, big_pow2, i;
2070 	volatile uint8_t *itable;
2071 
2072 	status = DDI_SUCCESS;
2073 	/* Allocate DMA resources and receive buffers */
2074 
2075 	status = myri10ge_reset(mgp);
2076 	if (status != 0) {
2077 		cmn_err(CE_WARN, "%s: failed reset\n", mgp->name);
2078 		return (DDI_FAILURE);
2079 	}
2080 
2081 	if (mgp->num_slices > 1) {
2082 		cmd.data0 = mgp->num_slices;
2083 		cmd.data1 = 1; /* use MSI-X */
2084 		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES,
2085 		    &cmd);
2086 		if (status != 0) {
2087 			cmn_err(CE_WARN,
2088 			    "%s: failed to set number of slices\n",
2089 			    mgp->name);
2090 			goto abort_with_nothing;
2091 		}
2092 		/* setup the indirection table */
2093 		cmd.data0 = mgp->num_slices;
2094 		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE,
2095 		    &cmd);
2096 
2097 		status |= myri10ge_send_cmd(mgp,
2098 		    MXGEFW_CMD_GET_RSS_TABLE_OFFSET, &cmd);
2099 		if (status != 0) {
2100 			cmn_err(CE_WARN,
2101 			    "%s: failed to setup rss tables\n", mgp->name);
2102 		}
2103 
2104 		/* just enable an identity mapping */
2105 		itable = mgp->sram + cmd.data0;
2106 		for (i = 0; i < mgp->num_slices; i++)
2107 			itable[i] = (uint8_t)i;
2108 
2109 		if (myri10ge_rss_hash & MYRI10GE_TOEPLITZ_HASH) {
2110 			status = myri10ge_init_toeplitz(mgp);
2111 			if (status != 0) {
2112 				cmn_err(CE_WARN, "%s: failed to setup "
2113 				    "toeplitz tx hash table", mgp->name);
2114 				goto abort_with_nothing;
2115 			}
2116 		}
2117 		cmd.data0 = 1;
2118 		cmd.data1 = myri10ge_rss_hash;
2119 		status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE,
2120 		    &cmd);
2121 		if (status != 0) {
2122 			cmn_err(CE_WARN,
2123 			    "%s: failed to enable slices\n", mgp->name);
2124 			goto abort_with_toeplitz;
2125 		}
2126 	}
2127 
2128 	for (i = 0; i < mgp->num_slices; i++) {
2129 		status = myri10ge_setup_slice(&mgp->ss[i]);
2130 		if (status != 0)
2131 			goto abort_with_slices;
2132 	}
2133 
2134 	/*
2135 	 * Tell the MCP how many buffers it has, and to
2136 	 *  bring the ethernet interface up
2137 	 *
2138 	 * Firmware needs the big buff size as a power of 2.  Lie and
2139 	 * tell it the buffer is larger, because we only use 1
2140 	 * buffer/pkt, and the mtu will prevent overruns
2141 	 */
2142 	big_pow2 = myri10ge_mtu + MXGEFW_PAD;
2143 	while (!ISP2(big_pow2))
2144 		big_pow2++;
2145 
2146 	/* now give firmware buffers sizes, and MTU */
2147 	cmd.data0 = myri10ge_mtu;
2148 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd);
2149 	cmd.data0 = myri10ge_small_bytes;
2150 	status |=
2151 	    myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd);
2152 	cmd.data0 = big_pow2;
2153 	status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd);
2154 	if (status) {
2155 		cmn_err(CE_WARN, "%s: Couldn't set buffer sizes\n", mgp->name);
2156 		goto abort_with_slices;
2157 	}
2158 
2159 
2160 	cmd.data0 = 1;
2161 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd);
2162 	if (status) {
2163 		cmn_err(CE_WARN, "%s: unable to setup TSO (%d)\n",
2164 		    mgp->name, status);
2165 	} else {
2166 		mgp->features |= MYRI10GE_TSO;
2167 	}
2168 
2169 	mgp->link_state = -1;
2170 	mgp->rdma_tags_available = 15;
2171 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd);
2172 	if (status) {
2173 		cmn_err(CE_WARN, "%s: unable to start ethernet\n", mgp->name);
2174 		goto abort_with_slices;
2175 	}
2176 	mgp->running = MYRI10GE_ETH_RUNNING;
2177 	return (DDI_SUCCESS);
2178 
2179 abort_with_slices:
2180 	for (i = 0; i < mgp->num_slices; i++)
2181 		myri10ge_teardown_slice(&mgp->ss[i]);
2182 
2183 	mgp->running = MYRI10GE_ETH_STOPPED;
2184 
2185 abort_with_toeplitz:
2186 	if (mgp->toeplitz_hash_table != NULL) {
2187 		kmem_free(mgp->toeplitz_hash_table,
2188 		    sizeof (uint32_t) * 12 * 256);
2189 		mgp->toeplitz_hash_table = NULL;
2190 	}
2191 
2192 abort_with_nothing:
2193 	return (DDI_FAILURE);
2194 }
2195 
2196 static void
myri10ge_stop_locked(struct myri10ge_priv * mgp)2197 myri10ge_stop_locked(struct myri10ge_priv *mgp)
2198 {
2199 	int status, old_down_cnt;
2200 	myri10ge_cmd_t cmd;
2201 	int wait_time = 10;
2202 	int i, polling;
2203 
2204 	old_down_cnt = mgp->down_cnt;
2205 	mb();
2206 	status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd);
2207 	if (status) {
2208 		cmn_err(CE_WARN, "%s: Couldn't bring down link\n", mgp->name);
2209 	}
2210 
2211 	while (old_down_cnt == *((volatile int *)&mgp->down_cnt)) {
2212 		delay(1 * drv_usectohz(1000000));
2213 		wait_time--;
2214 		if (wait_time == 0)
2215 			break;
2216 	}
2217 again:
2218 	if (old_down_cnt == *((volatile int *)&mgp->down_cnt)) {
2219 		cmn_err(CE_WARN, "%s: didn't get down irq\n", mgp->name);
2220 		for (i = 0; i < mgp->num_slices; i++) {
2221 			/*
2222 			 * take and release the rx lock to ensure
2223 			 * that no interrupt thread is blocked
2224 			 * elsewhere in the stack, preventing
2225 			 * completion
2226 			 */
2227 
2228 			mutex_enter(&mgp->ss[i].rx_lock);
2229 			printf("%s: slice %d rx irq idle\n",
2230 			    mgp->name, i);
2231 			mutex_exit(&mgp->ss[i].rx_lock);
2232 
2233 			/* verify that the poll handler is inactive */
2234 			mutex_enter(&mgp->ss->poll_lock);
2235 			polling = mgp->ss->rx_polling;
2236 			mutex_exit(&mgp->ss->poll_lock);
2237 			if (polling) {
2238 				printf("%s: slice %d is polling\n",
2239 				    mgp->name, i);
2240 				delay(1 * drv_usectohz(1000000));
2241 				goto again;
2242 			}
2243 		}
2244 		delay(1 * drv_usectohz(1000000));
2245 		if (old_down_cnt == *((volatile int *)&mgp->down_cnt)) {
2246 			cmn_err(CE_WARN, "%s: Never got down irq\n", mgp->name);
2247 		}
2248 	}
2249 
2250 	for (i = 0; i < mgp->num_slices; i++)
2251 		myri10ge_teardown_slice(&mgp->ss[i]);
2252 
2253 	if (mgp->toeplitz_hash_table != NULL) {
2254 		kmem_free(mgp->toeplitz_hash_table,
2255 		    sizeof (uint32_t) * 12 * 256);
2256 		mgp->toeplitz_hash_table = NULL;
2257 	}
2258 	mgp->running = MYRI10GE_ETH_STOPPED;
2259 }
2260 
2261 static int
myri10ge_m_start(void * arg)2262 myri10ge_m_start(void *arg)
2263 {
2264 	struct myri10ge_priv *mgp = arg;
2265 	int status;
2266 
2267 	mutex_enter(&mgp->intrlock);
2268 
2269 	if (mgp->running != MYRI10GE_ETH_STOPPED) {
2270 		mutex_exit(&mgp->intrlock);
2271 		return (DDI_FAILURE);
2272 	}
2273 	status = myri10ge_start_locked(mgp);
2274 	mutex_exit(&mgp->intrlock);
2275 
2276 	if (status != DDI_SUCCESS)
2277 		return (status);
2278 
2279 	/* start the watchdog timer */
2280 	mgp->timer_id = timeout(myri10ge_watchdog, mgp,
2281 	    mgp->timer_ticks);
2282 	return (DDI_SUCCESS);
2283 
2284 }
2285 
2286 static void
myri10ge_m_stop(void * arg)2287 myri10ge_m_stop(void *arg)
2288 {
2289 	struct myri10ge_priv *mgp = arg;
2290 
2291 	mutex_enter(&mgp->intrlock);
2292 	/* if the device not running give up */
2293 	if (mgp->running != MYRI10GE_ETH_RUNNING) {
2294 		mutex_exit(&mgp->intrlock);
2295 		return;
2296 	}
2297 
2298 	mgp->running = MYRI10GE_ETH_STOPPING;
2299 	mutex_exit(&mgp->intrlock);
2300 	(void) untimeout(mgp->timer_id);
2301 	mutex_enter(&mgp->intrlock);
2302 	myri10ge_stop_locked(mgp);
2303 	mutex_exit(&mgp->intrlock);
2304 
2305 }
2306 
2307 static inline void
myri10ge_rx_csum(mblk_t * mp,struct myri10ge_rx_ring_stats * s,uint32_t csum)2308 myri10ge_rx_csum(mblk_t *mp, struct myri10ge_rx_ring_stats *s, uint32_t csum)
2309 {
2310 	struct ether_header *eh;
2311 	struct ip *ip;
2312 	struct ip6_hdr *ip6;
2313 	uint32_t start, stuff, end, partial, hdrlen;
2314 
2315 
2316 	csum = ntohs((uint16_t)csum);
2317 	eh = (struct ether_header *)(void *)mp->b_rptr;
2318 	hdrlen = sizeof (*eh);
2319 	if (eh->ether_dhost.ether_addr_octet[0] & 1) {
2320 		if (0 == (bcmp(eh->ether_dhost.ether_addr_octet,
2321 		    myri10ge_broadcastaddr, sizeof (eh->ether_dhost))))
2322 			s->brdcstrcv++;
2323 		else
2324 			s->multircv++;
2325 	}
2326 
2327 	if (eh->ether_type == BE_16(ETHERTYPE_VLAN)) {
2328 		/*
2329 		 * fix checksum by subtracting 4 bytes after what the
2330 		 * firmware thought was the end of the ether hdr
2331 		 */
2332 		partial = *(uint32_t *)
2333 		    (void *)(mp->b_rptr + ETHERNET_HEADER_SIZE);
2334 		csum += ~partial;
2335 		csum +=  (csum < ~partial);
2336 		csum = (csum >> 16) + (csum & 0xFFFF);
2337 		csum = (csum >> 16) + (csum & 0xFFFF);
2338 		hdrlen += VLAN_TAGSZ;
2339 	}
2340 
2341 	if (eh->ether_type ==  BE_16(ETHERTYPE_IP)) {
2342 		ip = (struct ip *)(void *)(mp->b_rptr + hdrlen);
2343 		start = ip->ip_hl << 2;
2344 
2345 		if (ip->ip_p == IPPROTO_TCP)
2346 			stuff = start + offsetof(struct tcphdr, th_sum);
2347 		else if (ip->ip_p == IPPROTO_UDP)
2348 			stuff = start + offsetof(struct udphdr, uh_sum);
2349 		else
2350 			return;
2351 		end = ntohs(ip->ip_len);
2352 	} else if (eh->ether_type ==  BE_16(ETHERTYPE_IPV6)) {
2353 		ip6 = (struct ip6_hdr *)(void *)(mp->b_rptr + hdrlen);
2354 		start = sizeof (*ip6);
2355 		if (ip6->ip6_nxt == IPPROTO_TCP) {
2356 			stuff = start + offsetof(struct tcphdr, th_sum);
2357 		} else if (ip6->ip6_nxt == IPPROTO_UDP)
2358 			stuff = start + offsetof(struct udphdr, uh_sum);
2359 		else
2360 			return;
2361 		end = start + ntohs(ip6->ip6_plen);
2362 		/*
2363 		 * IPv6 headers do not contain a checksum, and hence
2364 		 * do not checksum to zero, so they don't "fall out"
2365 		 * of the partial checksum calculation like IPv4
2366 		 * headers do.  We need to fix the partial checksum by
2367 		 * subtracting the checksum of the IPv6 header.
2368 		 */
2369 
2370 		partial = myri10ge_csum_generic((uint16_t *)ip6, sizeof (*ip6));
2371 		csum += ~partial;
2372 		csum +=  (csum < ~partial);
2373 		csum = (csum >> 16) + (csum & 0xFFFF);
2374 		csum = (csum >> 16) + (csum & 0xFFFF);
2375 	} else {
2376 		return;
2377 	}
2378 
2379 	if (MBLKL(mp) > hdrlen + end) {
2380 		/* padded frame, so hw csum may be invalid */
2381 		return;
2382 	}
2383 
2384 	mac_hcksum_set(mp, start, stuff, end, csum, HCK_PARTIALCKSUM);
2385 }
2386 
2387 static mblk_t *
myri10ge_rx_done_small(struct myri10ge_slice_state * ss,uint32_t len,uint32_t csum)2388 myri10ge_rx_done_small(struct myri10ge_slice_state *ss, uint32_t len,
2389     uint32_t csum)
2390 {
2391 	mblk_t *mp;
2392 	myri10ge_rx_ring_t *rx;
2393 	int idx;
2394 
2395 	rx = &ss->rx_small;
2396 	idx = rx->cnt & rx->mask;
2397 	ss->rx_small.cnt++;
2398 
2399 	/* allocate a new buffer to pass up the stack */
2400 	mp = allocb(len + MXGEFW_PAD, 0);
2401 	if (mp == NULL) {
2402 		MYRI10GE_ATOMIC_SLICE_STAT_INC(rx_small_nobuf);
2403 		goto abort;
2404 	}
2405 	bcopy(ss->rx_small.info[idx].ptr,
2406 	    (caddr_t)mp->b_wptr, len + MXGEFW_PAD);
2407 	mp->b_wptr += len + MXGEFW_PAD;
2408 	mp->b_rptr += MXGEFW_PAD;
2409 
2410 	ss->rx_stats.ibytes += len;
2411 	ss->rx_stats.ipackets += 1;
2412 	myri10ge_rx_csum(mp, &ss->rx_stats, csum);
2413 
2414 abort:
2415 	if ((idx & 7) == 7) {
2416 		myri10ge_submit_8rx(&rx->lanai[idx - 7],
2417 		    &rx->shadow[idx - 7]);
2418 	}
2419 
2420 	return (mp);
2421 }
2422 
2423 
2424 static mblk_t *
myri10ge_rx_done_big(struct myri10ge_slice_state * ss,uint32_t len,uint32_t csum)2425 myri10ge_rx_done_big(struct myri10ge_slice_state *ss, uint32_t len,
2426     uint32_t csum)
2427 {
2428 	struct myri10ge_jpool_stuff *jpool;
2429 	struct myri10ge_jpool_entry *j;
2430 	mblk_t *mp;
2431 	int idx, num_owned_by_mcp;
2432 
2433 	jpool = &ss->jpool;
2434 	idx = ss->j_rx_cnt & ss->rx_big.mask;
2435 	j = ss->rx_big.info[idx].j;
2436 
2437 	if (j == NULL) {
2438 		printf("%s: null j at idx=%d, rx_big.cnt = %d, j_rx_cnt=%d\n",
2439 		    ss->mgp->name, idx, ss->rx_big.cnt, ss->j_rx_cnt);
2440 		return (NULL);
2441 	}
2442 
2443 
2444 	ss->rx_big.info[idx].j = NULL;
2445 	ss->j_rx_cnt++;
2446 
2447 
2448 	/*
2449 	 * Check to see if we are low on rx buffers.
2450 	 * Note that we must leave at least 8 free so there are
2451 	 * enough to free in a single 64-byte write.
2452 	 */
2453 	num_owned_by_mcp = ss->rx_big.cnt - ss->j_rx_cnt;
2454 	if (num_owned_by_mcp < jpool->low_water) {
2455 		mutex_enter(&jpool->mtx);
2456 		myri10ge_restock_jumbos(ss);
2457 		mutex_exit(&jpool->mtx);
2458 		num_owned_by_mcp = ss->rx_big.cnt - ss->j_rx_cnt;
2459 		/* if we are still low, then we have to copy */
2460 		if (num_owned_by_mcp < 16) {
2461 			MYRI10GE_ATOMIC_SLICE_STAT_INC(rx_copy);
2462 			/* allocate a new buffer to pass up the stack */
2463 			mp = allocb(len + MXGEFW_PAD, 0);
2464 			if (mp == NULL) {
2465 				goto abort;
2466 			}
2467 			bcopy(j->buf,
2468 			    (caddr_t)mp->b_wptr, len + MXGEFW_PAD);
2469 			myri10ge_jfree_rtn(j);
2470 			/* push buffer back to NIC */
2471 			mutex_enter(&jpool->mtx);
2472 			myri10ge_restock_jumbos(ss);
2473 			mutex_exit(&jpool->mtx);
2474 			goto set_len;
2475 		}
2476 	}
2477 
2478 	/* loan our buffer to the stack */
2479 	mp = desballoc((unsigned char *)j->buf, myri10ge_mtu, 0, &j->free_func);
2480 	if (mp == NULL) {
2481 		goto abort;
2482 	}
2483 
2484 set_len:
2485 	mp->b_rptr += MXGEFW_PAD;
2486 	mp->b_wptr = ((unsigned char *) mp->b_rptr + len);
2487 
2488 	ss->rx_stats.ibytes += len;
2489 	ss->rx_stats.ipackets += 1;
2490 	myri10ge_rx_csum(mp, &ss->rx_stats, csum);
2491 
2492 	return (mp);
2493 
2494 abort:
2495 	myri10ge_jfree_rtn(j);
2496 	MYRI10GE_ATOMIC_SLICE_STAT_INC(rx_big_nobuf);
2497 	return (NULL);
2498 }
2499 
2500 /*
2501  * Free all transmit buffers up until the specified index
2502  */
2503 static inline void
myri10ge_tx_done(struct myri10ge_slice_state * ss,uint32_t mcp_index)2504 myri10ge_tx_done(struct myri10ge_slice_state *ss, uint32_t mcp_index)
2505 {
2506 	myri10ge_tx_ring_t *tx;
2507 	struct myri10ge_tx_dma_handle_head handles;
2508 	int idx;
2509 	int limit = 0;
2510 
2511 	tx = &ss->tx;
2512 	handles.head = NULL;
2513 	handles.tail = NULL;
2514 	while (tx->pkt_done != (int)mcp_index) {
2515 		idx = tx->done & tx->mask;
2516 
2517 		/*
2518 		 * mblk & DMA handle attached only to first slot
2519 		 * per buffer in the packet
2520 		 */
2521 
2522 		if (tx->info[idx].m) {
2523 			(void) ddi_dma_unbind_handle(tx->info[idx].handle->h);
2524 			tx->info[idx].handle->next = handles.head;
2525 			handles.head = tx->info[idx].handle;
2526 			if (handles.tail == NULL)
2527 				handles.tail = tx->info[idx].handle;
2528 			freeb(tx->info[idx].m);
2529 			tx->info[idx].m = 0;
2530 			tx->info[idx].handle = 0;
2531 		}
2532 		if (tx->info[idx].ostat.opackets != 0) {
2533 			tx->stats.multixmt += tx->info[idx].ostat.multixmt;
2534 			tx->stats.brdcstxmt += tx->info[idx].ostat.brdcstxmt;
2535 			tx->stats.obytes += tx->info[idx].ostat.obytes;
2536 			tx->stats.opackets += tx->info[idx].ostat.opackets;
2537 			tx->info[idx].stat.un.all = 0;
2538 			tx->pkt_done++;
2539 		}
2540 
2541 		tx->done++;
2542 		/*
2543 		 * if we stalled the queue, wake it.  But Wait until
2544 		 * we have at least 1/2 our slots free.
2545 		 */
2546 		if ((tx->req - tx->done) < (tx->mask >> 1) &&
2547 		    tx->stall != tx->sched) {
2548 			mutex_enter(&ss->tx.lock);
2549 			tx->sched = tx->stall;
2550 			mutex_exit(&ss->tx.lock);
2551 			mac_tx_ring_update(ss->mgp->mh, tx->rh);
2552 		}
2553 
2554 		/* limit potential for livelock */
2555 		if (unlikely(++limit >  2 * tx->mask))
2556 			break;
2557 	}
2558 	if (tx->req == tx->done && tx->stop != NULL) {
2559 		/*
2560 		 * Nic has sent all pending requests, allow it
2561 		 * to stop polling this queue
2562 		 */
2563 		mutex_enter(&tx->lock);
2564 		if (tx->req == tx->done && tx->active) {
2565 			*(int *)(void *)tx->stop = 1;
2566 			tx->active = 0;
2567 			mb();
2568 		}
2569 		mutex_exit(&tx->lock);
2570 	}
2571 	if (handles.head != NULL)
2572 		myri10ge_free_tx_handles(tx, &handles);
2573 }
2574 
2575 static void
myri10ge_mbl_init(struct myri10ge_mblk_list * mbl)2576 myri10ge_mbl_init(struct myri10ge_mblk_list *mbl)
2577 {
2578 	mbl->head = NULL;
2579 	mbl->tail = &mbl->head;
2580 	mbl->cnt = 0;
2581 }
2582 
2583 /*ARGSUSED*/
2584 void
myri10ge_mbl_append(struct myri10ge_slice_state * ss,struct myri10ge_mblk_list * mbl,mblk_t * mp)2585 myri10ge_mbl_append(struct myri10ge_slice_state *ss,
2586     struct myri10ge_mblk_list *mbl, mblk_t *mp)
2587 {
2588 	*(mbl->tail) = mp;
2589 	mbl->tail = &mp->b_next;
2590 	mp->b_next = NULL;
2591 	mbl->cnt++;
2592 }
2593 
2594 
2595 static inline void
myri10ge_clean_rx_done(struct myri10ge_slice_state * ss,struct myri10ge_mblk_list * mbl,int limit,boolean_t * stop)2596 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss,
2597     struct myri10ge_mblk_list *mbl, int limit, boolean_t *stop)
2598 {
2599 	myri10ge_rx_done_t *rx_done = &ss->rx_done;
2600 	struct myri10ge_priv *mgp = ss->mgp;
2601 	mblk_t *mp;
2602 	struct lro_entry *lro;
2603 	uint16_t length;
2604 	uint16_t checksum;
2605 
2606 
2607 	while (rx_done->entry[rx_done->idx].length != 0) {
2608 		if (unlikely (*stop)) {
2609 			break;
2610 		}
2611 		length = ntohs(rx_done->entry[rx_done->idx].length);
2612 		length &= (~MXGEFW_RSS_HASH_MASK);
2613 
2614 		/* limit potential for livelock */
2615 		limit -= length;
2616 		if (unlikely(limit < 0))
2617 			break;
2618 
2619 		rx_done->entry[rx_done->idx].length = 0;
2620 		checksum = ntohs(rx_done->entry[rx_done->idx].checksum);
2621 		if (length <= myri10ge_small_bytes)
2622 			mp = myri10ge_rx_done_small(ss, length, checksum);
2623 		else
2624 			mp = myri10ge_rx_done_big(ss, length, checksum);
2625 		if (mp != NULL) {
2626 			if (!myri10ge_lro ||
2627 			    0 != myri10ge_lro_rx(ss, mp, checksum, mbl))
2628 				myri10ge_mbl_append(ss, mbl, mp);
2629 		}
2630 		rx_done->cnt++;
2631 		rx_done->idx = rx_done->cnt & (mgp->max_intr_slots - 1);
2632 	}
2633 	while (ss->lro_active != NULL) {
2634 		lro = ss->lro_active;
2635 		ss->lro_active = lro->next;
2636 		myri10ge_lro_flush(ss, lro, mbl);
2637 	}
2638 }
2639 
2640 static void
myri10ge_intr_rx(struct myri10ge_slice_state * ss)2641 myri10ge_intr_rx(struct myri10ge_slice_state *ss)
2642 {
2643 	uint64_t gen;
2644 	struct myri10ge_mblk_list mbl;
2645 
2646 	myri10ge_mbl_init(&mbl);
2647 	if (mutex_tryenter(&ss->rx_lock) == 0)
2648 		return;
2649 	gen = ss->rx_gen_num;
2650 	myri10ge_clean_rx_done(ss, &mbl, MYRI10GE_POLL_NULL,
2651 	    &ss->rx_polling);
2652 	if (mbl.head != NULL)
2653 		mac_rx_ring(ss->mgp->mh, ss->rx_rh, mbl.head, gen);
2654 	mutex_exit(&ss->rx_lock);
2655 
2656 }
2657 
2658 static mblk_t *
myri10ge_poll_rx(void * arg,int bytes)2659 myri10ge_poll_rx(void *arg, int bytes)
2660 {
2661 	struct myri10ge_slice_state *ss = arg;
2662 	struct myri10ge_mblk_list mbl;
2663 	boolean_t dummy = B_FALSE;
2664 
2665 	if (bytes == 0)
2666 		return (NULL);
2667 
2668 	myri10ge_mbl_init(&mbl);
2669 	mutex_enter(&ss->rx_lock);
2670 	if (ss->rx_polling)
2671 		myri10ge_clean_rx_done(ss, &mbl, bytes, &dummy);
2672 	else
2673 		printf("%d: poll_rx: token=%d, polling=%d\n", (int)(ss -
2674 		    ss->mgp->ss), ss->rx_token, ss->rx_polling);
2675 	mutex_exit(&ss->rx_lock);
2676 	return (mbl.head);
2677 }
2678 
2679 /*ARGSUSED*/
2680 static uint_t
myri10ge_intr(caddr_t arg0,caddr_t arg1)2681 myri10ge_intr(caddr_t arg0, caddr_t arg1)
2682 {
2683 	struct myri10ge_slice_state *ss =
2684 	    (struct myri10ge_slice_state *)(void *)arg0;
2685 	struct myri10ge_priv *mgp = ss->mgp;
2686 	mcp_irq_data_t *stats = ss->fw_stats;
2687 	myri10ge_tx_ring_t *tx = &ss->tx;
2688 	uint32_t send_done_count;
2689 	uint8_t valid;
2690 
2691 
2692 	/* make sure the DMA has finished */
2693 	if (!stats->valid) {
2694 		return (DDI_INTR_UNCLAIMED);
2695 	}
2696 	valid = stats->valid;
2697 
2698 	/* low bit indicates receives are present */
2699 	if (valid & 1)
2700 		myri10ge_intr_rx(ss);
2701 
2702 	if (mgp->ddi_intr_type == DDI_INTR_TYPE_FIXED) {
2703 		/* lower legacy IRQ  */
2704 		*mgp->irq_deassert = 0;
2705 		if (!myri10ge_deassert_wait)
2706 			/* don't wait for conf. that irq is low */
2707 			stats->valid = 0;
2708 		mb();
2709 	} else {
2710 		/* no need to wait for conf. that irq is low */
2711 		stats->valid = 0;
2712 	}
2713 
2714 	do {
2715 		/* check for transmit completes and receives */
2716 		send_done_count = ntohl(stats->send_done_count);
2717 		if (send_done_count != tx->pkt_done)
2718 			myri10ge_tx_done(ss, (int)send_done_count);
2719 	} while (*((volatile uint8_t *) &stats->valid));
2720 
2721 	if (stats->stats_updated) {
2722 		if (mgp->link_state != stats->link_up || stats->link_down) {
2723 			mgp->link_state = stats->link_up;
2724 			if (stats->link_down) {
2725 				mgp->down_cnt += stats->link_down;
2726 				mgp->link_state = 0;
2727 			}
2728 			if (mgp->link_state) {
2729 				if (myri10ge_verbose)
2730 					printf("%s: link up\n", mgp->name);
2731 				mac_link_update(mgp->mh, LINK_STATE_UP);
2732 			} else {
2733 				if (myri10ge_verbose)
2734 					printf("%s: link down\n", mgp->name);
2735 				mac_link_update(mgp->mh, LINK_STATE_DOWN);
2736 			}
2737 			MYRI10GE_NIC_STAT_INC(link_changes);
2738 		}
2739 		if (mgp->rdma_tags_available !=
2740 		    ntohl(ss->fw_stats->rdma_tags_available)) {
2741 			mgp->rdma_tags_available =
2742 			    ntohl(ss->fw_stats->rdma_tags_available);
2743 			cmn_err(CE_NOTE, "%s: RDMA timed out! "
2744 			    "%d tags left\n", mgp->name,
2745 			    mgp->rdma_tags_available);
2746 		}
2747 	}
2748 
2749 	mb();
2750 	/* check to see if we have rx token to pass back */
2751 	if (valid & 0x1) {
2752 		mutex_enter(&ss->poll_lock);
2753 		if (ss->rx_polling) {
2754 			ss->rx_token = 1;
2755 		} else {
2756 			*ss->irq_claim = BE_32(3);
2757 			ss->rx_token = 0;
2758 		}
2759 		mutex_exit(&ss->poll_lock);
2760 	}
2761 	*(ss->irq_claim + 1) = BE_32(3);
2762 	return (DDI_INTR_CLAIMED);
2763 }
2764 
2765 /*
2766  * Add or remove a multicast address.  This is called with our
2767  * macinfo's lock held by GLD, so we do not need to worry about
2768  * our own locking here.
2769  */
2770 static int
myri10ge_m_multicst(void * arg,boolean_t add,const uint8_t * multicastaddr)2771 myri10ge_m_multicst(void *arg, boolean_t add, const uint8_t *multicastaddr)
2772 {
2773 	myri10ge_cmd_t cmd;
2774 	struct myri10ge_priv *mgp = arg;
2775 	int status, join_leave;
2776 
2777 	if (add)
2778 		join_leave = MXGEFW_JOIN_MULTICAST_GROUP;
2779 	else
2780 		join_leave = MXGEFW_LEAVE_MULTICAST_GROUP;
2781 	(void) memcpy(&cmd.data0, multicastaddr, 4);
2782 	(void) memcpy(&cmd.data1, multicastaddr + 4, 2);
2783 	cmd.data0 = htonl(cmd.data0);
2784 	cmd.data1 = htonl(cmd.data1);
2785 	status = myri10ge_send_cmd(mgp, join_leave, &cmd);
2786 	if (status == 0)
2787 		return (0);
2788 
2789 	cmn_err(CE_WARN, "%s: failed to set multicast address\n",
2790 	    mgp->name);
2791 	return (status);
2792 }
2793 
2794 
2795 static int
myri10ge_m_promisc(void * arg,boolean_t on)2796 myri10ge_m_promisc(void *arg, boolean_t on)
2797 {
2798 	struct myri10ge_priv *mgp = arg;
2799 
2800 	myri10ge_change_promisc(mgp, on);
2801 	return (0);
2802 }
2803 
2804 /*
2805  * copy an array of mcp_kreq_ether_send_t's to the mcp.  Copy
2806  *  backwards one at a time and handle ring wraps
2807  */
2808 
2809 static inline void
myri10ge_submit_req_backwards(myri10ge_tx_ring_t * tx,mcp_kreq_ether_send_t * src,int cnt)2810 myri10ge_submit_req_backwards(myri10ge_tx_ring_t *tx,
2811     mcp_kreq_ether_send_t *src, int cnt)
2812 {
2813 	int idx, starting_slot;
2814 	starting_slot = tx->req;
2815 	while (cnt > 1) {
2816 		cnt--;
2817 		idx = (starting_slot + cnt) & tx->mask;
2818 		myri10ge_pio_copy(&tx->lanai[idx],
2819 		    &src[cnt], sizeof (*src));
2820 		mb();
2821 	}
2822 }
2823 
2824 /*
2825  * copy an array of mcp_kreq_ether_send_t's to the mcp.  Copy
2826  * at most 32 bytes at a time, so as to avoid involving the software
2827  * pio handler in the nic.   We re-write the first segment's flags
2828  * to mark them valid only after writing the entire chain
2829  */
2830 
2831 static inline void
myri10ge_submit_req(myri10ge_tx_ring_t * tx,mcp_kreq_ether_send_t * src,int cnt)2832 myri10ge_submit_req(myri10ge_tx_ring_t *tx, mcp_kreq_ether_send_t *src,
2833     int cnt)
2834 {
2835 	int idx, i;
2836 	uint32_t *src_ints, *dst_ints;
2837 	mcp_kreq_ether_send_t *srcp, *dstp, *dst;
2838 	uint8_t last_flags;
2839 
2840 	idx = tx->req & tx->mask;
2841 
2842 	last_flags = src->flags;
2843 	src->flags = 0;
2844 	mb();
2845 	dst = dstp = &tx->lanai[idx];
2846 	srcp = src;
2847 
2848 	if ((idx + cnt) < tx->mask) {
2849 		for (i = 0; i < (cnt - 1); i += 2) {
2850 			myri10ge_pio_copy(dstp, srcp, 2 * sizeof (*src));
2851 			mb(); /* force write every 32 bytes */
2852 			srcp += 2;
2853 			dstp += 2;
2854 		}
2855 	} else {
2856 		/*
2857 		 * submit all but the first request, and ensure
2858 		 *  that it is submitted below
2859 		 */
2860 		myri10ge_submit_req_backwards(tx, src, cnt);
2861 		i = 0;
2862 	}
2863 	if (i < cnt) {
2864 		/* submit the first request */
2865 		myri10ge_pio_copy(dstp, srcp, sizeof (*src));
2866 		mb(); /* barrier before setting valid flag */
2867 	}
2868 
2869 	/* re-write the last 32-bits with the valid flags */
2870 	src->flags |= last_flags;
2871 	src_ints = (uint32_t *)src;
2872 	src_ints += 3;
2873 	dst_ints = (uint32_t *)dst;
2874 	dst_ints += 3;
2875 	*dst_ints =  *src_ints;
2876 	tx->req += cnt;
2877 	mb();
2878 	/* notify NIC to poll this tx ring */
2879 	if (!tx->active && tx->go != NULL) {
2880 		*(int *)(void *)tx->go = 1;
2881 		tx->active = 1;
2882 		tx->activate++;
2883 		mb();
2884 	}
2885 }
2886 
2887 /* ARGSUSED */
2888 static inline void
myri10ge_lso_info_get(mblk_t * mp,uint32_t * mss,uint32_t * flags)2889 myri10ge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
2890 {
2891 	uint32_t lso_flag;
2892 	mac_lso_get(mp, mss, &lso_flag);
2893 	(*flags) |= lso_flag;
2894 }
2895 
2896 
2897 /* like pullupmsg, except preserve hcksum/LSO attributes */
2898 static int
myri10ge_pullup(struct myri10ge_slice_state * ss,mblk_t * mp)2899 myri10ge_pullup(struct myri10ge_slice_state *ss, mblk_t *mp)
2900 {
2901 	uint32_t start, stuff, tx_offload_flags, mss;
2902 	int ok;
2903 
2904 	mss = 0;
2905 	mac_hcksum_get(mp, &start, &stuff, NULL, NULL, &tx_offload_flags);
2906 	myri10ge_lso_info_get(mp, &mss, &tx_offload_flags);
2907 
2908 	ok = pullupmsg(mp, -1);
2909 	if (!ok) {
2910 		printf("pullupmsg failed");
2911 		return (DDI_FAILURE);
2912 	}
2913 	MYRI10GE_ATOMIC_SLICE_STAT_INC(xmit_pullup);
2914 	mac_hcksum_set(mp, start, stuff, 0, 0, tx_offload_flags);
2915 	if (tx_offload_flags & HW_LSO)
2916 		DB_LSOMSS(mp) = (uint16_t)mss;
2917 	lso_info_set(mp, mss, tx_offload_flags);
2918 	return (DDI_SUCCESS);
2919 }
2920 
2921 static inline void
myri10ge_tx_stat(struct myri10ge_tx_pkt_stats * s,struct ether_header * eh,int opackets,int obytes)2922 myri10ge_tx_stat(struct myri10ge_tx_pkt_stats *s, struct ether_header *eh,
2923     int opackets, int obytes)
2924 {
2925 	s->un.all = 0;
2926 	if (eh->ether_dhost.ether_addr_octet[0] & 1) {
2927 		if (0 == (bcmp(eh->ether_dhost.ether_addr_octet,
2928 		    myri10ge_broadcastaddr, sizeof (eh->ether_dhost))))
2929 			s->un.s.brdcstxmt = 1;
2930 		else
2931 			s->un.s.multixmt = 1;
2932 	}
2933 	s->un.s.opackets = (uint16_t)opackets;
2934 	s->un.s.obytes = obytes;
2935 }
2936 
2937 static int
myri10ge_tx_copy(struct myri10ge_slice_state * ss,mblk_t * mp,mcp_kreq_ether_send_t * req)2938 myri10ge_tx_copy(struct myri10ge_slice_state *ss, mblk_t *mp,
2939     mcp_kreq_ether_send_t *req)
2940 {
2941 	myri10ge_tx_ring_t *tx = &ss->tx;
2942 	caddr_t ptr;
2943 	struct myri10ge_tx_copybuf *cp;
2944 	mblk_t *bp;
2945 	int idx, mblen, avail;
2946 	uint16_t len;
2947 
2948 	mutex_enter(&tx->lock);
2949 	avail = tx->mask - (tx->req - tx->done);
2950 	if (avail <= 1) {
2951 		mutex_exit(&