1/*
2 * Copyright (c) 2008-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 *    this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 *    this list of conditions and the following disclaimer in the documentation
12 *    and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/types.h>
32#include <sys/ddi.h>
33#include <sys/sunddi.h>
34#include <sys/atomic.h>
35#include <sys/modctl.h>
36#include <sys/conf.h>
37#include <sys/ethernet.h>
38#include <sys/pci.h>
39#include <sys/pcie.h>
40
41#include "sfxge.h"
42
43#include "efx.h"
44
45
46/* Interrupt table DMA attributes */
47static ddi_device_acc_attr_t sfxge_intr_devacc = {
48
49	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
50	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
51	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
52};
53
54static ddi_dma_attr_t sfxge_intr_dma_attr = {
55	DMA_ATTR_V0,		/* dma_attr_version	*/
56	0,			/* dma_attr_addr_lo	*/
57	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
58	0xffffffffffffffffull,	/* dma_attr_count_max	*/
59	EFX_INTR_SIZE,		/* dma_attr_align	*/
60	0xffffffff,		/* dma_attr_burstsizes	*/
61	1,			/* dma_attr_minxfer	*/
62	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
63	0xffffffffffffffffull,	/* dma_attr_seg		*/
64	1,			/* dma_attr_sgllen	*/
65	1,			/* dma_attr_granular	*/
66	0			/* dma_attr_flags	*/
67};
68
69static unsigned int
70sfxge_intr_line(caddr_t arg1, caddr_t arg2)
71{
72	sfxge_t *sp = (void *)arg1;
73	efx_nic_t *enp = sp->s_enp;
74	sfxge_intr_t *sip = &(sp->s_intr);
75	unsigned int index;
76	boolean_t fatal;
77	uint32_t qmask;
78	int rc;
79
80	_NOTE(ARGUNUSED(arg2))
81
82	ASSERT3U(sip->si_type, ==, EFX_INTR_LINE);
83
84	if (sip->si_state != SFXGE_INTR_STARTED &&
85	    sip->si_state != SFXGE_INTR_TESTING) {
86		rc = DDI_INTR_UNCLAIMED;
87		goto done;
88	}
89
90	if (sip->si_state == SFXGE_INTR_TESTING) {
91		sip->si_mask |= 1;	/* only one interrupt */
92		rc = DDI_INTR_CLAIMED;
93		goto done;
94	}
95
96	efx_intr_status_line(enp, &fatal, &qmask);
97
98	if (fatal) {
99		sfxge_intr_fatal(sp);
100
101		rc = DDI_INTR_CLAIMED;
102		goto done;
103	}
104
105	if (qmask != 0) {
106		for (index = 0; index < EFX_INTR_NEVQS; index++) {
107			if (qmask & (1 << index))
108				(void) sfxge_ev_qpoll(sp, index);
109		}
110
111		sip->si_zero_count = 0;
112		sfxge_gld_rx_push(sp);
113		rc = DDI_INTR_CLAIMED;
114		goto done;
115	}
116
117	/*
118	 * bug15671/bug17203 workaround. Return CLAIMED for the first ISR=0
119	 * interrupt, and poll all evqs for work. For subsequent ISR=0
120	 * interrupts (the line must be shared in this case), just rearm the
121	 * event queues to ensure we don't miss an interrupt.
122	 */
123	if (sip->si_zero_count++ == 0) {
124		for (index = 0; index < EFX_INTR_NEVQS; index++) {
125			if (sp->s_sep[index] != NULL)
126				(void) sfxge_ev_qpoll(sp, index);
127		}
128
129		rc = DDI_INTR_CLAIMED;
130	} else {
131		for (index = 0; index < EFX_INTR_NEVQS; index++) {
132			if (sp->s_sep[index] != NULL)
133				(void) sfxge_ev_qprime(sp, index);
134		}
135
136		rc = DDI_INTR_UNCLAIMED;
137	}
138
139done:
140	return (rc);
141}
142
143static unsigned int
144sfxge_intr_message(caddr_t arg1, caddr_t arg2)
145{
146	sfxge_t *sp = (void *)arg1;
147	efx_nic_t *enp = sp->s_enp;
148	sfxge_intr_t *sip = &(sp->s_intr);
149	unsigned int index = (unsigned int)(uintptr_t)arg2;
150	boolean_t fatal;
151	int rc;
152
153	ASSERT3U(sip->si_type, ==, EFX_INTR_MESSAGE);
154
155	if (sip->si_state != SFXGE_INTR_STARTED &&
156	    sip->si_state != SFXGE_INTR_TESTING) {
157		rc = DDI_INTR_UNCLAIMED;
158		goto done;
159	}
160
161	if (sip->si_state == SFXGE_INTR_TESTING) {
162		uint64_t mask;
163
164		do {
165			mask = sip->si_mask;
166		} while (atomic_cas_64(&(sip->si_mask), mask,
167		    mask | (1 << index)) != mask);
168
169		rc = DDI_INTR_CLAIMED;
170		goto done;
171	}
172
173	efx_intr_status_message(enp, index, &fatal);
174
175	if (fatal) {
176		sfxge_intr_fatal(sp);
177
178		rc = DDI_INTR_CLAIMED;
179		goto done;
180	}
181
182	(void) sfxge_ev_qpoll(sp, index);
183
184	sfxge_gld_rx_push(sp);
185	rc = DDI_INTR_CLAIMED;
186
187done:
188	return (rc);
189}
190
191static int
192sfxge_intr_bus_enable(sfxge_t *sp)
193{
194	sfxge_intr_t *sip = &(sp->s_intr);
195	ddi_intr_handler_t *handler;
196	int add_index;
197	int en_index;
198	int err;
199	int rc;
200
201	/* Serialise all instances to avoid problems seen in bug31184. */
202	mutex_enter(&sfxge_global_lock);
203
204	switch (sip->si_type) {
205	case EFX_INTR_MESSAGE:
206		handler = sfxge_intr_message;
207		break;
208
209	case EFX_INTR_LINE:
210		handler = sfxge_intr_line;
211		break;
212
213	default:
214		dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
215		    "bus_enable: unknown intr type (si_type=%d nalloc=%d)",
216		    sip->si_type, sip->si_nalloc);
217		ASSERT(B_FALSE);
218		rc = EINVAL;
219		goto fail1;
220	}
221
222	/* Try to add the handlers */
223	for (add_index = 0; add_index < sip->si_nalloc; add_index++) {
224		unsigned int pri;
225
226		/* This cannot fail unless given invalid inputs. */
227		err = ddi_intr_get_pri(sip->si_table[add_index], &pri);
228		ASSERT(err == DDI_SUCCESS);
229
230		DTRACE_PROBE2(pri, unsigned int, add_index, unsigned int, pri);
231
232		err = ddi_intr_add_handler(sip->si_table[add_index], handler,
233		    (caddr_t)sp, (caddr_t)(uintptr_t)add_index);
234		if (err != DDI_SUCCESS) {
235			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
236			    "bus_enable: ddi_intr_add_handler failed"
237			    " err=%d (h=%p idx=%d nalloc=%d)",
238			    err, (void *)sip->si_table[add_index], add_index,
239			    sip->si_nalloc);
240
241			rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
242			goto fail2;
243		}
244	}
245
246	/* Get interrupt capabilities */
247	err = ddi_intr_get_cap(sip->si_table[0], &(sip->si_cap));
248	if (err != DDI_SUCCESS) {
249		dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
250		    "bus_enable: ddi_intr_get_cap failed"
251		    " err=%d (h=%p idx=%d nalloc=%d)",
252		    err, (void *)sip->si_table[0], 0, sip->si_nalloc);
253
254		if (err == DDI_EINVAL)
255			rc = EINVAL;
256		else if (err == DDI_ENOTSUP)
257			rc = ENOTSUP;
258		else
259			rc = EFAULT;
260
261		goto fail3;
262	}
263
264	/* Enable interrupts at the bus  */
265	if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
266		en_index = 0; /* Silence gcc */
267		err = ddi_intr_block_enable(sip->si_table, sip->si_nalloc);
268		if (err != DDI_SUCCESS) {
269			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
270			    "bus_enable: ddi_intr_block_enable failed"
271			    " err=%d (table=%p nalloc=%d)",
272			    err, (void *)sip->si_table, sip->si_nalloc);
273
274			rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
275			goto fail4;
276		}
277	} else {
278		for (en_index = 0; en_index < sip->si_nalloc; en_index++) {
279			err = ddi_intr_enable(sip->si_table[en_index]);
280			if (err != DDI_SUCCESS) {
281				dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
282				    "bus_enable: ddi_intr_enable failed"
283				    " err=%d (h=%p idx=%d nalloc=%d)",
284				    err, (void *)sip->si_table[en_index],
285				    en_index, sip->si_nalloc);
286
287				rc = (err == DDI_EINVAL) ? EINVAL : EFAULT;
288				goto fail4;
289			}
290		}
291	}
292
293	mutex_exit(&sfxge_global_lock);
294	return (0);
295
296fail4:
297	DTRACE_PROBE(fail4);
298
299	/* Disable the enabled handlers */
300	if (!(sip->si_cap & DDI_INTR_FLAG_BLOCK)) {
301		while (--en_index >= 0) {
302			err = ddi_intr_disable(sip->si_table[en_index]);
303			if (err != DDI_SUCCESS) {
304				dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
305				    "bus_enable: ddi_intr_disable"
306				    " failed err=%d (h=%p idx=%d nalloc=%d)",
307				    err, (void *)sip->si_table[en_index],
308				    en_index, sip->si_nalloc);
309			}
310		}
311	}
312
313fail3:
314	DTRACE_PROBE(fail3);
315
316	/* Remove all handlers */
317	add_index = sip->si_nalloc;
318
319fail2:
320	DTRACE_PROBE(fail2);
321
322	/* Remove remaining handlers */
323	while (--add_index >= 0) {
324		err = ddi_intr_remove_handler(sip->si_table[add_index]);
325		if (err != DDI_SUCCESS) {
326			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
327			    "bus_enable: ddi_intr_remove_handler"
328			    " failed err=%d (h=%p idx=%d nalloc=%d)",
329			    err, (void *)sip->si_table[add_index], add_index,
330			    sip->si_nalloc);
331		}
332	}
333
334fail1:
335	DTRACE_PROBE1(fail1, int, rc);
336
337	mutex_exit(&sfxge_global_lock);
338	return (rc);
339}
340
341static void
342sfxge_intr_bus_disable(sfxge_t *sp)
343{
344	sfxge_intr_t *sip = &(sp->s_intr);
345	int index;
346	int err;
347
348	/* Serialise all instances to avoid problems seen in bug31184. */
349	mutex_enter(&sfxge_global_lock);
350
351	/* Disable interrupts at the bus */
352	if (sip->si_cap & DDI_INTR_FLAG_BLOCK) {
353		err = ddi_intr_block_disable(sip->si_table, sip->si_nalloc);
354		if (err != DDI_SUCCESS) {
355			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
356			    "bus_disable: ddi_intr_block_disable"
357			    " failed err=%d (table=%p nalloc=%d)",
358			    err, (void *)sip->si_table, sip->si_nalloc);
359		}
360	} else {
361		index = sip->si_nalloc;
362		while (--index >= 0) {
363			err = ddi_intr_disable(sip->si_table[index]);
364			if (err != DDI_SUCCESS) {
365				dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
366				    "bus_disable: ddi_intr_disable"
367				    " failed err=%d (h=%p idx=%d nalloc=%d)",
368				    err, (void *)sip->si_table[index], index,
369				    sip->si_nalloc);
370			}
371		}
372	}
373
374	sip->si_cap = 0;
375
376	/* Remove all handlers */
377	index = sip->si_nalloc;
378	while (--index >= 0) {
379		err = ddi_intr_remove_handler(sip->si_table[index]);
380		if (err != DDI_SUCCESS) {
381			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
382			    "bus_disable: ddi_intr_remove_handler"
383			    " failed err=%d (h=%p idx=%d nalloc=%d)",
384			    err, (void *)sip->si_table[index], index,
385			    sip->si_nalloc);
386		}
387	}
388
389	mutex_exit(&sfxge_global_lock);
390}
391
392static int
393sfxge_intr_nic_enable(sfxge_t *sp)
394{
395	sfxge_intr_t *sip = &(sp->s_intr);
396	efsys_mem_t *esmp = &(sip->si_mem);
397	efx_nic_t *enp = sp->s_enp;
398	unsigned int index;
399	uint64_t mask;
400	unsigned int count;
401	int rc;
402
403	/* Zero the memory */
404	bzero(esmp->esm_base, EFX_INTR_SIZE);
405
406	/* Enable interrupts at the NIC */
407	if ((rc = efx_intr_init(enp, sip->si_type, esmp)) != 0)
408		goto fail1;
409
410	efx_intr_enable(enp);
411
412	/* FIXME FIXME FIXME */
413	if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
414		/* Disable interrupt test until supported on Huntington. */
415		return (0);
416	}
417	/* FIXME FIXME FIXME */
418
419	/* Test the interrupts */
420	mask = 0;
421	for (index = 0; index < sip->si_nalloc; index++) {
422		mask |= (1 << index);
423
424		rc = efx_intr_trigger(enp, index);
425		ASSERT3U(rc, ==, 0);
426	}
427
428	/* Wait for the tests to complete */
429	count = 0;
430	do {
431		DTRACE_PROBE1(wait, unsigned int, count);
432
433		/* Spin for 1 ms */
434		drv_usecwait(1000);
435
436		/*
437		 * Check to see that all the test interrupts have been
438		 * processed.
439		 */
440		if ((mask & sip->si_mask) == mask)
441			goto done;
442
443	} while (++count < 20);
444
445	rc = ETIMEDOUT;
446	goto fail2;
447
448done:
449	return (0);
450
451fail2:
452	DTRACE_PROBE(fail2);
453
454	dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
455	    "Interrupt test failed (mask=%"PRIx64" got=%"
456	    PRIx64"). NIC is disabled",
457	    mask, sip->si_mask);
458
459	DTRACE_PROBE2(int_test_fail, uint64_t, mask, uint64_t, sip->si_mask);
460
461	sip->si_mask = 0;
462
463	/* Disable interrupts at the NIC */
464	efx_intr_disable(enp);
465	efx_intr_fini(enp);
466
467fail1:
468	DTRACE_PROBE1(fail1, int, rc);
469
470	return (rc);
471}
472
473static void
474sfxge_intr_nic_disable(sfxge_t *sp)
475{
476	sfxge_intr_t *sip = &(sp->s_intr);
477	efx_nic_t *enp = sp->s_enp;
478
479	sip->si_mask = 0;
480
481	/* Disable interrupts at the NIC */
482	efx_intr_disable(enp);
483	efx_intr_fini(enp);
484}
485
486static inline unsigned
487pow2_le(unsigned long n)
488{
489	unsigned int order = 1;
490	ASSERT3U(n, >, 0);
491	while ((1ul << order) <= n) ++order;
492	return (1ul << (order - 1));
493}
494
495int
496sfxge_intr_init(sfxge_t *sp)
497{
498	dev_info_t *dip = sp->s_dip;
499	sfxge_intr_t *sip = &(sp->s_intr);
500	efsys_mem_t *esmp = &(sip->si_mem);
501	sfxge_dma_buffer_attr_t dma_attr;
502	int err;
503	int rc;
504	int types;
505	int type;
506	int index;
507	unsigned int nalloc;
508	int navail;
509
510	SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
511
512	ASSERT3U(sip->si_state, ==, SFXGE_INTR_UNINITIALIZED);
513
514#ifdef __sparc
515	/* PSARC 2007/453 */
516	(void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
517	    "#msix-request", NULL, 0);
518#endif
519
520	/* Get the map of supported interrupt types */
521	err = ddi_intr_get_supported_types(dip, &types);
522	if (err != DDI_SUCCESS) {
523		dev_err(dip, CE_WARN, SFXGE_CMN_ERR
524		    "intr_init: ddi_intr_get_supported_types failed err=%d",
525		    err);
526
527		if (err == DDI_EINVAL)
528			rc = EINVAL;
529		else if (err == DDI_INTR_NOTFOUND)
530			rc = ENOENT;
531		else
532			rc = EFAULT;
533
534		goto fail1;
535	}
536
537	/* Choose most favourable type */
538	if (types & DDI_INTR_TYPE_MSIX) {
539		DTRACE_PROBE(msix);
540
541		type = DDI_INTR_TYPE_MSIX;
542		sip->si_type = EFX_INTR_MESSAGE;
543	} else {
544		DTRACE_PROBE(fixed);
545
546		ASSERT(types & DDI_INTR_TYPE_FIXED);
547
548		type = DDI_INTR_TYPE_FIXED;
549		sip->si_type = EFX_INTR_LINE;
550	}
551
552	/* Get the number of available interrupts */
553	navail = 0;
554	err = ddi_intr_get_navail(dip, type, &navail);
555	if (err != DDI_SUCCESS) {
556		dev_err(dip, CE_WARN, SFXGE_CMN_ERR
557		    "intr_init: ddi_intr_get_navail failed err=%d", err);
558
559		if (err == DDI_EINVAL)
560			rc = EINVAL;
561		else if (err == DDI_INTR_NOTFOUND)
562			rc = ENOENT;
563		else
564			rc = EFAULT;
565
566		goto fail2;
567	}
568
569	/* Double-check */
570	if (navail == 0) {
571		rc = ENOENT;
572		goto fail2;
573	}
574
575	/*
576	 * Allow greater number of MSI-X interrupts than CPUs.
577	 * This can be useful to prevent RX no desc drops; See task 32179.
578	 * Limit non MSI-X interrupts to a single instance.
579	 */
580	if (type != DDI_INTR_TYPE_MSIX)
581		navail = 1;
582	else
583		navail = min(navail, sfxge_rx_scale_prop_get(sp));
584
585	DTRACE_PROBE1(navail, unsigned int, navail);
586
587	/* Allocate a handle table */
588	sip->si_table_size = navail * sizeof (ddi_intr_handle_t);
589	sip->si_table = kmem_zalloc(sip->si_table_size, KM_SLEEP);
590
591	/*
592	 * Allocate interrupt handles.
593	 * Serialise all device instances to avoid problems seen in bug31184.
594	 */
595	mutex_enter(&sfxge_global_lock);
596
597	err = ddi_intr_alloc(dip, sip->si_table, type, 0,
598	    navail, &(sip->si_nalloc), DDI_INTR_ALLOC_NORMAL);
599
600	mutex_exit(&sfxge_global_lock);
601
602	if (err != DDI_SUCCESS) {
603		dev_err(dip, CE_WARN, SFXGE_CMN_ERR
604		    "intr_init: ddi_intr_alloc failed err=%d"
605		    " (navail=%d nalloc=%d)",
606		    err, navail, sip->si_nalloc);
607
608		if (err == DDI_EINVAL)
609			rc = EINVAL;
610		else if (err == DDI_EAGAIN)
611			rc = EAGAIN;
612		else if (err == DDI_INTR_NOTFOUND)
613			rc = ENOENT;
614		else
615			rc = EFAULT;
616
617		goto fail3;
618	}
619
620	/* Double-check */
621	if (sip->si_nalloc == 0) {
622		rc = ENOENT;
623		goto fail3;
624	}
625
626	/* Round down to a power of 2 */
627	nalloc = pow2_le(sip->si_nalloc);
628
629	/* Free off any excess handles */
630	mutex_enter(&sfxge_global_lock);
631
632	index = sip->si_nalloc;
633	while (--index >= nalloc) {
634		(void) ddi_intr_free(sip->si_table[index]);
635		sip->si_table[index] = NULL;
636	}
637
638	mutex_exit(&sfxge_global_lock);
639
640	sip->si_nalloc = nalloc;
641	DTRACE_PROBE1(nalloc, unsigned int, sip->si_nalloc);
642
643	dma_attr.sdba_dip	 = sp->s_dip;
644	dma_attr.sdba_dattrp	 = &sfxge_intr_dma_attr;
645	dma_attr.sdba_callback	 = DDI_DMA_SLEEP;
646	dma_attr.sdba_length	 = EFX_INTR_SIZE;
647	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
648	dma_attr.sdba_devaccp	 = &sfxge_intr_devacc;
649	dma_attr.sdba_bindflags	 = DDI_DMA_RDWR | DDI_DMA_CONSISTENT;
650	dma_attr.sdba_maxcookies = 1;
651	dma_attr.sdba_zeroinit	 = B_TRUE;
652
653	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
654		goto fail4;
655
656	/* Store the highest priority for convenience */
657	sip->si_intr_pri = 0;
658	for (index = 0; index < sip->si_nalloc; index++) {
659		uint_t pri;
660		if ((rc = ddi_intr_get_pri(sip->si_table[index], &pri)) !=  0)
661			goto fail5;
662		if (pri > sip->si_intr_pri)
663			sip->si_intr_pri = pri;
664	}
665
666	sip->si_state = SFXGE_INTR_INITIALIZED;
667	return (0);
668
669fail5:
670	DTRACE_PROBE(fail5);
671
672fail4:
673	DTRACE_PROBE(fail4);
674
675	/* Free interrupt handles */
676	mutex_exit(&sfxge_global_lock);
677
678	index = sip->si_nalloc;
679	while (--index >= 0) {
680		err = ddi_intr_free(sip->si_table[index]);
681		if (err != DDI_SUCCESS) {
682			dev_err(dip, CE_WARN, SFXGE_CMN_ERR
683			    "intr_init: ddi_intr_free failed err=%d"
684			    " (h=%p idx=%d nalloc=%d)",
685			    err, (void *)sip->si_table[index], index,
686			    sip->si_nalloc);
687		}
688		sip->si_table[index] = NULL;
689	}
690	sip->si_nalloc = 0;
691
692	mutex_exit(&sfxge_global_lock);
693
694fail3:
695	DTRACE_PROBE(fail3);
696
697	/* Free the handle table */
698	kmem_free(sip->si_table, sip->si_table_size);
699	sip->si_table = NULL;
700	sip->si_table_size = 0;
701
702fail2:
703	DTRACE_PROBE(fail2);
704
705	/* Clear the interrupt type */
706	sip->si_type = EFX_INTR_INVALID;
707
708fail1:
709	DTRACE_PROBE1(fail1, int, rc);
710
711	SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
712
713	return (rc);
714}
715
716int
717sfxge_intr_start(sfxge_t *sp)
718{
719	sfxge_intr_t *sip = &(sp->s_intr);
720	int rc;
721
722	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
723
724	/* Enable interrupts at the bus */
725	if ((rc = sfxge_intr_bus_enable(sp)) != 0)
726		goto fail1;
727
728	sip->si_state = SFXGE_INTR_TESTING;
729
730	/* Enable interrupts at the NIC */
731	if ((rc = sfxge_intr_nic_enable(sp)) != 0)
732		goto fail2;
733
734	sip->si_state = SFXGE_INTR_STARTED;
735
736	return (0);
737
738fail2:
739	DTRACE_PROBE(fail2);
740
741	/* Disable interrupts at the bus */
742	sfxge_intr_bus_disable(sp);
743
744fail1:
745	DTRACE_PROBE1(fail1, int, rc);
746
747	sip->si_state = SFXGE_INTR_INITIALIZED;
748
749	return (rc);
750}
751
752void
753sfxge_intr_stop(sfxge_t *sp)
754{
755	sfxge_intr_t *sip = &(sp->s_intr);
756
757	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
758
759	sip->si_state = SFXGE_INTR_INITIALIZED;
760
761	/* Disable interrupts at the NIC */
762	sfxge_intr_nic_disable(sp);
763
764	/* Disable interrupts at the bus */
765	sfxge_intr_bus_disable(sp);
766}
767
768void
769sfxge_intr_fini(sfxge_t *sp)
770{
771	sfxge_intr_t *sip = &(sp->s_intr);
772	efsys_mem_t *esmp = &(sip->si_mem);
773	int index;
774	int err;
775
776	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
777
778	sip->si_state = SFXGE_INTR_UNINITIALIZED;
779
780	/* Tear down dma setup */
781	sfxge_dma_buffer_destroy(esmp);
782
783
784	/* Free interrupt handles */
785	mutex_enter(&sfxge_global_lock);
786
787	index = sip->si_nalloc;
788	while (--index >= 0) {
789		err = ddi_intr_free(sip->si_table[index]);
790		if (err != DDI_SUCCESS) {
791			dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
792			    "intr_fini: ddi_intr_free failed err=%d"
793			    " (h=%p idx=%d nalloc=%d)",
794			    err, (void *)sip->si_table[index],
795			    index, sip->si_nalloc);
796		}
797		sip->si_table[index] = NULL;
798	}
799	sip->si_nalloc = 0;
800
801	mutex_exit(&sfxge_global_lock);
802
803	/* Free the handle table */
804	kmem_free(sip->si_table, sip->si_table_size);
805	sip->si_table = NULL;
806	sip->si_table_size = 0;
807
808	/* Clear the interrupt type */
809	sip->si_type = EFX_INTR_INVALID;
810
811	SFXGE_OBJ_CHECK(sip, sfxge_intr_t);
812}
813