1/*
2 * Copyright (c) 2008-2016 Solarflare Communications Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 *    this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 *    this list of conditions and the following disclaimer in the documentation
12 *    and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
29 */
30
31#include <sys/types.h>
32#include <sys/sysmacros.h>
33#include <sys/ddi.h>
34#include <sys/sunddi.h>
35#include <sys/stream.h>
36#include <sys/strsun.h>
37#include <sys/strsubr.h>
38#include <sys/cpu.h>
39#include <sys/pghw.h>
40
41#include "sfxge.h"
42
43#include "efx.h"
44
45
46/* Timeout to wait for DRIVER_EV_START event at EVQ startup */
47#define	SFXGE_EV_QSTART_TIMEOUT_USEC	(2000000)
48
49
50/* Event queue DMA attributes */
51static ddi_device_acc_attr_t sfxge_evq_devacc = {
52
53	DDI_DEVICE_ATTR_V0,	/* devacc_attr_version */
54	DDI_NEVERSWAP_ACC,	/* devacc_attr_endian_flags */
55	DDI_STRICTORDER_ACC	/* devacc_attr_dataorder */
56};
57
58static ddi_dma_attr_t sfxge_evq_dma_attr = {
59	DMA_ATTR_V0,		/* dma_attr_version	*/
60	0,			/* dma_attr_addr_lo	*/
61	0xffffffffffffffffull,	/* dma_attr_addr_hi	*/
62	0xffffffffffffffffull,	/* dma_attr_count_max	*/
63	EFX_BUF_SIZE,		/* dma_attr_align	*/
64	0xffffffff,		/* dma_attr_burstsizes	*/
65	1,			/* dma_attr_minxfer	*/
66	0xffffffffffffffffull,	/* dma_attr_maxxfer	*/
67	0xffffffffffffffffull,	/* dma_attr_seg		*/
68	1,			/* dma_attr_sgllen	*/
69	1,			/* dma_attr_granular	*/
70	0			/* dma_attr_flags	*/
71};
72
73static int
74_sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size)
75{
76	efsys_mem_t *esmp = &(sep->se_mem);
77	sfxge_dma_buffer_attr_t dma_attr;
78	int rc;
79
80	/* Compile-time structure layout checks */
81	EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <=
82	    sizeof (sep->__se_u1.__se_pad));
83	EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <=
84	    sizeof (sep->__se_u2.__se_pad));
85	EFX_STATIC_ASSERT(sizeof (sep->__se_u3.__se_s3) <=
86	    sizeof (sep->__se_u3.__se_pad));
87
88	bzero(sep, sizeof (sfxge_evq_t));
89
90	sep->se_sp = sp;
91
92	dma_attr.sdba_dip	 = sp->s_dip;
93	dma_attr.sdba_dattrp	 = &sfxge_evq_dma_attr;
94	dma_attr.sdba_callback	 = (kmflags == KM_SLEEP) ?
95	    DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
96	dma_attr.sdba_length	 = EFX_EVQ_SIZE(evq_size);
97	dma_attr.sdba_memflags	 = DDI_DMA_CONSISTENT;
98	dma_attr.sdba_devaccp	 = &sfxge_evq_devacc;
99	dma_attr.sdba_bindflags	 = DDI_DMA_READ | DDI_DMA_CONSISTENT;
100	dma_attr.sdba_maxcookies = 1;
101	dma_attr.sdba_zeroinit	 = B_FALSE;
102
103	if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
104		goto fail1;
105
106	/* Allocate some buffer table entries */
107	if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_EVQ_NBUFS(evq_size),
108	    &(sep->se_id))) != 0)
109		goto fail2;
110
111	sep->se_stpp = &(sep->se_stp);
112
113	return (0);
114
115fail2:
116	DTRACE_PROBE(fail2);
117
118	/* Tear down DMA setup */
119	esmp->esm_addr = 0;
120	sfxge_dma_buffer_destroy(esmp);
121
122fail1:
123	DTRACE_PROBE1(fail1, int, rc);
124
125	sep->se_sp = NULL;
126
127	SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
128
129	return (-1);
130}
131
132static int
133sfxge_ev_q0ctor(void *buf, void *arg, int kmflags)
134{
135	sfxge_evq_t *sep = buf;
136	sfxge_t *sp = arg;
137	return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size));
138}
139
140static int
141sfxge_ev_qXctor(void *buf, void *arg, int kmflags)
142{
143	sfxge_evq_t *sep = buf;
144	sfxge_t *sp = arg;
145	return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size));
146}
147static void
148_sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size)
149{
150	efsys_mem_t *esmp = &(sep->se_mem);
151	ASSERT3P(sep->se_sp, ==, sp);
152	ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
153	sep->se_stpp = NULL;
154
155	/* Free the buffer table entries */
156	sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
157	sep->se_id = 0;
158
159	/* Tear down DMA setup */
160	sfxge_dma_buffer_destroy(esmp);
161
162	sep->se_sp = NULL;
163
164	SFXGE_OBJ_CHECK(sep, sfxge_evq_t);
165}
166
167static void
168sfxge_ev_q0dtor(void *buf, void *arg)
169{
170	sfxge_evq_t *sep = buf;
171	sfxge_t *sp = arg;
172	_sfxge_ev_qdtor(sp, sep, sp->s_evq0_size);
173}
174
175static void
176sfxge_ev_qXdtor(void *buf, void *arg)
177{
178	sfxge_evq_t *sep = buf;
179	sfxge_t *sp = arg;
180	_sfxge_ev_qdtor(sp, sep, sp->s_evqX_size);
181}
182
183static boolean_t
184sfxge_ev_initialized(void *arg)
185{
186	sfxge_evq_t *sep = arg;
187
188	ASSERT(mutex_owned(&(sep->se_lock)));
189
190	/* Init done events may be duplicated on 7xxx (see SFCbug31631) */
191	if (sep->se_state == SFXGE_EVQ_STARTED)
192		goto done;
193
194	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING);
195	sep->se_state = SFXGE_EVQ_STARTED;
196
197	cv_broadcast(&(sep->se_init_kv));
198
199done:
200	return (B_FALSE);
201}
202
203static void
204sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop)
205{
206	sfxge_t *sp = sep->se_sp;
207	unsigned int index = sep->se_index;
208	sfxge_rxq_t *srp = sp->s_srp[index];
209	sfxge_txq_t *stp;
210
211	if ((stp = sep->se_stp) != NULL) {
212		sep->se_stp = NULL;
213		sep->se_stpp = &(sep->se_stp);
214
215		do {
216			sfxge_txq_t *next;
217
218			next = stp->st_next;
219			stp->st_next = NULL;
220
221			ASSERT3U(stp->st_evq, ==, index);
222
223			if (stp->st_pending != stp->st_completed)
224				sfxge_tx_qcomplete(stp);
225
226			stp = next;
227		} while (stp != NULL);
228	}
229
230	if (srp != NULL) {
231		if (srp->sr_pending != srp->sr_completed)
232			sfxge_rx_qcomplete(srp, eop);
233	}
234}
235
236static boolean_t
237sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size,
238    uint16_t flags)
239{
240	sfxge_evq_t *sep = arg;
241	sfxge_t *sp = sep->se_sp;
242	sfxge_rxq_t *srp;
243	sfxge_rx_packet_t *srpp;
244	unsigned int prefetch;
245	unsigned int stop;
246	unsigned int delta;
247
248	ASSERT(mutex_owned(&(sep->se_lock)));
249
250	if (sep->se_exception)
251		goto done;
252
253	srp = sp->s_srp[label];
254	if (srp == NULL)
255		goto done;
256
257	ASSERT3U(sep->se_index, ==, srp->sr_index);
258	ASSERT3U(id, <, sp->s_rxq_size);
259
260	/*
261	 * Note that in sfxge_stop() EVQ stopped after RXQ, and will be reset
262	 * So the return missing srp->sr_pending increase is safe
263	 */
264	if (srp->sr_state != SFXGE_RXQ_STARTED)
265		goto done;
266
267	stop = (id + 1) & (sp->s_rxq_size - 1);
268	id = srp->sr_pending & (sp->s_rxq_size - 1);
269
270	delta = (stop >= id) ? (stop - id) : (sp->s_rxq_size - id + stop);
271	srp->sr_pending += delta;
272
273	if (delta != 1) {
274		if ((!efx_nic_cfg_get(sp->s_enp)->enc_rx_batching_enabled) ||
275		    (delta == 0) ||
276		    (delta > efx_nic_cfg_get(sp->s_enp)->enc_rx_batch_max)) {
277			/*
278			 * FIXME: This does not take into account scatter
279			 * aborts.  See Bug40811
280			 */
281			sep->se_exception = B_TRUE;
282
283			DTRACE_PROBE(restart_ev_rx_id);
284			/* sfxge_evq_t->se_lock held */
285			(void) sfxge_restart_dispatch(sp, DDI_SLEEP,
286			    SFXGE_HW_ERR, "Out of order RX event", delta);
287
288			goto done;
289		}
290	}
291
292	prefetch = (id + 4) & (sp->s_rxq_size - 1);
293	if ((srpp = srp->sr_srpp[prefetch]) != NULL)
294		prefetch_read_many(srpp);
295
296	srpp = srp->sr_srpp[id];
297	ASSERT(srpp != NULL);
298	prefetch_read_many(srpp->srp_mp);
299
300	for (; id != stop; id = (id + 1) & (sp->s_rxq_size - 1)) {
301		srpp = srp->sr_srpp[id];
302		ASSERT(srpp != NULL);
303
304		ASSERT3U(srpp->srp_flags, ==, EFX_DISCARD);
305		srpp->srp_flags = flags;
306
307		ASSERT3U(size, <, (1 << 16));
308		srpp->srp_size = (uint16_t)size;
309	}
310
311	sep->se_rx++;
312
313	DTRACE_PROBE2(qlevel, unsigned int, srp->sr_index,
314	    unsigned int, srp->sr_added - srp->sr_pending);
315
316	if (srp->sr_pending - srp->sr_completed >= SFXGE_RX_BATCH)
317		sfxge_ev_qcomplete(sep, B_FALSE);
318
319done:
320	/* returning B_TRUE makes efx_ev_qpoll() stop processing events */
321	return (sep->se_rx >= sep->se_ev_batch);
322}
323
324static boolean_t
325sfxge_ev_exception(void *arg, uint32_t code, uint32_t data)
326{
327	sfxge_evq_t *sep = arg;
328	sfxge_t *sp = sep->se_sp;
329
330	_NOTE(ARGUNUSED(code))
331	_NOTE(ARGUNUSED(data))
332
333	ASSERT(mutex_owned(&(sep->se_lock)));
334	sep->se_exception = B_TRUE;
335
336	if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) {
337
338		DTRACE_PROBE(restart_ev_exception);
339
340		/* sfxge_evq_t->se_lock held */
341		(void) sfxge_restart_dispatch(sp, DDI_SLEEP, SFXGE_HW_ERR,
342		    "Unknown EV", code);
343	}
344
345	return (B_FALSE);
346}
347
348static boolean_t
349sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index)
350{
351	sfxge_evq_t *sep_targetq, *sep = arg;
352	sfxge_t *sp = sep->se_sp;
353	sfxge_rxq_t *srp;
354	unsigned int index;
355	unsigned int label;
356	uint16_t magic;
357
358	ASSERT(mutex_owned(&(sep->se_lock)));
359
360	/* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
361	srp = sp->s_srp[rxq_index];
362	if (srp == NULL)
363		goto done;
364
365	/* Process right now if it is the correct event queue */
366	index = srp->sr_index;
367	if (index == sep->se_index) {
368		sfxge_rx_qflush_done(srp);
369		goto done;
370	}
371
372	/* Resend a software event on the correct queue */
373	sep_targetq = sp->s_sep[index];
374
375	if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
376		goto done; /* TBD: state test not under the lock */
377
378	label = rxq_index;
379	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
380	magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label;
381
382	efx_ev_qpost(sep_targetq->se_eep, magic);
383
384done:
385	return (B_FALSE);
386}
387
388static boolean_t
389sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index)
390{
391	sfxge_evq_t *sep_targetq, *sep = arg;
392	sfxge_t *sp = sep->se_sp;
393	sfxge_rxq_t *srp;
394	unsigned int index;
395	unsigned int label;
396	uint16_t magic;
397
398	ASSERT(mutex_owned(&(sep->se_lock)));
399
400	/* Ensure RXQ exists, as events may arrive after RXQ was destroyed */
401	srp = sp->s_srp[rxq_index];
402	if (srp == NULL)
403		goto done;
404
405	/* Process right now if it is the correct event queue */
406	index = srp->sr_index;
407	if (index == sep->se_index) {
408		sfxge_rx_qflush_failed(srp);
409		goto done;
410	}
411
412	/* Resend a software event on the correct queue */
413	sep_targetq = sp->s_sep[index];
414
415	label = rxq_index;
416	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
417	magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label;
418
419	if (sep_targetq->se_state != SFXGE_EVQ_STARTED)
420		goto done; /* TBD: state test not under the lock */
421
422	efx_ev_qpost(sep_targetq->se_eep, magic);
423
424done:
425	return (B_FALSE);
426}
427
428static boolean_t
429sfxge_ev_tx(void *arg, uint32_t label, uint32_t id)
430{
431	sfxge_evq_t *sep = arg;
432	sfxge_txq_t *stp;
433	unsigned int stop;
434	unsigned int delta;
435
436	ASSERT(mutex_owned(&(sep->se_lock)));
437
438	stp = sep->se_label_stp[label];
439	if (stp == NULL)
440		goto done;
441
442	if (stp->st_state != SFXGE_TXQ_STARTED)
443		goto done;
444
445	ASSERT3U(sep->se_index, ==, stp->st_evq);
446
447	stop = (id + 1) & (SFXGE_TX_NDESCS - 1);
448	id = stp->st_pending & (SFXGE_TX_NDESCS - 1);
449
450	delta = (stop >= id) ? (stop - id) : (SFXGE_TX_NDESCS - id + stop);
451	stp->st_pending += delta;
452
453	sep->se_tx++;
454
455	if (stp->st_next == NULL &&
456	    sep->se_stpp != &(stp->st_next)) {
457		*(sep->se_stpp) = stp;
458		sep->se_stpp = &(stp->st_next);
459	}
460
461	DTRACE_PROBE2(qlevel, unsigned int, stp->st_index,
462	    unsigned int, stp->st_added - stp->st_pending);
463
464	if (stp->st_pending - stp->st_completed >= SFXGE_TX_BATCH)
465		sfxge_tx_qcomplete(stp);
466
467done:
468	/* returning B_TRUE makes efx_ev_qpoll() stop processing events */
469	return (sep->se_tx >= sep->se_ev_batch);
470}
471
472static boolean_t
473sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index)
474{
475	sfxge_evq_t *sep = arg;
476	sfxge_t *sp = sep->se_sp;
477	sfxge_txq_t *stp;
478	unsigned int evq;
479	unsigned int label;
480	uint16_t magic;
481
482	ASSERT(mutex_owned(&(sep->se_lock)));
483
484	/* Ensure TXQ exists, as events may arrive after TXQ was destroyed */
485	stp = sp->s_stp[txq_index];
486	if (stp == NULL)
487		goto done;
488
489	/* Process right now if it is the correct event queue */
490	evq = stp->st_evq;
491	if (evq == sep->se_index) {
492		sfxge_tx_qflush_done(stp);
493		goto done;
494	}
495
496	/* Resend a software event on the correct queue */
497	sep = sp->s_sep[evq];
498
499	label = stp->st_label;
500
501	ASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label);
502	magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label;
503
504	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
505	efx_ev_qpost(sep->se_eep, magic);
506
507done:
508	return (B_FALSE);
509}
510
511static boolean_t
512sfxge_ev_software(void *arg, uint16_t magic)
513{
514	sfxge_evq_t *sep = arg;
515	sfxge_t *sp = sep->se_sp;
516	dev_info_t *dip = sp->s_dip;
517	unsigned int label;
518
519	ASSERT(mutex_owned(&(sep->se_lock)));
520
521	EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
522	    FSF_AZ_RX_EV_Q_LABEL_WIDTH);
523	EFX_STATIC_ASSERT(SFXGE_MAGIC_DMAQ_LABEL_WIDTH ==
524	    FSF_AZ_TX_EV_Q_LABEL_WIDTH);
525
526	label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK;
527	magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK;
528
529	switch (magic) {
530	case SFXGE_MAGIC_RX_QFLUSH_DONE: {
531		sfxge_rxq_t *srp = sp->s_srp[label];
532
533		if (srp != NULL) {
534			ASSERT3U(sep->se_index, ==, srp->sr_index);
535
536			sfxge_rx_qflush_done(srp);
537		}
538		break;
539	}
540	case SFXGE_MAGIC_RX_QFLUSH_FAILED: {
541		sfxge_rxq_t *srp = sp->s_srp[label];
542
543		if (srp != NULL) {
544			ASSERT3U(sep->se_index, ==, srp->sr_index);
545
546			sfxge_rx_qflush_failed(srp);
547		}
548		break;
549	}
550	case SFXGE_MAGIC_RX_QFPP_TRIM: {
551		sfxge_rxq_t *srp = sp->s_srp[label];
552
553		if (srp != NULL) {
554			ASSERT3U(sep->se_index, ==, srp->sr_index);
555
556			sfxge_rx_qfpp_trim(srp);
557		}
558		break;
559	}
560	case SFXGE_MAGIC_TX_QFLUSH_DONE: {
561		sfxge_txq_t *stp = sep->se_label_stp[label];
562
563		if (stp != NULL) {
564			ASSERT3U(sep->se_index, ==, stp->st_evq);
565
566			sfxge_tx_qflush_done(stp);
567		}
568		break;
569	}
570	default:
571		dev_err(dip, CE_NOTE,
572		    SFXGE_CMN_ERR "unknown software event 0x%x", magic);
573		break;
574	}
575
576	return (B_FALSE);
577}
578
579static boolean_t
580sfxge_ev_sram(void *arg, uint32_t code)
581{
582	_NOTE(ARGUNUSED(arg))
583
584	switch (code) {
585	case EFX_SRAM_UPDATE:
586		DTRACE_PROBE(sram_update);
587		break;
588
589	case EFX_SRAM_CLEAR:
590		DTRACE_PROBE(sram_clear);
591		break;
592
593	case EFX_SRAM_ILLEGAL_CLEAR:
594		DTRACE_PROBE(sram_illegal_clear);
595		break;
596
597	default:
598		ASSERT(B_FALSE);
599		break;
600	}
601
602	return (B_FALSE);
603}
604
605static boolean_t
606sfxge_ev_timer(void *arg, uint32_t index)
607{
608	_NOTE(ARGUNUSED(arg, index))
609
610	return (B_FALSE);
611}
612
613static boolean_t
614sfxge_ev_wake_up(void *arg, uint32_t index)
615{
616	_NOTE(ARGUNUSED(arg, index))
617
618	return (B_FALSE);
619}
620
621static boolean_t
622sfxge_ev_link_change(void *arg, efx_link_mode_t	link_mode)
623{
624	sfxge_evq_t *sep = arg;
625	sfxge_t *sp = sep->se_sp;
626
627	sfxge_mac_link_update(sp, link_mode);
628
629	return (B_FALSE);
630}
631
632static int
633sfxge_ev_kstat_update(kstat_t *ksp, int rw)
634{
635	sfxge_evq_t *sep = ksp->ks_private;
636	kstat_named_t *knp;
637	int rc;
638
639	if (rw != KSTAT_READ) {
640		rc = EACCES;
641		goto fail1;
642	}
643
644	ASSERT(mutex_owned(&(sep->se_lock)));
645
646	if (sep->se_state != SFXGE_EVQ_STARTED)
647		goto done;
648
649	efx_ev_qstats_update(sep->se_eep, sep->se_stat);
650
651	knp = ksp->ks_data;
652	knp += EV_NQSTATS;
653
654	knp->value.ui64 = sep->se_cpu_id;
655
656done:
657	return (0);
658
659fail1:
660	DTRACE_PROBE1(fail1, int, rc);
661
662	return (rc);
663}
664
665static int
666sfxge_ev_kstat_init(sfxge_evq_t *sep)
667{
668	sfxge_t *sp = sep->se_sp;
669	unsigned int index = sep->se_index;
670	dev_info_t *dip = sp->s_dip;
671	kstat_t *ksp;
672	kstat_named_t *knp;
673	char name[MAXNAMELEN];
674	unsigned int id;
675	int rc;
676
677	/* Determine the name */
678	(void) snprintf(name, MAXNAMELEN - 1, "%s_evq%04d",
679	    ddi_driver_name(dip), index);
680
681	/* Create the set */
682	if ((ksp = kstat_create((char *)ddi_driver_name(dip),
683	    ddi_get_instance(dip), name, "queue", KSTAT_TYPE_NAMED,
684	    EV_NQSTATS + 1, 0)) == NULL) {
685		rc = ENOMEM;
686		goto fail1;
687	}
688
689	sep->se_ksp = ksp;
690
691	ksp->ks_update = sfxge_ev_kstat_update;
692	ksp->ks_private = sep;
693	ksp->ks_lock = &(sep->se_lock);
694
695	/* Initialise the named stats */
696	sep->se_stat = knp = ksp->ks_data;
697	for (id = 0; id < EV_NQSTATS; id++) {
698		kstat_named_init(knp, (char *)efx_ev_qstat_name(sp->s_enp, id),
699		    KSTAT_DATA_UINT64);
700		knp++;
701	}
702
703	kstat_named_init(knp, "cpu", KSTAT_DATA_UINT64);
704
705	kstat_install(ksp);
706	return (0);
707
708fail1:
709	DTRACE_PROBE1(fail1, int, rc);
710
711	return (rc);
712}
713
714static void
715sfxge_ev_kstat_fini(sfxge_evq_t *sep)
716{
717	/* Destroy the set */
718	kstat_delete(sep->se_ksp);
719	sep->se_ksp = NULL;
720	sep->se_stat = NULL;
721}
722
723inline unsigned pow2_ge(unsigned int n) {
724	unsigned int order = 0;
725	ASSERT3U(n, >, 0);
726	while ((1ul << order) < n) ++order;
727	return (1ul << (order));
728}
729
730static int
731sfxge_ev_qinit(sfxge_t *sp, unsigned int index, unsigned int ev_batch)
732{
733	sfxge_evq_t *sep;
734	int rc;
735
736	ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);
737
738	sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP);
739	if (sep == NULL) {
740		rc = ENOMEM;
741		goto fail1;
742	}
743	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED);
744
745	sep->se_index = index;
746
747	mutex_init(&(sep->se_lock), NULL,
748	    MUTEX_DRIVER, DDI_INTR_PRI(sp->s_intr.si_intr_pri));
749
750	cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL);
751
752	/* Initialize the statistics */
753	if ((rc = sfxge_ev_kstat_init(sep)) != 0)
754		goto fail2;
755
756	sep->se_state = SFXGE_EVQ_INITIALIZED;
757	sep->se_ev_batch = (uint16_t)ev_batch;
758	sp->s_sep[index] = sep;
759
760	return (0);
761
762fail2:
763	DTRACE_PROBE(fail2);
764
765	sep->se_index = 0;
766
767	cv_destroy(&(sep->se_init_kv));
768	mutex_destroy(&(sep->se_lock));
769
770	kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
771
772fail1:
773	DTRACE_PROBE1(fail1, int, rc);
774
775	return (rc);
776}
777
778static int
779sfxge_ev_qstart(sfxge_t *sp, unsigned int index)
780{
781	sfxge_evq_t *sep = sp->s_sep[index];
782	sfxge_intr_t *sip = &(sp->s_intr);
783	efx_nic_t *enp = sp->s_enp;
784	efx_ev_callbacks_t *eecp;
785	efsys_mem_t *esmp;
786	clock_t timeout;
787	int rc;
788	uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
789
790	mutex_enter(&(sep->se_lock));
791	esmp = &(sep->se_mem);
792
793	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
794
795	/* Set the memory to all ones */
796	(void) memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq_size));
797
798	/* Program the buffer table */
799	if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp,
800	    EFX_EVQ_NBUFS(evq_size))) != 0)
801		goto fail1;
802
803	/* Set up the event callbacks */
804	eecp = &(sep->se_eec);
805	eecp->eec_initialized = sfxge_ev_initialized;
806	eecp->eec_rx = sfxge_ev_rx;
807	eecp->eec_tx = sfxge_ev_tx;
808	eecp->eec_exception = sfxge_ev_exception;
809	eecp->eec_rxq_flush_done = sfxge_ev_rxq_flush_done;
810	eecp->eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed;
811	eecp->eec_txq_flush_done = sfxge_ev_txq_flush_done;
812	eecp->eec_software = sfxge_ev_software;
813	eecp->eec_sram = sfxge_ev_sram;
814	eecp->eec_wake_up = sfxge_ev_wake_up;
815	eecp->eec_timer = sfxge_ev_timer;
816	eecp->eec_link_change = sfxge_ev_link_change;
817
818	/* Create the event queue */
819	if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id,
820	    &(sep->se_eep))) != 0)
821		goto fail2;
822
823	/* Set the default moderation */
824	if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0)
825		goto fail3;
826
827	/* Check that interrupts are enabled at the NIC */
828	if (sip->si_state != SFXGE_INTR_STARTED) {
829		rc = EINVAL;
830		goto fail4;
831	}
832
833	sep->se_state = SFXGE_EVQ_STARTING;
834
835	/* Prime the event queue for interrupts */
836	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
837		goto fail5;
838
839	/* Wait for the initialization event */
840	timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_EV_QSTART_TIMEOUT_USEC);
841	while (sep->se_state != SFXGE_EVQ_STARTED) {
842		if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock),
843		    timeout) < 0) {
844			/* Timeout waiting for initialization */
845			dev_info_t *dip = sp->s_dip;
846
847			DTRACE_PROBE(timeout);
848			dev_err(dip, CE_NOTE,
849			    SFXGE_CMN_ERR "evq[%d] qstart timeout", index);
850
851			rc = ETIMEDOUT;
852			goto fail6;
853		}
854	}
855
856	mutex_exit(&(sep->se_lock));
857	return (0);
858
859fail6:
860	DTRACE_PROBE(fail6);
861
862fail5:
863	DTRACE_PROBE(fail5);
864
865	sep->se_state = SFXGE_EVQ_INITIALIZED;
866
867fail4:
868	DTRACE_PROBE(fail4);
869
870fail3:
871	DTRACE_PROBE(fail3);
872
873	/* Destroy the event queue */
874	efx_ev_qdestroy(sep->se_eep);
875	sep->se_eep = NULL;
876
877fail2:
878	DTRACE_PROBE(fail2);
879
880	/* Zero out the event handlers */
881	bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
882
883	/* Clear entries from the buffer table */
884	sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
885
886fail1:
887	DTRACE_PROBE1(fail1, int, rc);
888
889	mutex_exit(&(sep->se_lock));
890
891	return (rc);
892}
893
894int
895sfxge_ev_qpoll(sfxge_t *sp, unsigned int index)
896{
897	sfxge_evq_t *sep = sp->s_sep[index];
898	processorid_t cpu_id;
899	int rc;
900	uint16_t evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
901
902	mutex_enter(&(sep->se_lock));
903
904	if (sep->se_state != SFXGE_EVQ_STARTING &&
905	    sep->se_state != SFXGE_EVQ_STARTED) {
906		rc = EINVAL;
907		goto fail1;
908	}
909
910	/* Make sure the CPU information is up to date */
911	cpu_id = CPU->cpu_id;
912
913	if (cpu_id != sep->se_cpu_id) {
914		sep->se_cpu_id = cpu_id;
915
916		/* sfxge_evq_t->se_lock held */
917		(void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
918		    DDI_NOSLEEP);
919	}
920
921	/* Synchronize the DMA memory for reading */
922	(void) ddi_dma_sync(sep->se_mem.esm_dma_handle,
923	    0,
924	    EFX_EVQ_SIZE(evq_size),
925	    DDI_DMA_SYNC_FORKERNEL);
926
927	ASSERT3U(sep->se_rx, ==, 0);
928	ASSERT3U(sep->se_tx, ==, 0);
929	ASSERT3P(sep->se_stp, ==, NULL);
930	ASSERT3P(sep->se_stpp, ==, &(sep->se_stp));
931
932	/* Poll the queue */
933	efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec),
934	    sep);
935
936	sep->se_rx = 0;
937	sep->se_tx = 0;
938
939	/* Perform any pending completion processing */
940	sfxge_ev_qcomplete(sep, B_TRUE);
941
942	/* Re-prime the event queue for interrupts */
943	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
944		goto fail2;
945
946	mutex_exit(&(sep->se_lock));
947
948	return (0);
949
950fail2:
951	DTRACE_PROBE(fail2);
952fail1:
953	DTRACE_PROBE1(fail1, int, rc);
954
955	mutex_exit(&(sep->se_lock));
956
957	return (rc);
958}
959
960int
961sfxge_ev_qprime(sfxge_t *sp, unsigned int index)
962{
963	sfxge_evq_t *sep = sp->s_sep[index];
964	int rc;
965
966	mutex_enter(&(sep->se_lock));
967
968	if (sep->se_state != SFXGE_EVQ_STARTING &&
969	    sep->se_state != SFXGE_EVQ_STARTED) {
970		rc = EINVAL;
971		goto fail1;
972	}
973
974	if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0)
975		goto fail2;
976
977	mutex_exit(&(sep->se_lock));
978
979	return (0);
980
981fail2:
982	DTRACE_PROBE(fail2);
983fail1:
984	DTRACE_PROBE1(fail1, int, rc);
985
986	mutex_exit(&(sep->se_lock));
987
988	return (rc);
989}
990
991
992int
993sfxge_ev_qmoderate(sfxge_t *sp, unsigned int index, unsigned int us)
994{
995	sfxge_evq_t *sep = sp->s_sep[index];
996	efx_evq_t *eep = sep->se_eep;
997
998	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
999
1000	return (efx_ev_qmoderate(eep, us));
1001}
1002
1003static void
1004sfxge_ev_qstop(sfxge_t *sp, unsigned int index)
1005{
1006	sfxge_evq_t *sep = sp->s_sep[index];
1007	uint16_t evq_size;
1008
1009	mutex_enter(&(sep->se_lock));
1010	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
1011	sep->se_state = SFXGE_EVQ_INITIALIZED;
1012	evq_size = index ? sp->s_evqX_size : sp->s_evq0_size;
1013
1014	/* Clear the CPU information */
1015	sep->se_cpu_id = 0;
1016
1017	/* Clear the event count */
1018	sep->se_count = 0;
1019
1020	/* Reset the exception flag */
1021	sep->se_exception = B_FALSE;
1022
1023	/* Destroy the event queue */
1024	efx_ev_qdestroy(sep->se_eep);
1025	sep->se_eep = NULL;
1026
1027	mutex_exit(&(sep->se_lock));
1028
1029	/* Zero out the event handlers */
1030	bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t));
1031
1032	/* Clear entries from the buffer table */
1033	sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size));
1034}
1035
1036static void
1037sfxge_ev_qfini(sfxge_t *sp, unsigned int index)
1038{
1039	sfxge_evq_t *sep = sp->s_sep[index];
1040
1041	ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED);
1042
1043	sp->s_sep[index] = NULL;
1044	sep->se_state = SFXGE_EVQ_UNINITIALIZED;
1045
1046	/* Tear down the statistics */
1047	sfxge_ev_kstat_fini(sep);
1048
1049	cv_destroy(&(sep->se_init_kv));
1050	mutex_destroy(&(sep->se_lock));
1051
1052	sep->se_index = 0;
1053
1054	kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep);
1055}
1056
1057int
1058sfxge_ev_txlabel_alloc(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
1059    unsigned int *labelp)
1060{
1061	sfxge_evq_t *sep = sp->s_sep[evq];
1062	sfxge_txq_t **stpp;
1063	unsigned int label;
1064	int rc;
1065
1066	mutex_enter(&(sep->se_lock));
1067
1068	if (stp == NULL || labelp == NULL) {
1069		rc = EINVAL;
1070		goto fail1;
1071	}
1072
1073	stpp = NULL;
1074	for (label = 0; label < SFXGE_TX_NLABELS; label++) {
1075		if (sep->se_label_stp[label] == stp) {
1076			rc = EEXIST;
1077			goto fail2;
1078		}
1079		if ((stpp == NULL) && (sep->se_label_stp[label] == NULL)) {
1080			stpp = &sep->se_label_stp[label];
1081		}
1082	}
1083	if (stpp == NULL) {
1084		rc = ENOSPC;
1085		goto fail3;
1086	}
1087	*stpp = stp;
1088	label = stpp - sep->se_label_stp;
1089
1090	ASSERT3U(label, <, SFXGE_TX_NLABELS);
1091	*labelp = label;
1092
1093	mutex_exit(&(sep->se_lock));
1094	return (0);
1095
1096fail3:
1097	DTRACE_PROBE(fail3);
1098fail2:
1099	DTRACE_PROBE(fail2);
1100fail1:
1101	DTRACE_PROBE1(fail1, int, rc);
1102
1103	mutex_exit(&(sep->se_lock));
1104
1105	return (rc);
1106}
1107
1108
1109int
1110sfxge_ev_txlabel_free(sfxge_t *sp, unsigned int evq, sfxge_txq_t *stp,
1111    unsigned int label)
1112{
1113	sfxge_evq_t *sep = sp->s_sep[evq];
1114	int rc;
1115
1116	mutex_enter(&(sep->se_lock));
1117
1118	if (stp == NULL || label > SFXGE_TX_NLABELS) {
1119		rc = EINVAL;
1120		goto fail1;
1121	}
1122
1123	if (sep->se_label_stp[label] != stp) {
1124		rc = EINVAL;
1125		goto fail2;
1126	}
1127	sep->se_label_stp[label] = NULL;
1128
1129	mutex_exit(&(sep->se_lock));
1130
1131	return (0);
1132
1133fail2:
1134	DTRACE_PROBE(fail2);
1135fail1:
1136	DTRACE_PROBE1(fail1, int, rc);
1137
1138	mutex_exit(&(sep->se_lock));
1139
1140	return (rc);
1141}
1142
1143
1144static 	kmem_cache_t *
1145sfxge_ev_kmem_cache_create(sfxge_t *sp, const char *qname,
1146    int (*ctor)(void *, void *, int), void (*dtor)(void *, void *))
1147{
1148	char name[MAXNAMELEN];
1149	kmem_cache_t *eqc;
1150
1151	(void) snprintf(name, MAXNAMELEN - 1, "%s%d_%s_cache",
1152	    ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip), qname);
1153
1154	eqc = kmem_cache_create(name, sizeof (sfxge_evq_t),
1155	    SFXGE_CPU_CACHE_SIZE, ctor, dtor, NULL, sp, NULL, 0);
1156	ASSERT(eqc != NULL);
1157	return (eqc);
1158}
1159
1160int
1161sfxge_ev_init(sfxge_t *sp)
1162{
1163	sfxge_intr_t *sip = &(sp->s_intr);
1164	unsigned int evq0_size;
1165	unsigned int evqX_size;
1166	unsigned int ev_batch;
1167	int index;
1168	int rc;
1169
1170	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
1171
1172	/*
1173	 * Must account for RXQ, TXQ(s); MCDI not event completed at present
1174	 * Note that common code does not completely fill descriptor queues
1175	 */
1176	evqX_size = sp->s_rxq_size + SFXGE_TX_NDESCS;
1177	evq0_size = evqX_size + SFXGE_TX_NDESCS; /* only IP checksum TXQ */
1178	evq0_size += SFXGE_TX_NDESCS; /* no checksums */
1179
1180	ASSERT3U(evqX_size, >=, EFX_EVQ_MINNEVS);
1181	ASSERT3U(evq0_size, >, evqX_size);
1182
1183	if (evq0_size > EFX_EVQ_MAXNEVS) {
1184		rc = EINVAL;
1185		goto fail1;
1186	}
1187
1188	sp->s_evq0_size = pow2_ge(evq0_size);
1189	sp->s_evqX_size = pow2_ge(evqX_size);
1190
1191	/* Read driver parameters */
1192	sp->s_ev_moderation = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
1193	    DDI_PROP_DONTPASS, "intr_moderation", SFXGE_DEFAULT_MODERATION);
1194
1195	ev_batch = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
1196	    DDI_PROP_DONTPASS, "ev_batch", SFXGE_EV_BATCH);
1197
1198	/*
1199	 * It is slightly peverse to have a cache for one item. But it allows
1200	 * for simple alignment control without increasing the allocation size
1201	 */
1202	sp->s_eq0c = sfxge_ev_kmem_cache_create(sp, "evq0", sfxge_ev_q0ctor,
1203	    sfxge_ev_q0dtor);
1204	sp->s_eqXc = sfxge_ev_kmem_cache_create(sp, "evqX", sfxge_ev_qXctor,
1205	    sfxge_ev_qXdtor);
1206
1207	/* Initialize the event queue(s) */
1208	for (index = 0; index < sip->si_nalloc; index++) {
1209		if ((rc = sfxge_ev_qinit(sp, index, ev_batch)) != 0)
1210			goto fail2;
1211	}
1212
1213	return (0);
1214
1215fail2:
1216	DTRACE_PROBE(fail2);
1217
1218	while (--index >= 0)
1219		sfxge_ev_qfini(sp, index);
1220	sp->s_ev_moderation = 0;
1221
1222fail1:
1223	DTRACE_PROBE1(fail1, int, rc);
1224
1225	kmem_cache_destroy(sp->s_eqXc);
1226	kmem_cache_destroy(sp->s_eq0c);
1227	sp->s_eqXc = NULL;
1228	sp->s_eq0c = NULL;
1229
1230	return (rc);
1231}
1232
1233int
1234sfxge_ev_start(sfxge_t *sp)
1235{
1236	sfxge_intr_t *sip = &(sp->s_intr);
1237	efx_nic_t *enp = sp->s_enp;
1238	int index;
1239	int rc;
1240
1241	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
1242
1243	/* Initialize the event module */
1244	if ((rc = efx_ev_init(enp)) != 0)
1245		goto fail1;
1246
1247	/* Start the event queues */
1248	for (index = 0; index < sip->si_nalloc; index++) {
1249		if ((rc = sfxge_ev_qstart(sp, index)) != 0)
1250			goto fail2;
1251	}
1252
1253	return (0);
1254
1255fail2:
1256	DTRACE_PROBE(fail2);
1257
1258	/* Stop the event queue(s) */
1259	while (--index >= 0)
1260		sfxge_ev_qstop(sp, index);
1261
1262	/* Tear down the event module */
1263	efx_ev_fini(enp);
1264
1265fail1:
1266	DTRACE_PROBE1(fail1, int, rc);
1267
1268	return (rc);
1269}
1270
1271void
1272sfxge_ev_moderation_get(sfxge_t *sp, unsigned int *usp)
1273{
1274	*usp = sp->s_ev_moderation;
1275}
1276
1277int
1278sfxge_ev_moderation_set(sfxge_t *sp, unsigned int us)
1279{
1280	sfxge_intr_t *sip = &(sp->s_intr);
1281	int index;
1282	int rc;
1283
1284	if (sip->si_state != SFXGE_INTR_STARTED)
1285		return (ENODEV);
1286
1287	for (index = 0; index < sip->si_nalloc; index++) {
1288		if ((rc = sfxge_ev_qmoderate(sp, index, us)) != 0)
1289			goto fail1;
1290	}
1291
1292	sp->s_ev_moderation = us;
1293	return (0);
1294
1295fail1:
1296	DTRACE_PROBE1(fail1, int, rc);
1297
1298	/*  The only error path is if the value to set to is invalid. */
1299	ASSERT3U(index, ==, 0);
1300
1301	return (rc);
1302}
1303
1304void
1305sfxge_ev_stop(sfxge_t *sp)
1306{
1307	sfxge_intr_t *sip = &(sp->s_intr);
1308	efx_nic_t *enp = sp->s_enp;
1309	int index;
1310
1311	ASSERT3U(sip->si_state, ==, SFXGE_INTR_STARTED);
1312
1313	/* Stop the event queue(s) */
1314	index = sip->si_nalloc;
1315	while (--index >= 0)
1316		sfxge_ev_qstop(sp, index);
1317
1318	/* Tear down the event module */
1319	efx_ev_fini(enp);
1320}
1321
1322void
1323sfxge_ev_fini(sfxge_t *sp)
1324{
1325	sfxge_intr_t *sip = &(sp->s_intr);
1326	int index;
1327
1328	ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
1329
1330	sp->s_ev_moderation = 0;
1331
1332	/* Tear down the event queue(s) */
1333	index = sip->si_nalloc;
1334	while (--index >= 0)
1335		sfxge_ev_qfini(sp, index);
1336
1337	kmem_cache_destroy(sp->s_eqXc);
1338	kmem_cache_destroy(sp->s_eq0c);
1339	sp->s_eqXc = NULL;
1340	sp->s_eq0c = NULL;
1341}
1342