Lines Matching refs:sep
74 _sfxge_ev_qctor(sfxge_t *sp, sfxge_evq_t *sep, int kmflags, uint16_t evq_size) in _sfxge_ev_qctor() argument
76 efsys_mem_t *esmp = &(sep->se_mem); in _sfxge_ev_qctor()
81 EFX_STATIC_ASSERT(sizeof (sep->__se_u1.__se_s1) <= in _sfxge_ev_qctor()
82 sizeof (sep->__se_u1.__se_pad)); in _sfxge_ev_qctor()
83 EFX_STATIC_ASSERT(sizeof (sep->__se_u2.__se_s2) <= in _sfxge_ev_qctor()
84 sizeof (sep->__se_u2.__se_pad)); in _sfxge_ev_qctor()
85 EFX_STATIC_ASSERT(sizeof (sep->__se_u3.__se_s3) <= in _sfxge_ev_qctor()
86 sizeof (sep->__se_u3.__se_pad)); in _sfxge_ev_qctor()
88 bzero(sep, sizeof (sfxge_evq_t)); in _sfxge_ev_qctor()
90 sep->se_sp = sp; in _sfxge_ev_qctor()
108 &(sep->se_id))) != 0) in _sfxge_ev_qctor()
111 sep->se_stpp = &(sep->se_stp); in _sfxge_ev_qctor()
125 sep->se_sp = NULL; in _sfxge_ev_qctor()
127 SFXGE_OBJ_CHECK(sep, sfxge_evq_t); in _sfxge_ev_qctor()
135 sfxge_evq_t *sep = buf; in sfxge_ev_q0ctor() local
137 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evq0_size)); in sfxge_ev_q0ctor()
143 sfxge_evq_t *sep = buf; in sfxge_ev_qXctor() local
145 return (_sfxge_ev_qctor(sp, sep, kmflags, sp->s_evqX_size)); in sfxge_ev_qXctor()
148 _sfxge_ev_qdtor(sfxge_t *sp, sfxge_evq_t *sep, uint16_t evq_size) in _sfxge_ev_qdtor() argument
150 efsys_mem_t *esmp = &(sep->se_mem); in _sfxge_ev_qdtor()
151 ASSERT3P(sep->se_sp, ==, sp); in _sfxge_ev_qdtor()
152 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp)); in _sfxge_ev_qdtor()
153 sep->se_stpp = NULL; in _sfxge_ev_qdtor()
156 sfxge_sram_buf_tbl_free(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); in _sfxge_ev_qdtor()
157 sep->se_id = 0; in _sfxge_ev_qdtor()
162 sep->se_sp = NULL; in _sfxge_ev_qdtor()
164 SFXGE_OBJ_CHECK(sep, sfxge_evq_t); in _sfxge_ev_qdtor()
170 sfxge_evq_t *sep = buf; in sfxge_ev_q0dtor() local
172 _sfxge_ev_qdtor(sp, sep, sp->s_evq0_size); in sfxge_ev_q0dtor()
178 sfxge_evq_t *sep = buf; in sfxge_ev_qXdtor() local
180 _sfxge_ev_qdtor(sp, sep, sp->s_evqX_size); in sfxge_ev_qXdtor()
186 sfxge_evq_t *sep = arg; in sfxge_ev_initialized() local
188 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_initialized()
191 if (sep->se_state == SFXGE_EVQ_STARTED) in sfxge_ev_initialized()
194 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTING); in sfxge_ev_initialized()
195 sep->se_state = SFXGE_EVQ_STARTED; in sfxge_ev_initialized()
197 cv_broadcast(&(sep->se_init_kv)); in sfxge_ev_initialized()
204 sfxge_ev_qcomplete(sfxge_evq_t *sep, boolean_t eop) in sfxge_ev_qcomplete() argument
206 sfxge_t *sp = sep->se_sp; in sfxge_ev_qcomplete()
207 unsigned int index = sep->se_index; in sfxge_ev_qcomplete()
211 if ((stp = sep->se_stp) != NULL) { in sfxge_ev_qcomplete()
212 sep->se_stp = NULL; in sfxge_ev_qcomplete()
213 sep->se_stpp = &(sep->se_stp); in sfxge_ev_qcomplete()
240 sfxge_evq_t *sep = arg; in sfxge_ev_rx() local
241 sfxge_t *sp = sep->se_sp; in sfxge_ev_rx()
248 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_rx()
250 if (sep->se_exception) in sfxge_ev_rx()
257 ASSERT3U(sep->se_index, ==, srp->sr_index); in sfxge_ev_rx()
281 sep->se_exception = B_TRUE; in sfxge_ev_rx()
311 sep->se_rx++; in sfxge_ev_rx()
317 sfxge_ev_qcomplete(sep, B_FALSE); in sfxge_ev_rx()
321 return (sep->se_rx >= sep->se_ev_batch); in sfxge_ev_rx()
327 sfxge_evq_t *sep = arg; in sfxge_ev_exception() local
328 sfxge_t *sp = sep->se_sp; in sfxge_ev_exception()
333 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_exception()
334 sep->se_exception = B_TRUE; in sfxge_ev_exception()
351 sfxge_evq_t *sep_targetq, *sep = arg; in sfxge_ev_rxq_flush_done() local
352 sfxge_t *sp = sep->se_sp; in sfxge_ev_rxq_flush_done()
358 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_rxq_flush_done()
367 if (index == sep->se_index) { in sfxge_ev_rxq_flush_done()
391 sfxge_evq_t *sep_targetq, *sep = arg; in sfxge_ev_rxq_flush_failed() local
392 sfxge_t *sp = sep->se_sp; in sfxge_ev_rxq_flush_failed()
398 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_rxq_flush_failed()
407 if (index == sep->se_index) { in sfxge_ev_rxq_flush_failed()
431 sfxge_evq_t *sep = arg; in sfxge_ev_tx() local
436 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_tx()
438 stp = sep->se_label_stp[label]; in sfxge_ev_tx()
445 ASSERT3U(sep->se_index, ==, stp->st_evq); in sfxge_ev_tx()
453 sep->se_tx++; in sfxge_ev_tx()
456 sep->se_stpp != &(stp->st_next)) { in sfxge_ev_tx()
457 *(sep->se_stpp) = stp; in sfxge_ev_tx()
458 sep->se_stpp = &(stp->st_next); in sfxge_ev_tx()
469 return (sep->se_tx >= sep->se_ev_batch); in sfxge_ev_tx()
475 sfxge_evq_t *sep = arg; in sfxge_ev_txq_flush_done() local
476 sfxge_t *sp = sep->se_sp; in sfxge_ev_txq_flush_done()
482 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_txq_flush_done()
491 if (evq == sep->se_index) { in sfxge_ev_txq_flush_done()
497 sep = sp->s_sep[evq]; in sfxge_ev_txq_flush_done()
504 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); in sfxge_ev_txq_flush_done()
505 efx_ev_qpost(sep->se_eep, magic); in sfxge_ev_txq_flush_done()
514 sfxge_evq_t *sep = arg; in sfxge_ev_software() local
515 sfxge_t *sp = sep->se_sp; in sfxge_ev_software()
519 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_software()
534 ASSERT3U(sep->se_index, ==, srp->sr_index); in sfxge_ev_software()
544 ASSERT3U(sep->se_index, ==, srp->sr_index); in sfxge_ev_software()
554 ASSERT3U(sep->se_index, ==, srp->sr_index); in sfxge_ev_software()
561 sfxge_txq_t *stp = sep->se_label_stp[label]; in sfxge_ev_software()
564 ASSERT3U(sep->se_index, ==, stp->st_evq); in sfxge_ev_software()
624 sfxge_evq_t *sep = arg; in sfxge_ev_link_change() local
625 sfxge_t *sp = sep->se_sp; in sfxge_ev_link_change()
635 sfxge_evq_t *sep = ksp->ks_private; in sfxge_ev_kstat_update() local
644 ASSERT(mutex_owned(&(sep->se_lock))); in sfxge_ev_kstat_update()
646 if (sep->se_state != SFXGE_EVQ_STARTED) in sfxge_ev_kstat_update()
649 efx_ev_qstats_update(sep->se_eep, sep->se_stat); in sfxge_ev_kstat_update()
654 knp->value.ui64 = sep->se_cpu_id; in sfxge_ev_kstat_update()
666 sfxge_ev_kstat_init(sfxge_evq_t *sep) in sfxge_ev_kstat_init() argument
668 sfxge_t *sp = sep->se_sp; in sfxge_ev_kstat_init()
669 unsigned int index = sep->se_index; in sfxge_ev_kstat_init()
689 sep->se_ksp = ksp; in sfxge_ev_kstat_init()
692 ksp->ks_private = sep; in sfxge_ev_kstat_init()
693 ksp->ks_lock = &(sep->se_lock); in sfxge_ev_kstat_init()
696 sep->se_stat = knp = ksp->ks_data; in sfxge_ev_kstat_init()
715 sfxge_ev_kstat_fini(sfxge_evq_t *sep) in sfxge_ev_kstat_fini() argument
718 kstat_delete(sep->se_ksp); in sfxge_ev_kstat_fini()
719 sep->se_ksp = NULL; in sfxge_ev_kstat_fini()
720 sep->se_stat = NULL; in sfxge_ev_kstat_fini()
733 sfxge_evq_t *sep; in sfxge_ev_qinit() local
738 sep = kmem_cache_alloc(index ? sp->s_eqXc : sp->s_eq0c, KM_SLEEP); in sfxge_ev_qinit()
739 if (sep == NULL) { in sfxge_ev_qinit()
743 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_UNINITIALIZED); in sfxge_ev_qinit()
745 sep->se_index = index; in sfxge_ev_qinit()
747 mutex_init(&(sep->se_lock), NULL, in sfxge_ev_qinit()
750 cv_init(&(sep->se_init_kv), NULL, CV_DRIVER, NULL); in sfxge_ev_qinit()
753 if ((rc = sfxge_ev_kstat_init(sep)) != 0) in sfxge_ev_qinit()
756 sep->se_state = SFXGE_EVQ_INITIALIZED; in sfxge_ev_qinit()
757 sep->se_ev_batch = (uint16_t)ev_batch; in sfxge_ev_qinit()
758 sp->s_sep[index] = sep; in sfxge_ev_qinit()
765 sep->se_index = 0; in sfxge_ev_qinit()
767 cv_destroy(&(sep->se_init_kv)); in sfxge_ev_qinit()
768 mutex_destroy(&(sep->se_lock)); in sfxge_ev_qinit()
770 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep); in sfxge_ev_qinit()
781 sfxge_evq_t *sep = sp->s_sep[index]; in sfxge_ev_qstart() local
790 mutex_enter(&(sep->se_lock)); in sfxge_ev_qstart()
791 esmp = &(sep->se_mem); in sfxge_ev_qstart()
793 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED); in sfxge_ev_qstart()
799 if ((rc = sfxge_sram_buf_tbl_set(sp, sep->se_id, esmp, in sfxge_ev_qstart()
804 eecp = &(sep->se_eec); in sfxge_ev_qstart()
819 if ((rc = efx_ev_qcreate(enp, index, esmp, evq_size, sep->se_id, in sfxge_ev_qstart()
820 &(sep->se_eep))) != 0) in sfxge_ev_qstart()
824 if ((rc = efx_ev_qmoderate(sep->se_eep, sp->s_ev_moderation)) != 0) in sfxge_ev_qstart()
833 sep->se_state = SFXGE_EVQ_STARTING; in sfxge_ev_qstart()
836 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) in sfxge_ev_qstart()
841 while (sep->se_state != SFXGE_EVQ_STARTED) { in sfxge_ev_qstart()
842 if (cv_timedwait(&(sep->se_init_kv), &(sep->se_lock), in sfxge_ev_qstart()
856 mutex_exit(&(sep->se_lock)); in sfxge_ev_qstart()
865 sep->se_state = SFXGE_EVQ_INITIALIZED; in sfxge_ev_qstart()
874 efx_ev_qdestroy(sep->se_eep); in sfxge_ev_qstart()
875 sep->se_eep = NULL; in sfxge_ev_qstart()
881 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t)); in sfxge_ev_qstart()
884 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); in sfxge_ev_qstart()
889 mutex_exit(&(sep->se_lock)); in sfxge_ev_qstart()
897 sfxge_evq_t *sep = sp->s_sep[index]; in sfxge_ev_qpoll() local
902 mutex_enter(&(sep->se_lock)); in sfxge_ev_qpoll()
904 if (sep->se_state != SFXGE_EVQ_STARTING && in sfxge_ev_qpoll()
905 sep->se_state != SFXGE_EVQ_STARTED) { in sfxge_ev_qpoll()
913 if (cpu_id != sep->se_cpu_id) { in sfxge_ev_qpoll()
914 sep->se_cpu_id = cpu_id; in sfxge_ev_qpoll()
922 (void) ddi_dma_sync(sep->se_mem.esm_dma_handle, in sfxge_ev_qpoll()
927 ASSERT3U(sep->se_rx, ==, 0); in sfxge_ev_qpoll()
928 ASSERT3U(sep->se_tx, ==, 0); in sfxge_ev_qpoll()
929 ASSERT3P(sep->se_stp, ==, NULL); in sfxge_ev_qpoll()
930 ASSERT3P(sep->se_stpp, ==, &(sep->se_stp)); in sfxge_ev_qpoll()
933 efx_ev_qpoll(sep->se_eep, &(sep->se_count), &(sep->se_eec), in sfxge_ev_qpoll()
934 sep); in sfxge_ev_qpoll()
936 sep->se_rx = 0; in sfxge_ev_qpoll()
937 sep->se_tx = 0; in sfxge_ev_qpoll()
940 sfxge_ev_qcomplete(sep, B_TRUE); in sfxge_ev_qpoll()
943 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) in sfxge_ev_qpoll()
946 mutex_exit(&(sep->se_lock)); in sfxge_ev_qpoll()
955 mutex_exit(&(sep->se_lock)); in sfxge_ev_qpoll()
963 sfxge_evq_t *sep = sp->s_sep[index]; in sfxge_ev_qprime() local
966 mutex_enter(&(sep->se_lock)); in sfxge_ev_qprime()
968 if (sep->se_state != SFXGE_EVQ_STARTING && in sfxge_ev_qprime()
969 sep->se_state != SFXGE_EVQ_STARTED) { in sfxge_ev_qprime()
974 if ((rc = efx_ev_qprime(sep->se_eep, sep->se_count)) != 0) in sfxge_ev_qprime()
977 mutex_exit(&(sep->se_lock)); in sfxge_ev_qprime()
986 mutex_exit(&(sep->se_lock)); in sfxge_ev_qprime()
995 sfxge_evq_t *sep = sp->s_sep[index]; in sfxge_ev_qmoderate() local
996 efx_evq_t *eep = sep->se_eep; in sfxge_ev_qmoderate()
998 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); in sfxge_ev_qmoderate()
1006 sfxge_evq_t *sep = sp->s_sep[index]; in sfxge_ev_qstop() local
1009 mutex_enter(&(sep->se_lock)); in sfxge_ev_qstop()
1010 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED); in sfxge_ev_qstop()
1011 sep->se_state = SFXGE_EVQ_INITIALIZED; in sfxge_ev_qstop()
1015 sep->se_cpu_id = 0; in sfxge_ev_qstop()
1018 sep->se_count = 0; in sfxge_ev_qstop()
1021 sep->se_exception = B_FALSE; in sfxge_ev_qstop()
1024 efx_ev_qdestroy(sep->se_eep); in sfxge_ev_qstop()
1025 sep->se_eep = NULL; in sfxge_ev_qstop()
1027 mutex_exit(&(sep->se_lock)); in sfxge_ev_qstop()
1030 bzero(&(sep->se_eec), sizeof (efx_ev_callbacks_t)); in sfxge_ev_qstop()
1033 sfxge_sram_buf_tbl_clear(sp, sep->se_id, EFX_EVQ_NBUFS(evq_size)); in sfxge_ev_qstop()
1039 sfxge_evq_t *sep = sp->s_sep[index]; in sfxge_ev_qfini() local
1041 ASSERT3U(sep->se_state, ==, SFXGE_EVQ_INITIALIZED); in sfxge_ev_qfini()
1044 sep->se_state = SFXGE_EVQ_UNINITIALIZED; in sfxge_ev_qfini()
1047 sfxge_ev_kstat_fini(sep); in sfxge_ev_qfini()
1049 cv_destroy(&(sep->se_init_kv)); in sfxge_ev_qfini()
1050 mutex_destroy(&(sep->se_lock)); in sfxge_ev_qfini()
1052 sep->se_index = 0; in sfxge_ev_qfini()
1054 kmem_cache_free(index ? sp->s_eqXc : sp->s_eq0c, sep); in sfxge_ev_qfini()
1061 sfxge_evq_t *sep = sp->s_sep[evq]; in sfxge_ev_txlabel_alloc() local
1066 mutex_enter(&(sep->se_lock)); in sfxge_ev_txlabel_alloc()
1075 if (sep->se_label_stp[label] == stp) { in sfxge_ev_txlabel_alloc()
1079 if ((stpp == NULL) && (sep->se_label_stp[label] == NULL)) { in sfxge_ev_txlabel_alloc()
1080 stpp = &sep->se_label_stp[label]; in sfxge_ev_txlabel_alloc()
1088 label = stpp - sep->se_label_stp; in sfxge_ev_txlabel_alloc()
1093 mutex_exit(&(sep->se_lock)); in sfxge_ev_txlabel_alloc()
1103 mutex_exit(&(sep->se_lock)); in sfxge_ev_txlabel_alloc()
1113 sfxge_evq_t *sep = sp->s_sep[evq]; in sfxge_ev_txlabel_free() local
1116 mutex_enter(&(sep->se_lock)); in sfxge_ev_txlabel_free()
1123 if (sep->se_label_stp[label] != stp) { in sfxge_ev_txlabel_free()
1127 sep->se_label_stp[label] = NULL; in sfxge_ev_txlabel_free()
1129 mutex_exit(&(sep->se_lock)); in sfxge_ev_txlabel_free()
1138 mutex_exit(&(sep->se_lock)); in sfxge_ev_txlabel_free()