Lines Matching refs:stp
130 sfxge_tx_dpl_get_pkt_max(sfxge_txq_t *stp) in sfxge_tx_dpl_get_pkt_max() argument
132 sfxge_tx_dpl_t *stdp = &(stp->st_dpl); in sfxge_tx_dpl_get_pkt_max()
253 sfxge_txq_t *stp = buf; in sfxge_tx_qctor() local
254 efsys_mem_t *esmp = &(stp->st_mem); in sfxge_tx_qctor()
261 EFX_STATIC_ASSERT(sizeof (stp->__st_u1.__st_s1) <= in sfxge_tx_qctor()
262 sizeof (stp->__st_u1.__st_pad)); in sfxge_tx_qctor()
263 EFX_STATIC_ASSERT(sizeof (stp->__st_u2.__st_s2) <= in sfxge_tx_qctor()
264 sizeof (stp->__st_u2.__st_pad)); in sfxge_tx_qctor()
265 EFX_STATIC_ASSERT(sizeof (stp->__st_u3.__st_s3) <= in sfxge_tx_qctor()
266 sizeof (stp->__st_u3.__st_pad)); in sfxge_tx_qctor()
267 EFX_STATIC_ASSERT(sizeof (stp->__st_u4.__st_s4) <= in sfxge_tx_qctor()
268 sizeof (stp->__st_u4.__st_pad)); in sfxge_tx_qctor()
272 stp->st_sp = sp; in sfxge_tx_qctor()
289 &(stp->st_id))) != 0) in sfxge_tx_qctor()
293 if ((stp->st_eb = kmem_zalloc(sizeof (efx_buffer_t) * in sfxge_tx_qctor()
300 if ((stp->st_stmp = kmem_zalloc(sizeof (sfxge_tx_mapping_t *) * in sfxge_tx_qctor()
306 if ((stp->st_stbp = kmem_zalloc(sizeof (sfxge_tx_buffer_t *) * in sfxge_tx_qctor()
312 if ((stp->st_mp = kmem_zalloc(sizeof (mblk_t *) * in sfxge_tx_qctor()
319 stdp = &(stp->st_dpl); in sfxge_tx_qctor()
322 stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED; in sfxge_tx_qctor()
329 kmem_free(stp->st_stbp, sizeof (sfxge_tx_buffer_t *) * SFXGE_TX_NDESCS); in sfxge_tx_qctor()
330 stp->st_stbp = NULL; in sfxge_tx_qctor()
335 kmem_free(stp->st_stmp, in sfxge_tx_qctor()
337 stp->st_stmp = NULL; in sfxge_tx_qctor()
343 kmem_free(stp->st_eb, sizeof (efx_buffer_t) * in sfxge_tx_qctor()
345 stp->st_eb = NULL; in sfxge_tx_qctor()
351 sfxge_sram_buf_tbl_free(sp, stp->st_id, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS)); in sfxge_tx_qctor()
352 stp->st_id = 0; in sfxge_tx_qctor()
363 stp->st_sp = NULL; in sfxge_tx_qctor()
365 SFXGE_OBJ_CHECK(stp, sfxge_txq_t); in sfxge_tx_qctor()
373 sfxge_txq_t *stp = buf; in sfxge_tx_qdtor() local
374 efsys_mem_t *esmp = &(stp->st_mem); in sfxge_tx_qdtor()
375 sfxge_t *sp = stp->st_sp; in sfxge_tx_qdtor()
380 stp->st_unblock = 0; in sfxge_tx_qdtor()
383 stdp = &(stp->st_dpl); in sfxge_tx_qdtor()
388 kmem_free(stp->st_mp, sizeof (mblk_t *) * SFXGE_TX_NDESCS); in sfxge_tx_qdtor()
389 stp->st_mp = NULL; in sfxge_tx_qdtor()
391 kmem_free(stp->st_stbp, sizeof (sfxge_tx_buffer_t *) * SFXGE_TX_NDESCS); in sfxge_tx_qdtor()
392 stp->st_stbp = NULL; in sfxge_tx_qdtor()
394 kmem_free(stp->st_stmp, in sfxge_tx_qdtor()
396 stp->st_stmp = NULL; in sfxge_tx_qdtor()
399 kmem_free(stp->st_eb, sizeof (efx_buffer_t) * in sfxge_tx_qdtor()
401 stp->st_eb = NULL; in sfxge_tx_qdtor()
404 sfxge_sram_buf_tbl_free(sp, stp->st_id, EFX_TXQ_NBUFS(SFXGE_TX_NDESCS)); in sfxge_tx_qdtor()
405 stp->st_id = 0; in sfxge_tx_qdtor()
410 stp->st_sp = NULL; in sfxge_tx_qdtor()
412 SFXGE_OBJ_CHECK(stp, sfxge_txq_t); in sfxge_tx_qdtor()
432 sfxge_tx_qfpp_put(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp) in sfxge_tx_qfpp_put() argument
434 sfxge_tx_fpp_t *stfp = &(stp->st_fpp); in sfxge_tx_qfpp_put()
436 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qfpp_put()
462 sfxge_tx_qfpp_get(sfxge_txq_t *stp) in sfxge_tx_qfpp_get() argument
465 sfxge_tx_fpp_t *stfp = &(stp->st_fpp); in sfxge_tx_qfpp_get()
467 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qfpp_get()
490 sfxge_tx_qfpp_empty(sfxge_txq_t *stp) in sfxge_tx_qfpp_empty() argument
492 sfxge_t *sp = stp->st_sp; in sfxge_tx_qfpp_empty()
493 sfxge_tx_fpp_t *stfp = &(stp->st_fpp); in sfxge_tx_qfpp_empty()
496 mutex_enter(&(stp->st_lock)); in sfxge_tx_qfpp_empty()
516 mutex_exit(&(stp->st_lock)); in sfxge_tx_qfpp_empty()
520 sfxge_tx_qfbp_put(sfxge_txq_t *stp, sfxge_tx_buffer_t *stbp) in sfxge_tx_qfbp_put() argument
522 sfxge_tx_fbp_t *stfp = &(stp->st_fbp); in sfxge_tx_qfbp_put()
535 sfxge_tx_qfbp_get(sfxge_txq_t *stp) in sfxge_tx_qfbp_get() argument
538 sfxge_tx_fbp_t *stfp = &(stp->st_fbp); in sfxge_tx_qfbp_get()
561 sfxge_tx_qfbp_empty(sfxge_txq_t *stp) in sfxge_tx_qfbp_empty() argument
563 sfxge_t *sp = stp->st_sp; in sfxge_tx_qfbp_empty()
564 sfxge_tx_fbp_t *stfp = &(stp->st_fbp); in sfxge_tx_qfbp_empty()
567 mutex_enter(&(stp->st_lock)); in sfxge_tx_qfbp_empty()
587 mutex_exit(&(stp->st_lock)); in sfxge_tx_qfbp_empty()
591 sfxge_tx_qfmp_put(sfxge_txq_t *stp, sfxge_tx_mapping_t *stmp) in sfxge_tx_qfmp_put() argument
593 sfxge_tx_fmp_t *stfp = &(stp->st_fmp); in sfxge_tx_qfmp_put()
607 sfxge_tx_qfmp_get(sfxge_txq_t *stp) in sfxge_tx_qfmp_get() argument
610 sfxge_tx_fmp_t *stfp = &(stp->st_fmp); in sfxge_tx_qfmp_get()
632 sfxge_tx_qfmp_empty(sfxge_txq_t *stp) in sfxge_tx_qfmp_empty() argument
634 sfxge_t *sp = stp->st_sp; in sfxge_tx_qfmp_empty()
635 sfxge_tx_fmp_t *stfp = &(stp->st_fmp); in sfxge_tx_qfmp_empty()
638 mutex_enter(&(stp->st_lock)); in sfxge_tx_qfmp_empty()
658 mutex_exit(&(stp->st_lock)); in sfxge_tx_qfmp_empty()
757 sfxge_tx_qreap(sfxge_txq_t *stp) in sfxge_tx_qreap() argument
761 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qreap()
763 reaped = stp->st_reaped; in sfxge_tx_qreap()
764 while (reaped != stp->st_completed) { in sfxge_tx_qreap()
771 ASSERT3P(stp->st_mp[id], ==, NULL); in sfxge_tx_qreap()
773 if ((stmp = stp->st_stmp[id]) != NULL) { in sfxge_tx_qreap()
774 stp->st_stmp[id] = NULL; in sfxge_tx_qreap()
783 sfxge_tx_qfmp_put(stp, stmp); in sfxge_tx_qreap()
789 if ((stbp = stp->st_stbp[id]) != NULL) { in sfxge_tx_qreap()
790 stp->st_stbp[id] = NULL; in sfxge_tx_qreap()
802 sfxge_tx_qfbp_put(stp, stbp); in sfxge_tx_qreap()
808 stp->st_reaped = reaped; in sfxge_tx_qreap()
812 sfxge_tx_qlist_abort(sfxge_txq_t *stp) in sfxge_tx_qlist_abort() argument
819 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qlist_abort()
821 id = stp->st_added & (SFXGE_TX_NDESCS - 1); in sfxge_tx_qlist_abort()
824 stmp = stp->st_stmp[id]; in sfxge_tx_qlist_abort()
825 stp->st_stmp[id] = NULL; in sfxge_tx_qlist_abort()
837 sfxge_tx_qfmp_put(stp, stmp); in sfxge_tx_qlist_abort()
842 stbp = stp->st_stbp[id]; in sfxge_tx_qlist_abort()
843 stp->st_stbp[id] = NULL; in sfxge_tx_qlist_abort()
855 sfxge_tx_qfbp_put(stp, stbp); in sfxge_tx_qlist_abort()
860 mp = stp->st_mp[id]; in sfxge_tx_qlist_abort()
861 stp->st_mp[id] = NULL; in sfxge_tx_qlist_abort()
867 stp->st_n = 0; in sfxge_tx_qlist_abort()
872 sfxge_tx_qlist_post(sfxge_txq_t *stp) in sfxge_tx_qlist_post() argument
879 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qlist_post()
881 ASSERT(stp->st_n != 0); in sfxge_tx_qlist_post()
884 level = stp->st_added - stp->st_reaped; in sfxge_tx_qlist_post()
887 id = stp->st_added & (SFXGE_TX_NDESCS - 1); in sfxge_tx_qlist_post()
889 if (available < stp->st_n) { in sfxge_tx_qlist_post()
894 ASSERT3U(available, >=, stp->st_n); in sfxge_tx_qlist_post()
897 if ((rc = efx_tx_qpost(stp->st_etp, stp->st_eb, stp->st_n, in sfxge_tx_qlist_post()
898 stp->st_reaped, &(stp->st_added))) != 0) in sfxge_tx_qlist_post()
906 if (((stp->st_added - 1) & (SFXGE_TX_NDESCS - 1)) != id) { in sfxge_tx_qlist_post()
911 stmp = stp->st_stmp[id]; in sfxge_tx_qlist_post()
912 stp->st_stmp[id] = NULL; in sfxge_tx_qlist_post()
914 stbp = stp->st_stbp[id]; in sfxge_tx_qlist_post()
915 stp->st_stbp[id] = NULL; in sfxge_tx_qlist_post()
917 mp = stp->st_mp[id]; in sfxge_tx_qlist_post()
918 stp->st_mp[id] = NULL; in sfxge_tx_qlist_post()
920 id = (stp->st_added - 1) & (SFXGE_TX_NDESCS - 1); in sfxge_tx_qlist_post()
922 ASSERT(stp->st_stmp[id] == NULL); in sfxge_tx_qlist_post()
923 stp->st_stmp[id] = stmp; in sfxge_tx_qlist_post()
925 ASSERT(stp->st_stbp[id] == NULL); in sfxge_tx_qlist_post()
926 stp->st_stbp[id] = stbp; in sfxge_tx_qlist_post()
928 ASSERT(stp->st_mp[id] == NULL); in sfxge_tx_qlist_post()
929 stp->st_mp[id] = mp; in sfxge_tx_qlist_post()
933 stp->st_n = 0; in sfxge_tx_qlist_post()
935 ASSERT3U(stp->st_unblock, ==, SFXGE_TXQ_NOT_BLOCKED); in sfxge_tx_qlist_post()
945 level = stp->st_added - stp->st_completed; in sfxge_tx_qlist_post()
953 if (stp->st_n <= available && in sfxge_tx_qlist_post()
954 stp->st_completed - stp->st_reaped >= SFXGE_TX_BATCH) { in sfxge_tx_qlist_post()
955 sfxge_tx_qreap(stp); in sfxge_tx_qlist_post()
961 if (stp->st_unblock == SFXGE_TXQ_NOT_BLOCKED) { in sfxge_tx_qlist_post()
962 stp->st_unblock = SFXGE_TXQ_UNBLOCK_LEVEL1; in sfxge_tx_qlist_post()
964 ASSERT(stp->st_unblock == SFXGE_TXQ_UNBLOCK_LEVEL1); in sfxge_tx_qlist_post()
966 stp->st_unblock = SFXGE_TXQ_UNBLOCK_LEVEL2; in sfxge_tx_qlist_post()
977 if (stp->st_pending == stp->st_added) { in sfxge_tx_qlist_post()
978 sfxge_tx_qreap(stp); in sfxge_tx_qlist_post()
980 stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED; in sfxge_tx_qlist_post()
984 ASSERT(stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED); in sfxge_tx_qlist_post()
990 sfxge_txq_t *stp = ksp->ks_private; in sfxge_tx_kstat_update() local
991 sfxge_tx_dpl_t *stdp = &(stp->st_dpl); in sfxge_tx_kstat_update()
995 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_kstat_update()
1002 if (stp->st_state != SFXGE_TXQ_STARTED) in sfxge_tx_kstat_update()
1005 efx_tx_qstats_update(stp->st_etp, stp->st_stat); in sfxge_tx_kstat_update()
1025 sfxge_tx_kstat_init(sfxge_txq_t *stp) in sfxge_tx_kstat_init() argument
1027 sfxge_t *sp = stp->st_sp; in sfxge_tx_kstat_init()
1028 unsigned int index = stp->st_index; in sfxge_tx_kstat_init()
1047 stp->st_ksp = ksp; in sfxge_tx_kstat_init()
1050 ksp->ks_private = stp; in sfxge_tx_kstat_init()
1051 ksp->ks_lock = &(stp->st_lock); in sfxge_tx_kstat_init()
1054 stp->st_stat = knp = ksp->ks_data; in sfxge_tx_kstat_init()
1078 sfxge_tx_kstat_fini(sfxge_txq_t *stp) in sfxge_tx_kstat_fini() argument
1081 kstat_delete(stp->st_ksp); in sfxge_tx_kstat_fini()
1082 stp->st_ksp = NULL; in sfxge_tx_kstat_fini()
1083 stp->st_stat = NULL; in sfxge_tx_kstat_fini()
1090 sfxge_txq_t *stp; in sfxge_tx_qinit() local
1098 if ((stp = kmem_cache_alloc(sp->s_tqc, KM_SLEEP)) == NULL) { in sfxge_tx_qinit()
1102 ASSERT3U(stp->st_state, ==, SFXGE_TXQ_UNINITIALIZED); in sfxge_tx_qinit()
1104 stdp = &(stp->st_dpl); in sfxge_tx_qinit()
1106 stp->st_index = index; in sfxge_tx_qinit()
1107 stp->st_type = type; in sfxge_tx_qinit()
1108 stp->st_evq = evq; in sfxge_tx_qinit()
1110 mutex_init(&(stp->st_lock), NULL, MUTEX_DRIVER, in sfxge_tx_qinit()
1114 if ((rc = sfxge_tx_kstat_init(stp)) != 0) in sfxge_tx_qinit()
1126 if ((rc = sfxge_ev_txlabel_alloc(sp, evq, stp, &(stp->st_label))) != 0) in sfxge_tx_qinit()
1129 stp->st_state = SFXGE_TXQ_INITIALIZED; in sfxge_tx_qinit()
1133 sp->s_stp[index] = stp; in sfxge_tx_qinit()
1141 sfxge_tx_kstat_fini(stp); in sfxge_tx_qinit()
1144 stp->st_evq = 0; in sfxge_tx_qinit()
1145 stp->st_type = 0; in sfxge_tx_qinit()
1146 stp->st_index = 0; in sfxge_tx_qinit()
1148 mutex_destroy(&(stp->st_lock)); in sfxge_tx_qinit()
1150 kmem_cache_free(sp->s_tqc, stp); in sfxge_tx_qinit()
1161 sfxge_txq_t *stp = sp->s_stp[index]; in sfxge_tx_qstart() local
1170 mutex_enter(&(stp->st_lock)); in sfxge_tx_qstart()
1172 esmp = &(stp->st_mem); in sfxge_tx_qstart()
1173 evq = stp->st_evq; in sfxge_tx_qstart()
1176 ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED); in sfxge_tx_qstart()
1183 if ((rc = sfxge_sram_buf_tbl_set(sp, stp->st_id, esmp, in sfxge_tx_qstart()
1187 switch (stp->st_type) { in sfxge_tx_qstart()
1208 if ((rc = efx_tx_qcreate(enp, index, stp->st_label, esmp, in sfxge_tx_qstart()
1209 SFXGE_TX_NDESCS, stp->st_id, flags, sep->se_eep, in sfxge_tx_qstart()
1210 &(stp->st_etp), &desc_index)) != 0) in sfxge_tx_qstart()
1214 stp->st_added = desc_index; in sfxge_tx_qstart()
1215 stp->st_pending = desc_index; in sfxge_tx_qstart()
1216 stp->st_completed = desc_index; in sfxge_tx_qstart()
1217 stp->st_reaped = desc_index; in sfxge_tx_qstart()
1220 efx_tx_qenable(stp->st_etp); in sfxge_tx_qstart()
1222 stp->st_state = SFXGE_TXQ_STARTED; in sfxge_tx_qstart()
1224 mutex_exit(&(stp->st_lock)); in sfxge_tx_qstart()
1232 sfxge_sram_buf_tbl_clear(sp, stp->st_id, in sfxge_tx_qstart()
1238 mutex_exit(&(stp->st_lock)); in sfxge_tx_qstart()
1244 sfxge_tx_qmapping_add(sfxge_txq_t *stp, sfxge_tx_mapping_t *stmp, in sfxge_tx_qmapping_add() argument
1278 ASSERT3U(stp->st_n, <=, in sfxge_tx_qmapping_add()
1280 if (stp->st_n == in sfxge_tx_qmapping_add()
1286 ebp = &(stp->st_eb[stp->st_n++]); in sfxge_tx_qmapping_add()
1301 unsigned int, stp->st_index, in sfxge_tx_qmapping_add()
1302 unsigned int, stp->st_n - 1, in sfxge_tx_qmapping_add()
1319 sfxge_tx_qbuffer_add(sfxge_txq_t *stp, sfxge_tx_buffer_t *stbp, boolean_t eop) in sfxge_tx_qbuffer_add() argument
1324 ASSERT3U(stp->st_n, <=, in sfxge_tx_qbuffer_add()
1326 if (stp->st_n == EFX_TXQ_LIMIT(SFXGE_TX_NDESCS)) { in sfxge_tx_qbuffer_add()
1331 ebp = &(stp->st_eb[stp->st_n++]); in sfxge_tx_qbuffer_add()
1343 unsigned int, stp->st_index, in sfxge_tx_qbuffer_add()
1344 unsigned int, stp->st_n - 1, in sfxge_tx_qbuffer_add()
1400 sfxge_tx_qpayload_fragment(sfxge_txq_t *stp, unsigned int id, mblk_t **mpp, in sfxge_tx_qpayload_fragment() argument
1403 sfxge_t *sp = stp->st_sp; in sfxge_tx_qpayload_fragment()
1410 stbp = stp->st_stbp[id]; in sfxge_tx_qpayload_fragment()
1413 stmp = stp->st_stmp[id]; in sfxge_tx_qpayload_fragment()
1449 stmp = sfxge_tx_qfmp_get(stp); in sfxge_tx_qpayload_fragment()
1461 stmp->stm_next = stp->st_stmp[id]; in sfxge_tx_qpayload_fragment()
1462 stp->st_stmp[id] = stmp; in sfxge_tx_qpayload_fragment()
1475 rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE); in sfxge_tx_qpayload_fragment()
1481 rc = sfxge_tx_qmapping_add(stp, stmp, &off, &size); in sfxge_tx_qpayload_fragment()
1502 stbp = sfxge_tx_qfbp_get(stp); in sfxge_tx_qpayload_fragment()
1516 stbp->stb_next = stp->st_stbp[id]; in sfxge_tx_qpayload_fragment()
1517 stp->st_stbp[id] = stbp; in sfxge_tx_qpayload_fragment()
1531 rc = sfxge_tx_qbuffer_add(stp, stbp, eop); in sfxge_tx_qpayload_fragment()
1564 sfxge_tx_qlso_fragment(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp, in sfxge_tx_qlso_fragment() argument
1567 sfxge_t *sp = stp->st_sp; in sfxge_tx_qlso_fragment()
1588 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qlso_fragment()
1595 id = stp->st_added & (SFXGE_TX_NDESCS - 1); in sfxge_tx_qlso_fragment()
1597 ASSERT(stp->st_n == 0); in sfxge_tx_qlso_fragment()
1598 ASSERT(stp->st_stbp[id] == NULL); in sfxge_tx_qlso_fragment()
1599 ASSERT(stp->st_stmp[id] == NULL); in sfxge_tx_qlso_fragment()
1685 stbp = sfxge_tx_qfbp_get(stp); in sfxge_tx_qlso_fragment()
1699 stbp->stb_next = stp->st_stbp[id]; in sfxge_tx_qlso_fragment()
1700 stp->st_stbp[id] = stbp; in sfxge_tx_qlso_fragment()
1709 rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE); in sfxge_tx_qlso_fragment()
1714 if ((rc = sfxge_tx_qpayload_fragment(stp, id, &mp, &off, in sfxge_tx_qlso_fragment()
1729 if (stp->st_stmp[id] == NULL) in sfxge_tx_qlso_fragment()
1732 stp->st_mp[id] = stpp->stp_mp; in sfxge_tx_qlso_fragment()
1765 ASSERT(stp->st_mp[id] == NULL); in sfxge_tx_qlso_fragment()
1778 sfxge_tx_qpacket_fragment(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp, in sfxge_tx_qpacket_fragment() argument
1781 sfxge_t *sp = stp->st_sp; in sfxge_tx_qpacket_fragment()
1790 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qpacket_fragment()
1792 ASSERT(stp->st_n == 0); in sfxge_tx_qpacket_fragment()
1794 id = stp->st_added & (SFXGE_TX_NDESCS - 1); in sfxge_tx_qpacket_fragment()
1796 ASSERT(stp->st_stbp[id] == NULL); in sfxge_tx_qpacket_fragment()
1797 ASSERT(stp->st_stmp[id] == NULL); in sfxge_tx_qpacket_fragment()
1835 stmp = sfxge_tx_qfmp_get(stp); in sfxge_tx_qpacket_fragment()
1847 stmp->stm_next = stp->st_stmp[id]; in sfxge_tx_qpacket_fragment()
1848 stp->st_stmp[id] = stmp; in sfxge_tx_qpacket_fragment()
1859 rc = sfxge_tx_qbuffer_add(stp, stbp, B_FALSE); in sfxge_tx_qpacket_fragment()
1865 rc = sfxge_tx_qmapping_add(stp, stmp, &off, &size); in sfxge_tx_qpacket_fragment()
1880 stbp = sfxge_tx_qfbp_get(stp); in sfxge_tx_qpacket_fragment()
1894 stbp->stb_next = stp->st_stbp[id]; in sfxge_tx_qpacket_fragment()
1895 stp->st_stbp[id] = stbp; in sfxge_tx_qpacket_fragment()
1909 rc = sfxge_tx_qbuffer_add(stp, stbp, eop); in sfxge_tx_qpacket_fragment()
1931 if (stp->st_stmp[id] == NULL) in sfxge_tx_qpacket_fragment()
1934 stp->st_mp[id] = stpp->stp_mp; in sfxge_tx_qpacket_fragment()
1949 ASSERT(stp->st_stmp[id] == NULL); in sfxge_tx_qpacket_fragment()
1959 sfxge_tx_qdpl_swizzle(sfxge_txq_t *stp) in sfxge_tx_qdpl_swizzle() argument
1961 sfxge_tx_dpl_t *stdp = &(stp->st_dpl); in sfxge_tx_qdpl_swizzle()
1969 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qdpl_swizzle()
1975 ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp)); in sfxge_tx_qdpl_swizzle()
2010 ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp)); in sfxge_tx_qdpl_swizzle()
2028 sfxge_tx_qdpl_add(sfxge_txq_t *stp, sfxge_tx_packet_t *stpp, int locked) in sfxge_tx_qdpl_add() argument
2030 sfxge_tx_dpl_t *stdp = &stp->st_dpl; in sfxge_tx_qdpl_add()
2035 ASSERT(mutex_owned(&stp->st_lock)); in sfxge_tx_qdpl_add()
2043 sfxge_tx_qdpl_swizzle(stp); in sfxge_tx_qdpl_add()
2049 ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp)); in sfxge_tx_qdpl_add()
2084 sfxge_tx_qdpl_drain(sfxge_txq_t *stp) in sfxge_tx_qdpl_drain() argument
2086 sfxge_t *sp = stp->st_sp; in sfxge_tx_qdpl_drain()
2087 sfxge_tx_dpl_t *stdp = &(stp->st_dpl); in sfxge_tx_qdpl_drain()
2088 unsigned int pushed = stp->st_added; in sfxge_tx_qdpl_drain()
2092 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qdpl_drain()
2095 prefetch_read_many(stp->st_etp); in sfxge_tx_qdpl_drain()
2114 if (stp->st_state != SFXGE_TXQ_STARTED) in sfxge_tx_qdpl_drain()
2122 rc = sfxge_tx_qlso_fragment(stp, stpp, copy); in sfxge_tx_qdpl_drain()
2124 rc = sfxge_tx_qpacket_fragment(stp, stpp, copy); in sfxge_tx_qdpl_drain()
2151 if (sfxge_tx_qfpp_put(stp, stpp) != 0) { in sfxge_tx_qdpl_drain()
2160 sfxge_tx_qlist_post(stp); in sfxge_tx_qdpl_drain()
2162 if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED) in sfxge_tx_qdpl_drain()
2165 if (stp->st_added - pushed >= SFXGE_TX_BATCH) { in sfxge_tx_qdpl_drain()
2166 efx_tx_qpush(stp->st_etp, stp->st_added, pushed); in sfxge_tx_qdpl_drain()
2167 pushed = stp->st_added; in sfxge_tx_qdpl_drain()
2174 sfxge_tx_qlist_abort(stp); in sfxge_tx_qdpl_drain()
2184 sfxge_tx_qlist_abort(stp); in sfxge_tx_qdpl_drain()
2199 if (sfxge_tx_qfpp_put(stp, stpp) != 0) { in sfxge_tx_qdpl_drain()
2208 DTRACE_PROBE1(defer, unsigned int, stp->st_index); in sfxge_tx_qdpl_drain()
2223 ASSERT3U(stdp->std_count, <=, sfxge_tx_dpl_get_pkt_max(stp)); in sfxge_tx_qdpl_drain()
2226 if (stp->st_added != pushed) in sfxge_tx_qdpl_drain()
2227 efx_tx_qpush(stp->st_etp, stp->st_added, pushed); in sfxge_tx_qdpl_drain()
2229 ASSERT(stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED || in sfxge_tx_qdpl_drain()
2235 sfxge_tx_qdpl_service(sfxge_txq_t *stp) in sfxge_tx_qdpl_service() argument
2238 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qdpl_service()
2240 if (SFXGE_TX_QDPL_PUT_PENDING(stp)) in sfxge_tx_qdpl_service()
2241 sfxge_tx_qdpl_swizzle(stp); in sfxge_tx_qdpl_service()
2243 if (stp->st_unblock == SFXGE_TXQ_NOT_BLOCKED) in sfxge_tx_qdpl_service()
2244 sfxge_tx_qdpl_drain(stp); in sfxge_tx_qdpl_service()
2246 mutex_exit(&(stp->st_lock)); in sfxge_tx_qdpl_service()
2248 if (!SFXGE_TX_QDPL_PUT_PENDING(stp)) in sfxge_tx_qdpl_service()
2250 } while (mutex_tryenter(&(stp->st_lock))); in sfxge_tx_qdpl_service()
2254 sfxge_tx_qdpl_flush_locked(sfxge_txq_t *stp) in sfxge_tx_qdpl_flush_locked() argument
2256 sfxge_t *sp = stp->st_sp; in sfxge_tx_qdpl_flush_locked()
2257 sfxge_tx_dpl_t *stdp = &(stp->st_dpl); in sfxge_tx_qdpl_flush_locked()
2261 ASSERT(mutex_owned(&(stp->st_lock))); in sfxge_tx_qdpl_flush_locked()
2264 sfxge_tx_qdpl_swizzle(stp); in sfxge_tx_qdpl_flush_locked()
2304 sfxge_tx_qdpl_flush(sfxge_txq_t *stp) in sfxge_tx_qdpl_flush() argument
2306 mutex_enter(&(stp->st_lock)); in sfxge_tx_qdpl_flush()
2307 sfxge_tx_qdpl_flush_locked(stp); in sfxge_tx_qdpl_flush()
2308 mutex_exit(&(stp->st_lock)); in sfxge_tx_qdpl_flush()
2313 sfxge_tx_qunblock(sfxge_txq_t *stp) in sfxge_tx_qunblock() argument
2315 sfxge_t *sp = stp->st_sp; in sfxge_tx_qunblock()
2316 unsigned int evq = stp->st_evq; in sfxge_tx_qunblock()
2321 mutex_enter(&(stp->st_lock)); in sfxge_tx_qunblock()
2323 if (stp->st_state != SFXGE_TXQ_STARTED) { in sfxge_tx_qunblock()
2324 mutex_exit(&(stp->st_lock)); in sfxge_tx_qunblock()
2328 if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED) { in sfxge_tx_qunblock()
2331 level = stp->st_added - stp->st_completed; in sfxge_tx_qunblock()
2332 if (level <= stp->st_unblock) { in sfxge_tx_qunblock()
2333 stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED; in sfxge_tx_qunblock()
2334 sfxge_tx_qlist_post(stp); in sfxge_tx_qunblock()
2338 sfxge_tx_qdpl_service(stp); in sfxge_tx_qunblock()
2343 sfxge_tx_qcomplete(sfxge_txq_t *stp) in sfxge_tx_qcomplete() argument
2345 sfxge_t *sp = stp->st_sp; in sfxge_tx_qcomplete()
2346 sfxge_tx_dpl_t *stdp = &(stp->st_dpl); in sfxge_tx_qcomplete()
2347 unsigned int evq = stp->st_evq; in sfxge_tx_qcomplete()
2353 completed = stp->st_completed; in sfxge_tx_qcomplete()
2354 while (completed != stp->st_pending) { in sfxge_tx_qcomplete()
2360 if ((stmp = stp->st_stmp[id]) != NULL) { in sfxge_tx_qcomplete()
2375 mp = stp->st_mp[id]; in sfxge_tx_qcomplete()
2376 stp->st_mp[id] = NULL; in sfxge_tx_qcomplete()
2382 stp->st_completed = completed; in sfxge_tx_qcomplete()
2385 if (stp->st_unblock != SFXGE_TXQ_NOT_BLOCKED) { in sfxge_tx_qcomplete()
2388 level = stp->st_added - stp->st_completed; in sfxge_tx_qcomplete()
2389 if (level <= stp->st_unblock) in sfxge_tx_qcomplete()
2390 sfxge_tx_qunblock(stp); in sfxge_tx_qcomplete()
2399 sfxge_tx_qflush_done(sfxge_txq_t *stp) in sfxge_tx_qflush_done() argument
2401 sfxge_t *sp = stp->st_sp; in sfxge_tx_qflush_done()
2404 ASSERT(mutex_owned(&(sp->s_sep[stp->st_evq]->se_lock))); in sfxge_tx_qflush_done()
2406 mutex_enter(&(stp->st_lock)); in sfxge_tx_qflush_done()
2408 switch (stp->st_state) { in sfxge_tx_qflush_done()
2415 stp->st_state = SFXGE_TXQ_FLUSH_DONE; in sfxge_tx_qflush_done()
2420 stp->st_state = SFXGE_TXQ_FLUSH_DONE; in sfxge_tx_qflush_done()
2428 stp->st_state = SFXGE_TXQ_FLUSH_DONE; in sfxge_tx_qflush_done()
2441 mutex_exit(&(stp->st_lock)); in sfxge_tx_qflush_done()
2460 sfxge_txq_t *stp = sp->s_stp[index]; in sfxge_tx_qflush() local
2466 mutex_enter(&(stp->st_lock)); in sfxge_tx_qflush()
2469 if (stp->st_state == SFXGE_TXQ_STARTED) { in sfxge_tx_qflush()
2471 if ((rc = efx_tx_qflush(stp->st_etp)) == EALREADY) { in sfxge_tx_qflush()
2473 stp->st_state = SFXGE_TXQ_FLUSH_DONE; in sfxge_tx_qflush()
2476 stp->st_state = SFXGE_TXQ_FLUSH_FAILED; in sfxge_tx_qflush()
2478 stp->st_state = SFXGE_TXQ_FLUSH_PENDING; in sfxge_tx_qflush()
2482 stp->st_state = SFXGE_TXQ_FLUSH_DONE; in sfxge_tx_qflush()
2486 mutex_exit(&(stp->st_lock)); in sfxge_tx_qflush()
2492 sfxge_txq_t *stp = sp->s_stp[index]; in sfxge_tx_qstop() local
2493 unsigned int evq = stp->st_evq; in sfxge_tx_qstop()
2497 mutex_enter(&(stp->st_lock)); in sfxge_tx_qstop()
2499 if (stp->st_state == SFXGE_TXQ_INITIALIZED) in sfxge_tx_qstop()
2502 ASSERT(stp->st_state == SFXGE_TXQ_FLUSH_PENDING || in sfxge_tx_qstop()
2503 stp->st_state == SFXGE_TXQ_FLUSH_DONE || in sfxge_tx_qstop()
2504 stp->st_state == SFXGE_TXQ_FLUSH_FAILED); in sfxge_tx_qstop()
2507 if (stp->st_sp->s_tx_flush_pending != 0) { in sfxge_tx_qstop()
2510 index, stp->st_sp->s_tx_flush_pending); in sfxge_tx_qstop()
2512 if (stp->st_state == SFXGE_TXQ_FLUSH_FAILED) { in sfxge_tx_qstop()
2518 efx_tx_qdestroy(stp->st_etp); in sfxge_tx_qstop()
2519 stp->st_etp = NULL; in sfxge_tx_qstop()
2522 sfxge_sram_buf_tbl_clear(sp, stp->st_id, in sfxge_tx_qstop()
2525 sfxge_tx_qlist_abort(stp); in sfxge_tx_qstop()
2526 ASSERT3U(stp->st_n, ==, 0); in sfxge_tx_qstop()
2528 stp->st_unblock = SFXGE_TXQ_NOT_BLOCKED; in sfxge_tx_qstop()
2530 stp->st_pending = stp->st_added; in sfxge_tx_qstop()
2532 sfxge_tx_qcomplete(stp); in sfxge_tx_qstop()
2533 ASSERT3U(stp->st_completed, ==, stp->st_pending); in sfxge_tx_qstop()
2535 sfxge_tx_qreap(stp); in sfxge_tx_qstop()
2536 ASSERT3U(stp->st_reaped, ==, stp->st_completed); in sfxge_tx_qstop()
2542 sfxge_tx_qdpl_flush_locked(stp); in sfxge_tx_qstop()
2544 stp->st_added = 0; in sfxge_tx_qstop()
2545 stp->st_pending = 0; in sfxge_tx_qstop()
2546 stp->st_completed = 0; in sfxge_tx_qstop()
2547 stp->st_reaped = 0; in sfxge_tx_qstop()
2549 stp->st_state = SFXGE_TXQ_INITIALIZED; in sfxge_tx_qstop()
2552 mutex_exit(&(stp->st_lock)); in sfxge_tx_qstop()
2559 sfxge_txq_t *stp = sp->s_stp[index]; in sfxge_tx_qfini() local
2560 sfxge_tx_dpl_t *stdp = &(stp->st_dpl); in sfxge_tx_qfini()
2562 ASSERT3U(stp->st_state, ==, SFXGE_TXQ_INITIALIZED); in sfxge_tx_qfini()
2563 stp->st_state = SFXGE_TXQ_UNINITIALIZED; in sfxge_tx_qfini()
2571 (void) sfxge_ev_txlabel_free(sp, stp->st_evq, stp, stp->st_label); in sfxge_tx_qfini()
2572 stp->st_label = 0; in sfxge_tx_qfini()
2575 sfxge_tx_kstat_fini(stp); in sfxge_tx_qfini()
2583 sfxge_tx_qfbp_empty(stp); in sfxge_tx_qfini()
2586 sfxge_tx_qfmp_empty(stp); in sfxge_tx_qfini()
2589 sfxge_tx_qfpp_empty(stp); in sfxge_tx_qfini()
2591 mutex_destroy(&(stp->st_lock)); in sfxge_tx_qfini()
2593 stp->st_evq = 0; in sfxge_tx_qfini()
2594 stp->st_type = 0; in sfxge_tx_qfini()
2595 stp->st_index = 0; in sfxge_tx_qfini()
2597 kmem_cache_free(sp->s_tqc, stp); in sfxge_tx_qfini()
2754 sfxge_txq_t *stp; in sfxge_tx_packet_add() local
2872 stp = sp->s_stp[txq]; in sfxge_tx_packet_add()
2873 ASSERT3U(stp->st_evq, ==, index); in sfxge_tx_packet_add()
2878 stp = sp->s_stp[txq]; in sfxge_tx_packet_add()
2884 stp = sp->s_stp[txq]; in sfxge_tx_packet_add()
2924 stp = sp->s_stp[txq]; in sfxge_tx_packet_add()
2925 ASSERT3U(stp->st_evq, ==, index); in sfxge_tx_packet_add()
2930 stp = sp->s_stp[txq]; in sfxge_tx_packet_add()
2935 ASSERT(stp != NULL); in sfxge_tx_packet_add()
2940 locked = mutex_tryenter(&(stp->st_lock)); in sfxge_tx_packet_add()
2944 stpp = sfxge_tx_qfpp_get(stp); in sfxge_tx_packet_add()
2969 rc = sfxge_tx_qdpl_add(stp, stpp, locked); in sfxge_tx_packet_add()
2987 locked = mutex_tryenter(&(stp->st_lock)); in sfxge_tx_packet_add()
2991 sfxge_tx_qdpl_service(stp); in sfxge_tx_packet_add()
3003 mutex_exit(&(stp->st_lock)); in sfxge_tx_packet_add()