16f443ebcSRyan Zezeski /*
26f443ebcSRyan Zezeski * This file and its contents are supplied under the terms of the
36f443ebcSRyan Zezeski * Common Development and Distribution License ("CDDL"), version 1.0.
46f443ebcSRyan Zezeski * You may only use this file in accordance with the terms of version
56f443ebcSRyan Zezeski * 1.0 of the CDDL.
66f443ebcSRyan Zezeski *
76f443ebcSRyan Zezeski * A full copy of the text of the CDDL should have accompanied this
86f443ebcSRyan Zezeski * source. A copy of the CDDL is also available via the Internet at
96f443ebcSRyan Zezeski * http://www.illumos.org/license/CDDL.
106f443ebcSRyan Zezeski */
116f443ebcSRyan Zezeski
126f443ebcSRyan Zezeski /*
13eebd18daSAndy Fiddaman * Copyright 2024 Oxide Computer Company
146f443ebcSRyan Zezeski */
15*c46e4de3SAndy Fiddaman
166f443ebcSRyan Zezeski #include "ena.h"
176f443ebcSRyan Zezeski
186f443ebcSRyan Zezeski void
ena_free_tx_dma(ena_txq_t * txq)196f443ebcSRyan Zezeski ena_free_tx_dma(ena_txq_t *txq)
206f443ebcSRyan Zezeski {
216f443ebcSRyan Zezeski if (txq->et_tcbs != NULL) {
226f443ebcSRyan Zezeski for (uint_t i = 0; i < txq->et_sq_num_descs; i++) {
236f443ebcSRyan Zezeski ena_tx_control_block_t *tcb = &txq->et_tcbs[i];
246f443ebcSRyan Zezeski ena_dma_free(&tcb->etcb_dma);
256f443ebcSRyan Zezeski }
266f443ebcSRyan Zezeski
276f443ebcSRyan Zezeski kmem_free(txq->et_tcbs,
286f443ebcSRyan Zezeski sizeof (*txq->et_tcbs) * txq->et_sq_num_descs);
296f443ebcSRyan Zezeski
306f443ebcSRyan Zezeski txq->et_tcbs = NULL;
316f443ebcSRyan Zezeski }
326f443ebcSRyan Zezeski
336f443ebcSRyan Zezeski ena_dma_free(&txq->et_cq_dma);
346f443ebcSRyan Zezeski txq->et_cq_descs = NULL;
356f443ebcSRyan Zezeski
366f443ebcSRyan Zezeski ena_dma_free(&txq->et_sq_dma);
376f443ebcSRyan Zezeski txq->et_sq_descs = NULL;
386f443ebcSRyan Zezeski
396f443ebcSRyan Zezeski txq->et_state &= ~ENA_TXQ_STATE_HOST_ALLOC;
406f443ebcSRyan Zezeski }
416f443ebcSRyan Zezeski
426f443ebcSRyan Zezeski static int
ena_alloc_tx_dma(ena_txq_t * txq)436f443ebcSRyan Zezeski ena_alloc_tx_dma(ena_txq_t *txq)
446f443ebcSRyan Zezeski {
456f443ebcSRyan Zezeski ena_t *ena = txq->et_ena;
466f443ebcSRyan Zezeski size_t cq_descs_sz;
476f443ebcSRyan Zezeski size_t sq_descs_sz;
486f443ebcSRyan Zezeski int err = 0;
496f443ebcSRyan Zezeski
506f443ebcSRyan Zezeski ASSERT0(txq->et_state & ENA_TXQ_STATE_HOST_ALLOC);
516f443ebcSRyan Zezeski ASSERT3P(ena, !=, NULL);
526f443ebcSRyan Zezeski
536f443ebcSRyan Zezeski cq_descs_sz = txq->et_cq_num_descs * sizeof (*txq->et_cq_descs);
546f443ebcSRyan Zezeski sq_descs_sz = txq->et_sq_num_descs * sizeof (*txq->et_sq_descs);
556f443ebcSRyan Zezeski
56*c46e4de3SAndy Fiddaman ena_dma_conf_t sq_conf = {
576f443ebcSRyan Zezeski .edc_size = sq_descs_sz,
586f443ebcSRyan Zezeski .edc_align = ENAHW_IO_SQ_DESC_BUF_ALIGNMENT,
596f443ebcSRyan Zezeski .edc_sgl = 1,
606f443ebcSRyan Zezeski .edc_endian = DDI_NEVERSWAP_ACC,
61*c46e4de3SAndy Fiddaman .edc_stream = false,
626f443ebcSRyan Zezeski };
636f443ebcSRyan Zezeski
64*c46e4de3SAndy Fiddaman if (!ena_dma_alloc(ena, &txq->et_sq_dma, &sq_conf, sq_descs_sz)) {
656f443ebcSRyan Zezeski return (ENOMEM);
666f443ebcSRyan Zezeski }
676f443ebcSRyan Zezeski
686f443ebcSRyan Zezeski txq->et_sq_descs = (void *)txq->et_sq_dma.edb_va;
696f443ebcSRyan Zezeski txq->et_tcbs = kmem_zalloc(sizeof (*txq->et_tcbs) *
706f443ebcSRyan Zezeski txq->et_sq_num_descs, KM_SLEEP);
716f443ebcSRyan Zezeski
726f443ebcSRyan Zezeski for (uint_t i = 0; i < txq->et_sq_num_descs; i++) {
736f443ebcSRyan Zezeski ena_tx_control_block_t *tcb = &txq->et_tcbs[i];
746f443ebcSRyan Zezeski ena_dma_conf_t buf_conf = {
756f443ebcSRyan Zezeski .edc_size = ena->ena_tx_buf_sz,
766f443ebcSRyan Zezeski .edc_align = 1,
776f443ebcSRyan Zezeski .edc_sgl = ena->ena_tx_sgl_max_sz,
786f443ebcSRyan Zezeski .edc_endian = DDI_NEVERSWAP_ACC,
79*c46e4de3SAndy Fiddaman .edc_stream = true,
806f443ebcSRyan Zezeski };
816f443ebcSRyan Zezeski
826f443ebcSRyan Zezeski if (!ena_dma_alloc(ena, &tcb->etcb_dma, &buf_conf,
836f443ebcSRyan Zezeski ena->ena_tx_buf_sz)) {
846f443ebcSRyan Zezeski err = ENOMEM;
856f443ebcSRyan Zezeski goto error;
866f443ebcSRyan Zezeski }
876f443ebcSRyan Zezeski }
886f443ebcSRyan Zezeski
89*c46e4de3SAndy Fiddaman ena_dma_conf_t cq_conf = {
906f443ebcSRyan Zezeski .edc_size = cq_descs_sz,
916f443ebcSRyan Zezeski .edc_align = ENAHW_IO_CQ_DESC_BUF_ALIGNMENT,
926f443ebcSRyan Zezeski .edc_sgl = 1,
936f443ebcSRyan Zezeski .edc_endian = DDI_NEVERSWAP_ACC,
94*c46e4de3SAndy Fiddaman .edc_stream = false,
956f443ebcSRyan Zezeski };
966f443ebcSRyan Zezeski
97*c46e4de3SAndy Fiddaman if (!ena_dma_alloc(ena, &txq->et_cq_dma, &cq_conf, cq_descs_sz)) {
986f443ebcSRyan Zezeski err = ENOMEM;
996f443ebcSRyan Zezeski goto error;
1006f443ebcSRyan Zezeski }
1016f443ebcSRyan Zezeski
1026f443ebcSRyan Zezeski txq->et_cq_descs = (void *)txq->et_cq_dma.edb_va;
1036f443ebcSRyan Zezeski txq->et_state |= ENA_TXQ_STATE_HOST_ALLOC;
1046f443ebcSRyan Zezeski return (0);
1056f443ebcSRyan Zezeski
1066f443ebcSRyan Zezeski error:
1076f443ebcSRyan Zezeski ena_free_tx_dma(txq);
1086f443ebcSRyan Zezeski return (err);
1096f443ebcSRyan Zezeski }
1106f443ebcSRyan Zezeski
111*c46e4de3SAndy Fiddaman bool
ena_alloc_txq(ena_txq_t * txq)1126f443ebcSRyan Zezeski ena_alloc_txq(ena_txq_t *txq)
1136f443ebcSRyan Zezeski {
1146f443ebcSRyan Zezeski int ret = 0;
1156f443ebcSRyan Zezeski ena_t *ena = txq->et_ena;
1166f443ebcSRyan Zezeski uint16_t cq_hw_idx, sq_hw_idx;
11713776732SAndy Fiddaman uint32_t *cq_unmask_addr, *cq_numanode;
1186f443ebcSRyan Zezeski uint32_t *sq_db_addr;
1196f443ebcSRyan Zezeski
1206f443ebcSRyan Zezeski ASSERT3U(txq->et_cq_num_descs, >, 0);
1216f443ebcSRyan Zezeski
1226f443ebcSRyan Zezeski /*
1236f443ebcSRyan Zezeski * First, allocate the Tx data buffers.
1246f443ebcSRyan Zezeski */
1256f443ebcSRyan Zezeski if ((ret = ena_alloc_tx_dma(txq)) != 0) {
1266f443ebcSRyan Zezeski ena_err(ena, "failed to allocate Tx queue %u data buffers: %d",
1276f443ebcSRyan Zezeski txq->et_txqs_idx, ret);
128*c46e4de3SAndy Fiddaman return (false);
1296f443ebcSRyan Zezeski }
1306f443ebcSRyan Zezeski
1316f443ebcSRyan Zezeski ASSERT(txq->et_state & ENA_TXQ_STATE_HOST_ALLOC);
1326f443ebcSRyan Zezeski
1336f443ebcSRyan Zezeski /*
1346f443ebcSRyan Zezeski * Second, create the Completion Queue.
1356f443ebcSRyan Zezeski */
1366f443ebcSRyan Zezeski ret = ena_create_cq(ena, txq->et_cq_num_descs,
137*c46e4de3SAndy Fiddaman txq->et_cq_dma.edb_cookie->dmac_laddress, true,
13813776732SAndy Fiddaman txq->et_intr_vector, &cq_hw_idx, &cq_unmask_addr, &cq_numanode);
1396f443ebcSRyan Zezeski
1406f443ebcSRyan Zezeski if (ret != 0) {
1416f443ebcSRyan Zezeski ena_err(ena, "failed to create Tx CQ %u: %d", txq->et_txqs_idx,
1426f443ebcSRyan Zezeski ret);
143*c46e4de3SAndy Fiddaman return (false);
1446f443ebcSRyan Zezeski }
1456f443ebcSRyan Zezeski
1466f443ebcSRyan Zezeski txq->et_cq_hw_idx = cq_hw_idx;
1476f443ebcSRyan Zezeski txq->et_cq_phase = 1;
1486f443ebcSRyan Zezeski txq->et_cq_unmask_addr = cq_unmask_addr;
1496f443ebcSRyan Zezeski txq->et_cq_numa_addr = cq_numanode;
1506f443ebcSRyan Zezeski txq->et_state |= ENA_TXQ_STATE_CQ_CREATED;
1516f443ebcSRyan Zezeski
1526f443ebcSRyan Zezeski /*
1536f443ebcSRyan Zezeski * Third, create the Submission Queue to match with the above
1546f443ebcSRyan Zezeski * CQ. At this time we force the SQ and CQ to have the same
1556f443ebcSRyan Zezeski * number of descriptors as we only use a 1:1 completion
1566f443ebcSRyan Zezeski * policy. However, in the future, we could loosen this and
1576f443ebcSRyan Zezeski * use an on-demand completion policy and the two could have a
1586f443ebcSRyan Zezeski * different number of descriptors.
1596f443ebcSRyan Zezeski */
1606f443ebcSRyan Zezeski ASSERT3U(txq->et_sq_num_descs, ==, txq->et_cq_num_descs);
1616f443ebcSRyan Zezeski
1626f443ebcSRyan Zezeski ret = ena_create_sq(ena, txq->et_sq_num_descs,
163*c46e4de3SAndy Fiddaman txq->et_sq_dma.edb_cookie->dmac_laddress, true, cq_hw_idx,
1646f443ebcSRyan Zezeski &sq_hw_idx, &sq_db_addr);
1656f443ebcSRyan Zezeski
1666f443ebcSRyan Zezeski if (ret != 0) {
1676f443ebcSRyan Zezeski ena_err(ena, "failed to create Tx SQ %u: %d", txq->et_txqs_idx,
1686f443ebcSRyan Zezeski ret);
169*c46e4de3SAndy Fiddaman return (false);
1706f443ebcSRyan Zezeski }
1716f443ebcSRyan Zezeski
1726f443ebcSRyan Zezeski txq->et_sq_hw_idx = sq_hw_idx;
1736f443ebcSRyan Zezeski txq->et_sq_db_addr = sq_db_addr;
1746f443ebcSRyan Zezeski /* The phase must always start on 1. */
1756f443ebcSRyan Zezeski txq->et_sq_phase = 1;
1766f443ebcSRyan Zezeski txq->et_sq_avail_descs = txq->et_sq_num_descs;
177*c46e4de3SAndy Fiddaman txq->et_blocked = false;
178*c46e4de3SAndy Fiddaman txq->et_stall_watchdog = 0;
1796f443ebcSRyan Zezeski txq->et_state |= ENA_TXQ_STATE_SQ_CREATED;
1806f443ebcSRyan Zezeski
181*c46e4de3SAndy Fiddaman return (true);
1826f443ebcSRyan Zezeski }
1836f443ebcSRyan Zezeski
1846f443ebcSRyan Zezeski void
ena_cleanup_txq(ena_txq_t * txq,bool resetting)185*c46e4de3SAndy Fiddaman ena_cleanup_txq(ena_txq_t *txq, bool resetting)
1866f443ebcSRyan Zezeski {
1876f443ebcSRyan Zezeski int ret = 0;
1886f443ebcSRyan Zezeski ena_t *ena = txq->et_ena;
1896f443ebcSRyan Zezeski
1906f443ebcSRyan Zezeski if ((txq->et_state & ENA_TXQ_STATE_SQ_CREATED) != 0) {
191*c46e4de3SAndy Fiddaman if (!resetting) {
192*c46e4de3SAndy Fiddaman ret = ena_destroy_sq(ena, txq->et_sq_hw_idx, true);
1936f443ebcSRyan Zezeski
194*c46e4de3SAndy Fiddaman if (ret != 0) {
195*c46e4de3SAndy Fiddaman ena_err(ena, "failed to destroy Tx SQ %u: %d",
196*c46e4de3SAndy Fiddaman txq->et_txqs_idx, ret);
197*c46e4de3SAndy Fiddaman }
1986f443ebcSRyan Zezeski }
1996f443ebcSRyan Zezeski
2006f443ebcSRyan Zezeski txq->et_sq_hw_idx = 0;
2016f443ebcSRyan Zezeski txq->et_sq_db_addr = NULL;
2026f443ebcSRyan Zezeski txq->et_sq_tail_idx = 0;
2036f443ebcSRyan Zezeski txq->et_sq_phase = 0;
2046f443ebcSRyan Zezeski txq->et_state &= ~ENA_TXQ_STATE_SQ_CREATED;
2056f443ebcSRyan Zezeski }
2066f443ebcSRyan Zezeski
2076f443ebcSRyan Zezeski if ((txq->et_state & ENA_TXQ_STATE_CQ_CREATED) != 0) {
208*c46e4de3SAndy Fiddaman if (!resetting) {
209*c46e4de3SAndy Fiddaman ret = ena_destroy_cq(ena, txq->et_cq_hw_idx);
2106f443ebcSRyan Zezeski
211*c46e4de3SAndy Fiddaman if (ret != 0) {
212*c46e4de3SAndy Fiddaman ena_err(ena, "failed to destroy Tx CQ %u: %d",
213*c46e4de3SAndy Fiddaman txq->et_txqs_idx, ret);
214*c46e4de3SAndy Fiddaman }
2156f443ebcSRyan Zezeski }
2166f443ebcSRyan Zezeski
2176f443ebcSRyan Zezeski txq->et_cq_hw_idx = 0;
2186f443ebcSRyan Zezeski txq->et_cq_head_idx = 0;
2196f443ebcSRyan Zezeski txq->et_cq_phase = 0;
2206f443ebcSRyan Zezeski txq->et_cq_unmask_addr = NULL;
2216f443ebcSRyan Zezeski txq->et_cq_numa_addr = NULL;
2226f443ebcSRyan Zezeski txq->et_state &= ~ENA_TXQ_STATE_CQ_CREATED;
2236f443ebcSRyan Zezeski }
2246f443ebcSRyan Zezeski
2256f443ebcSRyan Zezeski ena_free_tx_dma(txq);
2266f443ebcSRyan Zezeski VERIFY3S(txq->et_state, ==, ENA_TXQ_STATE_NONE);
2276f443ebcSRyan Zezeski }
2286f443ebcSRyan Zezeski
2296f443ebcSRyan Zezeski void
ena_ring_tx_stop(mac_ring_driver_t rh)2306f443ebcSRyan Zezeski ena_ring_tx_stop(mac_ring_driver_t rh)
2316f443ebcSRyan Zezeski {
2326f443ebcSRyan Zezeski ena_txq_t *txq = (ena_txq_t *)rh;
2336f443ebcSRyan Zezeski uint32_t intr_ctrl;
2346f443ebcSRyan Zezeski
2356f443ebcSRyan Zezeski intr_ctrl = ena_hw_abs_read32(txq->et_ena, txq->et_cq_unmask_addr);
2366f443ebcSRyan Zezeski ENAHW_REG_INTR_UNMASK(intr_ctrl);
2376f443ebcSRyan Zezeski ena_hw_abs_write32(txq->et_ena, txq->et_cq_unmask_addr, intr_ctrl);
2386f443ebcSRyan Zezeski
2396f443ebcSRyan Zezeski txq->et_state &= ~ENA_TXQ_STATE_RUNNING;
2406f443ebcSRyan Zezeski txq->et_state &= ~ENA_TXQ_STATE_READY;
2416f443ebcSRyan Zezeski }
2426f443ebcSRyan Zezeski
2436f443ebcSRyan Zezeski int
ena_ring_tx_start(mac_ring_driver_t rh,uint64_t gen_num)2446f443ebcSRyan Zezeski ena_ring_tx_start(mac_ring_driver_t rh, uint64_t gen_num)
2456f443ebcSRyan Zezeski {
2466f443ebcSRyan Zezeski ena_txq_t *txq = (ena_txq_t *)rh;
2476f443ebcSRyan Zezeski ena_t *ena = txq->et_ena;
2486f443ebcSRyan Zezeski uint32_t intr_ctrl;
2496f443ebcSRyan Zezeski
250*c46e4de3SAndy Fiddaman ena_dbg(ena, "ring_tx_start %p: state 0x%x", txq, txq->et_state);
251*c46e4de3SAndy Fiddaman
2526f443ebcSRyan Zezeski mutex_enter(&txq->et_lock);
2536f443ebcSRyan Zezeski txq->et_m_gen_num = gen_num;
2546f443ebcSRyan Zezeski mutex_exit(&txq->et_lock);
2556f443ebcSRyan Zezeski
2566f443ebcSRyan Zezeski txq->et_state |= ENA_TXQ_STATE_READY;
2576f443ebcSRyan Zezeski
2586f443ebcSRyan Zezeski intr_ctrl = ena_hw_abs_read32(ena, txq->et_cq_unmask_addr);
2596f443ebcSRyan Zezeski ENAHW_REG_INTR_UNMASK(intr_ctrl);
2606f443ebcSRyan Zezeski ena_hw_abs_write32(ena, txq->et_cq_unmask_addr, intr_ctrl);
2616f443ebcSRyan Zezeski txq->et_state |= ENA_TXQ_STATE_RUNNING;
262*c46e4de3SAndy Fiddaman
2636f443ebcSRyan Zezeski return (0);
2646f443ebcSRyan Zezeski }
2656f443ebcSRyan Zezeski
2666f443ebcSRyan Zezeski static void
ena_tx_copy_fragment(ena_tx_control_block_t * tcb,const mblk_t * mp,const size_t off,const size_t len)2676f443ebcSRyan Zezeski ena_tx_copy_fragment(ena_tx_control_block_t *tcb, const mblk_t *mp,
2686f443ebcSRyan Zezeski const size_t off, const size_t len)
2696f443ebcSRyan Zezeski {
2706f443ebcSRyan Zezeski const void *soff = mp->b_rptr + off;
2716f443ebcSRyan Zezeski void *doff =
2726f443ebcSRyan Zezeski (void *)(tcb->etcb_dma.edb_va + tcb->etcb_dma.edb_used_len);
2736f443ebcSRyan Zezeski
2746f443ebcSRyan Zezeski VERIFY3U(len, >, 0);
2756f443ebcSRyan Zezeski VERIFY3P(soff, >=, mp->b_rptr);
2766f443ebcSRyan Zezeski VERIFY3P(soff, <=, mp->b_wptr);
2776f443ebcSRyan Zezeski VERIFY3U(len, <=, MBLKL(mp));
2786f443ebcSRyan Zezeski VERIFY3U((uintptr_t)soff + len, <=, (uintptr_t)mp->b_wptr);
2796f443ebcSRyan Zezeski VERIFY3U(tcb->etcb_dma.edb_used_len + len, <, tcb->etcb_dma.edb_len);
2806f443ebcSRyan Zezeski
2816f443ebcSRyan Zezeski bcopy(soff, doff, len);
2826f443ebcSRyan Zezeski tcb->etcb_type = ENA_TCB_COPY;
2836f443ebcSRyan Zezeski tcb->etcb_dma.edb_used_len += len;
2846f443ebcSRyan Zezeski }
2856f443ebcSRyan Zezeski
2866f443ebcSRyan Zezeski ena_tx_control_block_t *
ena_pull_tcb(const ena_txq_t * txq,mblk_t * mp)2876f443ebcSRyan Zezeski ena_pull_tcb(const ena_txq_t *txq, mblk_t *mp)
2886f443ebcSRyan Zezeski {
2896f443ebcSRyan Zezeski mblk_t *nmp = mp;
2906f443ebcSRyan Zezeski ena_t *ena = txq->et_ena;
2916f443ebcSRyan Zezeski ena_tx_control_block_t *tcb = NULL;
2926f443ebcSRyan Zezeski const uint16_t tail_mod =
2936f443ebcSRyan Zezeski txq->et_sq_tail_idx & (txq->et_sq_num_descs - 1);
2946f443ebcSRyan Zezeski
2956f443ebcSRyan Zezeski ASSERT(MUTEX_HELD(&txq->et_lock));
2966f443ebcSRyan Zezeski VERIFY3U(msgsize(mp), <, ena->ena_tx_buf_sz);
2976f443ebcSRyan Zezeski
2986f443ebcSRyan Zezeski while (nmp != NULL) {
2996f443ebcSRyan Zezeski const size_t nmp_len = MBLKL(nmp);
3006f443ebcSRyan Zezeski
3016f443ebcSRyan Zezeski if (nmp_len == 0) {
3026f443ebcSRyan Zezeski nmp = nmp->b_cont;
3036f443ebcSRyan Zezeski continue;
3046f443ebcSRyan Zezeski }
3056f443ebcSRyan Zezeski
3066f443ebcSRyan Zezeski /* For now TCB is bound to SQ desc. */
3076f443ebcSRyan Zezeski if (tcb == NULL) {
3086f443ebcSRyan Zezeski tcb = &txq->et_tcbs[tail_mod];
3096f443ebcSRyan Zezeski }
3106f443ebcSRyan Zezeski
3116f443ebcSRyan Zezeski ena_tx_copy_fragment(tcb, nmp, 0, nmp_len);
3126f443ebcSRyan Zezeski nmp = nmp->b_cont;
3136f443ebcSRyan Zezeski }
3146f443ebcSRyan Zezeski
3156f443ebcSRyan Zezeski ENA_DMA_SYNC(tcb->etcb_dma, DDI_DMA_SYNC_FORDEV);
3166f443ebcSRyan Zezeski VERIFY3P(nmp, ==, NULL);
3176f443ebcSRyan Zezeski VERIFY3P(tcb, !=, NULL);
3186f443ebcSRyan Zezeski return (tcb);
3196f443ebcSRyan Zezeski }
3206f443ebcSRyan Zezeski
3216f443ebcSRyan Zezeski static void
ena_fill_tx_data_desc(ena_txq_t * txq,ena_tx_control_block_t * tcb,uint16_t tail,uint8_t phase,enahw_tx_data_desc_t * desc,mac_ether_offload_info_t * meo,size_t mlen)3226f443ebcSRyan Zezeski ena_fill_tx_data_desc(ena_txq_t *txq, ena_tx_control_block_t *tcb,
3236f443ebcSRyan Zezeski uint16_t tail, uint8_t phase, enahw_tx_data_desc_t *desc,
3246f443ebcSRyan Zezeski mac_ether_offload_info_t *meo, size_t mlen)
3256f443ebcSRyan Zezeski {
3266f443ebcSRyan Zezeski VERIFY3U(mlen, <=, ENAHW_TX_DESC_LENGTH_MASK);
3276f443ebcSRyan Zezeski
3286f443ebcSRyan Zezeski #ifdef DEBUG
3296f443ebcSRyan Zezeski /*
3306f443ebcSRyan Zezeski * If there is no header for the specific layer it will be set
3316f443ebcSRyan Zezeski * to zero, thus we elide the meoi_flags check here.
3326f443ebcSRyan Zezeski */
3336f443ebcSRyan Zezeski size_t hdr_len = meo->meoi_l2hlen + meo->meoi_l3hlen + meo->meoi_l4hlen;
3346f443ebcSRyan Zezeski ASSERT3U(hdr_len, <=, txq->et_ena->ena_tx_max_hdr_len);
3356f443ebcSRyan Zezeski #endif
3366f443ebcSRyan Zezeski
3376f443ebcSRyan Zezeski bzero(desc, sizeof (*desc));
3386f443ebcSRyan Zezeski ENAHW_TX_DESC_FIRST_ON(desc);
3396f443ebcSRyan Zezeski ENAHW_TX_DESC_LENGTH(desc, mlen);
3406f443ebcSRyan Zezeski ENAHW_TX_DESC_REQID_HI(desc, tail);
3416f443ebcSRyan Zezeski ENAHW_TX_DESC_REQID_LO(desc, tail);
3426f443ebcSRyan Zezeski ENAHW_TX_DESC_PHASE(desc, phase);
3436f443ebcSRyan Zezeski ENAHW_TX_DESC_DF_ON(desc);
3446f443ebcSRyan Zezeski ENAHW_TX_DESC_LAST_ON(desc);
3456f443ebcSRyan Zezeski ENAHW_TX_DESC_COMP_REQ_ON(desc);
3466f443ebcSRyan Zezeski ENAHW_TX_DESC_META_DESC_OFF(desc);
3476f443ebcSRyan Zezeski ENAHW_TX_DESC_ADDR_LO(desc, tcb->etcb_dma.edb_cookie->dmac_laddress);
3486f443ebcSRyan Zezeski ENAHW_TX_DESC_ADDR_HI(desc, tcb->etcb_dma.edb_cookie->dmac_laddress);
3496f443ebcSRyan Zezeski /*
3506f443ebcSRyan Zezeski * NOTE: Please see the block comment above
3516f443ebcSRyan Zezeski * etd_buff_addr_hi_hdr_sz to see why this is set to 0.
3526f443ebcSRyan Zezeski */
3536f443ebcSRyan Zezeski ENAHW_TX_DESC_HEADER_LENGTH(desc, 0);
3546f443ebcSRyan Zezeski ENAHW_TX_DESC_TSO_OFF(desc);
3556f443ebcSRyan Zezeski ENAHW_TX_DESC_L3_CSUM_OFF(desc);
3566f443ebcSRyan Zezeski ENAHW_TX_DESC_L4_CSUM_OFF(desc);
3576f443ebcSRyan Zezeski /*
3586f443ebcSRyan Zezeski * Enabling this bit tells the device NOT to calculate the
3596f443ebcSRyan Zezeski * pseudo header checksum.
3606f443ebcSRyan Zezeski */
3616f443ebcSRyan Zezeski ENAHW_TX_DESC_L4_CSUM_PARTIAL_ON(desc);
3626f443ebcSRyan Zezeski }
3636f443ebcSRyan Zezeski
3646f443ebcSRyan Zezeski static void
ena_submit_tx(ena_txq_t * txq,uint16_t desc_idx)3656f443ebcSRyan Zezeski ena_submit_tx(ena_txq_t *txq, uint16_t desc_idx)
3666f443ebcSRyan Zezeski {
3676f443ebcSRyan Zezeski ena_hw_abs_write32(txq->et_ena, txq->et_sq_db_addr, desc_idx);
3686f443ebcSRyan Zezeski }
3696f443ebcSRyan Zezeski
3706f443ebcSRyan Zezeski /*
3716f443ebcSRyan Zezeski * For now we do the simplest thing possible. All Tx uses bcopy to
3726f443ebcSRyan Zezeski * pre-allocated buffers, no checksum, no TSO, etc.
3736f443ebcSRyan Zezeski */
3746f443ebcSRyan Zezeski mblk_t *
ena_ring_tx(void * arg,mblk_t * mp)3756f443ebcSRyan Zezeski ena_ring_tx(void *arg, mblk_t *mp)
3766f443ebcSRyan Zezeski {
3776f443ebcSRyan Zezeski ena_txq_t *txq = arg;
3786f443ebcSRyan Zezeski ena_t *ena = txq->et_ena;
3796f443ebcSRyan Zezeski mac_ether_offload_info_t meo;
3806f443ebcSRyan Zezeski enahw_tx_data_desc_t *desc;
3816f443ebcSRyan Zezeski ena_tx_control_block_t *tcb;
382*c46e4de3SAndy Fiddaman const uint16_t modulo_mask = txq->et_sq_num_descs - 1;
383*c46e4de3SAndy Fiddaman uint16_t tail_mod;
3846f443ebcSRyan Zezeski
3856f443ebcSRyan Zezeski VERIFY3P(mp->b_next, ==, NULL);
3866f443ebcSRyan Zezeski
3876f443ebcSRyan Zezeski /*
3886f443ebcSRyan Zezeski * The ena_state value is written by atomic operations. The
3896f443ebcSRyan Zezeski * et_state value is currently Write Once, but if that changes
3906f443ebcSRyan Zezeski * it should also be written with atomics.
3916f443ebcSRyan Zezeski */
392*c46e4de3SAndy Fiddaman if (!(ena->ena_state & ENA_STATE_STARTED) ||
3936f443ebcSRyan Zezeski !(txq->et_state & ENA_TXQ_STATE_RUNNING)) {
3946f443ebcSRyan Zezeski freemsg(mp);
3956f443ebcSRyan Zezeski return (NULL);
3966f443ebcSRyan Zezeski }
3976f443ebcSRyan Zezeski
3986f443ebcSRyan Zezeski if (mac_ether_offload_info(mp, &meo) != 0) {
3996f443ebcSRyan Zezeski freemsg(mp);
4006f443ebcSRyan Zezeski mutex_enter(&txq->et_stat_lock);
4016f443ebcSRyan Zezeski txq->et_stat.ets_hck_meoifail.value.ui64++;
4026f443ebcSRyan Zezeski mutex_exit(&txq->et_stat_lock);
4036f443ebcSRyan Zezeski return (NULL);
4046f443ebcSRyan Zezeski }
4056f443ebcSRyan Zezeski
4066f443ebcSRyan Zezeski mutex_enter(&txq->et_lock);
4076f443ebcSRyan Zezeski
4086f443ebcSRyan Zezeski /*
4096f443ebcSRyan Zezeski * For the moment there is a 1:1 mapping between Tx descs and
4106f443ebcSRyan Zezeski * Tx contexts. Currently Tx is copy only, and each context
4116f443ebcSRyan Zezeski * buffer is guaranteed to be as large as MTU + frame header,
4126f443ebcSRyan Zezeski * see ena_update_buf_sizes().
4136f443ebcSRyan Zezeski */
414eebd18daSAndy Fiddaman if (txq->et_blocked || txq->et_sq_avail_descs == 0) {
415*c46e4de3SAndy Fiddaman txq->et_blocked = true;
4166f443ebcSRyan Zezeski mutex_enter(&txq->et_stat_lock);
4176f443ebcSRyan Zezeski txq->et_stat.ets_blocked.value.ui64++;
4186f443ebcSRyan Zezeski mutex_exit(&txq->et_stat_lock);
4196f443ebcSRyan Zezeski mutex_exit(&txq->et_lock);
4206f443ebcSRyan Zezeski return (mp);
4216f443ebcSRyan Zezeski }
4226f443ebcSRyan Zezeski
4236f443ebcSRyan Zezeski ASSERT3U(meo.meoi_len, <=, ena->ena_max_frame_total);
4246f443ebcSRyan Zezeski tcb = ena_pull_tcb(txq, mp);
4256f443ebcSRyan Zezeski ASSERT3P(tcb, !=, NULL);
4266f443ebcSRyan Zezeski tcb->etcb_mp = mp;
4276f443ebcSRyan Zezeski txq->et_sq_avail_descs--;
4286f443ebcSRyan Zezeski
4296f443ebcSRyan Zezeski /* Fill in the Tx descriptor. */
430*c46e4de3SAndy Fiddaman tail_mod = txq->et_sq_tail_idx & modulo_mask;
431*c46e4de3SAndy Fiddaman desc = &txq->et_sq_descs[tail_mod].etd_data;
4326f443ebcSRyan Zezeski ena_fill_tx_data_desc(txq, tcb, tail_mod, txq->et_sq_phase, desc, &meo,
4336f443ebcSRyan Zezeski meo.meoi_len);
4346f443ebcSRyan Zezeski DTRACE_PROBE3(tx__submit, ena_tx_control_block_t *, tcb, uint16_t,
4356f443ebcSRyan Zezeski tail_mod, enahw_tx_data_desc_t *, desc);
4366f443ebcSRyan Zezeski
4376f443ebcSRyan Zezeski /*
4386f443ebcSRyan Zezeski * Remember, we submit the raw tail value to the device, the
4396f443ebcSRyan Zezeski * hardware performs its own modulo (like we did to get
4406f443ebcSRyan Zezeski * tail_mod).
4416f443ebcSRyan Zezeski */
4426f443ebcSRyan Zezeski txq->et_sq_tail_idx++;
4436f443ebcSRyan Zezeski ena_submit_tx(txq, txq->et_sq_tail_idx);
4446f443ebcSRyan Zezeski
4456f443ebcSRyan Zezeski mutex_enter(&txq->et_stat_lock);
4466f443ebcSRyan Zezeski txq->et_stat.ets_packets.value.ui64++;
4476f443ebcSRyan Zezeski txq->et_stat.ets_bytes.value.ui64 += meo.meoi_len;
4486f443ebcSRyan Zezeski mutex_exit(&txq->et_stat_lock);
4496f443ebcSRyan Zezeski
450*c46e4de3SAndy Fiddaman if ((txq->et_sq_tail_idx & modulo_mask) == 0)
451eebd18daSAndy Fiddaman txq->et_sq_phase ^= 1;
4526f443ebcSRyan Zezeski
4536f443ebcSRyan Zezeski mutex_exit(&txq->et_lock);
454*c46e4de3SAndy Fiddaman
4556f443ebcSRyan Zezeski return (NULL);
4566f443ebcSRyan Zezeski }
4576f443ebcSRyan Zezeski
4586f443ebcSRyan Zezeski void
ena_tx_intr_work(ena_txq_t * txq)4596f443ebcSRyan Zezeski ena_tx_intr_work(ena_txq_t *txq)
4606f443ebcSRyan Zezeski {
4616f443ebcSRyan Zezeski uint16_t head_mod;
4626f443ebcSRyan Zezeski enahw_tx_cdesc_t *cdesc;
4636f443ebcSRyan Zezeski ena_tx_control_block_t *tcb;
4646f443ebcSRyan Zezeski uint16_t req_id;
4656f443ebcSRyan Zezeski uint64_t recycled = 0;
466*c46e4de3SAndy Fiddaman bool unblocked = false;
467*c46e4de3SAndy Fiddaman const uint16_t modulo_mask = txq->et_cq_num_descs - 1;
468*c46e4de3SAndy Fiddaman ena_t *ena = txq->et_ena;
4696f443ebcSRyan Zezeski
4706f443ebcSRyan Zezeski mutex_enter(&txq->et_lock);
471*c46e4de3SAndy Fiddaman head_mod = txq->et_cq_head_idx & modulo_mask;
4726f443ebcSRyan Zezeski ENA_DMA_SYNC(txq->et_cq_dma, DDI_DMA_SYNC_FORKERNEL);
4736f443ebcSRyan Zezeski cdesc = &txq->et_cq_descs[head_mod];
4746f443ebcSRyan Zezeski
4756f443ebcSRyan Zezeski /* Recycle any completed descriptors. */
4766f443ebcSRyan Zezeski while (ENAHW_TX_CDESC_GET_PHASE(cdesc) == txq->et_cq_phase) {
4776f443ebcSRyan Zezeski mblk_t *mp;
4786f443ebcSRyan Zezeski
4796f443ebcSRyan Zezeski /* Get the corresponding TCB. */
4806f443ebcSRyan Zezeski req_id = cdesc->etc_req_id;
481*c46e4de3SAndy Fiddaman if (req_id > txq->et_sq_num_descs) {
482*c46e4de3SAndy Fiddaman ena_err(ena, "invalid Tx request ID: 0x%x", req_id);
483*c46e4de3SAndy Fiddaman ena_trigger_reset(ena, ENAHW_RESET_INV_TX_REQ_ID);
484*c46e4de3SAndy Fiddaman break;
485*c46e4de3SAndy Fiddaman }
4866f443ebcSRyan Zezeski tcb = &txq->et_tcbs[req_id];
4876f443ebcSRyan Zezeski DTRACE_PROBE2(tx__complete, uint16_t, req_id,
4886f443ebcSRyan Zezeski ena_tx_control_block_t *, tcb);
4896f443ebcSRyan Zezeski
4906f443ebcSRyan Zezeski /* Free the associated mblk. */
4916f443ebcSRyan Zezeski tcb->etcb_dma.edb_used_len = 0;
4926f443ebcSRyan Zezeski mp = tcb->etcb_mp;
4936f443ebcSRyan Zezeski VERIFY3P(mp, !=, NULL);
4946f443ebcSRyan Zezeski freemsg(mp);
4956f443ebcSRyan Zezeski tcb->etcb_mp = NULL;
4966f443ebcSRyan Zezeski
4976f443ebcSRyan Zezeski /* Add this descriptor back to the free list. */
4986f443ebcSRyan Zezeski txq->et_sq_avail_descs++;
4996f443ebcSRyan Zezeski txq->et_cq_head_idx++;
5006f443ebcSRyan Zezeski
5016f443ebcSRyan Zezeski /* Check for phase rollover. */
502*c46e4de3SAndy Fiddaman head_mod = txq->et_cq_head_idx & modulo_mask;
503*c46e4de3SAndy Fiddaman if (head_mod == 0)
504eebd18daSAndy Fiddaman txq->et_cq_phase ^= 1;
5056f443ebcSRyan Zezeski
5066f443ebcSRyan Zezeski if (txq->et_blocked) {
507*c46e4de3SAndy Fiddaman txq->et_blocked = false;
508*c46e4de3SAndy Fiddaman txq->et_stall_watchdog = 0;
509*c46e4de3SAndy Fiddaman unblocked = true;
510*c46e4de3SAndy Fiddaman mac_tx_ring_update(ena->ena_mh, txq->et_mrh);
5116f443ebcSRyan Zezeski }
5126f443ebcSRyan Zezeski
5136f443ebcSRyan Zezeski recycled++;
5146f443ebcSRyan Zezeski cdesc = &txq->et_cq_descs[head_mod];
5156f443ebcSRyan Zezeski }
5166f443ebcSRyan Zezeski
5176f443ebcSRyan Zezeski mutex_exit(&txq->et_lock);
5186f443ebcSRyan Zezeski
519*c46e4de3SAndy Fiddaman if (recycled == 0)
520*c46e4de3SAndy Fiddaman return;
521*c46e4de3SAndy Fiddaman
5226f443ebcSRyan Zezeski /* Update stats. */
5236f443ebcSRyan Zezeski mutex_enter(&txq->et_stat_lock);
5246f443ebcSRyan Zezeski txq->et_stat.ets_recycled.value.ui64 += recycled;
5256f443ebcSRyan Zezeski if (unblocked) {
5266f443ebcSRyan Zezeski txq->et_stat.ets_unblocked.value.ui64++;
5276f443ebcSRyan Zezeski }
5286f443ebcSRyan Zezeski mutex_exit(&txq->et_stat_lock);
5296f443ebcSRyan Zezeski }
530