/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, v.1, (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://opensource.org/licenses/CDDL-1.0. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2014-2017 Cavium, Inc. * The contents of this file are subject to the terms of the Common Development * and Distribution License, v.1, (the "License"). * You may not use this file except in compliance with the License. * You can obtain a copy of the License at available * at http://opensource.org/licenses/CDDL-1.0 * See the License for the specific language governing permissions and * limitations under the License. */ #include "qede.h" static qede_dma_handle_entry_t * qede_get_dmah_entry(qede_tx_ring_t *tx_ring) { qede_dma_handles_list_t *list = &tx_ring->dmah_list; qede_dma_handle_entry_t *dmah; mutex_enter(&list->lock); dmah = list->free_list[list->head]; list->free_list[list->head] = NULL; list->head = (list->head + 1) & TX_RING_MASK; mutex_exit(&list->lock); return (dmah); } static void qede_put_dmah_entries(qede_tx_ring_t *tx_ring, qede_dma_handle_entry_t *dmah) { qede_dma_handles_list_t *list = &tx_ring->dmah_list; qede_dma_handle_entry_t *next; u16 index; mutex_enter(&list->lock); index = list->tail; while (dmah != NULL) { next = dmah->next; dmah->next = NULL; list->free_list[index] = dmah; index = (index + 1) & TX_RING_MASK; dmah = next; } list->tail = index; mutex_exit(&list->lock); } static qede_tx_bcopy_pkt_t * qede_get_bcopy_pkt(qede_tx_ring_t *tx_ring) { qede_tx_bcopy_list_t *list = &tx_ring->bcopy_list; qede_tx_bcopy_pkt_t *pkt; mutex_enter(&list->lock); pkt = list->free_list[list->head]; list->free_list[list->head] = NULL; list->head = (list->head + 1) & TX_RING_MASK; mutex_exit(&list->lock); return (pkt); } static void qede_put_bcopy_pkt(qede_tx_ring_t *tx_ring, qede_tx_bcopy_pkt_t *pkt) { qede_tx_bcopy_list_t *list = &tx_ring->bcopy_list; mutex_enter(&list->lock); list->free_list[list->tail] = pkt; list->tail = (list->tail + 1) & TX_RING_MASK; mutex_exit(&list->lock); } void qede_print_tx_indexes(qede_tx_ring_t *tx_ring) { uint16_t hw_consumer = LE_16(*tx_ring->hw_cons_ptr); uint16_t chain_idx = ecore_chain_get_cons_idx(&tx_ring->tx_bd_ring); hw_consumer &= TX_RING_MASK; chain_idx &= TX_RING_MASK; qede_print_err("!indices: hw_cons %d, chain_cons = %d, sw_prod = %d", hw_consumer, chain_idx, tx_ring->sw_tx_prod); } void qede_print_rx_indexes(qede_rx_ring_t *rx_ring) { u16 hw_bd_cons = HOST_TO_LE_16(*rx_ring->hw_cons_ptr); u16 sw_bd_cons = ecore_chain_get_cons_idx(&rx_ring->rx_cqe_ring); hw_bd_cons &= (rx_ring->qede->rx_ring_size - 1); sw_bd_cons &= (rx_ring->qede->rx_ring_size - 1); qede_print_err("!RX indices: hw_cons %d, chain_cons = %d", hw_bd_cons, sw_bd_cons); } /* * Called from tx_completion intr handler. * NOTE: statu_block dma mem. must be sync'ed * in the interrupt handler */ int qede_process_tx_completions(qede_tx_ring_t *tx_ring) { int count = 0; u16 hw_consumer; struct eth_tx_bd *tx_bd; uint16_t chain_idx; u16 nbd, sw_consumer = tx_ring->sw_tx_cons; struct eth_tx_1st_bd *first_bd; u16 bd_consumed = 0; qede_tx_recycle_list_t *recycle_entry; qede_dma_handle_entry_t *dmah, *head = NULL, *tail = NULL; qede_tx_bcopy_pkt_t *bcopy_pkt; hw_consumer = LE_16(*tx_ring->hw_cons_ptr); chain_idx = ecore_chain_get_cons_idx(&tx_ring->tx_bd_ring); while (hw_consumer != chain_idx) { nbd = 0; bd_consumed = 0; first_bd = NULL; recycle_entry = &tx_ring->tx_recycle_list[sw_consumer]; if (recycle_entry->dmah_entry != NULL) { dmah = recycle_entry->dmah_entry; head = dmah; if (head->mp) { freemsg(head->mp); } while (dmah != NULL) { (void) ddi_dma_unbind_handle(dmah->dma_handle); dmah = dmah->next; } qede_put_dmah_entries(tx_ring, head); recycle_entry->dmah_entry = NULL; } else if (recycle_entry->bcopy_pkt != NULL) { bcopy_pkt = recycle_entry->bcopy_pkt; qede_put_bcopy_pkt(tx_ring, bcopy_pkt); recycle_entry->bcopy_pkt = NULL; } else { qede_warn(tx_ring->qede, "Invalid completion at index %d", sw_consumer); } sw_consumer = (sw_consumer + 1) & TX_RING_MASK; first_bd = (struct eth_tx_1st_bd *)ecore_chain_consume( &tx_ring->tx_bd_ring); bd_consumed++; nbd = first_bd->data.nbds; while (bd_consumed++ < nbd) { ecore_chain_consume(&tx_ring->tx_bd_ring); } chain_idx = ecore_chain_get_cons_idx(&tx_ring->tx_bd_ring); count++; } tx_ring->sw_tx_cons = sw_consumer; if (count && tx_ring->tx_q_sleeping) { tx_ring->tx_q_sleeping = 0; #ifndef NO_CROSSBOW RESUME_TX(tx_ring); #else mac_tx_update(tx_ring->qede->mac_handle); #endif } return (count); } static int qede_has_tx_work(qede_tx_ring_t *tx_ring) { u16 hw_bd_cons = LE_16(*tx_ring->hw_cons_ptr); u16 sw_bd_cons = ecore_chain_get_cons_idx(&tx_ring->tx_bd_ring); if (sw_bd_cons == (hw_bd_cons + 1)) { return (0); } return (hw_bd_cons != sw_bd_cons); } static int qede_has_rx_work(qede_rx_ring_t *rx_ring) { u16 hw_bd_cons = HOST_TO_LE_16(*rx_ring->hw_cons_ptr); u16 sw_bd_cons = ecore_chain_get_cons_idx(&rx_ring->rx_cqe_ring); return (hw_bd_cons != sw_bd_cons); } static void qede_set_cksum_flags(mblk_t *mp, uint16_t parse_flags) { uint32_t cksum_flags = 0; int error = 0; bool l4_is_calc, l4_csum_err, iphdr_len_err; l4_is_calc = (parse_flags >> PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT) & PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK; l4_csum_err = (parse_flags >> PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT) & PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK; iphdr_len_err = (parse_flags >> PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT) & PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK; if (l4_is_calc) { if (l4_csum_err) { error = 1; } else if (iphdr_len_err) { error = 2; } else { cksum_flags = HCK_FULLCKSUM_OK | HCK_IPV4_HDRCKSUM_OK; } } if (error == 1) { qede_print_err("!%s: got L4 csum error",__func__); } else if (error == 2) { qede_print_err("!%s: got IPHDER csum error" ,__func__); } mac_hcksum_set(mp, 0, 0, 0, 0, cksum_flags); } static qede_rx_buffer_t * qede_get_next_rx_buffer(qede_rx_ring_t *rx_ring, uint32_t *free_buffer_count) { qede_rx_buffer_t *rx_buffer; uint32_t num_entries; rx_buffer = qede_get_from_active_list(rx_ring, &num_entries); ASSERT(rx_buffer != NULL); ecore_chain_consume(&rx_ring->rx_bd_ring); *free_buffer_count = num_entries; return (rx_buffer); } static uint32_t qede_get_next_lro_buffer(qede_rx_ring_t *rx_ring, qede_lro_info_t *lro_info) { lro_info->rx_buffer[lro_info->bd_count] = qede_get_next_rx_buffer(rx_ring, &lro_info->free_buffer_count); lro_info->bd_count++; return (DDI_SUCCESS); } #ifdef DEBUG_LRO int agg_count = 0; bool agg_print = B_TRUE; #endif static void qede_lro_start(qede_rx_ring_t *rx_ring, struct eth_fast_path_rx_tpa_start_cqe *cqe) { qede_lro_info_t *lro_info; int i, len_on_first_bd, seg_len; lro_info = &rx_ring->lro_info[cqe->tpa_agg_index]; /* ASSERT(lro_info->agg_state != QEDE_AGG_STATE_NONE); */ #ifdef DEBUG_LRO if (agg_count++ < 30) { qede_dump_start_lro_cqe(cqe); } else { agg_print = B_FALSE; } #endif memset(lro_info, 0, sizeof (qede_lro_info_t)); lro_info->agg_state = QEDE_AGG_STATE_START; rx_ring->lro_active_count++; /* Parsing and error flags from the parser */; lro_info->pars_flags = LE_16(cqe->pars_flags.flags); lro_info->pad = LE_16(cqe->placement_offset); lro_info->header_len = (uint32_t)cqe->header_len; lro_info->vlan_tag = LE_16(cqe->vlan_tag); lro_info->rss_hash = LE_32(cqe->rss_hash); seg_len = (int)LE_16(cqe->seg_len); len_on_first_bd = (int)LE_16(cqe->len_on_first_bd); /* * Get the first bd */ qede_get_next_lro_buffer(rx_ring, lro_info); if (len_on_first_bd < seg_len) { /* * We end up here with jumbo frames * since a TCP segment can span * multiple buffer descriptors. */ for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) { if (cqe->ext_bd_len_list[i] == 0) { break; } qede_get_next_lro_buffer(rx_ring, lro_info); } } } static void qede_lro_cont(qede_rx_ring_t *rx_ring, struct eth_fast_path_rx_tpa_cont_cqe *cqe) { qede_lro_info_t *lro_info; int i; lro_info = &rx_ring->lro_info[cqe->tpa_agg_index]; /* ASSERT(lro_info->agg_state != QEDE_AGG_STATE_START); */ #ifdef DEBUG_LRO if (agg_print) { qede_dump_cont_lro_cqe(cqe); } #endif for (i = 0; i < ETH_TPA_CQE_CONT_LEN_LIST_SIZE; i++) { if (cqe->len_list[i] == 0) { break; } qede_get_next_lro_buffer(rx_ring, lro_info); } } static mblk_t * qede_lro_end(qede_rx_ring_t *rx_ring, struct eth_fast_path_rx_tpa_end_cqe *cqe, int *pkt_bytes) { qede_lro_info_t *lro_info; mblk_t *head = NULL, *tail = NULL, *mp = NULL; qede_rx_buffer_t *rx_buffer; int i, bd_len; uint16_t work_length, total_packet_length; uint32_t rx_buf_size = rx_ring->rx_buf_size; qede_dma_info_t *dma_info; lro_info = &rx_ring->lro_info[cqe->tpa_agg_index]; /* ASSERT(lro_info->agg_state != QEDE_AGG_STATE_START); */ #ifdef DEBUG_LRO if (agg_print) { qede_dump_end_lro_cqe(cqe); } #endif work_length = total_packet_length = LE_16(cqe->total_packet_len); /* * Get any buffer descriptors for this cqe */ for (i=0; ilen_list[i] == 0) { break; } qede_get_next_lro_buffer(rx_ring, lro_info); } /* ASSERT(lro_info->bd_count != cqe->num_of_bds); */ if (lro_info->free_buffer_count < rx_ring->rx_low_buffer_threshold) { for (i = 0; i < lro_info->bd_count; i++) { qede_recycle_copied_rx_buffer( lro_info->rx_buffer[i]); lro_info->rx_buffer[i] = NULL; } rx_ring->rx_low_water_cnt++; lro_info->agg_state = QEDE_AGG_STATE_NONE; return (NULL); } /* * Loop through list of buffers for this * aggregation. For each one: * 1. Calculate the buffer length * 2. Adjust the mblk read/write pointers * 3. Link the mblk to the local chain using * b_cont pointers. * Note: each buffer will be rx_buf_size except * the first (subtract the placement_offset) * and the last which contains the remainder * of cqe_end->total_packet_len minus length * of all other buffers. */ for (i = 0; i < lro_info->bd_count; i++) { rx_buffer = lro_info->rx_buffer[i]; bd_len = (work_length > rx_buf_size) ? rx_buf_size : work_length; if (i == 0 && (cqe->num_of_bds > 1)) { bd_len -= lro_info->pad; } dma_info = &rx_buffer->dma_info; ddi_dma_sync(dma_info->dma_handle, dma_info->offset, rx_buf_size, DDI_DMA_SYNC_FORKERNEL); mp = rx_buffer->mp; mp->b_next = mp->b_cont = NULL; if (head == NULL) { head = tail = mp; mp->b_rptr += lro_info->pad; } else { tail->b_cont = mp; tail = mp; } mp->b_wptr = (uchar_t *)((unsigned long)mp->b_rptr + bd_len); work_length -= bd_len; } qede_set_cksum_flags(head, lro_info->pars_flags); rx_ring->rx_lro_pkt_cnt++; rx_ring->lro_active_count--; lro_info->agg_state = QEDE_AGG_STATE_NONE; #ifdef DEBUG_LRO if (agg_print) { qede_dump_mblk_chain_bcont_ptr(rx_ring->qede, head); } #endif *pkt_bytes = (int)total_packet_length; return (head); } #ifdef DEBUG_JUMBO int jumbo_count = 0; bool jumbo_print = B_TRUE; #endif static mblk_t * qede_reg_jumbo_cqe(qede_rx_ring_t *rx_ring, struct eth_fast_path_rx_reg_cqe *cqe) { int i; qede_rx_buffer_t *rx_buf, *rx_buffer[ETH_RX_MAX_BUFF_PER_PKT]; mblk_t *mp = NULL, *head = NULL, *tail = NULL; uint32_t free_buffer_count = 0; uint16_t work_length; uint32_t rx_buf_size = rx_ring->rx_buf_size, bd_len; qede_dma_info_t *dma_info; u8 pad = cqe->placement_offset; #ifdef DEBUG_JUMBO if (jumbo_count++ < 8) { qede_dump_reg_cqe(cqe); } else { jumbo_print = B_FALSE; } #endif work_length = HOST_TO_LE_16(cqe->pkt_len); /* * Get the buffers/mps for this cqe */ for (i = 0; i < cqe->bd_num; i++) { rx_buffer[i] = qede_get_next_rx_buffer(rx_ring, &free_buffer_count); } /* * If the buffer ring is running low, drop the * packet and return these buffers. */ if (free_buffer_count < rx_ring->rx_low_buffer_threshold) { for (i = 0; i < cqe->bd_num; i++) { qede_recycle_copied_rx_buffer(rx_buffer[i]); } rx_ring->rx_low_water_cnt++; return (NULL); } for (i = 0; i < cqe->bd_num; i++) { rx_buf = rx_buffer[i]; bd_len = (work_length > rx_buf_size) ? rx_buf_size : work_length; /* * Adjust for placement offset * on first bufffer. */ if (i == 0) { bd_len -= pad; } dma_info = &rx_buf->dma_info; ddi_dma_sync(dma_info->dma_handle, dma_info->offset, rx_buf_size, DDI_DMA_SYNC_FORKERNEL); mp = rx_buf->mp; mp->b_next = mp->b_cont = NULL; /* * Adjust for placement offset * on first bufffer. */ if (i == 0) { mp->b_rptr += pad; } mp->b_wptr = (uchar_t *)((unsigned long)mp->b_rptr + bd_len); if (head == NULL) { head = tail = mp; } else { tail->b_cont = mp; tail = mp; } work_length -= bd_len; } qede_set_cksum_flags(head, HOST_TO_LE_16(cqe->pars_flags.flags)); #ifdef DEBUG_JUMBO if (jumbo_print) { qede_dump_mblk_chain_bcont_ptr(rx_ring->qede, head); } #endif rx_ring->rx_jumbo_pkt_cnt++; return (head); } static mblk_t * qede_reg_cqe(qede_rx_ring_t *rx_ring, struct eth_fast_path_rx_reg_cqe *cqe, int *pkt_bytes) { qede_t *qede = rx_ring->qede; qede_rx_buffer_t *rx_buffer; uint32_t free_buffer_count; mblk_t *mp; uint16_t pkt_len = HOST_TO_LE_16(cqe->pkt_len); u8 pad = cqe->placement_offset; qede_dma_info_t *dma_info; ddi_dma_handle_t dma_handle; char *virt_addr; /* * Update the byte count as it will * be the same for normal and jumbo */ *pkt_bytes = (int)pkt_len; if (cqe->bd_num > 1) { /* * If this cqe uses more than one * rx buffer then it must be * jumbo. Call another handler * for this because the process is * quite different. */ return (qede_reg_jumbo_cqe(rx_ring, cqe)); } rx_buffer = qede_get_next_rx_buffer(rx_ring, &free_buffer_count); if (free_buffer_count < rx_ring->rx_low_buffer_threshold) { qede_recycle_copied_rx_buffer(rx_buffer); rx_ring->rx_low_water_cnt++; *pkt_bytes = 0; return (NULL); } dma_info = &rx_buffer->dma_info; virt_addr = dma_info->virt_addr; dma_handle = dma_info->dma_handle; ddi_dma_sync(dma_handle, 0, 0, DDI_DMA_SYNC_FORKERNEL); if (pkt_len <= rx_ring->rx_copy_threshold) { mp = allocb(pkt_len + 2, 0); /* IP HDR_ALIGN */ if (mp != NULL) { virt_addr += pad; bcopy(virt_addr, mp->b_rptr, pkt_len); } else { /* * Post the buffer back to fw and * drop packet */ qede_print_err("!%s(%d): allocb failed", __func__, rx_ring->qede->instance); qede->allocbFailures++; goto freebuf; } /* * We've copied it (or not) and are done with it * so put it back into the passive list. */ ddi_dma_sync(dma_handle, 0, 0, DDI_DMA_SYNC_FORDEV); qede_recycle_copied_rx_buffer(rx_buffer); rx_ring->rx_copy_cnt++; } else { /* * We are going to send this mp/buffer * up to the mac layer. Adjust the * pointeres and link it to our chain. * the rx_buffer is returned to us in * the recycle function so we drop it * here. */ mp = rx_buffer->mp; mp->b_rptr += pad; } mp->b_cont = mp->b_next = NULL; mp->b_wptr = (uchar_t *)((unsigned long)mp->b_rptr + pkt_len); qede_set_cksum_flags(mp, HOST_TO_LE_16(cqe->pars_flags.flags)); #ifdef DEBUG_JUMBO if (jumbo_print) { qede_dump_mblk_chain_bnext_ptr(rx_ring->qede, mp); } #endif rx_ring->rx_reg_pkt_cnt++; return (mp); freebuf: qede_recycle_copied_rx_buffer(rx_buffer); return (NULL); } /* * Routine to process the rx packets on the * passed rx_ring. Can be called for intr or * poll context/routines */ static mblk_t * qede_process_rx_ring(qede_rx_ring_t *rx_ring, int nbytes, int npkts) { union eth_rx_cqe *cqe; u16 last_cqe_consumer = rx_ring->last_cqe_consumer; enum eth_rx_cqe_type cqe_type; u16 sw_comp_cons, hw_comp_cons; mblk_t *mp = NULL, *first_mp = NULL, *last_mp = NULL; int pkt_bytes = 0, byte_cnt = 0, pkt_cnt = 0; hw_comp_cons = HOST_TO_LE_16(*rx_ring->hw_cons_ptr); /* Completion ring sw consumer */ sw_comp_cons = ecore_chain_get_cons_idx(&rx_ring->rx_cqe_ring); while (sw_comp_cons != hw_comp_cons) { if ((byte_cnt >= nbytes) || (pkt_cnt >= npkts)) { break; } cqe = (union eth_rx_cqe *) ecore_chain_consume(&rx_ring->rx_cqe_ring); /* Get next element and increment the cons_idx */ (void) ddi_dma_sync(rx_ring->rx_cqe_dmah, last_cqe_consumer, sizeof (*cqe), DDI_DMA_SYNC_FORKERNEL); cqe_type = cqe->fast_path_regular.type; switch (cqe_type) { case ETH_RX_CQE_TYPE_SLOW_PATH: ecore_eth_cqe_completion(&rx_ring->qede->edev.hwfns[0], (struct eth_slow_path_rx_cqe *)cqe); goto next_cqe; case ETH_RX_CQE_TYPE_REGULAR: mp = qede_reg_cqe(rx_ring, &cqe->fast_path_regular, &pkt_bytes); break; case ETH_RX_CQE_TYPE_TPA_START: qede_lro_start(rx_ring, &cqe->fast_path_tpa_start); goto next_cqe; case ETH_RX_CQE_TYPE_TPA_CONT: qede_lro_cont(rx_ring, &cqe->fast_path_tpa_cont); goto next_cqe; case ETH_RX_CQE_TYPE_TPA_END: mp = qede_lro_end(rx_ring, &cqe->fast_path_tpa_end, &pkt_bytes); break; default: if (cqe_type != 0) { qede_print_err("!%s(%d): cqe_type %x not " "supported", __func__, rx_ring->qede->instance, cqe_type); } goto exit_rx; } /* * If we arrive here with no mp, * then we hit an RX buffer threshold * where we had to drop the packet and * give the buffers back to the device. */ if (mp == NULL) { rx_ring->rx_drop_cnt++; goto next_cqe; } if (first_mp) { last_mp->b_next = mp; } else { first_mp = mp; } last_mp = mp; pkt_cnt++; byte_cnt += pkt_bytes; next_cqe: ecore_chain_recycle_consumed(&rx_ring->rx_cqe_ring); last_cqe_consumer = sw_comp_cons; sw_comp_cons = ecore_chain_get_cons_idx(&rx_ring->rx_cqe_ring); if (!(qede_has_rx_work(rx_ring))) { ecore_sb_update_sb_idx(rx_ring->fp->sb_info); } hw_comp_cons = HOST_TO_LE_16(*rx_ring->hw_cons_ptr); } rx_ring->rx_pkt_cnt += pkt_cnt; rx_ring->rx_byte_cnt += byte_cnt; exit_rx: if (first_mp) { last_mp->b_next = NULL; } /* * Since prod update will result in * reading of the bd's, do a dma_sync */ qede_replenish_rx_buffers(rx_ring); qede_update_rx_q_producer(rx_ring); rx_ring->last_cqe_consumer = last_cqe_consumer; return (first_mp); } mblk_t * qede_process_fastpath(qede_fastpath_t *fp, int nbytes, int npkts, int *work_done) { int i = 0; qede_tx_ring_t *tx_ring; qede_rx_ring_t *rx_ring; mblk_t *mp = NULL; rx_ring = fp->rx_ring; for (i = 0; i < fp->qede->num_tc; i++) { tx_ring = fp->tx_ring[i]; if (qede_has_tx_work(tx_ring)) { /* process tx completions */ if (mutex_tryenter(&tx_ring->tx_lock) != 0) { *work_done += qede_process_tx_completions(tx_ring); mutex_exit(&tx_ring->tx_lock); } } } if (!(qede_has_rx_work(rx_ring))) { ecore_sb_update_sb_idx(fp->sb_info); } rx_ring = fp->rx_ring; if (qede_has_rx_work(rx_ring)) { mutex_enter(&rx_ring->rx_lock); mp = qede_process_rx_ring(rx_ring, nbytes, npkts); if (mp) { *work_done += 1; } mutex_exit(&rx_ring->rx_lock); } return (mp); } /* * Parse the mblk to extract information * from the protocol headers. * The routine assumes that the l4 header is tcp. Also * it does not account for ipv6 headers since ipv6 lso is * unsupported */ static void qede_pkt_parse_lso_headers(qede_tx_pktinfo_t *pktinfo, mblk_t *mp) { struct ether_header *eth_hdr = (struct ether_header *)(void *)mp->b_rptr; ipha_t *ip_hdr; struct tcphdr *tcp_hdr; /* mac header type and len */ if (ntohs(eth_hdr->ether_type) == ETHERTYPE_IP) { pktinfo->ether_type = ntohs(eth_hdr->ether_type); pktinfo->mac_hlen = sizeof (struct ether_header); } else if (ntohs(eth_hdr->ether_type) == ETHERTYPE_VLAN) { struct ether_vlan_header *vlan_hdr = (struct ether_vlan_header *)(void *)mp->b_rptr; pktinfo->ether_type = ntohs(vlan_hdr->ether_type); pktinfo->mac_hlen = sizeof (struct ether_vlan_header); } /* ip header type and len */ ip_hdr = (ipha_t *)(void *)((u8 *)mp->b_rptr + pktinfo->mac_hlen); pktinfo->ip_hlen = IPH_HDR_LENGTH(ip_hdr); /* Assume TCP protocol */ pktinfo->l4_proto = 0x06; tcp_hdr = (struct tcphdr *)(void *) ((u8 *)mp->b_rptr + pktinfo->mac_hlen + pktinfo->ip_hlen); pktinfo->l4_hlen = TCP_HDR_LENGTH(tcp_hdr); pktinfo->total_hlen = pktinfo->mac_hlen + pktinfo->ip_hlen + pktinfo->l4_hlen; } static void qede_get_pkt_offload_info(qede_t *qede, mblk_t *mp, u32 *use_cksum, boolean_t *use_lso, uint16_t *mss) { u32 pflags; mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &pflags); *use_cksum = pflags; if (qede->lso_enable) { u32 pkt_mss = 0; u32 lso_flags = 0; mac_lso_get(mp, &pkt_mss, &lso_flags); *use_lso = (lso_flags == HW_LSO); *mss = (u16)pkt_mss; } } static void /* LINTED E_FUNC_ARG_UNUSED */ qede_get_pkt_info(qede_t *qede, mblk_t *mp, qede_tx_pktinfo_t *pktinfo) { mblk_t *bp; size_t size; struct ether_header *eth_hdr = (struct ether_header *)(void *)mp->b_rptr; pktinfo->total_len = 0; pktinfo->mblk_no = 0; /* * Count the total length and the number of * chained mblks in the packet */ for (bp = mp; bp != NULL; bp = bp->b_cont) { size = MBLKL(bp); if (size == 0) { continue; } pktinfo->total_len += size; pktinfo->mblk_no++; } /* mac header type and len */ if (ntohs(eth_hdr->ether_type) == ETHERTYPE_IP) { pktinfo->ether_type = ntohs(eth_hdr->ether_type); pktinfo->mac_hlen = sizeof (struct ether_header); } else if (ntohs(eth_hdr->ether_type) == ETHERTYPE_VLAN) { struct ether_vlan_header *vlan_hdr = (struct ether_vlan_header *)(void *)mp->b_rptr; pktinfo->ether_type = ntohs(vlan_hdr->ether_type); pktinfo->mac_hlen = sizeof (struct ether_vlan_header); } } /* * Routine to sync dma mem for multiple * descriptors in a chain */ void qede_desc_dma_mem_sync(ddi_dma_handle_t *dma_handle, uint_t start, uint_t count, uint_t range, uint_t unit_size, uint_t direction) { if ((start + count) < range) { (void) ddi_dma_sync(*dma_handle, start * unit_size, count * unit_size, direction); } else { (void) ddi_dma_sync(*dma_handle, start * unit_size, 0, direction); (void) ddi_dma_sync(*dma_handle, 0, (start + count - range) * unit_size, direction); } } /* * Send tx pkt by copying incoming packet in a * preallocated and mapped dma buffer * Not designed to handle lso for now */ static enum qede_xmit_status qede_tx_bcopy(qede_tx_ring_t *tx_ring, mblk_t *mp, qede_tx_pktinfo_t *pktinfo) { qede_tx_bcopy_pkt_t *bcopy_pkt = NULL; /* Only one bd will be needed for bcopy packets */ struct eth_tx_1st_bd *first_bd; u16 last_producer = tx_ring->sw_tx_prod; uint8_t *txb; mblk_t *bp; u32 mblen; bcopy_pkt = qede_get_bcopy_pkt(tx_ring); if (bcopy_pkt == NULL) { qede_print_err("!%s(%d): entry NULL at _tx_ bcopy_list head", __func__, tx_ring->qede->instance); return (XMIT_FAILED); } /* * Copy the packet data to our copy * buffer */ txb = bcopy_pkt->virt_addr; for (bp = mp; bp != NULL; bp = bp->b_cont) { mblen = MBLKL(bp); if (mblen == 0) { continue; } bcopy(bp->b_rptr, txb, mblen); txb += mblen; } (void) ddi_dma_sync(bcopy_pkt->dma_handle, 0, pktinfo->total_len, DDI_DMA_SYNC_FORDEV); mutex_enter(&tx_ring->tx_lock); if (ecore_chain_get_elem_left(&tx_ring->tx_bd_ring)< QEDE_TX_COPY_PATH_PAUSE_THRESHOLD) { tx_ring->tx_q_sleeping = 1; qede_put_bcopy_pkt(tx_ring, bcopy_pkt); mutex_exit(&tx_ring->tx_lock); #ifdef DEBUG_TX_RECYCLE qede_print_err("!%s(%d): Pausing tx queue", __func__, tx_ring->qede->instance); #endif return (XMIT_PAUSE_QUEUE); } first_bd = ecore_chain_produce(&tx_ring->tx_bd_ring); bzero(first_bd, sizeof (*first_bd)); first_bd->data.nbds = 1; first_bd->data.bd_flags.bitfields = (1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT); if (pktinfo->cksum_flags & HCK_IPV4_HDRCKSUM) { first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT); } if (pktinfo->cksum_flags & HCK_FULLCKSUM) { first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT); } BD_SET_ADDR_LEN(first_bd, bcopy_pkt->phys_addr, pktinfo->total_len); first_bd->data.bitfields |= (pktinfo->total_len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; tx_ring->tx_db.data.bd_prod = HOST_TO_LE_16(ecore_chain_get_prod_idx(&tx_ring->tx_bd_ring)); tx_ring->tx_recycle_list[tx_ring->sw_tx_prod].bcopy_pkt = bcopy_pkt; tx_ring->tx_recycle_list[tx_ring->sw_tx_prod].dmah_entry = NULL; tx_ring->sw_tx_prod++; tx_ring->sw_tx_prod &= TX_RING_MASK; (void) ddi_dma_sync(tx_ring->tx_bd_dmah, last_producer, sizeof (*first_bd), DDI_DMA_SYNC_FORDEV); QEDE_DOORBELL_WR(tx_ring, tx_ring->tx_db.raw); mutex_exit(&tx_ring->tx_lock); freemsg(mp); return (XMIT_DONE); } /* * Send tx packet by mapping the mp(kernel addr) * to an existing dma_handle in the driver */ static enum qede_xmit_status qede_tx_mapped(qede_tx_ring_t *tx_ring, mblk_t *mp, qede_tx_pktinfo_t *pktinfo) { enum qede_xmit_status status = XMIT_FAILED; int ret; qede_dma_handle_entry_t *dmah_entry = NULL; qede_dma_handle_entry_t *head = NULL, *tail = NULL, *hdl; struct eth_tx_1st_bd *first_bd; struct eth_tx_2nd_bd *second_bd = 0; struct eth_tx_3rd_bd *third_bd = 0; struct eth_tx_bd *tx_data_bd; struct eth_tx_bd local_bd[64] = { 0 }; ddi_dma_cookie_t cookie[64]; u32 ncookies, total_cookies = 0, max_cookies = 0, index = 0; ddi_dma_handle_t dma_handle; mblk_t *bp; u32 mblen; bool is_premapped = B_FALSE; u64 dma_premapped = 0, dma_bound = 0; u32 hdl_reserved = 0; u8 nbd = 0; int i, bd_index; u16 last_producer; qede_tx_recycle_list_t *tx_recycle_list = tx_ring->tx_recycle_list; u64 data_addr; size_t data_size; if (pktinfo->use_lso) { /* * For tso pkt, we can use as many as 255 bds */ max_cookies = ETH_TX_MAX_BDS_PER_NON_LSO_PACKET - 1; qede_pkt_parse_lso_headers(pktinfo, mp); } else { /* * For non-tso packet, only 18 bds can be used */ max_cookies = ETH_TX_MAX_BDS_PER_NON_LSO_PACKET - 1; } for (bp = mp; bp != NULL; bp = bp->b_cont) { mblen = MBLKL(bp); if (mblen == 0) { continue; } is_premapped = B_FALSE; /* * If the mblk is premapped then get the * dma_handle and sync the dma mem. otherwise * reserve an handle from the driver dma * handles list */ #ifdef DBLK_DMA_PREMAP if (bp->b_datap->db_flags & DBLK_DMA_PREMAP) { #ifdef DEBUG_PREMAP qede_info(tx_ring->qede, "mp is premapped"); #endif tx_ring->tx_premap_count++; ret = dblk_dma_info_get(tx_ring->pm_handle, bp->b_rptr, mblen, bp->b_datap, &cookie[index], &ncookies, &dma_handle); if (ret == DDI_DMA_MAPPED) { is_premapped = B_TRUE; dma_premapped++; (void) ddi_dma_sync(dma_handle, 0, 0, DDI_DMA_SYNC_FORDEV); } else { tx_ring->tx_premap_fail++; } } #endif /* DBLK_DMA_PREMAP */ if (!is_premapped) { dmah_entry = qede_get_dmah_entry(tx_ring); if (dmah_entry == NULL) { qede_info(tx_ring->qede, "dmah_entry NULL, " "Fallback to copy mode..."); status = XMIT_FAILED; goto err_map; } if (ddi_dma_addr_bind_handle(dmah_entry->dma_handle, NULL, (caddr_t)bp->b_rptr, mblen, DDI_DMA_STREAMING | DDI_DMA_WRITE, DDI_DMA_DONTWAIT, NULL, &cookie[index], &ncookies) != DDI_DMA_MAPPED) { #ifdef DEBUG_PULLUP qede_info(tx_ring->qede, "addr_bind() failed for " "handle %p, len %d mblk_no %d tot_len 0x%x" " use_lso %d", dmah_entry->dma_handle, mblen, pktinfo->mblk_no, pktinfo->total_len, pktinfo->use_lso); qede_info(tx_ring->qede, "Falling back to pullup"); #endif status = XMIT_FALLBACK_PULLUP; tx_ring->tx_bind_fail++; goto err_map; } tx_ring->tx_bind_count++; if (index == 0) { dmah_entry->mp = mp; } else { dmah_entry->mp = NULL; } /* queue into recycle list for tx completion routine */ if (tail == NULL) { head = tail = dmah_entry; } else { tail->next = dmah_entry; tail = dmah_entry; } hdl_reserved++; dma_bound++; } total_cookies += ncookies; if (total_cookies > max_cookies) { tx_ring->tx_too_many_cookies++; #ifdef DEBUG_PULLUP qede_info(tx_ring->qede, "total_cookies > max_cookies, " "pktlen %d, mb num %d", pktinfo->total_len, pktinfo->mblk_no); #endif status = XMIT_TOO_MANY_COOKIES; goto err_map_sec; } if (is_premapped) { index += ncookies; } else { index++; /* * Dec. ncookies since we already stored cookie[0] */ ncookies--; for (i = 0; i < ncookies; i++, index++) ddi_dma_nextcookie(dmah_entry->dma_handle, &cookie[index]); } } /* * Guard against the case where we get a series of mblks that cause us * not to end up with any mapped data. */ if (total_cookies == 0) { status = XMIT_FAILED; goto err_map_sec; } if (total_cookies > max_cookies) { tx_ring->tx_too_many_cookies++; status = XMIT_TOO_MANY_COOKIES; goto err_map_sec; } first_bd = (struct eth_tx_1st_bd *)&local_bd[0]; /* * Mark this bd as start bd */ first_bd->data.bd_flags.bitfields = (1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT); if (pktinfo->cksum_flags & HCK_IPV4_HDRCKSUM) { first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT); } if (pktinfo->cksum_flags & HCK_FULLCKSUM) { first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT); } /* Fill-up local bds with the tx data and flags */ for (i = 0, bd_index = 0; i < total_cookies; i++, bd_index++) { if (bd_index == 0) { BD_SET_ADDR_LEN(first_bd, cookie[i].dmac_laddress, cookie[i].dmac_size); if (pktinfo->use_lso) { first_bd->data.bd_flags.bitfields |= 1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT; second_bd = (struct eth_tx_2nd_bd *)&local_bd[1]; /* * If the fisrt bd contains * hdr + data (partial or full data), then spilt * the hdr and data between 1st and 2nd * bd respectively */ if (first_bd->nbytes > pktinfo->total_hlen) { data_addr = cookie[0].dmac_laddress + pktinfo->total_hlen; data_size = cookie[i].dmac_size - pktinfo->total_hlen; BD_SET_ADDR_LEN(second_bd, data_addr, data_size); /* * First bd already contains the addr to * to start of pkt, just adjust the dma * len of first_bd */ first_bd->nbytes = pktinfo->total_hlen; bd_index++; } else if (first_bd->nbytes < pktinfo->total_hlen) { #ifdef DEBUG_PULLUP qede_info(tx_ring->qede, "Headers not in single bd"); #endif status = XMIT_FALLBACK_PULLUP; goto err_map_sec; } /* * Third bd is used to indicates to fw * that tso needs to be performed. It should * be present even if only two cookies are * needed for the mblk */ third_bd = (struct eth_tx_3rd_bd *)&local_bd[2]; third_bd->data.lso_mss |= HOST_TO_LE_16(pktinfo->mss); third_bd->data.bitfields |= 1 << ETH_TX_DATA_3RD_BD_HDR_NBD_SHIFT; } continue; } tx_data_bd = &local_bd[bd_index]; BD_SET_ADDR_LEN(tx_data_bd, cookie[i].dmac_laddress, cookie[i].dmac_size); } if (pktinfo->use_lso) { if (bd_index < 3) { nbd = 3; } else { nbd = bd_index; } } else { nbd = total_cookies; first_bd->data.bitfields |= (pktinfo->total_len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; } first_bd->data.nbds = nbd; mutex_enter(&tx_ring->tx_lock); /* * Before copying the local bds into actual, * check if we have enough on the bd_chain */ if (ecore_chain_get_elem_left(&tx_ring->tx_bd_ring) < nbd) { tx_ring->tx_q_sleeping = 1; status = XMIT_PAUSE_QUEUE; #ifdef DEBUG_TX_RECYCLE qede_info(tx_ring->qede, "Pausing tx queue..."); #endif mutex_exit(&tx_ring->tx_lock); goto err_map_sec ; } /* Copy the local_bd(s) into the actual bds */ for (i = 0; i < nbd; i++) { tx_data_bd = ecore_chain_produce(&tx_ring->tx_bd_ring); bcopy(&local_bd[i], tx_data_bd, sizeof (*tx_data_bd)); } last_producer = tx_ring->sw_tx_prod; tx_ring->tx_recycle_list[tx_ring->sw_tx_prod].dmah_entry = head; tx_ring->tx_recycle_list[tx_ring->sw_tx_prod].bcopy_pkt = NULL; tx_ring->sw_tx_prod = (tx_ring->sw_tx_prod + 1) & TX_RING_MASK; tx_ring->tx_db.data.bd_prod = HOST_TO_LE_16(ecore_chain_get_prod_idx(&tx_ring->tx_bd_ring)); /* Sync the tx_bd dma mem */ qede_desc_dma_mem_sync(&tx_ring->tx_bd_dmah, last_producer, nbd, tx_ring->tx_ring_size, sizeof (struct eth_tx_bd), DDI_DMA_SYNC_FORDEV); /* * Write to doorbell bar */ QEDE_DOORBELL_WR(tx_ring, tx_ring->tx_db.raw); mutex_exit(&tx_ring->tx_lock); return (XMIT_DONE); err_map: if (dmah_entry != NULL) { if (tail == NULL) { head = tail = dmah_entry; } else { tail->next = dmah_entry; tail = dmah_entry; } hdl_reserved++; } err_map_sec: hdl = head; while (hdl != NULL) { (void) ddi_dma_unbind_handle(hdl->dma_handle); hdl = hdl->next; } if (head != NULL) { qede_put_dmah_entries(tx_ring, head); } return (status); } static enum qede_xmit_status qede_send_tx_packet(qede_t *qede, qede_tx_ring_t *tx_ring, mblk_t *mp) { boolean_t force_pullup = B_FALSE; enum qede_xmit_status status = XMIT_FAILED; enum qede_xmit_mode xmit_mode = USE_BCOPY; qede_tx_pktinfo_t pktinfo; mblk_t *original_mp = NULL, *pulled_up_mp = NULL; struct ether_vlan_header *ethvhdr; mutex_enter(&tx_ring->tx_lock); if (ecore_chain_get_elem_left(&tx_ring->tx_bd_ring) < qede->tx_recycle_threshold) { #ifdef DEBUG_TX_RECYCLE qede_info(qede, "Recyclycling from tx routine"); #endif if (qede_process_tx_completions(tx_ring) < qede->tx_recycle_threshold) { #ifdef DEBUG_TX_RECYCLE qede_info(qede, "Still not enough bd after cleanup, " "pausing tx queue..."); #endif tx_ring->tx_q_sleeping = 1; mutex_exit(&tx_ring->tx_lock); return (XMIT_PAUSE_QUEUE); } } mutex_exit(&tx_ring->tx_lock); bzero(&pktinfo, sizeof (pktinfo)); /* Get the offload reqd. on the pkt */ qede_get_pkt_offload_info(qede, mp, &pktinfo.cksum_flags, &pktinfo.use_lso, &pktinfo.mss); do_pullup: if (force_pullup) { tx_ring->tx_pullup_count++; #ifdef DEBUG_PULLUP qede_info(qede, "Pulling up original mp %p", mp); #endif /* * Try to accumulate all mblks of this pkt * into a single mblk */ original_mp = mp; if ((pulled_up_mp = msgpullup(mp, -1)) != NULL) { #ifdef DEBUG_PULLUP qede_info(qede, "New mp %p, ori %p", pulled_up_mp, mp); #endif /* * Proceed with the new single * mp */ mp = pulled_up_mp; xmit_mode = XMIT_MODE_UNUSED; pktinfo.pulled_up = B_TRUE; } else { #ifdef DEBUG_PULLUP qede_info(tx_ring->qede, "Pullup failed"); #endif status = XMIT_FAILED; goto exit; } } qede_get_pkt_info(qede, mp, &pktinfo); if ((!pktinfo.use_lso) && (pktinfo.total_len > (qede->mtu + pktinfo.mac_hlen))) { qede_info(tx_ring->qede, "Packet drop as packet len 0x%x > 0x%x", pktinfo.total_len, (qede->mtu + QEDE_MAX_ETHER_HDR)); status = XMIT_FAILED; goto exit; } #ifdef DEBUG_PULLUP if (force_pullup) { qede_print_err("!%s: mp %p, pktinfo : total_len %d," " mblk_no %d, ether_type %d\n" "mac_hlen %d, ip_hlen %d, l4_hlen %d\n" "l4_proto %d, use_cksum:use_lso %d:%d mss %d", __func__, mp, pktinfo.total_len, pktinfo.mblk_no, pktinfo.ether_type, pktinfo.mac_hlen, pktinfo.ip_hlen, pktinfo.l4_hlen, pktinfo.l4_proto, pktinfo.cksum_flags, pktinfo.use_lso, pktinfo.mss); } #endif #ifdef DEBUG_PREMAP if (DBLK_IS_PREMAPPED(mp->b_datap)) { qede_print_err("!%s(%d): mp %p id PREMAPPMED", __func__, qede->instance); } #endif #ifdef DBLK_DMA_PREMAP if (DBLK_IS_PREMAPPED(mp->b_datap) || pktinfo.total_len > qede->tx_bcopy_threshold) { xmit_mode = USE_DMA_BIND; } #else if (pktinfo.total_len > qede->tx_bcopy_threshold) { xmit_mode = USE_DMA_BIND; } #endif if (pktinfo.total_len <= qede->tx_bcopy_threshold) { xmit_mode = USE_BCOPY; } /* * if mac + ip hdr not in one contiguous block, * use copy mode */ if (MBLKL(mp) < (ETHER_HEADER_LEN + IP_HEADER_LEN)) { /*qede_info(qede, "mblk too small, using copy mode, len = %d", MBLKL(mp));*/ xmit_mode = USE_BCOPY; } if ((uintptr_t)mp->b_rptr & 1) { xmit_mode = USE_BCOPY; } /* * if too many mblks and hence the dma cookies, needed * for tx, then use bcopy or pullup on packet * currently, ETH_TX_MAX_BDS_PER_NON_LSO_PACKET = 18 */ if (pktinfo.mblk_no > (ETH_TX_MAX_BDS_PER_NON_LSO_PACKET - 1)) { if (force_pullup) { tx_ring->tx_too_many_mblks++; status = XMIT_FAILED; goto exit; } else { xmit_mode = USE_PULLUP; } } #ifdef TX_FORCE_COPY_MODE xmit_mode = USE_BCOPY; #elif TX_FORCE_MAPPED_MODE xmit_mode = USE_DMA_BIND; #endif #ifdef DEBUG_PULLUP if (force_pullup) { qede_info(qede, "using mode %d on pulled mp %p", xmit_mode, mp); } #endif /* * Use Mapped mode for the packet */ if (xmit_mode == USE_DMA_BIND) { status = qede_tx_mapped(tx_ring, mp, &pktinfo); if (status == XMIT_DONE) { if (pktinfo.use_lso) { tx_ring->tx_lso_pkt_count++; } else if(pktinfo.total_len > 1518) { tx_ring->tx_jumbo_pkt_count++; } tx_ring->tx_mapped_pkts++; goto exit; } else if ((status == XMIT_TOO_MANY_COOKIES || (status == XMIT_FALLBACK_PULLUP)) && !force_pullup) { xmit_mode = USE_PULLUP; } else { status = XMIT_FAILED; goto exit; } } if (xmit_mode == USE_BCOPY) { status = qede_tx_bcopy(tx_ring, mp, &pktinfo); if (status == XMIT_DONE) { tx_ring->tx_copy_count++; goto exit; } else if ((status == XMIT_FALLBACK_PULLUP) && !force_pullup) { xmit_mode = USE_PULLUP; } else { goto exit; } } if (xmit_mode == USE_PULLUP) { force_pullup = B_TRUE; tx_ring->tx_pullup_count++; goto do_pullup; } exit: if (status != XMIT_DONE) { /* * if msgpullup succeeded, but something else failed, * free the pulled-up msg and return original mblk to * stack, indicating tx failure */ if (pulled_up_mp) { qede_info(qede, "tx failed, free pullup pkt %p", mp); freemsg(pulled_up_mp); mp = original_mp; } } else { tx_ring->tx_byte_count += pktinfo.total_len; /* * If tx was successfull after a pullup, then free the * original mp. The pulled-up will be freed as part of * tx completions processing */ if (pulled_up_mp) { #ifdef DEBUG_PULLUP qede_info(qede, "success, free ori mp %p", original_mp); #endif freemsg(original_mp); } } return (status); } typedef uint32_t ub4; /* unsigned 4-byte quantities */ typedef uint8_t ub1; #define hashsize(n) ((ub4)1<<(n)) #define hashmask(n) (hashsize(n)-1) #define mix(a, b, c) \ { \ a -= b; a -= c; a ^= (c>>13); \ b -= c; b -= a; b ^= (a<<8); \ c -= a; c -= b; c ^= (b>>13); \ a -= b; a -= c; a ^= (c>>12); \ b -= c; b -= a; b ^= (a<<16); \ c -= a; c -= b; c ^= (b>>5); \ a -= b; a -= c; a ^= (c>>3); \ b -= c; b -= a; b ^= (a<<10); \ c -= a; c -= b; c ^= (b>>15); \ } ub4 hash(k, length, initval) register ub1 *k; /* the key */ register ub4 length; /* the length of the key */ register ub4 initval; /* the previous hash, or an arbitrary value */ { register ub4 a, b, c, len; /* Set up the internal state */ len = length; a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ c = initval; /* the previous hash value */ /* handle most of the key */ while (len >= 12) { a += (k[0] +((ub4)k[1]<<8) +((ub4)k[2]<<16) +((ub4)k[3]<<24)); b += (k[4] +((ub4)k[5]<<8) +((ub4)k[6]<<16) +((ub4)k[7]<<24)); c += (k[8] +((ub4)k[9]<<8) +((ub4)k[10]<<16)+((ub4)k[11]<<24)); mix(a, b, c); k += 12; len -= 12; } /* handle the last 11 bytes */ c += length; /* all the case statements fall through */ switch (len) { /* FALLTHRU */ case 11: c += ((ub4)k[10]<<24); /* FALLTHRU */ case 10: c += ((ub4)k[9]<<16); /* FALLTHRU */ case 9 : c += ((ub4)k[8]<<8); /* the first byte of c is reserved for the length */ /* FALLTHRU */ case 8 : b += ((ub4)k[7]<<24); /* FALLTHRU */ case 7 : b += ((ub4)k[6]<<16); /* FALLTHRU */ case 6 : b += ((ub4)k[5]<<8); /* FALLTHRU */ case 5 : b += k[4]; /* FALLTHRU */ case 4 : a += ((ub4)k[3]<<24); /* FALLTHRU */ case 3 : a += ((ub4)k[2]<<16); /* FALLTHRU */ case 2 : a += ((ub4)k[1]<<8); /* FALLTHRU */ case 1 : a += k[0]; /* case 0: nothing left to add */ } mix(a, b, c); /* report the result */ return (c); } #ifdef NO_CROSSBOW static uint8_t qede_hash_get_txq(qede_t *qede, caddr_t bp) { struct ip *iphdr = NULL; struct ether_header *ethhdr; struct ether_vlan_header *ethvhdr; struct tcphdr *tcp_hdr; struct udphdr *udp_hdr; uint32_t etherType; int mac_hdr_len, ip_hdr_len; uint32_t h = 0; /* 0 by default */ uint8_t tx_ring_id = 0; uint32_t ip_src_addr = 0; uint32_t ip_desc_addr = 0; uint16_t src_port = 0; uint16_t dest_port = 0; uint8_t key[12]; if (qede->num_fp == 1) { return (tx_ring_id); } ethhdr = (struct ether_header *)((void *)bp); ethvhdr = (struct ether_vlan_header *)((void *)bp); /* Is this vlan packet? */ if (ntohs(ethvhdr->ether_tpid) == ETHERTYPE_VLAN) { mac_hdr_len = sizeof (struct ether_vlan_header); etherType = ntohs(ethvhdr->ether_type); } else { mac_hdr_len = sizeof (struct ether_header); etherType = ntohs(ethhdr->ether_type); } /* Is this IPv4 or IPv6 packet? */ if (etherType == ETHERTYPE_IP /* 0800 */) { if (IPH_HDR_VERSION((ipha_t *)(void *)(bp+mac_hdr_len)) == IPV4_VERSION) { iphdr = (struct ip *)(void *)(bp+mac_hdr_len); } if (((unsigned long)iphdr) & 0x3) { /* IP hdr not 4-byte aligned */ return (tx_ring_id); } } /* ipV4 packets */ if (iphdr) { ip_hdr_len = IPH_HDR_LENGTH(iphdr); ip_src_addr = iphdr->ip_src.s_addr; ip_desc_addr = iphdr->ip_dst.s_addr; if (iphdr->ip_p == IPPROTO_TCP) { tcp_hdr = (struct tcphdr *)(void *) ((uint8_t *)iphdr + ip_hdr_len); src_port = tcp_hdr->th_sport; dest_port = tcp_hdr->th_dport; } else if (iphdr->ip_p == IPPROTO_UDP) { udp_hdr = (struct udphdr *)(void *) ((uint8_t *)iphdr + ip_hdr_len); src_port = udp_hdr->uh_sport; dest_port = udp_hdr->uh_dport; } key[0] = (uint8_t)((ip_src_addr) &0xFF); key[1] = (uint8_t)((ip_src_addr >> 8) &0xFF); key[2] = (uint8_t)((ip_src_addr >> 16) &0xFF); key[3] = (uint8_t)((ip_src_addr >> 24) &0xFF); key[4] = (uint8_t)((ip_desc_addr) &0xFF); key[5] = (uint8_t)((ip_desc_addr >> 8) &0xFF); key[6] = (uint8_t)((ip_desc_addr >> 16) &0xFF); key[7] = (uint8_t)((ip_desc_addr >> 24) &0xFF); key[8] = (uint8_t)((src_port) &0xFF); key[9] = (uint8_t)((src_port >> 8) &0xFF); key[10] = (uint8_t)((dest_port) &0xFF); key[11] = (uint8_t)((dest_port >> 8) &0xFF); h = hash(key, 12, 0); /* return 32 bit */ tx_ring_id = (h & (qede->num_fp - 1)); if (tx_ring_id >= qede->num_fp) { cmn_err(CE_WARN, "%s bad tx_ring_id %d\n", __func__, tx_ring_id); tx_ring_id = 0; } } return (tx_ring_id); } #endif mblk_t * qede_ring_tx(void *arg, mblk_t *mp) { qede_fastpath_t *fp = (qede_fastpath_t *)arg; qede_t *qede = fp->qede; #ifndef NO_CROSSBOW qede_tx_ring_t *tx_ring = fp->tx_ring[0]; #else qede_tx_ring_t *tx_ring; #endif uint32_t ring_id; mblk_t *next = NULL; enum qede_xmit_status status = XMIT_FAILED; caddr_t bp; ASSERT(mp->b_next == NULL); #ifndef NO_CROSSBOW if (!fp || !tx_ring) { qede_print_err("!%s: error, fp %p, tx_ring %p", __func__, fp, tx_ring); goto exit; } #endif if (qede->qede_state != QEDE_STATE_STARTED) { qede_print_err("!%s(%d): qede_state %d invalid", __func__, qede->instance, qede->qede_state); goto exit; } if (!qede->params.link_state) { goto exit; } while (mp != NULL) { #ifdef NO_CROSSBOW /* * Figure out which tx ring to send this packet to. * Currently multiple rings are not exposed to mac layer * and fanout done by driver */ bp = (caddr_t)mp->b_rptr; ring_id = qede_hash_get_txq(qede, bp); fp = &qede->fp_array[ring_id]; tx_ring = fp->tx_ring[0]; if (qede->num_tc > 1) { qede_info(qede, "Traffic classes(%d) > 1 not supported", qede->num_tc); goto exit; } #endif next = mp->b_next; mp->b_next = NULL; status = qede_send_tx_packet(qede, tx_ring, mp); if (status == XMIT_DONE) { tx_ring->tx_pkt_count++; mp = next; } else if (status == XMIT_PAUSE_QUEUE) { tx_ring->tx_ring_pause++; mp->b_next = next; break; } else if (status == XMIT_FAILED) { goto exit; } } return (mp); exit: tx_ring->tx_pkt_dropped++; freemsgchain(mp); mp = NULL; return (mp); }