1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/mac_provider.h>
27 #include <sys/nxge/nxge_impl.h>
28 #include <sys/nxge/nxge_hio.h>
29 #include <npi_tx_wr64.h>
30
31 /* Software LSO required header files */
32 #include <netinet/tcp.h>
33 #include <inet/ip_impl.h>
34 #include <inet/tcp.h>
35
36 extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy,
37 boolean_t is_outbound);
38
39 static mblk_t *nxge_lso_eliminate(mblk_t *);
40 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss);
41 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *);
42 static void nxge_hcksum_retrieve(mblk_t *,
43 uint32_t *, uint32_t *, uint32_t *,
44 uint32_t *, uint32_t *);
45 static uint32_t nxge_csgen(uint16_t *, int);
46
47 extern uint32_t nxge_reclaim_pending;
48 extern uint32_t nxge_bcopy_thresh;
49 extern uint32_t nxge_dvma_thresh;
50 extern uint32_t nxge_dma_stream_thresh;
51 extern uint32_t nxge_tx_minfree;
52 extern uint32_t nxge_tx_intr_thres;
53 extern uint32_t nxge_tx_max_gathers;
54 extern uint32_t nxge_tx_tiny_pack;
55 extern uint32_t nxge_tx_use_bcopy;
56 extern nxge_tx_mode_t nxge_tx_scheme;
57 uint32_t nxge_lso_kick_cnt = 2;
58
59
60 void
nxge_tx_ring_task(void * arg)61 nxge_tx_ring_task(void *arg)
62 {
63 p_tx_ring_t ring = (p_tx_ring_t)arg;
64
65 ASSERT(ring->tx_ring_handle != NULL);
66
67 MUTEX_ENTER(&ring->lock);
68 (void) nxge_txdma_reclaim(ring->nxgep, ring, 0);
69 MUTEX_EXIT(&ring->lock);
70
71 if (!ring->tx_ring_offline) {
72 mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle);
73 }
74 }
75
76 static void
nxge_tx_ring_dispatch(p_tx_ring_t ring)77 nxge_tx_ring_dispatch(p_tx_ring_t ring)
78 {
79 /*
80 * Kick the ring task to reclaim some buffers.
81 */
82 (void) ddi_taskq_dispatch(ring->taskq,
83 nxge_tx_ring_task, (void *)ring, DDI_SLEEP);
84 }
85
86 mblk_t *
nxge_tx_ring_send(void * arg,mblk_t * mp)87 nxge_tx_ring_send(void *arg, mblk_t *mp)
88 {
89 p_nxge_ring_handle_t nrhp = (p_nxge_ring_handle_t)arg;
90 p_nxge_t nxgep;
91 p_tx_ring_t tx_ring_p;
92 int status, channel;
93
94 ASSERT(nrhp != NULL);
95 nxgep = nrhp->nxgep;
96 channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index;
97 tx_ring_p = nxgep->tx_rings->rings[channel];
98
99 /*
100 * We may be in a transition from offlined DMA to onlined
101 * DMA.
102 */
103 if (tx_ring_p == NULL) {
104 ASSERT(tx_ring_p != NULL);
105 freemsg(mp);
106 return ((mblk_t *)NULL);
107 }
108
109 /*
110 * Valid DMA?
111 */
112 ASSERT(nxgep == tx_ring_p->nxgep);
113
114 /*
115 * Make sure DMA is not offlined.
116 */
117 if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) {
118 ASSERT(!tx_ring_p->tx_ring_offline);
119 freemsg(mp);
120 return ((mblk_t *)NULL);
121 }
122
123 /*
124 * Transmit the packet.
125 */
126 status = nxge_start(nxgep, tx_ring_p, mp);
127 if (status) {
128 nxge_tx_ring_dispatch(tx_ring_p);
129 return (mp);
130 }
131
132 return ((mblk_t *)NULL);
133 }
134
135 int
nxge_start(p_nxge_t nxgep,p_tx_ring_t tx_ring_p,p_mblk_t mp)136 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
137 {
138 int dma_status, status = 0;
139 p_tx_desc_t tx_desc_ring_vp;
140 npi_handle_t npi_desc_handle;
141 nxge_os_dma_handle_t tx_desc_dma_handle;
142 p_tx_desc_t tx_desc_p;
143 p_tx_msg_t tx_msg_ring;
144 p_tx_msg_t tx_msg_p = NULL;
145 tx_desc_t tx_desc, *tmp_desc_p;
146 tx_desc_t sop_tx_desc, *sop_tx_desc_p;
147 p_tx_pkt_header_t hdrp = NULL;
148 tx_pkt_hdr_all_t tmp_hdrp;
149 p_tx_pkt_hdr_all_t pkthdrp;
150 uint8_t npads = 0;
151 uint64_t dma_ioaddr;
152 uint32_t dma_flags;
153 int last_bidx;
154 uint8_t *b_rptr;
155 caddr_t kaddr;
156 uint32_t nmblks;
157 uint32_t ngathers;
158 uint32_t clen;
159 int len;
160 uint32_t pkt_len, pack_len, min_len;
161 uint32_t bcopy_thresh;
162 int i, cur_index, sop_index;
163 uint16_t tail_index;
164 boolean_t tail_wrap = B_FALSE;
165 nxge_dma_common_t desc_area;
166 nxge_os_dma_handle_t dma_handle;
167 ddi_dma_cookie_t dma_cookie;
168 npi_handle_t npi_handle;
169 p_mblk_t nmp;
170 p_mblk_t t_mp;
171 uint32_t ncookies;
172 boolean_t good_packet;
173 boolean_t mark_mode = B_FALSE;
174 p_nxge_stats_t statsp;
175 p_nxge_tx_ring_stats_t tdc_stats;
176 t_uscalar_t start_offset = 0;
177 t_uscalar_t stuff_offset = 0;
178 t_uscalar_t end_offset = 0;
179 t_uscalar_t value = 0;
180 t_uscalar_t cksum_flags = 0;
181 boolean_t cksum_on = B_FALSE;
182 uint32_t boff = 0;
183 uint64_t tot_xfer_len = 0;
184 boolean_t header_set = B_FALSE;
185 #ifdef NXGE_DEBUG
186 p_tx_desc_t tx_desc_ring_pp;
187 p_tx_desc_t tx_desc_pp;
188 tx_desc_t *save_desc_p;
189 int dump_len;
190 int sad_len;
191 uint64_t sad;
192 int xfer_len;
193 uint32_t msgsize;
194 #endif
195 p_mblk_t mp_chain = NULL;
196 boolean_t is_lso = B_FALSE;
197 boolean_t lso_again;
198 int cur_index_lso;
199 p_mblk_t nmp_lso_save = NULL;
200 uint32_t lso_ngathers;
201 boolean_t lso_tail_wrap = B_FALSE;
202
203 NXGE_DEBUG_MSG((nxgep, TX_CTL,
204 "==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
205 NXGE_DEBUG_MSG((nxgep, TX_CTL,
206 "==> nxge_start: Starting tdc %d desc pending %d",
207 tx_ring_p->tdc, tx_ring_p->descs_pending));
208
209 statsp = nxgep->statsp;
210
211 if (!isLDOMguest(nxgep)) {
212 switch (nxgep->mac.portmode) {
213 default:
214 if (nxgep->statsp->port_stats.lb_mode ==
215 nxge_lb_normal) {
216 if (!statsp->mac_stats.link_up) {
217 freemsg(mp);
218 NXGE_DEBUG_MSG((nxgep, TX_CTL,
219 "==> nxge_start: "
220 "link not up"));
221 goto nxge_start_fail1;
222 }
223 }
224 break;
225 case PORT_10G_FIBER:
226 /*
227 * For the following modes, check the link status
228 * before sending the packet out:
229 * nxge_lb_normal,
230 * nxge_lb_ext10g,
231 * nxge_lb_ext1000,
232 * nxge_lb_ext100,
233 * nxge_lb_ext10.
234 */
235 if (nxgep->statsp->port_stats.lb_mode <
236 nxge_lb_phy10g) {
237 if (!statsp->mac_stats.link_up) {
238 freemsg(mp);
239 NXGE_DEBUG_MSG((nxgep, TX_CTL,
240 "==> nxge_start: "
241 "link not up"));
242 goto nxge_start_fail1;
243 }
244 }
245 break;
246 }
247 }
248
249 if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
250 (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
251 NXGE_DEBUG_MSG((nxgep, TX_CTL,
252 "==> nxge_start: hardware not initialized or stopped"));
253 freemsg(mp);
254 goto nxge_start_fail1;
255 }
256
257 if (nxgep->soft_lso_enable) {
258 mp_chain = nxge_lso_eliminate(mp);
259 NXGE_DEBUG_MSG((nxgep, TX_CTL,
260 "==> nxge_start(0): LSO mp $%p mp_chain $%p",
261 mp, mp_chain));
262 if (mp_chain == NULL) {
263 NXGE_ERROR_MSG((nxgep, TX_CTL,
264 "==> nxge_send(0): NULL mp_chain $%p != mp $%p",
265 mp_chain, mp));
266 goto nxge_start_fail1;
267 }
268 if (mp_chain != mp) {
269 NXGE_DEBUG_MSG((nxgep, TX_CTL,
270 "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p",
271 mp_chain, mp));
272 is_lso = B_TRUE;
273 mp = mp_chain;
274 mp_chain = mp_chain->b_next;
275 mp->b_next = NULL;
276 }
277 }
278
279 mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset,
280 &value, &cksum_flags);
281 if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
282 start_offset += sizeof (ether_header_t);
283 stuff_offset += sizeof (ether_header_t);
284 } else {
285 start_offset += sizeof (struct ether_vlan_header);
286 stuff_offset += sizeof (struct ether_vlan_header);
287 }
288
289 if (cksum_flags & HCK_PARTIALCKSUM) {
290 NXGE_DEBUG_MSG((nxgep, TX_CTL,
291 "==> nxge_start: mp $%p len %d "
292 "cksum_flags 0x%x (partial checksum) ",
293 mp, MBLKL(mp), cksum_flags));
294 cksum_on = B_TRUE;
295 }
296
297 pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp;
298 pkthdrp->reserved = 0;
299 tmp_hdrp.pkthdr.value = 0;
300 nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
301 0, 0, pkthdrp,
302 start_offset, stuff_offset);
303
304 lso_again = B_FALSE;
305 lso_ngathers = 0;
306
307 MUTEX_ENTER(&tx_ring_p->lock);
308
309 if (isLDOMservice(nxgep)) {
310 tx_ring_p->tx_ring_busy = B_TRUE;
311 if (tx_ring_p->tx_ring_offline) {
312 freemsg(mp);
313 tx_ring_p->tx_ring_busy = B_FALSE;
314 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
315 NXGE_TX_RING_OFFLINED);
316 MUTEX_EXIT(&tx_ring_p->lock);
317 return (status);
318 }
319 }
320
321 cur_index_lso = tx_ring_p->wr_index;
322 lso_tail_wrap = tx_ring_p->wr_index_wrap;
323 start_again:
324 ngathers = 0;
325 sop_index = tx_ring_p->wr_index;
326 #ifdef NXGE_DEBUG
327 if (tx_ring_p->descs_pending) {
328 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
329 "desc pending %d ", tx_ring_p->descs_pending));
330 }
331
332 dump_len = (int)(MBLKL(mp));
333 dump_len = (dump_len > 128) ? 128: dump_len;
334
335 NXGE_DEBUG_MSG((nxgep, TX_CTL,
336 "==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
337 "(Before header reserve: ORIGINAL LEN %d)",
338 tx_ring_p->tdc,
339 mp->b_rptr,
340 dump_len));
341
342 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
343 "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
344 nxge_dump_packet((char *)mp->b_rptr, dump_len)));
345 #endif
346
347 tdc_stats = tx_ring_p->tdc_stats;
348 mark_mode = (tx_ring_p->descs_pending &&
349 (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) <
350 (int)nxge_tx_minfree));
351
352 NXGE_DEBUG_MSG((nxgep, TX_CTL,
353 "TX Descriptor ring is channel %d mark mode %d",
354 tx_ring_p->tdc, mark_mode));
355
356 if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) {
357 if (!nxge_txdma_reclaim(nxgep, tx_ring_p,
358 (nxge_tx_minfree + lso_ngathers))) {
359 NXGE_DEBUG_MSG((nxgep, TX_CTL,
360 "TX Descriptor ring is full: channel %d",
361 tx_ring_p->tdc));
362 NXGE_DEBUG_MSG((nxgep, TX_CTL,
363 "TX Descriptor ring is full: channel %d",
364 tx_ring_p->tdc));
365 if (is_lso) {
366 /*
367 * free the current mp and mp_chain if not FULL.
368 */
369 tdc_stats->tx_no_desc++;
370 NXGE_DEBUG_MSG((nxgep, TX_CTL,
371 "LSO packet: TX Descriptor ring is full: "
372 "channel %d",
373 tx_ring_p->tdc));
374 goto nxge_start_fail_lso;
375 } else {
376 (void) atomic_cas_32(
377 (uint32_t *)&tx_ring_p->queueing, 0, 1);
378 tdc_stats->tx_no_desc++;
379
380 if (isLDOMservice(nxgep)) {
381 tx_ring_p->tx_ring_busy = B_FALSE;
382 if (tx_ring_p->tx_ring_offline) {
383 (void) atomic_swap_32(
384 &tx_ring_p->tx_ring_offline,
385 NXGE_TX_RING_OFFLINED);
386 }
387 }
388
389 MUTEX_EXIT(&tx_ring_p->lock);
390 status = 1;
391 goto nxge_start_fail1;
392 }
393 }
394 }
395
396 nmp = mp;
397 i = sop_index = tx_ring_p->wr_index;
398 nmblks = 0;
399 ngathers = 0;
400 pkt_len = 0;
401 pack_len = 0;
402 clen = 0;
403 last_bidx = -1;
404 good_packet = B_TRUE;
405
406 desc_area = tx_ring_p->tdc_desc;
407 npi_handle = desc_area.npi_handle;
408 npi_desc_handle.regh = (nxge_os_acc_handle_t)
409 DMA_COMMON_ACC_HANDLE(desc_area);
410 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
411 tx_desc_dma_handle = (nxge_os_dma_handle_t)
412 DMA_COMMON_HANDLE(desc_area);
413 tx_msg_ring = tx_ring_p->tx_msg_ring;
414
415 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
416 sop_index, i));
417
418 #ifdef NXGE_DEBUG
419 msgsize = msgdsize(nmp);
420 NXGE_DEBUG_MSG((nxgep, TX_CTL,
421 "==> nxge_start(1): wr_index %d i %d msgdsize %d",
422 sop_index, i, msgsize));
423 #endif
424 /*
425 * The first 16 bytes of the premapped buffer are reserved
426 * for header. No padding will be used.
427 */
428 pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
429 if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
430 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
431 } else {
432 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
433 }
434 while (nmp) {
435 good_packet = B_TRUE;
436 b_rptr = nmp->b_rptr;
437 len = MBLKL(nmp);
438 if (len <= 0) {
439 nmp = nmp->b_cont;
440 continue;
441 }
442 nmblks++;
443
444 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
445 "len %d pkt_len %d pack_len %d",
446 nmblks, len, pkt_len, pack_len));
447 /*
448 * Hardware limits the transfer length to 4K for NIU and
449 * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
450 * use TX_MAX_TRANSFER_LENGTH as the limit for both.
451 * If len is longer than the limit, then we break nmp into
452 * two chunks: Make the first chunk equal to the limit and
453 * the second chunk for the remaining data. If the second
454 * chunk is still larger than the limit, then it will be
455 * broken into two in the next pass.
456 */
457 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
458 if ((t_mp = dupb(nmp)) != NULL) {
459 nmp->b_wptr = nmp->b_rptr +
460 (TX_MAX_TRANSFER_LENGTH
461 - TX_PKT_HEADER_SIZE);
462 t_mp->b_rptr = nmp->b_wptr;
463 t_mp->b_cont = nmp->b_cont;
464 nmp->b_cont = t_mp;
465 len = MBLKL(nmp);
466 } else {
467 if (is_lso) {
468 NXGE_DEBUG_MSG((nxgep, TX_CTL,
469 "LSO packet: dupb failed: "
470 "channel %d",
471 tx_ring_p->tdc));
472 mp = nmp;
473 goto nxge_start_fail_lso;
474 } else {
475 good_packet = B_FALSE;
476 goto nxge_start_fail2;
477 }
478 }
479 }
480 tx_desc.value = 0;
481 tx_desc_p = &tx_desc_ring_vp[i];
482 #ifdef NXGE_DEBUG
483 tx_desc_pp = &tx_desc_ring_pp[i];
484 #endif
485 tx_msg_p = &tx_msg_ring[i];
486 npi_desc_handle.regp = (uint64_t)tx_desc_p;
487 if (!header_set &&
488 ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
489 (len >= bcopy_thresh))) {
490 header_set = B_TRUE;
491 bcopy_thresh += TX_PKT_HEADER_SIZE;
492 boff = 0;
493 pack_len = 0;
494 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
495 hdrp = (p_tx_pkt_header_t)kaddr;
496 clen = pkt_len;
497 dma_handle = tx_msg_p->buf_dma_handle;
498 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
499 (void) ddi_dma_sync(dma_handle,
500 i * nxge_bcopy_thresh, nxge_bcopy_thresh,
501 DDI_DMA_SYNC_FORDEV);
502
503 tx_msg_p->flags.dma_type = USE_BCOPY;
504 goto nxge_start_control_header_only;
505 }
506
507 pkt_len += len;
508 pack_len += len;
509
510 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
511 "desc entry %d "
512 "DESC IOADDR $%p "
513 "desc_vp $%p tx_desc_p $%p "
514 "desc_pp $%p tx_desc_pp $%p "
515 "len %d pkt_len %d pack_len %d",
516 i,
517 DMA_COMMON_IOADDR(desc_area),
518 tx_desc_ring_vp, tx_desc_p,
519 tx_desc_ring_pp, tx_desc_pp,
520 len, pkt_len, pack_len));
521
522 if (len < bcopy_thresh) {
523 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
524 "USE BCOPY: "));
525 if (nxge_tx_tiny_pack) {
526 uint32_t blst =
527 TXDMA_DESC_NEXT_INDEX(i, -1,
528 tx_ring_p->tx_wrap_mask);
529 NXGE_DEBUG_MSG((nxgep, TX_CTL,
530 "==> nxge_start(5): pack"));
531 if ((pack_len <= bcopy_thresh) &&
532 (last_bidx == blst)) {
533 NXGE_DEBUG_MSG((nxgep, TX_CTL,
534 "==> nxge_start: pack(6) "
535 "(pkt_len %d pack_len %d)",
536 pkt_len, pack_len));
537 i = blst;
538 tx_desc_p = &tx_desc_ring_vp[i];
539 #ifdef NXGE_DEBUG
540 tx_desc_pp = &tx_desc_ring_pp[i];
541 #endif
542 tx_msg_p = &tx_msg_ring[i];
543 boff = pack_len - len;
544 ngathers--;
545 } else if (pack_len > bcopy_thresh &&
546 header_set) {
547 pack_len = len;
548 boff = 0;
549 bcopy_thresh = nxge_bcopy_thresh;
550 NXGE_DEBUG_MSG((nxgep, TX_CTL,
551 "==> nxge_start(7): > max NEW "
552 "bcopy thresh %d "
553 "pkt_len %d pack_len %d(next)",
554 bcopy_thresh,
555 pkt_len, pack_len));
556 }
557 last_bidx = i;
558 }
559 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
560 if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
561 hdrp = (p_tx_pkt_header_t)kaddr;
562 header_set = B_TRUE;
563 NXGE_DEBUG_MSG((nxgep, TX_CTL,
564 "==> nxge_start(7_x2): "
565 "pkt_len %d pack_len %d (new hdrp $%p)",
566 pkt_len, pack_len, hdrp));
567 }
568 tx_msg_p->flags.dma_type = USE_BCOPY;
569 kaddr += boff;
570 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
571 "USE BCOPY: before bcopy "
572 "DESC IOADDR $%p entry %d "
573 "bcopy packets %d "
574 "bcopy kaddr $%p "
575 "bcopy ioaddr (SAD) $%p "
576 "bcopy clen %d "
577 "bcopy boff %d",
578 DMA_COMMON_IOADDR(desc_area), i,
579 tdc_stats->tx_hdr_pkts,
580 kaddr,
581 dma_ioaddr,
582 clen,
583 boff));
584 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
585 "1USE BCOPY: "));
586 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
587 "2USE BCOPY: "));
588 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
589 "last USE BCOPY: copy from b_rptr $%p "
590 "to KADDR $%p (len %d offset %d",
591 b_rptr, kaddr, len, boff));
592
593 bcopy(b_rptr, kaddr, len);
594
595 #ifdef NXGE_DEBUG
596 dump_len = (len > 128) ? 128: len;
597 NXGE_DEBUG_MSG((nxgep, TX_CTL,
598 "==> nxge_start: dump packets "
599 "(After BCOPY len %d)"
600 "(b_rptr $%p): %s", len, nmp->b_rptr,
601 nxge_dump_packet((char *)nmp->b_rptr,
602 dump_len)));
603 #endif
604
605 dma_handle = tx_msg_p->buf_dma_handle;
606 dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
607 (void) ddi_dma_sync(dma_handle,
608 i * nxge_bcopy_thresh, nxge_bcopy_thresh,
609 DDI_DMA_SYNC_FORDEV);
610 clen = len + boff;
611 tdc_stats->tx_hdr_pkts++;
612 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
613 "USE BCOPY: "
614 "DESC IOADDR $%p entry %d "
615 "bcopy packets %d "
616 "bcopy kaddr $%p "
617 "bcopy ioaddr (SAD) $%p "
618 "bcopy clen %d "
619 "bcopy boff %d",
620 DMA_COMMON_IOADDR(desc_area),
621 i,
622 tdc_stats->tx_hdr_pkts,
623 kaddr,
624 dma_ioaddr,
625 clen,
626 boff));
627 } else {
628 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
629 "USE DVMA: len %d", len));
630 tx_msg_p->flags.dma_type = USE_DMA;
631 dma_flags = DDI_DMA_WRITE;
632 if (len < nxge_dma_stream_thresh) {
633 dma_flags |= DDI_DMA_CONSISTENT;
634 } else {
635 dma_flags |= DDI_DMA_STREAMING;
636 }
637
638 dma_handle = tx_msg_p->dma_handle;
639 dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL,
640 (caddr_t)b_rptr, len, dma_flags,
641 DDI_DMA_DONTWAIT, NULL,
642 &dma_cookie, &ncookies);
643 if (dma_status == DDI_DMA_MAPPED) {
644 dma_ioaddr = dma_cookie.dmac_laddress;
645 len = (int)dma_cookie.dmac_size;
646 clen = (uint32_t)dma_cookie.dmac_size;
647 NXGE_DEBUG_MSG((nxgep, TX_CTL,
648 "==> nxge_start(12_1): "
649 "USE DVMA: len %d clen %d "
650 "ngathers %d",
651 len, clen,
652 ngathers));
653 npi_desc_handle.regp = (uint64_t)tx_desc_p;
654 while (ncookies > 1) {
655 ngathers++;
656 /*
657 * this is the fix for multiple
658 * cookies, which are basically
659 * a descriptor entry, we don't set
660 * SOP bit as well as related fields
661 */
662
663 (void) npi_txdma_desc_gather_set(
664 npi_desc_handle,
665 &tx_desc,
666 (ngathers -1),
667 mark_mode,
668 ngathers,
669 dma_ioaddr,
670 clen);
671
672 tx_msg_p->tx_msg_size = clen;
673 NXGE_DEBUG_MSG((nxgep, TX_CTL,
674 "==> nxge_start: DMA "
675 "ncookie %d "
676 "ngathers %d "
677 "dma_ioaddr $%p len %d"
678 "desc $%p descp $%p (%d)",
679 ncookies,
680 ngathers,
681 dma_ioaddr, clen,
682 *tx_desc_p, tx_desc_p, i));
683
684 ddi_dma_nextcookie(dma_handle,
685 &dma_cookie);
686 dma_ioaddr =
687 dma_cookie.dmac_laddress;
688
689 len = (int)dma_cookie.dmac_size;
690 clen = (uint32_t)dma_cookie.dmac_size;
691 NXGE_DEBUG_MSG((nxgep, TX_CTL,
692 "==> nxge_start(12_2): "
693 "USE DVMA: len %d clen %d ",
694 len, clen));
695
696 i = TXDMA_DESC_NEXT_INDEX(i, 1,
697 tx_ring_p->tx_wrap_mask);
698 tx_desc_p = &tx_desc_ring_vp[i];
699
700 npi_desc_handle.regp =
701 (uint64_t)tx_desc_p;
702 tx_msg_p = &tx_msg_ring[i];
703 tx_msg_p->flags.dma_type = USE_NONE;
704 tx_desc.value = 0;
705
706 ncookies--;
707 }
708 tdc_stats->tx_ddi_pkts++;
709 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
710 "DMA: ddi packets %d",
711 tdc_stats->tx_ddi_pkts));
712 } else {
713 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
714 "dma mapping failed for %d "
715 "bytes addr $%p flags %x (%d)",
716 len, b_rptr, status, status));
717 good_packet = B_FALSE;
718 tdc_stats->tx_dma_bind_fail++;
719 tx_msg_p->flags.dma_type = USE_NONE;
720 if (is_lso) {
721 mp = nmp;
722 goto nxge_start_fail_lso;
723 } else {
724 status = 1;
725 goto nxge_start_fail2;
726 }
727 }
728 } /* ddi dvma */
729
730 if (is_lso) {
731 nmp_lso_save = nmp;
732 }
733 nmp = nmp->b_cont;
734 nxge_start_control_header_only:
735 npi_desc_handle.regp = (uint64_t)tx_desc_p;
736 ngathers++;
737
738 if (ngathers == 1) {
739 #ifdef NXGE_DEBUG
740 save_desc_p = &sop_tx_desc;
741 #endif
742 sop_tx_desc_p = &sop_tx_desc;
743 sop_tx_desc_p->value = 0;
744 sop_tx_desc_p->bits.hdw.tr_len = clen;
745 sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
746 sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
747 } else {
748 #ifdef NXGE_DEBUG
749 save_desc_p = &tx_desc;
750 #endif
751 tmp_desc_p = &tx_desc;
752 tmp_desc_p->value = 0;
753 tmp_desc_p->bits.hdw.tr_len = clen;
754 tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
755 tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
756
757 tx_desc_p->value = tmp_desc_p->value;
758 }
759
760 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
761 "Desc_entry %d ngathers %d "
762 "desc_vp $%p tx_desc_p $%p "
763 "len %d clen %d pkt_len %d pack_len %d nmblks %d "
764 "dma_ioaddr (SAD) $%p mark %d",
765 i, ngathers,
766 tx_desc_ring_vp, tx_desc_p,
767 len, clen, pkt_len, pack_len, nmblks,
768 dma_ioaddr, mark_mode));
769
770 #ifdef NXGE_DEBUG
771 npi_desc_handle.nxgep = nxgep;
772 npi_desc_handle.function.function = nxgep->function_num;
773 npi_desc_handle.function.instance = nxgep->instance;
774 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
775 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
776 TX_PKT_DESC_TR_LEN_SHIFT);
777
778
779 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
780 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
781 "mark %d sop %d\n",
782 save_desc_p->value,
783 sad,
784 save_desc_p->bits.hdw.tr_len,
785 xfer_len,
786 save_desc_p->bits.hdw.num_ptr,
787 save_desc_p->bits.hdw.mark,
788 save_desc_p->bits.hdw.sop));
789
790 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
791 #endif
792
793 tx_msg_p->tx_msg_size = clen;
794 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
795 if (ngathers > nxge_tx_max_gathers) {
796 good_packet = B_FALSE;
797 mac_hcksum_get(mp, &start_offset,
798 &stuff_offset, &end_offset, &value,
799 &cksum_flags);
800
801 NXGE_DEBUG_MSG((NULL, TX_CTL,
802 "==> nxge_start(14): pull msg - "
803 "len %d pkt_len %d ngathers %d",
804 len, pkt_len, ngathers));
805
806 /*
807 * Just give up on this packet.
808 */
809 if (is_lso) {
810 mp = nmp_lso_save;
811 goto nxge_start_fail_lso;
812 }
813 status = 0;
814 goto nxge_start_fail2;
815 }
816 } /* while (nmp) */
817
818 tx_msg_p->tx_message = mp;
819 tx_desc_p = &tx_desc_ring_vp[sop_index];
820 npi_desc_handle.regp = (uint64_t)tx_desc_p;
821
822 pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
823 pkthdrp->reserved = 0;
824 hdrp->value = 0;
825 bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t));
826
827 if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
828 tdc_stats->tx_jumbo_pkts++;
829 }
830
831 min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2));
832 if (pkt_len < min_len) {
833 /* Assume we use bcopy to premapped buffers */
834 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
835 NXGE_DEBUG_MSG((NULL, TX_CTL,
836 "==> nxge_start(14-1): < (msg_min + 16)"
837 "len %d pkt_len %d min_len %d bzero %d ngathers %d",
838 len, pkt_len, min_len, (min_len - pkt_len), ngathers));
839 bzero((kaddr + pkt_len), (min_len - pkt_len));
840 pkt_len = tx_msg_p->tx_msg_size = min_len;
841
842 sop_tx_desc_p->bits.hdw.tr_len = min_len;
843
844 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
845 tx_desc_p->value = sop_tx_desc_p->value;
846
847 NXGE_DEBUG_MSG((NULL, TX_CTL,
848 "==> nxge_start(14-2): < msg_min - "
849 "len %d pkt_len %d min_len %d ngathers %d",
850 len, pkt_len, min_len, ngathers));
851 }
852
853 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
854 cksum_flags));
855 {
856 uint64_t tmp_len;
857
858 /* pkt_len already includes 16 + paddings!! */
859 /* Update the control header length */
860 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
861 tmp_len = hdrp->value |
862 (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
863
864 NXGE_DEBUG_MSG((nxgep, TX_CTL,
865 "==> nxge_start(15_x1): setting SOP "
866 "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
867 "0x%llx hdrp->value 0x%llx",
868 tot_xfer_len, tot_xfer_len, pkt_len,
869 tmp_len, hdrp->value));
870 #if defined(_BIG_ENDIAN)
871 hdrp->value = ddi_swap64(tmp_len);
872 #else
873 hdrp->value = tmp_len;
874 #endif
875 NXGE_DEBUG_MSG((nxgep,
876 TX_CTL, "==> nxge_start(15_x2): setting SOP "
877 "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
878 "tmp_len 0x%llx hdrp->value 0x%llx",
879 tot_xfer_len, pkt_len,
880 tmp_len, hdrp->value));
881 }
882
883 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
884 "wr_index %d "
885 "tot_xfer_len (%d) pkt_len %d npads %d",
886 sop_index,
887 tot_xfer_len, pkt_len,
888 npads));
889
890 sop_tx_desc_p->bits.hdw.sop = 1;
891 sop_tx_desc_p->bits.hdw.mark = mark_mode;
892 sop_tx_desc_p->bits.hdw.num_ptr = ngathers;
893
894 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
895
896 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));
897
898 #ifdef NXGE_DEBUG
899 npi_desc_handle.nxgep = nxgep;
900 npi_desc_handle.function.function = nxgep->function_num;
901 npi_desc_handle.function.instance = nxgep->instance;
902
903 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
904 "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
905 save_desc_p->value,
906 sad,
907 save_desc_p->bits.hdw.tr_len,
908 xfer_len,
909 save_desc_p->bits.hdw.num_ptr,
910 save_desc_p->bits.hdw.mark,
911 save_desc_p->bits.hdw.sop));
912 (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);
913
914 dump_len = (pkt_len > 128) ? 128: pkt_len;
915 NXGE_DEBUG_MSG((nxgep, TX_CTL,
916 "==> nxge_start: dump packets(17) (after sop set, len "
917 " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
918 "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
919 (char *)hdrp,
920 nxge_dump_packet((char *)hdrp, dump_len)));
921 NXGE_DEBUG_MSG((nxgep, TX_CTL,
922 "==> nxge_start(18): TX desc sync: sop_index %d",
923 sop_index));
924 #endif
925
926 if ((ngathers == 1) || tx_ring_p->wr_index < i) {
927 (void) ddi_dma_sync(tx_desc_dma_handle,
928 sop_index * sizeof (tx_desc_t),
929 ngathers * sizeof (tx_desc_t),
930 DDI_DMA_SYNC_FORDEV);
931
932 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
933 "cs_off = 0x%02X cs_s_off = 0x%02X "
934 "pkt_len %d ngathers %d sop_index %d\n",
935 stuff_offset, start_offset,
936 pkt_len, ngathers, sop_index));
937 } else { /* more than one descriptor and wrap around */
938 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
939 (void) ddi_dma_sync(tx_desc_dma_handle,
940 sop_index * sizeof (tx_desc_t),
941 nsdescs * sizeof (tx_desc_t),
942 DDI_DMA_SYNC_FORDEV);
943 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
944 "cs_off = 0x%02X cs_s_off = 0x%02X "
945 "pkt_len %d ngathers %d sop_index %d\n",
946 stuff_offset, start_offset,
947 pkt_len, ngathers, sop_index));
948
949 (void) ddi_dma_sync(tx_desc_dma_handle,
950 0,
951 (ngathers - nsdescs) * sizeof (tx_desc_t),
952 DDI_DMA_SYNC_FORDEV);
953 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
954 "cs_off = 0x%02X cs_s_off = 0x%02X "
955 "pkt_len %d ngathers %d sop_index %d\n",
956 stuff_offset, start_offset,
957 pkt_len, ngathers, sop_index));
958 }
959
960 tail_index = tx_ring_p->wr_index;
961 tail_wrap = tx_ring_p->wr_index_wrap;
962
963 tx_ring_p->wr_index = i;
964 if (tx_ring_p->wr_index <= tail_index) {
965 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
966 B_FALSE : B_TRUE);
967 }
968
969 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: "
970 "channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
971 tx_ring_p->tdc,
972 tx_ring_p->wr_index,
973 tx_ring_p->wr_index_wrap,
974 ngathers,
975 tx_ring_p->descs_pending));
976
977 if (is_lso) {
978 lso_ngathers += ngathers;
979 if (mp_chain != NULL) {
980 mp = mp_chain;
981 mp_chain = mp_chain->b_next;
982 mp->b_next = NULL;
983 if (nxge_lso_kick_cnt == lso_ngathers) {
984 tx_ring_p->descs_pending += lso_ngathers;
985 {
986 tx_ring_kick_t kick;
987
988 kick.value = 0;
989 kick.bits.ldw.wrap =
990 tx_ring_p->wr_index_wrap;
991 kick.bits.ldw.tail =
992 (uint16_t)tx_ring_p->wr_index;
993
994 /* Kick the Transmit kick register */
995 TXDMA_REG_WRITE64(
996 NXGE_DEV_NPI_HANDLE(nxgep),
997 TX_RING_KICK_REG,
998 (uint8_t)tx_ring_p->tdc,
999 kick.value);
1000 tdc_stats->tx_starts++;
1001
1002 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1003 "==> nxge_start: more LSO: "
1004 "LSO_CNT %d",
1005 lso_ngathers));
1006 }
1007 lso_ngathers = 0;
1008 ngathers = 0;
1009 cur_index_lso = sop_index = tx_ring_p->wr_index;
1010 lso_tail_wrap = tx_ring_p->wr_index_wrap;
1011 }
1012 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1013 "==> nxge_start: lso again: "
1014 "lso_gathers %d ngathers %d cur_index_lso %d "
1015 "wr_index %d sop_index %d",
1016 lso_ngathers, ngathers, cur_index_lso,
1017 tx_ring_p->wr_index, sop_index));
1018
1019 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1020 "==> nxge_start: next : count %d",
1021 lso_ngathers));
1022 lso_again = B_TRUE;
1023 goto start_again;
1024 }
1025 ngathers = lso_ngathers;
1026 }
1027
1028 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: "));
1029
1030 {
1031 tx_ring_kick_t kick;
1032
1033 kick.value = 0;
1034 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap;
1035 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index;
1036
1037 /* Kick start the Transmit kick register */
1038 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep),
1039 TX_RING_KICK_REG,
1040 (uint8_t)tx_ring_p->tdc,
1041 kick.value);
1042 }
1043
1044 tx_ring_p->descs_pending += ngathers;
1045 tdc_stats->tx_starts++;
1046
1047 if (isLDOMservice(nxgep)) {
1048 tx_ring_p->tx_ring_busy = B_FALSE;
1049 if (tx_ring_p->tx_ring_offline) {
1050 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1051 NXGE_TX_RING_OFFLINED);
1052 }
1053 }
1054
1055 MUTEX_EXIT(&tx_ring_p->lock);
1056
1057 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1058 return (status);
1059
1060 nxge_start_fail_lso:
1061 status = 0;
1062 good_packet = B_FALSE;
1063 if (mp != NULL)
1064 freemsg(mp);
1065 if (mp_chain != NULL)
1066 freemsgchain(mp_chain);
1067
1068 if (!lso_again && !ngathers) {
1069 if (isLDOMservice(nxgep)) {
1070 tx_ring_p->tx_ring_busy = B_FALSE;
1071 if (tx_ring_p->tx_ring_offline) {
1072 (void) atomic_swap_32(
1073 &tx_ring_p->tx_ring_offline,
1074 NXGE_TX_RING_OFFLINED);
1075 }
1076 }
1077
1078 MUTEX_EXIT(&tx_ring_p->lock);
1079 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1080 "==> nxge_start: lso exit (nothing changed)"));
1081 goto nxge_start_fail1;
1082 }
1083
1084 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1085 "==> nxge_start (channel %d): before lso "
1086 "lso_gathers %d ngathers %d cur_index_lso %d "
1087 "wr_index %d sop_index %d lso_again %d",
1088 tx_ring_p->tdc,
1089 lso_ngathers, ngathers, cur_index_lso,
1090 tx_ring_p->wr_index, sop_index, lso_again));
1091
1092 if (lso_again) {
1093 lso_ngathers += ngathers;
1094 ngathers = lso_ngathers;
1095 sop_index = cur_index_lso;
1096 tx_ring_p->wr_index = sop_index;
1097 tx_ring_p->wr_index_wrap = lso_tail_wrap;
1098 }
1099
1100 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1101 "==> nxge_start (channel %d): after lso "
1102 "lso_gathers %d ngathers %d cur_index_lso %d "
1103 "wr_index %d sop_index %d lso_again %d",
1104 tx_ring_p->tdc,
1105 lso_ngathers, ngathers, cur_index_lso,
1106 tx_ring_p->wr_index, sop_index, lso_again));
1107
1108 nxge_start_fail2:
1109 if (good_packet == B_FALSE) {
1110 cur_index = sop_index;
1111 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up"));
1112 for (i = 0; i < ngathers; i++) {
1113 tx_desc_p = &tx_desc_ring_vp[cur_index];
1114 npi_handle.regp = (uint64_t)tx_desc_p;
1115 tx_msg_p = &tx_msg_ring[cur_index];
1116 (void) npi_txdma_desc_set_zero(npi_handle, 1);
1117 if (tx_msg_p->flags.dma_type == USE_DVMA) {
1118 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1119 "tx_desc_p = %X index = %d",
1120 tx_desc_p, tx_ring_p->rd_index));
1121 (void) dvma_unload(tx_msg_p->dvma_handle,
1122 0, -1);
1123 tx_msg_p->dvma_handle = NULL;
1124 if (tx_ring_p->dvma_wr_index ==
1125 tx_ring_p->dvma_wrap_mask)
1126 tx_ring_p->dvma_wr_index = 0;
1127 else
1128 tx_ring_p->dvma_wr_index++;
1129 tx_ring_p->dvma_pending--;
1130 } else if (tx_msg_p->flags.dma_type == USE_DMA) {
1131 if (ddi_dma_unbind_handle(
1132 tx_msg_p->dma_handle)) {
1133 cmn_err(CE_WARN, "!nxge_start: "
1134 "ddi_dma_unbind_handle failed");
1135 }
1136 }
1137 tx_msg_p->flags.dma_type = USE_NONE;
1138 cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
1139 tx_ring_p->tx_wrap_mask);
1140
1141 }
1142 }
1143
1144 if (isLDOMservice(nxgep)) {
1145 tx_ring_p->tx_ring_busy = B_FALSE;
1146 if (tx_ring_p->tx_ring_offline) {
1147 (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1148 NXGE_TX_RING_OFFLINED);
1149 }
1150 }
1151
1152 MUTEX_EXIT(&tx_ring_p->lock);
1153
1154 nxge_start_fail1:
1155 /* Add FMA to check the access handle nxge_hregh */
1156
1157 NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1158 return (status);
1159 }
1160
1161 /* Software LSO starts here */
1162 static void
nxge_hcksum_retrieve(mblk_t * mp,uint32_t * start,uint32_t * stuff,uint32_t * end,uint32_t * value,uint32_t * flags)1163 nxge_hcksum_retrieve(mblk_t *mp,
1164 uint32_t *start, uint32_t *stuff, uint32_t *end,
1165 uint32_t *value, uint32_t *flags)
1166 {
1167 if (mp->b_datap->db_type == M_DATA) {
1168 if (flags != NULL) {
1169 *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM |
1170 HCK_PARTIALCKSUM | HCK_FULLCKSUM |
1171 HCK_FULLCKSUM_OK);
1172 if ((*flags & (HCK_PARTIALCKSUM |
1173 HCK_FULLCKSUM)) != 0) {
1174 if (value != NULL)
1175 *value = (uint32_t)DB_CKSUM16(mp);
1176 if ((*flags & HCK_PARTIALCKSUM) != 0) {
1177 if (start != NULL)
1178 *start =
1179 (uint32_t)DB_CKSUMSTART(mp);
1180 if (stuff != NULL)
1181 *stuff =
1182 (uint32_t)DB_CKSUMSTUFF(mp);
1183 if (end != NULL)
1184 *end =
1185 (uint32_t)DB_CKSUMEND(mp);
1186 }
1187 }
1188 }
1189 }
1190 }
1191
1192 static void
nxge_lso_info_get(mblk_t * mp,uint32_t * mss,uint32_t * flags)1193 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
1194 {
1195 ASSERT(DB_TYPE(mp) == M_DATA);
1196
1197 if (mss == NULL || flags == NULL)
1198 return;
1199
1200 *mss = 0;
1201 if (flags != NULL) {
1202 *flags = DB_CKSUMFLAGS(mp) & HW_LSO;
1203 if (*flags != 0) {
1204 *mss = (uint32_t)DB_LSOMSS(mp);
1205 }
1206 NXGE_DEBUG_MSG((NULL, TX_CTL,
1207 "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x",
1208 *mss, *flags));
1209 }
1210
1211 NXGE_DEBUG_MSG((NULL, TX_CTL,
1212 "<== nxge_lso_info_get: mss %d", *mss));
1213 }
1214
1215 /*
1216 * Do Soft LSO on the oversized packet.
1217 *
1218 * 1. Create a chain of message for headers.
1219 * 2. Fill up header messages with proper information.
1220 * 3. Copy Eithernet, IP, and TCP headers from the original message to
1221 * each new message with necessary adjustments.
1222 * * Unchange the ethernet header for DIX frames. (by default)
1223 * * IP Total Length field is updated to MSS or less(only for the last one).
1224 * * IP Identification value is incremented by one for each packet.
1225 * * TCP sequence Number is recalculated according to the payload length.
1226 * * Set FIN and/or PSH flags for the *last* packet if applied.
1227 * * TCP partial Checksum
1228 * 4. Update LSO information in the first message header.
1229 * 5. Release the original message header.
1230 */
1231 static mblk_t *
nxge_do_softlso(mblk_t * mp,uint32_t mss)1232 nxge_do_softlso(mblk_t *mp, uint32_t mss)
1233 {
1234 uint32_t hckflags;
1235 int pktlen;
1236 int hdrlen;
1237 int segnum;
1238 int i;
1239 struct ether_vlan_header *evh;
1240 int ehlen, iphlen, tcphlen;
1241 struct ip *oiph, *niph;
1242 struct tcphdr *otcph, *ntcph;
1243 int available, len, left;
1244 uint16_t ip_id;
1245 uint32_t tcp_seq;
1246 #ifdef __sparc
1247 uint32_t tcp_seq_tmp;
1248 #endif
1249 mblk_t *datamp;
1250 uchar_t *rptr;
1251 mblk_t *nmp;
1252 mblk_t *cmp;
1253 mblk_t *mp_chain;
1254 boolean_t do_cleanup = B_FALSE;
1255 t_uscalar_t start_offset = 0;
1256 t_uscalar_t stuff_offset = 0;
1257 t_uscalar_t value = 0;
1258 uint16_t l4_len;
1259 ipaddr_t src, dst;
1260 uint32_t cksum, sum, l4cksum;
1261
1262 NXGE_DEBUG_MSG((NULL, TX_CTL,
1263 "==> nxge_do_softlso"));
1264 /*
1265 * check the length of LSO packet payload and calculate the number of
1266 * segments to be generated.
1267 */
1268 pktlen = msgsize(mp);
1269 evh = (struct ether_vlan_header *)mp->b_rptr;
1270
1271 /* VLAN? */
1272 if (evh->ether_tpid == htons(ETHERTYPE_VLAN))
1273 ehlen = sizeof (struct ether_vlan_header);
1274 else
1275 ehlen = sizeof (struct ether_header);
1276 oiph = (struct ip *)(mp->b_rptr + ehlen);
1277 iphlen = oiph->ip_hl * 4;
1278 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1279 tcphlen = otcph->th_off * 4;
1280
1281 l4_len = pktlen - ehlen - iphlen;
1282
1283 NXGE_DEBUG_MSG((NULL, TX_CTL,
1284 "==> nxge_do_softlso: mss %d oiph $%p "
1285 "original ip_sum oiph->ip_sum 0x%x "
1286 "original tcp_sum otcph->th_sum 0x%x "
1287 "oiph->ip_len %d pktlen %d ehlen %d "
1288 "l4_len %d (0x%x) ip_len - iphlen %d ",
1289 mss,
1290 oiph,
1291 oiph->ip_sum,
1292 otcph->th_sum,
1293 ntohs(oiph->ip_len), pktlen,
1294 ehlen,
1295 l4_len,
1296 l4_len,
1297 ntohs(oiph->ip_len) - iphlen));
1298
1299 /* IPv4 + TCP */
1300 if (!(oiph->ip_v == IPV4_VERSION)) {
1301 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1302 "<== nxge_do_softlso: not IPV4 "
1303 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1304 ntohs(oiph->ip_len), pktlen, ehlen,
1305 tcphlen));
1306 freemsg(mp);
1307 return (NULL);
1308 }
1309
1310 if (!(oiph->ip_p == IPPROTO_TCP)) {
1311 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1312 "<== nxge_do_softlso: not TCP "
1313 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1314 ntohs(oiph->ip_len), pktlen, ehlen,
1315 tcphlen));
1316 freemsg(mp);
1317 return (NULL);
1318 }
1319
1320 if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) {
1321 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1322 "<== nxge_do_softlso: len not matched "
1323 "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1324 ntohs(oiph->ip_len), pktlen, ehlen,
1325 tcphlen));
1326 freemsg(mp);
1327 return (NULL);
1328 }
1329
1330 otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1331 tcphlen = otcph->th_off * 4;
1332
1333 /* TCP flags can not include URG, RST, or SYN */
1334 VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0);
1335
1336 hdrlen = ehlen + iphlen + tcphlen;
1337
1338 VERIFY(MBLKL(mp) >= hdrlen);
1339
1340 if (MBLKL(mp) > hdrlen) {
1341 datamp = mp;
1342 rptr = mp->b_rptr + hdrlen;
1343 } else { /* = */
1344 datamp = mp->b_cont;
1345 rptr = datamp->b_rptr;
1346 }
1347
1348 NXGE_DEBUG_MSG((NULL, TX_CTL,
1349 "nxge_do_softlso: otcph $%p pktlen: %d, "
1350 "hdrlen %d ehlen %d iphlen %d tcphlen %d "
1351 "mblkl(mp): %d, mblkl(datamp): %d",
1352 otcph,
1353 pktlen, hdrlen, ehlen, iphlen, tcphlen,
1354 (int)MBLKL(mp), (int)MBLKL(datamp)));
1355
1356 hckflags = 0;
1357 nxge_hcksum_retrieve(mp,
1358 &start_offset, &stuff_offset, &value, NULL, &hckflags);
1359
1360 dst = oiph->ip_dst.s_addr;
1361 src = oiph->ip_src.s_addr;
1362
1363 cksum = (dst >> 16) + (dst & 0xFFFF) +
1364 (src >> 16) + (src & 0xFFFF);
1365 l4cksum = cksum + IP_TCP_CSUM_COMP;
1366
1367 sum = l4_len + l4cksum;
1368 sum = (sum & 0xFFFF) + (sum >> 16);
1369
1370 NXGE_DEBUG_MSG((NULL, TX_CTL,
1371 "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x "
1372 "hckflags 0x%x start_offset %d stuff_offset %d "
1373 "value (original) 0x%x th_sum 0x%x "
1374 "pktlen %d l4_len %d (0x%x) "
1375 "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s",
1376 dst, src,
1377 (sum & 0xffff), (~sum & 0xffff),
1378 hckflags, start_offset, stuff_offset,
1379 value, otcph->th_sum,
1380 pktlen,
1381 l4_len,
1382 l4_len,
1383 ntohs(oiph->ip_len) - (int)MBLKL(mp),
1384 (int)MBLKL(datamp),
1385 nxge_dump_packet((char *)evh, 12)));
1386
1387 /*
1388 * Start to process.
1389 */
1390 available = pktlen - hdrlen;
1391 segnum = (available - 1) / mss + 1;
1392
1393 NXGE_DEBUG_MSG((NULL, TX_CTL,
1394 "==> nxge_do_softlso: pktlen %d "
1395 "MBLKL(mp): %d, MBLKL(datamp): %d "
1396 "available %d mss %d segnum %d",
1397 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp),
1398 available,
1399 mss,
1400 segnum));
1401
1402 VERIFY(segnum >= 2);
1403
1404 /*
1405 * Try to pre-allocate all header messages
1406 */
1407 mp_chain = NULL;
1408 for (i = 0; i < segnum; i++) {
1409 if ((nmp = allocb(hdrlen, 0)) == NULL) {
1410 /* Clean up the mp_chain */
1411 while (mp_chain != NULL) {
1412 nmp = mp_chain;
1413 mp_chain = mp_chain->b_next;
1414 freemsg(nmp);
1415 }
1416 NXGE_DEBUG_MSG((NULL, TX_CTL,
1417 "<== nxge_do_softlso: "
1418 "Could not allocate enough messages for headers!"));
1419 freemsg(mp);
1420 return (NULL);
1421 }
1422 nmp->b_next = mp_chain;
1423 mp_chain = nmp;
1424
1425 NXGE_DEBUG_MSG((NULL, TX_CTL,
1426 "==> nxge_do_softlso: "
1427 "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p",
1428 mp, nmp, mp_chain, mp_chain->b_next));
1429 }
1430
1431 NXGE_DEBUG_MSG((NULL, TX_CTL,
1432 "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p",
1433 mp, nmp, mp_chain));
1434
1435 /*
1436 * Associate payload with new packets
1437 */
1438 cmp = mp_chain;
1439 left = available;
1440 while (cmp != NULL) {
1441 nmp = dupb(datamp);
1442 if (nmp == NULL) {
1443 do_cleanup = B_TRUE;
1444 NXGE_DEBUG_MSG((NULL, TX_CTL,
1445 "==>nxge_do_softlso: "
1446 "Can not dupb(datamp), have to do clean up"));
1447 goto cleanup_allocated_msgs;
1448 }
1449
1450 NXGE_DEBUG_MSG((NULL, TX_CTL,
1451 "==> nxge_do_softlso: (loop) before mp $%p cmp $%p "
1452 "dupb nmp $%p len %d left %d msd %d ",
1453 mp, cmp, nmp, len, left, mss));
1454
1455 cmp->b_cont = nmp;
1456 nmp->b_rptr = rptr;
1457 len = (left < mss) ? left : mss;
1458 left -= len;
1459
1460 NXGE_DEBUG_MSG((NULL, TX_CTL,
1461 "==> nxge_do_softlso: (loop) after mp $%p cmp $%p "
1462 "dupb nmp $%p len %d left %d mss %d ",
1463 mp, cmp, nmp, len, left, mss));
1464 NXGE_DEBUG_MSG((NULL, TX_CTL,
1465 "nxge_do_softlso: before available: %d, "
1466 "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1467 available, left, len, segnum, (int)MBLKL(nmp)));
1468
1469 len -= MBLKL(nmp);
1470 NXGE_DEBUG_MSG((NULL, TX_CTL,
1471 "nxge_do_softlso: after available: %d, "
1472 "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1473 available, left, len, segnum, (int)MBLKL(nmp)));
1474
1475 while (len > 0) {
1476 mblk_t *mmp = NULL;
1477
1478 NXGE_DEBUG_MSG((NULL, TX_CTL,
1479 "nxge_do_softlso: (4) len > 0 available: %d, "
1480 "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1481 available, left, len, segnum, (int)MBLKL(nmp)));
1482
1483 if (datamp->b_cont != NULL) {
1484 datamp = datamp->b_cont;
1485 rptr = datamp->b_rptr;
1486 mmp = dupb(datamp);
1487 if (mmp == NULL) {
1488 do_cleanup = B_TRUE;
1489 NXGE_DEBUG_MSG((NULL, TX_CTL,
1490 "==> nxge_do_softlso: "
1491 "Can not dupb(datamp) (1), :"
1492 "have to do clean up"));
1493 NXGE_DEBUG_MSG((NULL, TX_CTL,
1494 "==> nxge_do_softlso: "
1495 "available: %d, left: %d, "
1496 "len: %d, MBLKL(nmp): %d",
1497 available, left, len,
1498 (int)MBLKL(nmp)));
1499 goto cleanup_allocated_msgs;
1500 }
1501 } else {
1502 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1503 "==> nxge_do_softlso: "
1504 "(1)available: %d, left: %d, "
1505 "len: %d, MBLKL(nmp): %d",
1506 available, left, len,
1507 (int)MBLKL(nmp)));
1508 cmn_err(CE_PANIC,
1509 "==> nxge_do_softlso: "
1510 "Pointers must have been corrupted!\n"
1511 "datamp: $%p, nmp: $%p, rptr: $%p",
1512 (void *)datamp,
1513 (void *)nmp,
1514 (void *)rptr);
1515 }
1516 nmp->b_cont = mmp;
1517 nmp = mmp;
1518 len -= MBLKL(nmp);
1519 }
1520 if (len < 0) {
1521 nmp->b_wptr += len;
1522 rptr = nmp->b_wptr;
1523 NXGE_DEBUG_MSG((NULL, TX_CTL,
1524 "(5) len < 0 (less than 0)"
1525 "available: %d, left: %d, len: %d, MBLKL(nmp): %d",
1526 available, left, len, (int)MBLKL(nmp)));
1527
1528 } else if (len == 0) {
1529 if (datamp->b_cont != NULL) {
1530 NXGE_DEBUG_MSG((NULL, TX_CTL,
1531 "(5) len == 0"
1532 "available: %d, left: %d, len: %d, "
1533 "MBLKL(nmp): %d",
1534 available, left, len, (int)MBLKL(nmp)));
1535 datamp = datamp->b_cont;
1536 rptr = datamp->b_rptr;
1537 } else {
1538 NXGE_DEBUG_MSG((NULL, TX_CTL,
1539 "(6)available b_cont == NULL : %d, "
1540 "left: %d, len: %d, MBLKL(nmp): %d",
1541 available, left, len, (int)MBLKL(nmp)));
1542
1543 VERIFY(cmp->b_next == NULL);
1544 VERIFY(left == 0);
1545 break; /* Done! */
1546 }
1547 }
1548 cmp = cmp->b_next;
1549
1550 NXGE_DEBUG_MSG((NULL, TX_CTL,
1551 "(7) do_softlso: "
1552 "next mp in mp_chain available len != 0 : %d, "
1553 "left: %d, len: %d, MBLKL(nmp): %d",
1554 available, left, len, (int)MBLKL(nmp)));
1555 }
1556
1557 /*
1558 * From now, start to fill up all headers for the first message
1559 * Hardware checksum flags need to be updated separately for FULLCKSUM
1560 * and PARTIALCKSUM cases. For full checksum, copy the original flags
1561 * into every new packet is enough. But for HCK_PARTIALCKSUM, all
1562 * required fields need to be updated properly.
1563 */
1564 nmp = mp_chain;
1565 bcopy(mp->b_rptr, nmp->b_rptr, hdrlen);
1566 nmp->b_wptr = nmp->b_rptr + hdrlen;
1567 niph = (struct ip *)(nmp->b_rptr + ehlen);
1568 niph->ip_len = htons(mss + iphlen + tcphlen);
1569 ip_id = ntohs(niph->ip_id);
1570 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1571 #ifdef __sparc
1572 bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4);
1573 tcp_seq = ntohl(tcp_seq_tmp);
1574 #else
1575 tcp_seq = ntohl(ntcph->th_seq);
1576 #endif
1577
1578 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST);
1579
1580 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1581 DB_CKSUMSTART(nmp) = start_offset;
1582 DB_CKSUMSTUFF(nmp) = stuff_offset;
1583
1584 /* calculate IP checksum and TCP pseudo header checksum */
1585 niph->ip_sum = 0;
1586 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1587
1588 l4_len = mss + tcphlen;
1589 sum = htons(l4_len) + l4cksum;
1590 sum = (sum & 0xFFFF) + (sum >> 16);
1591 ntcph->th_sum = (sum & 0xffff);
1592
1593 NXGE_DEBUG_MSG((NULL, TX_CTL,
1594 "==> nxge_do_softlso: first mp $%p (mp_chain $%p) "
1595 "mss %d pktlen %d l4_len %d (0x%x) "
1596 "MBLKL(mp): %d, MBLKL(datamp): %d "
1597 "ip_sum 0x%x "
1598 "th_sum 0x%x sum 0x%x ) "
1599 "dump first ip->tcp %s",
1600 nmp, mp_chain,
1601 mss,
1602 pktlen,
1603 l4_len,
1604 l4_len,
1605 (int)MBLKL(mp), (int)MBLKL(datamp),
1606 niph->ip_sum,
1607 ntcph->th_sum,
1608 sum,
1609 nxge_dump_packet((char *)niph, 52)));
1610
1611 cmp = nmp;
1612 while ((nmp = nmp->b_next)->b_next != NULL) {
1613 NXGE_DEBUG_MSG((NULL, TX_CTL,
1614 "==>nxge_do_softlso: middle l4_len %d ", l4_len));
1615 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1616 nmp->b_wptr = nmp->b_rptr + hdrlen;
1617 niph = (struct ip *)(nmp->b_rptr + ehlen);
1618 niph->ip_id = htons(++ip_id);
1619 niph->ip_len = htons(mss + iphlen + tcphlen);
1620 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1621 tcp_seq += mss;
1622
1623 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG);
1624
1625 #ifdef __sparc
1626 tcp_seq_tmp = htonl(tcp_seq);
1627 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1628 #else
1629 ntcph->th_seq = htonl(tcp_seq);
1630 #endif
1631 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1632 DB_CKSUMSTART(nmp) = start_offset;
1633 DB_CKSUMSTUFF(nmp) = stuff_offset;
1634
1635 /* calculate IP checksum and TCP pseudo header checksum */
1636 niph->ip_sum = 0;
1637 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1638 ntcph->th_sum = (sum & 0xffff);
1639
1640 NXGE_DEBUG_MSG((NULL, TX_CTL,
1641 "==> nxge_do_softlso: middle ip_sum 0x%x "
1642 "th_sum 0x%x "
1643 " mp $%p (mp_chain $%p) pktlen %d "
1644 "MBLKL(mp): %d, MBLKL(datamp): %d ",
1645 niph->ip_sum,
1646 ntcph->th_sum,
1647 nmp, mp_chain,
1648 pktlen, (int)MBLKL(mp), (int)MBLKL(datamp)));
1649 }
1650
1651 /* Last segment */
1652 /*
1653 * Set FIN and/or PSH flags if present only in the last packet.
1654 * The ip_len could be different from prior packets.
1655 */
1656 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1657 nmp->b_wptr = nmp->b_rptr + hdrlen;
1658 niph = (struct ip *)(nmp->b_rptr + ehlen);
1659 niph->ip_id = htons(++ip_id);
1660 niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen);
1661 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1662 tcp_seq += mss;
1663 #ifdef __sparc
1664 tcp_seq_tmp = htonl(tcp_seq);
1665 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1666 #else
1667 ntcph->th_seq = htonl(tcp_seq);
1668 #endif
1669 ntcph->th_flags = (otcph->th_flags & ~TH_URG);
1670
1671 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1672 DB_CKSUMSTART(nmp) = start_offset;
1673 DB_CKSUMSTUFF(nmp) = stuff_offset;
1674
1675 /* calculate IP checksum and TCP pseudo header checksum */
1676 niph->ip_sum = 0;
1677 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1678
1679 l4_len = ntohs(niph->ip_len) - iphlen;
1680 sum = htons(l4_len) + l4cksum;
1681 sum = (sum & 0xFFFF) + (sum >> 16);
1682 ntcph->th_sum = (sum & 0xffff);
1683
1684 NXGE_DEBUG_MSG((NULL, TX_CTL,
1685 "==> nxge_do_softlso: last next "
1686 "niph->ip_sum 0x%x "
1687 "ntcph->th_sum 0x%x sum 0x%x "
1688 "dump last ip->tcp %s "
1689 "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) "
1690 "l4_len %d (0x%x) "
1691 "MBLKL(mp): %d, MBLKL(datamp): %d ",
1692 niph->ip_sum,
1693 ntcph->th_sum, sum,
1694 nxge_dump_packet((char *)niph, 52),
1695 cmp, nmp, mp_chain,
1696 pktlen, pktlen,
1697 l4_len,
1698 l4_len,
1699 (int)MBLKL(mp), (int)MBLKL(datamp)));
1700
1701 cleanup_allocated_msgs:
1702 if (do_cleanup) {
1703 NXGE_DEBUG_MSG((NULL, TX_CTL,
1704 "==> nxge_do_softlso: "
1705 "Failed allocating messages, "
1706 "have to clean up and fail!"));
1707 while (mp_chain != NULL) {
1708 nmp = mp_chain;
1709 mp_chain = mp_chain->b_next;
1710 freemsg(nmp);
1711 }
1712 }
1713 /*
1714 * We're done here, so just free the original message and return the
1715 * new message chain, that could be NULL if failed, back to the caller.
1716 */
1717 freemsg(mp);
1718
1719 NXGE_DEBUG_MSG((NULL, TX_CTL,
1720 "<== nxge_do_softlso:mp_chain $%p", mp_chain));
1721 return (mp_chain);
1722 }
1723
1724 /*
1725 * Will be called before NIC driver do further operation on the message.
1726 * The input message may include LSO information, if so, go to softlso logic
1727 * to eliminate the oversized LSO packet for the incapable underlying h/w.
1728 * The return could be the same non-LSO message or a message chain for LSO case.
1729 *
1730 * The driver needs to call this function per packet and process the whole chain
1731 * if applied.
1732 */
1733 static mblk_t *
nxge_lso_eliminate(mblk_t * mp)1734 nxge_lso_eliminate(mblk_t *mp)
1735 {
1736 uint32_t lsoflags;
1737 uint32_t mss;
1738
1739 NXGE_DEBUG_MSG((NULL, TX_CTL,
1740 "==>nxge_lso_eliminate:"));
1741 nxge_lso_info_get(mp, &mss, &lsoflags);
1742
1743 if (lsoflags & HW_LSO) {
1744 mblk_t *nmp;
1745
1746 NXGE_DEBUG_MSG((NULL, TX_CTL,
1747 "==>nxge_lso_eliminate:"
1748 "HW_LSO:mss %d mp $%p",
1749 mss, mp));
1750 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) {
1751 NXGE_DEBUG_MSG((NULL, TX_CTL,
1752 "<== nxge_lso_eliminate: "
1753 "LSO: nmp not NULL nmp $%p mss %d mp $%p",
1754 nmp, mss, mp));
1755 return (nmp);
1756 } else {
1757 NXGE_DEBUG_MSG((NULL, TX_CTL,
1758 "<== nxge_lso_eliminate_ "
1759 "LSO: failed nmp NULL nmp $%p mss %d mp $%p",
1760 nmp, mss, mp));
1761 return (NULL);
1762 }
1763 }
1764
1765 NXGE_DEBUG_MSG((NULL, TX_CTL,
1766 "<== nxge_lso_eliminate"));
1767 return (mp);
1768 }
1769
1770 static uint32_t
nxge_csgen(uint16_t * adr,int len)1771 nxge_csgen(uint16_t *adr, int len)
1772 {
1773 int i, odd;
1774 uint32_t sum = 0;
1775 uint32_t c = 0;
1776
1777 odd = len % 2;
1778 for (i = 0; i < (len / 2); i++) {
1779 sum += (adr[i] & 0xffff);
1780 }
1781 if (odd) {
1782 sum += adr[len / 2] & 0xff00;
1783 }
1784 while ((c = ((sum & 0xffff0000) >> 16)) != 0) {
1785 sum &= 0xffff;
1786 sum += c;
1787 }
1788 return (~sum & 0xffff);
1789 }
1790