1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <hxge_impl.h>
27 #include <hxge_rxdma.h>
28 #include <hpi.h>
29 #include <hpi_vir.h>
30
31 /*
32 * Number of blocks to accumulate before re-enabling DMA
33 * when we get RBR empty.
34 */
35 #define HXGE_RBR_EMPTY_THRESHOLD 64
36
37 /*
38 * Globals: tunable parameters (/etc/system or adb)
39 *
40 */
41 extern uint32_t hxge_rbr_size;
42 extern uint32_t hxge_rcr_size;
43 extern uint32_t hxge_rbr_spare_size;
44 extern uint32_t hxge_mblks_pending;
45
46 /*
47 * Tunables to manage the receive buffer blocks.
48 *
49 * hxge_rx_threshold_hi: copy all buffers.
50 * hxge_rx_bcopy_size_type: receive buffer block size type.
51 * hxge_rx_threshold_lo: copy only up to tunable block size type.
52 */
53 extern hxge_rxbuf_threshold_t hxge_rx_threshold_hi;
54 extern hxge_rxbuf_type_t hxge_rx_buf_size_type;
55 extern hxge_rxbuf_threshold_t hxge_rx_threshold_lo;
56
57 /*
58 * Static local functions.
59 */
60 static hxge_status_t hxge_map_rxdma(p_hxge_t hxgep);
61 static void hxge_unmap_rxdma(p_hxge_t hxgep);
62 static hxge_status_t hxge_rxdma_hw_start_common(p_hxge_t hxgep);
63 static hxge_status_t hxge_rxdma_hw_start(p_hxge_t hxgep);
64 static void hxge_rxdma_hw_stop(p_hxge_t hxgep);
65 static hxge_status_t hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
66 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p,
67 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p,
68 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p,
69 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p);
70 static void hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
71 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p);
72 static hxge_status_t hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep,
73 uint16_t dma_channel, p_hxge_dma_common_t *dma_rbr_cntl_p,
74 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p,
75 p_rx_rbr_ring_t *rbr_p, p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p);
76 static void hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep,
77 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p);
78 static hxge_status_t hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep,
79 uint16_t channel, p_hxge_dma_common_t *dma_buf_p,
80 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks);
81 static void hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep,
82 p_rx_rbr_ring_t rbr_p);
83 static hxge_status_t hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel,
84 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p,
85 int n_init_kick);
86 static hxge_status_t hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel);
87 static mblk_t *hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp,
88 p_rx_rcr_ring_t rcr_p, rdc_stat_t cs, int bytes_to_read);
89 static uint32_t hxge_scan_for_last_eop(p_rx_rcr_ring_t rcr_p,
90 p_rcr_entry_t rcr_desc_rd_head_p, uint32_t num_rcrs);
91 static void hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p,
92 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p,
93 mblk_t ** mp, mblk_t ** mp_cont, uint32_t *invalid_rcr_entry);
94 static hxge_status_t hxge_disable_rxdma_channel(p_hxge_t hxgep,
95 uint16_t channel);
96 static p_rx_msg_t hxge_allocb(size_t, uint32_t, p_hxge_dma_common_t);
97 static void hxge_freeb(p_rx_msg_t);
98 static hxge_status_t hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index,
99 p_hxge_ldv_t ldvp, rdc_stat_t cs);
100 static hxge_status_t hxge_rxbuf_index_info_init(p_hxge_t hxgep,
101 p_rx_rbr_ring_t rx_dmap);
102 static hxge_status_t hxge_rxdma_fatal_err_recover(p_hxge_t hxgep,
103 uint16_t channel);
104 static hxge_status_t hxge_rx_port_fatal_err_recover(p_hxge_t hxgep);
105 static void hxge_rbr_empty_restore(p_hxge_t hxgep,
106 p_rx_rbr_ring_t rx_rbr_p);
107
108 hxge_status_t
hxge_init_rxdma_channels(p_hxge_t hxgep)109 hxge_init_rxdma_channels(p_hxge_t hxgep)
110 {
111 hxge_status_t status = HXGE_OK;
112 block_reset_t reset_reg;
113 int i;
114
115 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_init_rxdma_channels"));
116
117 for (i = 0; i < HXGE_MAX_RDCS; i++)
118 hxgep->rdc_first_intr[i] = B_TRUE;
119
120 /* Reset RDC block from PEU to clear any previous state */
121 reset_reg.value = 0;
122 reset_reg.bits.rdc_rst = 1;
123 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
124 HXGE_DELAY(1000);
125
126 status = hxge_map_rxdma(hxgep);
127 if (status != HXGE_OK) {
128 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
129 "<== hxge_init_rxdma: status 0x%x", status));
130 return (status);
131 }
132
133 status = hxge_rxdma_hw_start_common(hxgep);
134 if (status != HXGE_OK) {
135 hxge_unmap_rxdma(hxgep);
136 }
137
138 status = hxge_rxdma_hw_start(hxgep);
139 if (status != HXGE_OK) {
140 hxge_unmap_rxdma(hxgep);
141 }
142
143 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
144 "<== hxge_init_rxdma_channels: status 0x%x", status));
145 return (status);
146 }
147
148 void
hxge_uninit_rxdma_channels(p_hxge_t hxgep)149 hxge_uninit_rxdma_channels(p_hxge_t hxgep)
150 {
151 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_uninit_rxdma_channels"));
152
153 hxge_rxdma_hw_stop(hxgep);
154 hxge_unmap_rxdma(hxgep);
155
156 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_uinit_rxdma_channels"));
157 }
158
159 hxge_status_t
hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep,uint16_t channel,rdc_stat_t * cs_p)160 hxge_init_rxdma_channel_cntl_stat(p_hxge_t hxgep, uint16_t channel,
161 rdc_stat_t *cs_p)
162 {
163 hpi_handle_t handle;
164 hpi_status_t rs = HPI_SUCCESS;
165 hxge_status_t status = HXGE_OK;
166
167 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
168 "<== hxge_init_rxdma_channel_cntl_stat"));
169
170 handle = HXGE_DEV_HPI_HANDLE(hxgep);
171 rs = hpi_rxdma_control_status(handle, OP_SET, channel, cs_p);
172
173 if (rs != HPI_SUCCESS) {
174 status = HXGE_ERROR | rs;
175 }
176 return (status);
177 }
178
179
180 hxge_status_t
hxge_enable_rxdma_channel(p_hxge_t hxgep,uint16_t channel,p_rx_rbr_ring_t rbr_p,p_rx_rcr_ring_t rcr_p,p_rx_mbox_t mbox_p,int n_init_kick)181 hxge_enable_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
182 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p,
183 int n_init_kick)
184 {
185 hpi_handle_t handle;
186 rdc_desc_cfg_t rdc_desc;
187 rdc_rcr_cfg_b_t *cfgb_p;
188 hpi_status_t rs = HPI_SUCCESS;
189
190 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel"));
191 handle = HXGE_DEV_HPI_HANDLE(hxgep);
192
193 /*
194 * Use configuration data composed at init time. Write to hardware the
195 * receive ring configurations.
196 */
197 rdc_desc.mbox_enable = 1;
198 rdc_desc.mbox_addr = mbox_p->mbox_addr;
199 HXGE_DEBUG_MSG((hxgep, RX_CTL,
200 "==> hxge_enable_rxdma_channel: mboxp $%p($%p)",
201 mbox_p->mbox_addr, rdc_desc.mbox_addr));
202
203 rdc_desc.rbr_len = rbr_p->rbb_max;
204 rdc_desc.rbr_addr = rbr_p->rbr_addr;
205
206 switch (hxgep->rx_bksize_code) {
207 case RBR_BKSIZE_4K:
208 rdc_desc.page_size = SIZE_4KB;
209 break;
210 case RBR_BKSIZE_8K:
211 rdc_desc.page_size = SIZE_8KB;
212 break;
213 }
214
215 rdc_desc.size0 = rbr_p->hpi_pkt_buf_size0;
216 rdc_desc.valid0 = 1;
217
218 rdc_desc.size1 = rbr_p->hpi_pkt_buf_size1;
219 rdc_desc.valid1 = 1;
220
221 rdc_desc.size2 = rbr_p->hpi_pkt_buf_size2;
222 rdc_desc.valid2 = 1;
223
224 rdc_desc.full_hdr = rcr_p->full_hdr_flag;
225 rdc_desc.offset = rcr_p->sw_priv_hdr_len;
226
227 rdc_desc.rcr_len = rcr_p->comp_size;
228 rdc_desc.rcr_addr = rcr_p->rcr_addr;
229
230 cfgb_p = &(rcr_p->rcr_cfgb);
231 rdc_desc.rcr_threshold = cfgb_p->bits.pthres;
232 rdc_desc.rcr_timeout = cfgb_p->bits.timeout;
233 rdc_desc.rcr_timeout_enable = cfgb_p->bits.entout;
234
235 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: "
236 "rbr_len qlen %d pagesize code %d rcr_len %d",
237 rdc_desc.rbr_len, rdc_desc.page_size, rdc_desc.rcr_len));
238 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_enable_rxdma_channel: "
239 "size 0 %d size 1 %d size 2 %d",
240 rbr_p->hpi_pkt_buf_size0, rbr_p->hpi_pkt_buf_size1,
241 rbr_p->hpi_pkt_buf_size2));
242
243 rs = hpi_rxdma_cfg_rdc_ring(handle, rbr_p->rdc, &rdc_desc);
244 if (rs != HPI_SUCCESS) {
245 return (HXGE_ERROR | rs);
246 }
247
248 /*
249 * Enable the timeout and threshold.
250 */
251 rs = hpi_rxdma_cfg_rdc_rcr_threshold(handle, channel,
252 rdc_desc.rcr_threshold);
253 if (rs != HPI_SUCCESS) {
254 return (HXGE_ERROR | rs);
255 }
256
257 rs = hpi_rxdma_cfg_rdc_rcr_timeout(handle, channel,
258 rdc_desc.rcr_timeout);
259 if (rs != HPI_SUCCESS) {
260 return (HXGE_ERROR | rs);
261 }
262
263 /* Kick the DMA engine */
264 hpi_rxdma_rdc_rbr_kick(handle, channel, n_init_kick);
265
266 /* Clear the rbr empty bit */
267 (void) hpi_rxdma_channel_rbr_empty_clear(handle, channel);
268
269 /*
270 * Enable the DMA
271 */
272 rs = hpi_rxdma_cfg_rdc_enable(handle, channel);
273 if (rs != HPI_SUCCESS) {
274 return (HXGE_ERROR | rs);
275 }
276
277 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_enable_rxdma_channel"));
278
279 return (HXGE_OK);
280 }
281
282 static hxge_status_t
hxge_disable_rxdma_channel(p_hxge_t hxgep,uint16_t channel)283 hxge_disable_rxdma_channel(p_hxge_t hxgep, uint16_t channel)
284 {
285 hpi_handle_t handle;
286 hpi_status_t rs = HPI_SUCCESS;
287
288 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_disable_rxdma_channel"));
289
290 handle = HXGE_DEV_HPI_HANDLE(hxgep);
291
292 /* disable the DMA */
293 rs = hpi_rxdma_cfg_rdc_disable(handle, channel);
294 if (rs != HPI_SUCCESS) {
295 HXGE_DEBUG_MSG((hxgep, RX_CTL,
296 "<== hxge_disable_rxdma_channel:failed (0x%x)", rs));
297 return (HXGE_ERROR | rs);
298 }
299 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_disable_rxdma_channel"));
300 return (HXGE_OK);
301 }
302
303 hxge_status_t
hxge_rxdma_channel_rcrflush(p_hxge_t hxgep,uint8_t channel)304 hxge_rxdma_channel_rcrflush(p_hxge_t hxgep, uint8_t channel)
305 {
306 hpi_handle_t handle;
307 hxge_status_t status = HXGE_OK;
308
309 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
310 "==> hxge_rxdma_channel_rcrflush"));
311
312 handle = HXGE_DEV_HPI_HANDLE(hxgep);
313 hpi_rxdma_rdc_rcr_flush(handle, channel);
314
315 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
316 "<== hxge_rxdma_channel_rcrflush"));
317 return (status);
318
319 }
320
321 #define MID_INDEX(l, r) ((r + l + 1) >> 1)
322
323 #define TO_LEFT -1
324 #define TO_RIGHT 1
325 #define BOTH_RIGHT (TO_RIGHT + TO_RIGHT)
326 #define BOTH_LEFT (TO_LEFT + TO_LEFT)
327 #define IN_MIDDLE (TO_RIGHT + TO_LEFT)
328 #define NO_HINT 0xffffffff
329
330 /*ARGSUSED*/
331 hxge_status_t
hxge_rxbuf_pp_to_vp(p_hxge_t hxgep,p_rx_rbr_ring_t rbr_p,uint8_t pktbufsz_type,uint64_t * pkt_buf_addr_pp,uint64_t ** pkt_buf_addr_p,uint32_t * bufoffset,uint32_t * msg_index)332 hxge_rxbuf_pp_to_vp(p_hxge_t hxgep, p_rx_rbr_ring_t rbr_p,
333 uint8_t pktbufsz_type, uint64_t *pkt_buf_addr_pp,
334 uint64_t **pkt_buf_addr_p, uint32_t *bufoffset, uint32_t *msg_index)
335 {
336 int bufsize;
337 uint64_t pktbuf_pp;
338 uint64_t dvma_addr;
339 rxring_info_t *ring_info;
340 int base_side, end_side;
341 int r_index, l_index, anchor_index;
342 int found, search_done;
343 uint32_t offset, chunk_size, block_size, page_size_mask;
344 uint32_t chunk_index, block_index, total_index;
345 int max_iterations, iteration;
346 rxbuf_index_info_t *bufinfo;
347
348 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_rxbuf_pp_to_vp"));
349
350 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
351 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d",
352 pkt_buf_addr_pp, pktbufsz_type));
353
354 pktbuf_pp = (uint64_t)pkt_buf_addr_pp;
355
356 switch (pktbufsz_type) {
357 case 0:
358 bufsize = rbr_p->pkt_buf_size0;
359 break;
360 case 1:
361 bufsize = rbr_p->pkt_buf_size1;
362 break;
363 case 2:
364 bufsize = rbr_p->pkt_buf_size2;
365 break;
366 case RCR_SINGLE_BLOCK:
367 bufsize = 0;
368 anchor_index = 0;
369 break;
370 default:
371 return (HXGE_ERROR);
372 }
373
374 if (rbr_p->num_blocks == 1) {
375 anchor_index = 0;
376 ring_info = rbr_p->ring_info;
377 bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
378
379 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
380 "==> hxge_rxbuf_pp_to_vp: (found, 1 block) "
381 "buf_pp $%p btype %d anchor_index %d bufinfo $%p",
382 pkt_buf_addr_pp, pktbufsz_type, anchor_index, bufinfo));
383
384 goto found_index;
385 }
386
387 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
388 "==> hxge_rxbuf_pp_to_vp: buf_pp $%p btype %d anchor_index %d",
389 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
390
391 ring_info = rbr_p->ring_info;
392 found = B_FALSE;
393 bufinfo = (rxbuf_index_info_t *)ring_info->buffer;
394 iteration = 0;
395 max_iterations = ring_info->max_iterations;
396
397 /*
398 * First check if this block have been seen recently. This is indicated
399 * by a hint which is initialized when the first buffer of the block is
400 * seen. The hint is reset when the last buffer of the block has been
401 * processed. As three block sizes are supported, three hints are kept.
402 * The idea behind the hints is that once the hardware uses a block
403 * for a buffer of that size, it will use it exclusively for that size
404 * and will use it until it is exhausted. It is assumed that there
405 * would a single block being used for the same buffer sizes at any
406 * given time.
407 */
408 if (ring_info->hint[pktbufsz_type] != NO_HINT) {
409 anchor_index = ring_info->hint[pktbufsz_type];
410 dvma_addr = bufinfo[anchor_index].dvma_addr;
411 chunk_size = bufinfo[anchor_index].buf_size;
412 if ((pktbuf_pp >= dvma_addr) &&
413 (pktbuf_pp < (dvma_addr + chunk_size))) {
414 found = B_TRUE;
415 /*
416 * check if this is the last buffer in the block If so,
417 * then reset the hint for the size;
418 */
419
420 if ((pktbuf_pp + bufsize) >= (dvma_addr + chunk_size))
421 ring_info->hint[pktbufsz_type] = NO_HINT;
422 }
423 }
424
425 if (found == B_FALSE) {
426 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
427 "==> hxge_rxbuf_pp_to_vp: (!found)"
428 "buf_pp $%p btype %d anchor_index %d",
429 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
430
431 /*
432 * This is the first buffer of the block of this size. Need to
433 * search the whole information array. the search algorithm
434 * uses a binary tree search algorithm. It assumes that the
435 * information is already sorted with increasing order info[0]
436 * < info[1] < info[2] .... < info[n-1] where n is the size of
437 * the information array
438 */
439 r_index = rbr_p->num_blocks - 1;
440 l_index = 0;
441 search_done = B_FALSE;
442 anchor_index = MID_INDEX(r_index, l_index);
443 while (search_done == B_FALSE) {
444 if ((r_index == l_index) ||
445 (iteration >= max_iterations))
446 search_done = B_TRUE;
447
448 end_side = TO_RIGHT; /* to the right */
449 base_side = TO_LEFT; /* to the left */
450 /* read the DVMA address information and sort it */
451 dvma_addr = bufinfo[anchor_index].dvma_addr;
452 chunk_size = bufinfo[anchor_index].buf_size;
453
454 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
455 "==> hxge_rxbuf_pp_to_vp: (searching)"
456 "buf_pp $%p btype %d "
457 "anchor_index %d chunk_size %d dvmaaddr $%p",
458 pkt_buf_addr_pp, pktbufsz_type, anchor_index,
459 chunk_size, dvma_addr));
460
461 if (pktbuf_pp >= dvma_addr)
462 base_side = TO_RIGHT; /* to the right */
463 if (pktbuf_pp < (dvma_addr + chunk_size))
464 end_side = TO_LEFT; /* to the left */
465
466 switch (base_side + end_side) {
467 case IN_MIDDLE:
468 /* found */
469 found = B_TRUE;
470 search_done = B_TRUE;
471 if ((pktbuf_pp + bufsize) <
472 (dvma_addr + chunk_size))
473 ring_info->hint[pktbufsz_type] =
474 bufinfo[anchor_index].buf_index;
475 break;
476 case BOTH_RIGHT:
477 /* not found: go to the right */
478 l_index = anchor_index + 1;
479 anchor_index = MID_INDEX(r_index, l_index);
480 break;
481
482 case BOTH_LEFT:
483 /* not found: go to the left */
484 r_index = anchor_index - 1;
485 anchor_index = MID_INDEX(r_index, l_index);
486 break;
487 default: /* should not come here */
488 return (HXGE_ERROR);
489 }
490 iteration++;
491 }
492
493 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
494 "==> hxge_rxbuf_pp_to_vp: (search done)"
495 "buf_pp $%p btype %d anchor_index %d",
496 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
497 }
498
499 if (found == B_FALSE) {
500 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
501 "==> hxge_rxbuf_pp_to_vp: (search failed)"
502 "buf_pp $%p btype %d anchor_index %d",
503 pkt_buf_addr_pp, pktbufsz_type, anchor_index));
504 return (HXGE_ERROR);
505 }
506
507 found_index:
508 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
509 "==> hxge_rxbuf_pp_to_vp: (FOUND1)"
510 "buf_pp $%p btype %d bufsize %d anchor_index %d",
511 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index));
512
513 /* index of the first block in this chunk */
514 chunk_index = bufinfo[anchor_index].start_index;
515 dvma_addr = bufinfo[anchor_index].dvma_addr;
516 page_size_mask = ring_info->block_size_mask;
517
518 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
519 "==> hxge_rxbuf_pp_to_vp: (FOUND3), get chunk)"
520 "buf_pp $%p btype %d bufsize %d "
521 "anchor_index %d chunk_index %d dvma $%p",
522 pkt_buf_addr_pp, pktbufsz_type, bufsize,
523 anchor_index, chunk_index, dvma_addr));
524
525 offset = pktbuf_pp - dvma_addr; /* offset within the chunk */
526 block_size = rbr_p->block_size; /* System block(page) size */
527
528 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
529 "==> hxge_rxbuf_pp_to_vp: (FOUND4), get chunk)"
530 "buf_pp $%p btype %d bufsize %d "
531 "anchor_index %d chunk_index %d dvma $%p "
532 "offset %d block_size %d",
533 pkt_buf_addr_pp, pktbufsz_type, bufsize, anchor_index,
534 chunk_index, dvma_addr, offset, block_size));
535 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> getting total index"));
536
537 block_index = (offset / block_size); /* index within chunk */
538 total_index = chunk_index + block_index;
539
540 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
541 "==> hxge_rxbuf_pp_to_vp: "
542 "total_index %d dvma_addr $%p "
543 "offset %d block_size %d "
544 "block_index %d ",
545 total_index, dvma_addr, offset, block_size, block_index));
546
547 *pkt_buf_addr_p = (uint64_t *)((uint64_t)bufinfo[anchor_index].kaddr +
548 offset);
549
550 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
551 "==> hxge_rxbuf_pp_to_vp: "
552 "total_index %d dvma_addr $%p "
553 "offset %d block_size %d "
554 "block_index %d "
555 "*pkt_buf_addr_p $%p",
556 total_index, dvma_addr, offset, block_size,
557 block_index, *pkt_buf_addr_p));
558
559 *msg_index = total_index;
560 *bufoffset = (offset & page_size_mask);
561
562 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
563 "==> hxge_rxbuf_pp_to_vp: get msg index: "
564 "msg_index %d bufoffset_index %d",
565 *msg_index, *bufoffset));
566 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "<== hxge_rxbuf_pp_to_vp"));
567
568 return (HXGE_OK);
569 }
570
571
572 /*
573 * used by quick sort (qsort) function
574 * to perform comparison
575 */
576 static int
hxge_sort_compare(const void * p1,const void * p2)577 hxge_sort_compare(const void *p1, const void *p2)
578 {
579
580 rxbuf_index_info_t *a, *b;
581
582 a = (rxbuf_index_info_t *)p1;
583 b = (rxbuf_index_info_t *)p2;
584
585 if (a->dvma_addr > b->dvma_addr)
586 return (1);
587 if (a->dvma_addr < b->dvma_addr)
588 return (-1);
589 return (0);
590 }
591
592 /*
593 * Grabbed this sort implementation from common/syscall/avl.c
594 *
595 * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
596 * v = Ptr to array/vector of objs
597 * n = # objs in the array
598 * s = size of each obj (must be multiples of a word size)
599 * f = ptr to function to compare two objs
600 * returns (-1 = less than, 0 = equal, 1 = greater than
601 */
602 void
hxge_ksort(caddr_t v,int n,int s,int (* f)())603 hxge_ksort(caddr_t v, int n, int s, int (*f) ())
604 {
605 int g, i, j, ii;
606 unsigned int *p1, *p2;
607 unsigned int tmp;
608
609 /* No work to do */
610 if (v == NULL || n <= 1)
611 return;
612 /* Sanity check on arguments */
613 ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
614 ASSERT(s > 0);
615
616 for (g = n / 2; g > 0; g /= 2) {
617 for (i = g; i < n; i++) {
618 for (j = i - g; j >= 0 &&
619 (*f) (v + j * s, v + (j + g) * s) == 1; j -= g) {
620 p1 = (unsigned *)(v + j * s);
621 p2 = (unsigned *)(v + (j + g) * s);
622 for (ii = 0; ii < s / 4; ii++) {
623 tmp = *p1;
624 *p1++ = *p2;
625 *p2++ = tmp;
626 }
627 }
628 }
629 }
630 }
631
632 /*
633 * Initialize data structures required for rxdma
634 * buffer dvma->vmem address lookup
635 */
636 /*ARGSUSED*/
637 static hxge_status_t
hxge_rxbuf_index_info_init(p_hxge_t hxgep,p_rx_rbr_ring_t rbrp)638 hxge_rxbuf_index_info_init(p_hxge_t hxgep, p_rx_rbr_ring_t rbrp)
639 {
640 int index;
641 rxring_info_t *ring_info;
642 int max_iteration = 0, max_index = 0;
643
644 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "==> hxge_rxbuf_index_info_init"));
645
646 ring_info = rbrp->ring_info;
647 ring_info->hint[0] = NO_HINT;
648 ring_info->hint[1] = NO_HINT;
649 ring_info->hint[2] = NO_HINT;
650 ring_info->hint[3] = NO_HINT;
651 max_index = rbrp->num_blocks;
652
653 /* read the DVMA address information and sort it */
654 /* do init of the information array */
655
656 HXGE_DEBUG_MSG((hxgep, DMA2_CTL,
657 " hxge_rxbuf_index_info_init Sort ptrs"));
658
659 /* sort the array */
660 hxge_ksort((void *) ring_info->buffer, max_index,
661 sizeof (rxbuf_index_info_t), hxge_sort_compare);
662
663 for (index = 0; index < max_index; index++) {
664 HXGE_DEBUG_MSG((hxgep, DMA2_CTL,
665 " hxge_rxbuf_index_info_init: sorted chunk %d "
666 " ioaddr $%p kaddr $%p size %x",
667 index, ring_info->buffer[index].dvma_addr,
668 ring_info->buffer[index].kaddr,
669 ring_info->buffer[index].buf_size));
670 }
671
672 max_iteration = 0;
673 while (max_index >= (1ULL << max_iteration))
674 max_iteration++;
675 ring_info->max_iterations = max_iteration + 1;
676
677 HXGE_DEBUG_MSG((hxgep, DMA2_CTL,
678 " hxge_rxbuf_index_info_init Find max iter %d",
679 ring_info->max_iterations));
680 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_rxbuf_index_info_init"));
681
682 return (HXGE_OK);
683 }
684
685 /*ARGSUSED*/
686 void
hxge_dump_rcr_entry(p_hxge_t hxgep,p_rcr_entry_t entry_p)687 hxge_dump_rcr_entry(p_hxge_t hxgep, p_rcr_entry_t entry_p)
688 {
689 #ifdef HXGE_DEBUG
690
691 uint32_t bptr;
692 uint64_t pp;
693
694 bptr = entry_p->bits.pkt_buf_addr;
695
696 HXGE_DEBUG_MSG((hxgep, RX_CTL,
697 "\trcr entry $%p "
698 "\trcr entry 0x%0llx "
699 "\trcr entry 0x%08x "
700 "\trcr entry 0x%08x "
701 "\tvalue 0x%0llx\n"
702 "\tmulti = %d\n"
703 "\tpkt_type = 0x%x\n"
704 "\terror = 0x%04x\n"
705 "\tl2_len = %d\n"
706 "\tpktbufsize = %d\n"
707 "\tpkt_buf_addr = $%p\n"
708 "\tpkt_buf_addr (<< 6) = $%p\n",
709 entry_p,
710 *(int64_t *)entry_p,
711 *(int32_t *)entry_p,
712 *(int32_t *)((char *)entry_p + 32),
713 entry_p->value,
714 entry_p->bits.multi,
715 entry_p->bits.pkt_type,
716 entry_p->bits.error,
717 entry_p->bits.l2_len,
718 entry_p->bits.pktbufsz,
719 bptr,
720 entry_p->bits.pkt_buf_addr_l));
721
722 pp = (entry_p->value & RCR_PKT_BUF_ADDR_MASK) <<
723 RCR_PKT_BUF_ADDR_SHIFT;
724
725 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rcr pp 0x%llx l2 len %d",
726 pp, (*(int64_t *)entry_p >> 40) & 0x3fff));
727 #endif
728 }
729
730 /*ARGSUSED*/
731 void
hxge_rxdma_stop(p_hxge_t hxgep)732 hxge_rxdma_stop(p_hxge_t hxgep)
733 {
734 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop"));
735
736 MUTEX_ENTER(&hxgep->vmac_lock);
737 (void) hxge_rx_vmac_disable(hxgep);
738 (void) hxge_rxdma_hw_mode(hxgep, HXGE_DMA_STOP);
739 MUTEX_EXIT(&hxgep->vmac_lock);
740
741 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop"));
742 }
743
744 void
hxge_rxdma_stop_reinit(p_hxge_t hxgep)745 hxge_rxdma_stop_reinit(p_hxge_t hxgep)
746 {
747 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_reinit"));
748
749 (void) hxge_rxdma_stop(hxgep);
750 (void) hxge_uninit_rxdma_channels(hxgep);
751 (void) hxge_init_rxdma_channels(hxgep);
752
753 MUTEX_ENTER(&hxgep->vmac_lock);
754 (void) hxge_rx_vmac_enable(hxgep);
755 MUTEX_EXIT(&hxgep->vmac_lock);
756
757 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_reinit"));
758 }
759
760 hxge_status_t
hxge_rxdma_hw_mode(p_hxge_t hxgep,boolean_t enable)761 hxge_rxdma_hw_mode(p_hxge_t hxgep, boolean_t enable)
762 {
763 int i, ndmas;
764 uint16_t channel;
765 p_rx_rbr_rings_t rx_rbr_rings;
766 p_rx_rbr_ring_t *rbr_rings;
767 hpi_handle_t handle;
768 hpi_status_t rs = HPI_SUCCESS;
769 hxge_status_t status = HXGE_OK;
770
771 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
772 "==> hxge_rxdma_hw_mode: mode %d", enable));
773
774 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) {
775 HXGE_DEBUG_MSG((hxgep, RX_CTL,
776 "<== hxge_rxdma_mode: not initialized"));
777 return (HXGE_ERROR);
778 }
779
780 rx_rbr_rings = hxgep->rx_rbr_rings;
781 if (rx_rbr_rings == NULL) {
782 HXGE_DEBUG_MSG((hxgep, RX_CTL,
783 "<== hxge_rxdma_mode: NULL ring pointer"));
784 return (HXGE_ERROR);
785 }
786
787 if (rx_rbr_rings->rbr_rings == NULL) {
788 HXGE_DEBUG_MSG((hxgep, RX_CTL,
789 "<== hxge_rxdma_mode: NULL rbr rings pointer"));
790 return (HXGE_ERROR);
791 }
792
793 ndmas = rx_rbr_rings->ndmas;
794 if (!ndmas) {
795 HXGE_DEBUG_MSG((hxgep, RX_CTL,
796 "<== hxge_rxdma_mode: no channel"));
797 return (HXGE_ERROR);
798 }
799
800 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
801 "==> hxge_rxdma_mode (ndmas %d)", ndmas));
802
803 rbr_rings = rx_rbr_rings->rbr_rings;
804
805 handle = HXGE_DEV_HPI_HANDLE(hxgep);
806
807 for (i = 0; i < ndmas; i++) {
808 if (rbr_rings == NULL || rbr_rings[i] == NULL) {
809 continue;
810 }
811 channel = rbr_rings[i]->rdc;
812 if (enable) {
813 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
814 "==> hxge_rxdma_hw_mode: channel %d (enable)",
815 channel));
816 rs = hpi_rxdma_cfg_rdc_enable(handle, channel);
817 } else {
818 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
819 "==> hxge_rxdma_hw_mode: channel %d (disable)",
820 channel));
821 rs = hpi_rxdma_cfg_rdc_disable(handle, channel);
822 }
823 }
824
825 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
826 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
827 "<== hxge_rxdma_hw_mode: status 0x%x", status));
828
829 return (status);
830 }
831
832 /*
833 * Static functions start here.
834 */
835 static p_rx_msg_t
hxge_allocb(size_t size,uint32_t pri,p_hxge_dma_common_t dmabuf_p)836 hxge_allocb(size_t size, uint32_t pri, p_hxge_dma_common_t dmabuf_p)
837 {
838 p_rx_msg_t hxge_mp = NULL;
839 p_hxge_dma_common_t dmamsg_p;
840 uchar_t *buffer;
841
842 hxge_mp = KMEM_ZALLOC(sizeof (rx_msg_t), KM_NOSLEEP);
843 if (hxge_mp == NULL) {
844 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL,
845 "Allocation of a rx msg failed."));
846 goto hxge_allocb_exit;
847 }
848
849 hxge_mp->use_buf_pool = B_FALSE;
850 if (dmabuf_p) {
851 hxge_mp->use_buf_pool = B_TRUE;
852
853 dmamsg_p = (p_hxge_dma_common_t)&hxge_mp->buf_dma;
854 *dmamsg_p = *dmabuf_p;
855 dmamsg_p->nblocks = 1;
856 dmamsg_p->block_size = size;
857 dmamsg_p->alength = size;
858 buffer = (uchar_t *)dmabuf_p->kaddrp;
859
860 dmabuf_p->kaddrp = (void *)((char *)dmabuf_p->kaddrp + size);
861 dmabuf_p->ioaddr_pp = (void *)
862 ((char *)dmabuf_p->ioaddr_pp + size);
863
864 dmabuf_p->alength -= size;
865 dmabuf_p->offset += size;
866 dmabuf_p->dma_cookie.dmac_laddress += size;
867 dmabuf_p->dma_cookie.dmac_size -= size;
868 } else {
869 buffer = KMEM_ALLOC(size, KM_NOSLEEP);
870 if (buffer == NULL) {
871 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL,
872 "Allocation of a receive page failed."));
873 goto hxge_allocb_fail1;
874 }
875 }
876
877 hxge_mp->rx_mblk_p = desballoc(buffer, size, pri, &hxge_mp->freeb);
878 if (hxge_mp->rx_mblk_p == NULL) {
879 HXGE_ERROR_MSG((NULL, HXGE_ERR_CTL, "desballoc failed."));
880 goto hxge_allocb_fail2;
881 }
882 hxge_mp->buffer = buffer;
883 hxge_mp->block_size = size;
884 hxge_mp->freeb.free_func = (void (*) ()) hxge_freeb;
885 hxge_mp->freeb.free_arg = (caddr_t)hxge_mp;
886 hxge_mp->ref_cnt = 1;
887 hxge_mp->free = B_TRUE;
888 hxge_mp->rx_use_bcopy = B_FALSE;
889
890 atomic_inc_32(&hxge_mblks_pending);
891
892 goto hxge_allocb_exit;
893
894 hxge_allocb_fail2:
895 if (!hxge_mp->use_buf_pool) {
896 KMEM_FREE(buffer, size);
897 }
898 hxge_allocb_fail1:
899 KMEM_FREE(hxge_mp, sizeof (rx_msg_t));
900 hxge_mp = NULL;
901
902 hxge_allocb_exit:
903 return (hxge_mp);
904 }
905
906 p_mblk_t
hxge_dupb(p_rx_msg_t hxge_mp,uint_t offset,size_t size)907 hxge_dupb(p_rx_msg_t hxge_mp, uint_t offset, size_t size)
908 {
909 p_mblk_t mp;
910
911 HXGE_DEBUG_MSG((NULL, MEM_CTL, "==> hxge_dupb"));
912 HXGE_DEBUG_MSG((NULL, MEM_CTL, "hxge_mp = $%p "
913 "offset = 0x%08X " "size = 0x%08X", hxge_mp, offset, size));
914
915 mp = desballoc(&hxge_mp->buffer[offset], size, 0, &hxge_mp->freeb);
916 if (mp == NULL) {
917 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
918 goto hxge_dupb_exit;
919 }
920
921 atomic_inc_32(&hxge_mp->ref_cnt);
922
923 hxge_dupb_exit:
924 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp));
925 return (mp);
926 }
927
928 p_mblk_t
hxge_dupb_bcopy(p_rx_msg_t hxge_mp,uint_t offset,size_t size)929 hxge_dupb_bcopy(p_rx_msg_t hxge_mp, uint_t offset, size_t size)
930 {
931 p_mblk_t mp;
932 uchar_t *dp;
933
934 mp = allocb(size + HXGE_RXBUF_EXTRA, 0);
935 if (mp == NULL) {
936 HXGE_DEBUG_MSG((NULL, RX_CTL, "desballoc failed"));
937 goto hxge_dupb_bcopy_exit;
938 }
939 dp = mp->b_rptr = mp->b_rptr + HXGE_RXBUF_EXTRA;
940 bcopy((void *) &hxge_mp->buffer[offset], dp, size);
941 mp->b_wptr = dp + size;
942
943 hxge_dupb_bcopy_exit:
944
945 HXGE_DEBUG_MSG((NULL, MEM_CTL, "<== hxge_dupb mp = $%p", hxge_mp));
946
947 return (mp);
948 }
949
950 void hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p,
951 p_rx_msg_t rx_msg_p);
952
953 void
hxge_post_page(p_hxge_t hxgep,p_rx_rbr_ring_t rx_rbr_p,p_rx_msg_t rx_msg_p)954 hxge_post_page(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p, p_rx_msg_t rx_msg_p)
955 {
956 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_post_page"));
957
958 /* Reuse this buffer */
959 rx_msg_p->free = B_FALSE;
960 rx_msg_p->cur_usage_cnt = 0;
961 rx_msg_p->max_usage_cnt = 0;
962 rx_msg_p->pkt_buf_size = 0;
963
964 if (rx_rbr_p->rbr_use_bcopy) {
965 rx_msg_p->rx_use_bcopy = B_FALSE;
966 atomic_dec_32(&rx_rbr_p->rbr_consumed);
967 }
968 atomic_dec_32(&rx_rbr_p->rbr_used);
969
970 /*
971 * Get the rbr header pointer and its offset index.
972 */
973 rx_rbr_p->rbr_wr_index = ((rx_rbr_p->rbr_wr_index + 1) &
974 rx_rbr_p->rbr_wrap_mask);
975 rx_rbr_p->rbr_desc_vp[rx_rbr_p->rbr_wr_index] = rx_msg_p->shifted_addr;
976
977 /*
978 * Accumulate some buffers in the ring before re-enabling the
979 * DMA channel, if rbr empty was signaled.
980 */
981 hpi_rxdma_rdc_rbr_kick(HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc, 1);
982 if (rx_rbr_p->rbr_is_empty && (rx_rbr_p->rbb_max -
983 rx_rbr_p->rbr_used) >= HXGE_RBR_EMPTY_THRESHOLD) {
984 hxge_rbr_empty_restore(hxgep, rx_rbr_p);
985 }
986
987 HXGE_DEBUG_MSG((hxgep, RX_CTL,
988 "<== hxge_post_page (channel %d post_next_index %d)",
989 rx_rbr_p->rdc, rx_rbr_p->rbr_wr_index));
990 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_post_page"));
991 }
992
993 void
hxge_freeb(p_rx_msg_t rx_msg_p)994 hxge_freeb(p_rx_msg_t rx_msg_p)
995 {
996 size_t size;
997 uchar_t *buffer = NULL;
998 int ref_cnt;
999 boolean_t free_state = B_FALSE;
1000 rx_rbr_ring_t *ring = rx_msg_p->rx_rbr_p;
1001
1002 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "==> hxge_freeb"));
1003 HXGE_DEBUG_MSG((NULL, MEM2_CTL,
1004 "hxge_freeb:rx_msg_p = $%p (block pending %d)",
1005 rx_msg_p, hxge_mblks_pending));
1006
1007 if (ring == NULL)
1008 return;
1009
1010 /*
1011 * This is to prevent posting activities while we are recovering
1012 * from fatal errors. This should not be a performance drag since
1013 * ref_cnt != 0 most times.
1014 */
1015 if (ring->rbr_state == RBR_POSTING)
1016 MUTEX_ENTER(&ring->post_lock);
1017
1018 /*
1019 * First we need to get the free state, then
1020 * atomic decrement the reference count to prevent
1021 * the race condition with the interrupt thread that
1022 * is processing a loaned up buffer block.
1023 */
1024 free_state = rx_msg_p->free;
1025 ref_cnt = atomic_dec_32_nv(&rx_msg_p->ref_cnt);
1026 if (!ref_cnt) {
1027 atomic_dec_32(&hxge_mblks_pending);
1028
1029 buffer = rx_msg_p->buffer;
1030 size = rx_msg_p->block_size;
1031
1032 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "hxge_freeb: "
1033 "will free: rx_msg_p = $%p (block pending %d)",
1034 rx_msg_p, hxge_mblks_pending));
1035
1036 if (!rx_msg_p->use_buf_pool) {
1037 KMEM_FREE(buffer, size);
1038 }
1039
1040 KMEM_FREE(rx_msg_p, sizeof (rx_msg_t));
1041 /*
1042 * Decrement the receive buffer ring's reference
1043 * count, too.
1044 */
1045 atomic_dec_32(&ring->rbr_ref_cnt);
1046
1047 /*
1048 * Free the receive buffer ring, iff
1049 * 1. all the receive buffers have been freed
1050 * 2. and we are in the proper state (that is,
1051 * we are not UNMAPPING).
1052 */
1053 if (ring->rbr_ref_cnt == 0 &&
1054 ring->rbr_state == RBR_UNMAPPED) {
1055 KMEM_FREE(ring, sizeof (*ring));
1056 /* post_lock has been destroyed already */
1057 return;
1058 }
1059 }
1060
1061 /*
1062 * Repost buffer.
1063 */
1064 if (free_state && (ref_cnt == 1)) {
1065 HXGE_DEBUG_MSG((NULL, RX_CTL,
1066 "hxge_freeb: post page $%p:", rx_msg_p));
1067 if (ring->rbr_state == RBR_POSTING)
1068 hxge_post_page(rx_msg_p->hxgep, ring, rx_msg_p);
1069 }
1070
1071 if (ring->rbr_state == RBR_POSTING)
1072 MUTEX_EXIT(&ring->post_lock);
1073
1074 HXGE_DEBUG_MSG((NULL, MEM2_CTL, "<== hxge_freeb"));
1075 }
1076
1077 uint_t
hxge_rx_intr(caddr_t arg1,caddr_t arg2)1078 hxge_rx_intr(caddr_t arg1, caddr_t arg2)
1079 {
1080 p_hxge_ring_handle_t rhp;
1081 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1;
1082 p_hxge_t hxgep = (p_hxge_t)arg2;
1083 p_hxge_ldg_t ldgp;
1084 uint8_t channel;
1085 hpi_handle_t handle;
1086 rdc_stat_t cs;
1087 p_rx_rcr_ring_t ring;
1088 p_rx_rbr_ring_t rbrp;
1089 mblk_t *mp = NULL;
1090
1091 if (ldvp == NULL) {
1092 HXGE_DEBUG_MSG((NULL, RX_INT_CTL,
1093 "<== hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp));
1094 return (DDI_INTR_UNCLAIMED);
1095 }
1096
1097 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) {
1098 hxgep = ldvp->hxgep;
1099 }
1100
1101 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1102 "==> hxge_rx_intr: arg2 $%p arg1 $%p", hxgep, ldvp));
1103
1104 /*
1105 * This interrupt handler is for a specific receive dma channel.
1106 */
1107 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1108
1109 /*
1110 * Get the control and status for this channel.
1111 */
1112 channel = ldvp->vdma_index;
1113 ring = hxgep->rx_rcr_rings->rcr_rings[channel];
1114 rhp = &hxgep->rx_ring_handles[channel];
1115 ldgp = ldvp->ldgp;
1116
1117 ASSERT(ring != NULL);
1118 #if defined(DEBUG)
1119 if (rhp->started) {
1120 ASSERT(ring->ldgp == ldgp);
1121 ASSERT(ring->ldvp == ldvp);
1122 }
1123 #endif
1124
1125 MUTEX_ENTER(&ring->lock);
1126
1127 if (!ring->poll_flag) {
1128 RXDMA_REG_READ64(handle, RDC_STAT, channel, &cs.value);
1129 cs.bits.ptrread = 0;
1130 cs.bits.pktread = 0;
1131 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value);
1132
1133 /*
1134 * Process packets, if we are not in polling mode, the ring is
1135 * started and the interface is started. The MAC layer under
1136 * load will be operating in polling mode for RX traffic.
1137 */
1138 if ((rhp->started) &&
1139 (hxgep->hxge_mac_state == HXGE_MAC_STARTED)) {
1140 mp = hxge_rx_pkts(hxgep, ldvp->vdma_index,
1141 ldvp, ring, cs, -1);
1142 }
1143
1144 /* Process error events. */
1145 if (cs.value & RDC_STAT_ERROR) {
1146 MUTEX_EXIT(&ring->lock);
1147 (void) hxge_rx_err_evnts(hxgep, channel, ldvp, cs);
1148 MUTEX_ENTER(&ring->lock);
1149 }
1150
1151 /*
1152 * Enable the mailbox update interrupt if we want to use
1153 * mailbox. We probably don't need to use mailbox as it only
1154 * saves us one pio read. Also write 1 to rcrthres and
1155 * rcrto to clear these two edge triggered bits.
1156 */
1157 rbrp = hxgep->rx_rbr_rings->rbr_rings[channel];
1158 MUTEX_ENTER(&rbrp->post_lock);
1159 if (!rbrp->rbr_is_empty) {
1160 cs.value = 0;
1161 cs.bits.mex = 1;
1162 cs.bits.ptrread = 0;
1163 cs.bits.pktread = 0;
1164 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, cs.value);
1165 }
1166 MUTEX_EXIT(&rbrp->post_lock);
1167
1168 if (ldgp->nldvs == 1) {
1169 /*
1170 * Re-arm the group.
1171 */
1172 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_TRUE,
1173 ldgp->ldg_timer);
1174 }
1175 } else if ((ldgp->nldvs == 1) && (ring->poll_flag)) {
1176 /*
1177 * Disarm the group, if we are not a shared interrupt.
1178 */
1179 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_FALSE, 0);
1180 } else if (ring->poll_flag) {
1181 /*
1182 * Mask-off this device from the group.
1183 */
1184 (void) hpi_intr_mask_set(handle, ldvp->ldv, 1);
1185 }
1186
1187 MUTEX_EXIT(&ring->lock);
1188
1189 /*
1190 * Send the packets up the stack.
1191 */
1192 if (mp != NULL) {
1193 mac_rx_ring(hxgep->mach, ring->rcr_mac_handle, mp,
1194 ring->rcr_gen_num);
1195 }
1196
1197 HXGE_DEBUG_MSG((NULL, RX_INT_CTL, "<== hxge_rx_intr"));
1198 return (DDI_INTR_CLAIMED);
1199 }
1200
1201 /*
1202 * Enable polling for a ring. Interrupt for the ring is disabled when
1203 * the hxge interrupt comes (see hxge_rx_intr).
1204 */
1205 int
hxge_enable_poll(void * arg)1206 hxge_enable_poll(void *arg)
1207 {
1208 p_hxge_ring_handle_t ring_handle = (p_hxge_ring_handle_t)arg;
1209 p_rx_rcr_ring_t ringp;
1210 p_hxge_t hxgep;
1211 p_hxge_ldg_t ldgp;
1212
1213 if (ring_handle == NULL) {
1214 ASSERT(ring_handle != NULL);
1215 return (1);
1216 }
1217
1218
1219 hxgep = ring_handle->hxgep;
1220 ringp = hxgep->rx_rcr_rings->rcr_rings[ring_handle->index];
1221
1222 MUTEX_ENTER(&ringp->lock);
1223
1224 /*
1225 * Are we already polling ?
1226 */
1227 if (ringp->poll_flag) {
1228 MUTEX_EXIT(&ringp->lock);
1229 return (1);
1230 }
1231
1232 ldgp = ringp->ldgp;
1233 if (ldgp == NULL) {
1234 MUTEX_EXIT(&ringp->lock);
1235 return (1);
1236 }
1237
1238 /*
1239 * Enable polling
1240 */
1241 ringp->poll_flag = B_TRUE;
1242
1243 MUTEX_EXIT(&ringp->lock);
1244 return (0);
1245 }
1246
1247 /*
1248 * Disable polling for a ring and enable its interrupt.
1249 */
1250 int
hxge_disable_poll(void * arg)1251 hxge_disable_poll(void *arg)
1252 {
1253 p_hxge_ring_handle_t ring_handle = (p_hxge_ring_handle_t)arg;
1254 p_rx_rcr_ring_t ringp;
1255 p_hxge_t hxgep;
1256
1257 if (ring_handle == NULL) {
1258 ASSERT(ring_handle != NULL);
1259 return (0);
1260 }
1261
1262 hxgep = ring_handle->hxgep;
1263 ringp = hxgep->rx_rcr_rings->rcr_rings[ring_handle->index];
1264
1265 MUTEX_ENTER(&ringp->lock);
1266
1267 /*
1268 * Disable polling: enable interrupt
1269 */
1270 if (ringp->poll_flag) {
1271 hpi_handle_t handle;
1272 rdc_stat_t cs;
1273 p_hxge_ldg_t ldgp;
1274
1275 /*
1276 * Get the control and status for this channel.
1277 */
1278 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1279
1280 /*
1281 * Rearm this logical group if this is a single device
1282 * group.
1283 */
1284 ldgp = ringp->ldgp;
1285 if (ldgp == NULL) {
1286 MUTEX_EXIT(&ringp->lock);
1287 return (1);
1288 }
1289
1290 ringp->poll_flag = B_FALSE;
1291
1292 /*
1293 * Enable mailbox update, to start interrupts again.
1294 */
1295 cs.value = 0ULL;
1296 cs.bits.mex = 1;
1297 cs.bits.pktread = 0;
1298 cs.bits.ptrread = 0;
1299 RXDMA_REG_WRITE64(handle, RDC_STAT, ringp->rdc, cs.value);
1300
1301 if (ldgp->nldvs == 1) {
1302 /*
1303 * Re-arm the group, since it is the only member
1304 * of the group.
1305 */
1306 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_TRUE,
1307 ldgp->ldg_timer);
1308 } else {
1309 /*
1310 * Mask-on interrupts for the device and re-arm
1311 * the group.
1312 */
1313 (void) hpi_intr_mask_set(handle, ringp->ldvp->ldv, 0);
1314 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg, B_TRUE,
1315 ldgp->ldg_timer);
1316 }
1317 }
1318 MUTEX_EXIT(&ringp->lock);
1319 return (0);
1320 }
1321
1322 /*
1323 * Poll 'bytes_to_pickup' bytes of message from the rx ring.
1324 */
1325 mblk_t *
hxge_rx_poll(void * arg,int bytes_to_pickup)1326 hxge_rx_poll(void *arg, int bytes_to_pickup)
1327 {
1328 p_hxge_ring_handle_t rhp = (p_hxge_ring_handle_t)arg;
1329 p_rx_rcr_ring_t ring;
1330 p_hxge_t hxgep;
1331 hpi_handle_t handle;
1332 rdc_stat_t cs;
1333 mblk_t *mblk;
1334 p_hxge_ldv_t ldvp;
1335
1336 hxgep = rhp->hxgep;
1337
1338 /*
1339 * Get the control and status for this channel.
1340 */
1341 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1342 ring = hxgep->rx_rcr_rings->rcr_rings[rhp->index];
1343
1344 MUTEX_ENTER(&ring->lock);
1345 ASSERT(ring->poll_flag == B_TRUE);
1346 ASSERT(rhp->started);
1347
1348 if (!ring->poll_flag) {
1349 MUTEX_EXIT(&ring->lock);
1350 return ((mblk_t *)NULL);
1351 }
1352
1353 /*
1354 * Get the control and status bits for the ring.
1355 */
1356 RXDMA_REG_READ64(handle, RDC_STAT, rhp->index, &cs.value);
1357 cs.bits.ptrread = 0;
1358 cs.bits.pktread = 0;
1359 RXDMA_REG_WRITE64(handle, RDC_STAT, rhp->index, cs.value);
1360
1361 /*
1362 * Process packets.
1363 */
1364 mblk = hxge_rx_pkts(hxgep, ring->ldvp->vdma_index,
1365 ring->ldvp, ring, cs, bytes_to_pickup);
1366 ldvp = ring->ldvp;
1367
1368 /*
1369 * Process Error Events.
1370 */
1371 if (ldvp && (cs.value & RDC_STAT_ERROR)) {
1372 /*
1373 * Recovery routines will grab the RCR ring lock.
1374 */
1375 MUTEX_EXIT(&ring->lock);
1376 (void) hxge_rx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs);
1377 MUTEX_ENTER(&ring->lock);
1378 }
1379
1380 MUTEX_EXIT(&ring->lock);
1381 return (mblk);
1382 }
1383
1384 /*ARGSUSED*/
1385 mblk_t *
hxge_rx_pkts(p_hxge_t hxgep,uint_t vindex,p_hxge_ldv_t ldvp,p_rx_rcr_ring_t rcrp,rdc_stat_t cs,int bytes_to_read)1386 hxge_rx_pkts(p_hxge_t hxgep, uint_t vindex, p_hxge_ldv_t ldvp,
1387 p_rx_rcr_ring_t rcrp, rdc_stat_t cs, int bytes_to_read)
1388 {
1389 hpi_handle_t handle;
1390 uint8_t channel;
1391 uint32_t comp_rd_index;
1392 p_rcr_entry_t rcr_desc_rd_head_p;
1393 p_rcr_entry_t rcr_desc_rd_head_pp;
1394 p_mblk_t nmp, mp_cont, head_mp, *tail_mp;
1395 uint16_t qlen, nrcr_read, npkt_read;
1396 uint32_t qlen_hw, npkts, num_rcrs;
1397 uint32_t invalid_rcr_entry;
1398 boolean_t multi;
1399 rdc_stat_t pktcs;
1400 rdc_rcr_cfg_b_t rcr_cfg_b;
1401 uint64_t rcr_head_index, rcr_tail_index;
1402 uint64_t rcr_tail;
1403 rdc_rcr_tail_t rcr_tail_reg;
1404 p_hxge_rx_ring_stats_t rdc_stats;
1405 int totallen = 0;
1406
1407 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:vindex %d "
1408 "channel %d", vindex, ldvp->channel));
1409
1410 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1411 channel = rcrp->rdc;
1412 if (channel != ldvp->channel) {
1413 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "==> hxge_rx_pkts:index %d "
1414 "channel %d, and rcr channel %d not matched.",
1415 vindex, ldvp->channel, channel));
1416 return (NULL);
1417 }
1418
1419 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1420 "==> hxge_rx_pkts: START: rcr channel %d "
1421 "head_p $%p head_pp $%p index %d ",
1422 channel, rcrp->rcr_desc_rd_head_p,
1423 rcrp->rcr_desc_rd_head_pp, rcrp->comp_rd_index));
1424
1425 (void) hpi_rxdma_rdc_rcr_qlen_get(handle, channel, &qlen);
1426 RXDMA_REG_READ64(handle, RDC_RCR_TAIL, channel, &rcr_tail_reg.value);
1427 rcr_tail = rcr_tail_reg.bits.tail;
1428
1429 if (!qlen) {
1430 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1431 "<== hxge_rx_pkts:rcr channel %d qlen %d (no pkts)",
1432 channel, qlen));
1433 return (NULL);
1434 }
1435
1436 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_pkts:rcr channel %d "
1437 "qlen %d", channel, qlen));
1438
1439 comp_rd_index = rcrp->comp_rd_index;
1440
1441 rcr_desc_rd_head_p = rcrp->rcr_desc_rd_head_p;
1442 rcr_desc_rd_head_pp = rcrp->rcr_desc_rd_head_pp;
1443 nrcr_read = npkt_read = 0;
1444
1445 if (hxgep->rdc_first_intr[channel])
1446 qlen_hw = qlen;
1447 else
1448 qlen_hw = qlen - 1;
1449
1450 head_mp = NULL;
1451 tail_mp = &head_mp;
1452 nmp = mp_cont = NULL;
1453 multi = B_FALSE;
1454
1455 rcr_head_index = rcrp->rcr_desc_rd_head_p - rcrp->rcr_desc_first_p;
1456 rcr_tail_index = rcr_tail - rcrp->rcr_tail_begin;
1457
1458 if (rcr_tail_index >= rcr_head_index) {
1459 num_rcrs = rcr_tail_index - rcr_head_index;
1460 } else {
1461 /* rcr_tail has wrapped around */
1462 num_rcrs = (rcrp->comp_size - rcr_head_index) + rcr_tail_index;
1463 }
1464
1465 npkts = hxge_scan_for_last_eop(rcrp, rcr_desc_rd_head_p, num_rcrs);
1466 if (!npkts)
1467 return (NULL);
1468
1469 if (qlen_hw > npkts) {
1470 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1471 "Channel %d, rcr_qlen from reg %d and from rcr_tail %d\n",
1472 channel, qlen_hw, qlen_sw));
1473 qlen_hw = npkts;
1474 }
1475
1476 while (qlen_hw) {
1477 #ifdef HXGE_DEBUG
1478 hxge_dump_rcr_entry(hxgep, rcr_desc_rd_head_p);
1479 #endif
1480 /*
1481 * Process one completion ring entry.
1482 */
1483 invalid_rcr_entry = 0;
1484 hxge_receive_packet(hxgep,
1485 rcrp, rcr_desc_rd_head_p, &multi, &nmp, &mp_cont,
1486 &invalid_rcr_entry);
1487 if (invalid_rcr_entry != 0) {
1488 rdc_stats = rcrp->rdc_stats;
1489 rdc_stats->rcr_invalids++;
1490 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1491 "Channel %d could only read 0x%x packets, "
1492 "but 0x%x pending\n", channel, npkt_read, qlen_hw));
1493 break;
1494 }
1495
1496 /*
1497 * message chaining modes (nemo msg chaining)
1498 */
1499 if (nmp) {
1500 nmp->b_next = NULL;
1501 if (!multi && !mp_cont) { /* frame fits a partition */
1502 *tail_mp = nmp;
1503 tail_mp = &nmp->b_next;
1504 nmp = NULL;
1505 } else if (multi && !mp_cont) { /* first segment */
1506 *tail_mp = nmp;
1507 tail_mp = &nmp->b_cont;
1508 } else if (multi && mp_cont) { /* mid of multi segs */
1509 *tail_mp = mp_cont;
1510 tail_mp = &mp_cont->b_cont;
1511 } else if (!multi && mp_cont) { /* last segment */
1512 *tail_mp = mp_cont;
1513 tail_mp = &nmp->b_next;
1514 totallen += MBLKL(mp_cont);
1515 nmp = NULL;
1516 }
1517 }
1518
1519 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1520 "==> hxge_rx_pkts: loop: rcr channel %d "
1521 "before updating: multi %d "
1522 "nrcr_read %d "
1523 "npk read %d "
1524 "head_pp $%p index %d ",
1525 channel, multi,
1526 nrcr_read, npkt_read, rcr_desc_rd_head_pp, comp_rd_index));
1527
1528 if (!multi) {
1529 qlen_hw--;
1530 npkt_read++;
1531 }
1532
1533 /*
1534 * Update the next read entry.
1535 */
1536 comp_rd_index = NEXT_ENTRY(comp_rd_index,
1537 rcrp->comp_wrap_mask);
1538
1539 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p,
1540 rcrp->rcr_desc_first_p, rcrp->rcr_desc_last_p);
1541
1542 nrcr_read++;
1543
1544 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1545 "<== hxge_rx_pkts: (SAM, process one packet) "
1546 "nrcr_read %d", nrcr_read));
1547 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1548 "==> hxge_rx_pkts: loop: rcr channel %d "
1549 "multi %d nrcr_read %d npk read %d head_pp $%p index %d ",
1550 channel, multi, nrcr_read, npkt_read, rcr_desc_rd_head_pp,
1551 comp_rd_index));
1552
1553 if ((bytes_to_read != -1) &&
1554 (totallen >= bytes_to_read)) {
1555 break;
1556 }
1557 }
1558
1559 rcrp->rcr_desc_rd_head_pp = rcr_desc_rd_head_pp;
1560 rcrp->comp_rd_index = comp_rd_index;
1561 rcrp->rcr_desc_rd_head_p = rcr_desc_rd_head_p;
1562
1563 if ((hxgep->intr_timeout != rcrp->intr_timeout) ||
1564 (hxgep->intr_threshold != rcrp->intr_threshold)) {
1565 rcrp->intr_timeout = hxgep->intr_timeout;
1566 rcrp->intr_threshold = hxgep->intr_threshold;
1567 rcr_cfg_b.value = 0x0ULL;
1568 if (rcrp->intr_timeout)
1569 rcr_cfg_b.bits.entout = 1;
1570 rcr_cfg_b.bits.timeout = rcrp->intr_timeout;
1571 rcr_cfg_b.bits.pthres = rcrp->intr_threshold;
1572 RXDMA_REG_WRITE64(handle, RDC_RCR_CFG_B,
1573 channel, rcr_cfg_b.value);
1574 }
1575
1576 pktcs.value = 0;
1577 if (hxgep->rdc_first_intr[channel] && (npkt_read > 0)) {
1578 hxgep->rdc_first_intr[channel] = B_FALSE;
1579 pktcs.bits.pktread = npkt_read - 1;
1580 } else
1581 pktcs.bits.pktread = npkt_read;
1582 pktcs.bits.ptrread = nrcr_read;
1583 RXDMA_REG_WRITE64(handle, RDC_STAT, channel, pktcs.value);
1584
1585 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL,
1586 "==> hxge_rx_pkts: EXIT: rcr channel %d "
1587 "head_pp $%p index %016llx ",
1588 channel, rcrp->rcr_desc_rd_head_pp, rcrp->comp_rd_index));
1589
1590 HXGE_DEBUG_MSG((hxgep, RX_INT_CTL, "<== hxge_rx_pkts"));
1591 return (head_mp);
1592 }
1593
1594 #define RCR_ENTRY_PATTERN 0x5a5a6b6b7c7c8d8dULL
1595 #define NO_PORT_BIT 0x20
1596 #define L4_CS_EQ_BIT 0x40
1597
hxge_scan_for_last_eop(p_rx_rcr_ring_t rcrp,p_rcr_entry_t rcr_desc_rd_head_p,uint32_t num_rcrs)1598 static uint32_t hxge_scan_for_last_eop(p_rx_rcr_ring_t rcrp,
1599 p_rcr_entry_t rcr_desc_rd_head_p, uint32_t num_rcrs)
1600 {
1601 uint64_t rcr_entry;
1602 uint32_t rcrs = 0;
1603 uint32_t pkts = 0;
1604
1605 while (rcrs < num_rcrs) {
1606 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p);
1607
1608 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN))
1609 break;
1610
1611 if (!(rcr_entry & RCR_MULTI_MASK))
1612 pkts++;
1613
1614 rcr_desc_rd_head_p = NEXT_ENTRY_PTR(rcr_desc_rd_head_p,
1615 rcrp->rcr_desc_first_p, rcrp->rcr_desc_last_p);
1616
1617 rcrs++;
1618 }
1619
1620 return (pkts);
1621 }
1622
1623 /*ARGSUSED*/
1624 void
hxge_receive_packet(p_hxge_t hxgep,p_rx_rcr_ring_t rcr_p,p_rcr_entry_t rcr_desc_rd_head_p,boolean_t * multi_p,mblk_t ** mp,mblk_t ** mp_cont,uint32_t * invalid_rcr_entry)1625 hxge_receive_packet(p_hxge_t hxgep, p_rx_rcr_ring_t rcr_p,
1626 p_rcr_entry_t rcr_desc_rd_head_p, boolean_t *multi_p, mblk_t **mp,
1627 mblk_t **mp_cont, uint32_t *invalid_rcr_entry)
1628 {
1629 p_mblk_t nmp = NULL;
1630 uint64_t multi;
1631 uint8_t channel;
1632 boolean_t first_entry = B_TRUE;
1633 boolean_t is_tcp_udp = B_FALSE;
1634 boolean_t buffer_free = B_FALSE;
1635 boolean_t error_send_up = B_FALSE;
1636 uint8_t error_type;
1637 uint16_t l2_len;
1638 uint16_t skip_len;
1639 uint8_t pktbufsz_type;
1640 uint64_t rcr_entry;
1641 uint64_t *pkt_buf_addr_pp;
1642 uint64_t *pkt_buf_addr_p;
1643 uint32_t buf_offset;
1644 uint32_t bsize;
1645 uint32_t msg_index;
1646 p_rx_rbr_ring_t rx_rbr_p;
1647 p_rx_msg_t *rx_msg_ring_p;
1648 p_rx_msg_t rx_msg_p;
1649 uint16_t sw_offset_bytes = 0, hdr_size = 0;
1650 hxge_status_t status = HXGE_OK;
1651 boolean_t is_valid = B_FALSE;
1652 p_hxge_rx_ring_stats_t rdc_stats;
1653 uint32_t bytes_read;
1654 uint8_t header0 = 0;
1655 uint8_t header1 = 0;
1656 uint64_t pkt_type;
1657 uint8_t no_port_bit = 0;
1658 uint8_t l4_cs_eq_bit = 0;
1659
1660 channel = rcr_p->rdc;
1661
1662 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "==> hxge_receive_packet"));
1663
1664 first_entry = (*mp == NULL) ? B_TRUE : B_FALSE;
1665 rcr_entry = *((uint64_t *)rcr_desc_rd_head_p);
1666
1667 /* Verify the content of the rcr_entry for a hardware bug workaround */
1668 if ((rcr_entry == 0x0) || (rcr_entry == RCR_ENTRY_PATTERN)) {
1669 *invalid_rcr_entry = 1;
1670 HXGE_DEBUG_MSG((hxgep, RX2_CTL, "hxge_receive_packet "
1671 "Channel %d invalid RCR entry 0x%llx found, returning\n",
1672 channel, (long long) rcr_entry));
1673 return;
1674 }
1675 *((uint64_t *)rcr_desc_rd_head_p) = RCR_ENTRY_PATTERN;
1676
1677 multi = (rcr_entry & RCR_MULTI_MASK);
1678 pkt_type = (rcr_entry & RCR_PKT_TYPE_MASK);
1679
1680 error_type = ((rcr_entry & RCR_ERROR_MASK) >> RCR_ERROR_SHIFT);
1681 l2_len = ((rcr_entry & RCR_L2_LEN_MASK) >> RCR_L2_LEN_SHIFT);
1682
1683 /*
1684 * Hardware does not strip the CRC due bug ID 11451 where
1685 * the hardware mis handles minimum size packets.
1686 */
1687 l2_len -= ETHERFCSL;
1688
1689 pktbufsz_type = ((rcr_entry & RCR_PKTBUFSZ_MASK) >>
1690 RCR_PKTBUFSZ_SHIFT);
1691 pkt_buf_addr_pp = (uint64_t *)((rcr_entry & RCR_PKT_BUF_ADDR_MASK) <<
1692 RCR_PKT_BUF_ADDR_SHIFT);
1693
1694 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1695 "==> hxge_receive_packet: entryp $%p entry 0x%0llx "
1696 "pkt_buf_addr_pp $%p l2_len %d multi %d "
1697 "error_type 0x%x pktbufsz_type %d ",
1698 rcr_desc_rd_head_p, rcr_entry, pkt_buf_addr_pp, l2_len,
1699 multi, error_type, pktbufsz_type));
1700
1701 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1702 "==> hxge_receive_packet: entryp $%p entry 0x%0llx "
1703 "pkt_buf_addr_pp $%p l2_len %d multi %d "
1704 "error_type 0x%x ", rcr_desc_rd_head_p,
1705 rcr_entry, pkt_buf_addr_pp, l2_len, multi, error_type));
1706
1707 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1708 "==> (rbr) hxge_receive_packet: entry 0x%0llx "
1709 "full pkt_buf_addr_pp $%p l2_len %d",
1710 rcr_entry, pkt_buf_addr_pp, l2_len));
1711
1712 /* get the stats ptr */
1713 rdc_stats = rcr_p->rdc_stats;
1714
1715 if (!l2_len) {
1716 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1717 "<== hxge_receive_packet: failed: l2 length is 0."));
1718 return;
1719 }
1720
1721 /* shift 6 bits to get the full io address */
1722 pkt_buf_addr_pp = (uint64_t *)((uint64_t)pkt_buf_addr_pp <<
1723 RCR_PKT_BUF_ADDR_SHIFT_FULL);
1724 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1725 "==> (rbr) hxge_receive_packet: entry 0x%0llx "
1726 "full pkt_buf_addr_pp $%p l2_len %d",
1727 rcr_entry, pkt_buf_addr_pp, l2_len));
1728
1729 rx_rbr_p = rcr_p->rx_rbr_p;
1730 rx_msg_ring_p = rx_rbr_p->rx_msg_ring;
1731
1732 if (first_entry) {
1733 hdr_size = (rcr_p->full_hdr_flag ? RXDMA_HDR_SIZE_FULL :
1734 RXDMA_HDR_SIZE_DEFAULT);
1735
1736 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1737 "==> hxge_receive_packet: first entry 0x%016llx "
1738 "pkt_buf_addr_pp $%p l2_len %d hdr %d",
1739 rcr_entry, pkt_buf_addr_pp, l2_len, hdr_size));
1740 }
1741
1742 MUTEX_ENTER(&rx_rbr_p->lock);
1743
1744 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1745 "==> (rbr 1) hxge_receive_packet: entry 0x%0llx "
1746 "full pkt_buf_addr_pp $%p l2_len %d",
1747 rcr_entry, pkt_buf_addr_pp, l2_len));
1748
1749 /*
1750 * Packet buffer address in the completion entry points to the starting
1751 * buffer address (offset 0). Use the starting buffer address to locate
1752 * the corresponding kernel address.
1753 */
1754 status = hxge_rxbuf_pp_to_vp(hxgep, rx_rbr_p,
1755 pktbufsz_type, pkt_buf_addr_pp, &pkt_buf_addr_p,
1756 &buf_offset, &msg_index);
1757
1758 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1759 "==> (rbr 2) hxge_receive_packet: entry 0x%0llx "
1760 "full pkt_buf_addr_pp $%p l2_len %d",
1761 rcr_entry, pkt_buf_addr_pp, l2_len));
1762
1763 if (status != HXGE_OK) {
1764 MUTEX_EXIT(&rx_rbr_p->lock);
1765 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1766 "<== hxge_receive_packet: found vaddr failed %d", status));
1767 return;
1768 }
1769
1770 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1771 "==> (rbr 3) hxge_receive_packet: entry 0x%0llx "
1772 "full pkt_buf_addr_pp $%p l2_len %d",
1773 rcr_entry, pkt_buf_addr_pp, l2_len));
1774 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1775 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx "
1776 "full pkt_buf_addr_pp $%p l2_len %d",
1777 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
1778
1779 if (msg_index >= rx_rbr_p->tnblocks) {
1780 MUTEX_EXIT(&rx_rbr_p->lock);
1781 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1782 "==> hxge_receive_packet: FATAL msg_index (%d) "
1783 "should be smaller than tnblocks (%d)\n",
1784 msg_index, rx_rbr_p->tnblocks));
1785 return;
1786 }
1787
1788 rx_msg_p = rx_msg_ring_p[msg_index];
1789
1790 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1791 "==> (rbr 4 msgindex %d) hxge_receive_packet: entry 0x%0llx "
1792 "full pkt_buf_addr_pp $%p l2_len %d",
1793 msg_index, rcr_entry, pkt_buf_addr_pp, l2_len));
1794
1795 switch (pktbufsz_type) {
1796 case RCR_PKTBUFSZ_0:
1797 bsize = rx_rbr_p->pkt_buf_size0_bytes;
1798 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1799 "==> hxge_receive_packet: 0 buf %d", bsize));
1800 break;
1801 case RCR_PKTBUFSZ_1:
1802 bsize = rx_rbr_p->pkt_buf_size1_bytes;
1803 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1804 "==> hxge_receive_packet: 1 buf %d", bsize));
1805 break;
1806 case RCR_PKTBUFSZ_2:
1807 bsize = rx_rbr_p->pkt_buf_size2_bytes;
1808 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1809 "==> hxge_receive_packet: 2 buf %d", bsize));
1810 break;
1811 case RCR_SINGLE_BLOCK:
1812 bsize = rx_msg_p->block_size;
1813 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1814 "==> hxge_receive_packet: single %d", bsize));
1815
1816 break;
1817 default:
1818 MUTEX_EXIT(&rx_rbr_p->lock);
1819 return;
1820 }
1821
1822 DMA_COMMON_SYNC_OFFSET(rx_msg_p->buf_dma,
1823 (buf_offset + sw_offset_bytes), (hdr_size + l2_len),
1824 DDI_DMA_SYNC_FORCPU);
1825
1826 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1827 "==> hxge_receive_packet: after first dump:usage count"));
1828
1829 if (rx_msg_p->cur_usage_cnt == 0) {
1830 atomic_inc_32(&rx_rbr_p->rbr_used);
1831 if (rx_rbr_p->rbr_use_bcopy) {
1832 atomic_inc_32(&rx_rbr_p->rbr_consumed);
1833 if (rx_rbr_p->rbr_consumed <
1834 rx_rbr_p->rbr_threshold_hi) {
1835 if (rx_rbr_p->rbr_threshold_lo == 0 ||
1836 ((rx_rbr_p->rbr_consumed >=
1837 rx_rbr_p->rbr_threshold_lo) &&
1838 (rx_rbr_p->rbr_bufsize_type >=
1839 pktbufsz_type))) {
1840 rx_msg_p->rx_use_bcopy = B_TRUE;
1841 }
1842 } else {
1843 rx_msg_p->rx_use_bcopy = B_TRUE;
1844 }
1845 }
1846 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1847 "==> hxge_receive_packet: buf %d (new block) ", bsize));
1848
1849 rx_msg_p->pkt_buf_size_code = pktbufsz_type;
1850 rx_msg_p->pkt_buf_size = bsize;
1851 rx_msg_p->cur_usage_cnt = 1;
1852 if (pktbufsz_type == RCR_SINGLE_BLOCK) {
1853 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1854 "==> hxge_receive_packet: buf %d (single block) ",
1855 bsize));
1856 /*
1857 * Buffer can be reused once the free function is
1858 * called.
1859 */
1860 rx_msg_p->max_usage_cnt = 1;
1861 buffer_free = B_TRUE;
1862 } else {
1863 rx_msg_p->max_usage_cnt = rx_msg_p->block_size / bsize;
1864 if (rx_msg_p->max_usage_cnt == 1) {
1865 buffer_free = B_TRUE;
1866 }
1867 }
1868 } else {
1869 rx_msg_p->cur_usage_cnt++;
1870 if (rx_msg_p->cur_usage_cnt == rx_msg_p->max_usage_cnt) {
1871 buffer_free = B_TRUE;
1872 }
1873 }
1874
1875 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1876 "msgbuf index = %d l2len %d bytes usage %d max_usage %d ",
1877 msg_index, l2_len,
1878 rx_msg_p->cur_usage_cnt, rx_msg_p->max_usage_cnt));
1879
1880 if (error_type) {
1881 rdc_stats->ierrors++;
1882 /* Update error stats */
1883 rdc_stats->errlog.compl_err_type = error_type;
1884 HXGE_FM_REPORT_ERROR(hxgep, 0, HXGE_FM_EREPORT_RDMC_RCR_ERR);
1885
1886 if (error_type & RCR_CTRL_FIFO_DED) {
1887 rdc_stats->ctrl_fifo_ecc_err++;
1888 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1889 " hxge_receive_packet: "
1890 " channel %d RCR ctrl_fifo_ded error", channel));
1891 } else if (error_type & RCR_DATA_FIFO_DED) {
1892 rdc_stats->data_fifo_ecc_err++;
1893 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1894 " hxge_receive_packet: channel %d"
1895 " RCR data_fifo_ded error", channel));
1896 }
1897
1898 /*
1899 * Update and repost buffer block if max usage count is
1900 * reached.
1901 */
1902 if (error_send_up == B_FALSE) {
1903 atomic_inc_32(&rx_msg_p->ref_cnt);
1904 if (buffer_free == B_TRUE) {
1905 rx_msg_p->free = B_TRUE;
1906 }
1907
1908 MUTEX_EXIT(&rx_rbr_p->lock);
1909 hxge_freeb(rx_msg_p);
1910 return;
1911 }
1912 }
1913
1914 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
1915 "==> hxge_receive_packet: DMA sync second "));
1916
1917 bytes_read = rcr_p->rcvd_pkt_bytes;
1918 skip_len = sw_offset_bytes + hdr_size;
1919
1920 if (first_entry) {
1921 header0 = rx_msg_p->buffer[buf_offset];
1922 no_port_bit = header0 & NO_PORT_BIT;
1923 header1 = rx_msg_p->buffer[buf_offset + 1];
1924 l4_cs_eq_bit = header1 & L4_CS_EQ_BIT;
1925 }
1926
1927 if (!rx_msg_p->rx_use_bcopy) {
1928 /*
1929 * For loaned up buffers, the driver reference count
1930 * will be incremented first and then the free state.
1931 */
1932 if ((nmp = hxge_dupb(rx_msg_p, buf_offset, bsize)) != NULL) {
1933 if (first_entry) {
1934 nmp->b_rptr = &nmp->b_rptr[skip_len];
1935 if (l2_len < bsize - skip_len) {
1936 nmp->b_wptr = &nmp->b_rptr[l2_len];
1937 } else {
1938 nmp->b_wptr = &nmp->b_rptr[bsize
1939 - skip_len];
1940 }
1941 } else {
1942 if (l2_len - bytes_read < bsize) {
1943 nmp->b_wptr =
1944 &nmp->b_rptr[l2_len - bytes_read];
1945 } else {
1946 nmp->b_wptr = &nmp->b_rptr[bsize];
1947 }
1948 }
1949 }
1950 } else {
1951 if (first_entry) {
1952 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset + skip_len,
1953 l2_len < bsize - skip_len ?
1954 l2_len : bsize - skip_len);
1955 } else {
1956 nmp = hxge_dupb_bcopy(rx_msg_p, buf_offset,
1957 l2_len - bytes_read < bsize ?
1958 l2_len - bytes_read : bsize);
1959 }
1960 }
1961
1962 if (nmp != NULL) {
1963 if (first_entry)
1964 bytes_read = nmp->b_wptr - nmp->b_rptr;
1965 else
1966 bytes_read += nmp->b_wptr - nmp->b_rptr;
1967
1968 HXGE_DEBUG_MSG((hxgep, RX_CTL,
1969 "==> hxge_receive_packet after dupb: "
1970 "rbr consumed %d "
1971 "pktbufsz_type %d "
1972 "nmp $%p rptr $%p wptr $%p "
1973 "buf_offset %d bzise %d l2_len %d skip_len %d",
1974 rx_rbr_p->rbr_consumed,
1975 pktbufsz_type,
1976 nmp, nmp->b_rptr, nmp->b_wptr,
1977 buf_offset, bsize, l2_len, skip_len));
1978 } else {
1979 cmn_err(CE_WARN, "!hxge_receive_packet: update stats (error)");
1980
1981 atomic_inc_32(&rx_msg_p->ref_cnt);
1982 if (buffer_free == B_TRUE) {
1983 rx_msg_p->free = B_TRUE;
1984 }
1985
1986 MUTEX_EXIT(&rx_rbr_p->lock);
1987 hxge_freeb(rx_msg_p);
1988 return;
1989 }
1990
1991 if (buffer_free == B_TRUE) {
1992 rx_msg_p->free = B_TRUE;
1993 }
1994
1995 /*
1996 * ERROR, FRAG and PKT_TYPE are only reported in the first entry. If a
1997 * packet is not fragmented and no error bit is set, then L4 checksum
1998 * is OK.
1999 */
2000 is_valid = (nmp != NULL);
2001 if (first_entry) {
2002 rdc_stats->ipackets++; /* count only 1st seg for jumbo */
2003 if (l2_len > (STD_FRAME_SIZE - ETHERFCSL))
2004 rdc_stats->jumbo_pkts++;
2005 rdc_stats->ibytes += skip_len + l2_len < bsize ?
2006 l2_len : bsize;
2007 } else {
2008 /*
2009 * Add the current portion of the packet to the kstats.
2010 * The current portion of the packet is calculated by using
2011 * length of the packet and the previously received portion.
2012 */
2013 rdc_stats->ibytes += l2_len - rcr_p->rcvd_pkt_bytes < bsize ?
2014 l2_len - rcr_p->rcvd_pkt_bytes : bsize;
2015 }
2016
2017 rcr_p->rcvd_pkt_bytes = bytes_read;
2018
2019 if (rx_msg_p->free && rx_msg_p->rx_use_bcopy) {
2020 atomic_inc_32(&rx_msg_p->ref_cnt);
2021 MUTEX_EXIT(&rx_rbr_p->lock);
2022 hxge_freeb(rx_msg_p);
2023 } else
2024 MUTEX_EXIT(&rx_rbr_p->lock);
2025
2026 if (is_valid) {
2027 nmp->b_cont = NULL;
2028 if (first_entry) {
2029 *mp = nmp;
2030 *mp_cont = NULL;
2031 } else {
2032 *mp_cont = nmp;
2033 }
2034 }
2035
2036 /*
2037 * Update stats and hardware checksuming.
2038 */
2039 if (is_valid && !multi) {
2040 is_tcp_udp = ((pkt_type == RCR_PKT_IS_TCP ||
2041 pkt_type == RCR_PKT_IS_UDP) ? B_TRUE : B_FALSE);
2042
2043 if (!no_port_bit && l4_cs_eq_bit && is_tcp_udp && !error_type) {
2044 mac_hcksum_set(nmp, 0, 0, 0, 0, HCK_FULLCKSUM_OK);
2045
2046 HXGE_DEBUG_MSG((hxgep, RX_CTL,
2047 "==> hxge_receive_packet: Full tcp/udp cksum "
2048 "is_valid 0x%x multi %d error %d",
2049 is_valid, multi, error_type));
2050 }
2051 }
2052
2053 HXGE_DEBUG_MSG((hxgep, RX2_CTL,
2054 "==> hxge_receive_packet: *mp 0x%016llx", *mp));
2055
2056 *multi_p = (multi == RCR_MULTI_MASK);
2057
2058 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_receive_packet: "
2059 "multi %d nmp 0x%016llx *mp 0x%016llx *mp_cont 0x%016llx",
2060 *multi_p, nmp, *mp, *mp_cont));
2061 }
2062
2063 static void
hxge_rx_rbr_empty_recover(p_hxge_t hxgep,uint8_t channel)2064 hxge_rx_rbr_empty_recover(p_hxge_t hxgep, uint8_t channel)
2065 {
2066 hpi_handle_t handle;
2067 p_rx_rcr_ring_t rcrp;
2068 p_rx_rbr_ring_t rbrp;
2069
2070 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel];
2071 rbrp = rcrp->rx_rbr_p;
2072 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2073
2074 /*
2075 * Wait for the channel to be quiet
2076 */
2077 (void) hpi_rxdma_cfg_rdc_wait_for_qst(handle, channel);
2078
2079 /*
2080 * Post page will accumulate some buffers before re-enabling
2081 * the DMA channel.
2082 */
2083
2084 MUTEX_ENTER(&rbrp->post_lock);
2085 if ((rbrp->rbb_max - rbrp->rbr_used) >= HXGE_RBR_EMPTY_THRESHOLD) {
2086 hxge_rbr_empty_restore(hxgep, rbrp);
2087 } else {
2088 rbrp->rbr_is_empty = B_TRUE;
2089 }
2090 MUTEX_EXIT(&rbrp->post_lock);
2091 }
2092
2093
2094 /*ARGSUSED*/
2095 static hxge_status_t
hxge_rx_err_evnts(p_hxge_t hxgep,uint_t index,p_hxge_ldv_t ldvp,rdc_stat_t cs)2096 hxge_rx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp,
2097 rdc_stat_t cs)
2098 {
2099 p_hxge_rx_ring_stats_t rdc_stats;
2100 hpi_handle_t handle;
2101 boolean_t rxchan_fatal = B_FALSE;
2102 uint8_t channel;
2103 hxge_status_t status = HXGE_OK;
2104
2105 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_rx_err_evnts"));
2106
2107 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2108 channel = ldvp->channel;
2109
2110 rdc_stats = &hxgep->statsp->rdc_stats[ldvp->vdma_index];
2111
2112 if (cs.bits.rbr_cpl_to) {
2113 rdc_stats->rbr_tmout++;
2114 HXGE_FM_REPORT_ERROR(hxgep, channel,
2115 HXGE_FM_EREPORT_RDMC_RBR_CPL_TO);
2116 rxchan_fatal = B_TRUE;
2117 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2118 "==> hxge_rx_err_evnts(channel %d): "
2119 "fatal error: rx_rbr_timeout", channel));
2120 }
2121
2122 if ((cs.bits.rcr_shadow_par_err) || (cs.bits.rbr_prefetch_par_err)) {
2123 (void) hpi_rxdma_ring_perr_stat_get(handle,
2124 &rdc_stats->errlog.pre_par, &rdc_stats->errlog.sha_par);
2125 }
2126
2127 if (cs.bits.rcr_shadow_par_err) {
2128 rdc_stats->rcr_sha_par++;
2129 HXGE_FM_REPORT_ERROR(hxgep, channel,
2130 HXGE_FM_EREPORT_RDMC_RCR_SHA_PAR);
2131 rxchan_fatal = B_TRUE;
2132 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2133 "==> hxge_rx_err_evnts(channel %d): "
2134 "fatal error: rcr_shadow_par_err", channel));
2135 }
2136
2137 if (cs.bits.rbr_prefetch_par_err) {
2138 rdc_stats->rbr_pre_par++;
2139 HXGE_FM_REPORT_ERROR(hxgep, channel,
2140 HXGE_FM_EREPORT_RDMC_RBR_PRE_PAR);
2141 rxchan_fatal = B_TRUE;
2142 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2143 "==> hxge_rx_err_evnts(channel %d): "
2144 "fatal error: rbr_prefetch_par_err", channel));
2145 }
2146
2147 if (cs.bits.rbr_pre_empty) {
2148 rdc_stats->rbr_pre_empty++;
2149 HXGE_FM_REPORT_ERROR(hxgep, channel,
2150 HXGE_FM_EREPORT_RDMC_RBR_PRE_EMPTY);
2151 rxchan_fatal = B_TRUE;
2152 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2153 "==> hxge_rx_err_evnts(channel %d): "
2154 "fatal error: rbr_pre_empty", channel));
2155 }
2156
2157 if (cs.bits.peu_resp_err) {
2158 rdc_stats->peu_resp_err++;
2159 HXGE_FM_REPORT_ERROR(hxgep, channel,
2160 HXGE_FM_EREPORT_RDMC_PEU_RESP_ERR);
2161 rxchan_fatal = B_TRUE;
2162 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2163 "==> hxge_rx_err_evnts(channel %d): "
2164 "fatal error: peu_resp_err", channel));
2165 }
2166
2167 if (cs.bits.rcr_thres) {
2168 rdc_stats->rcr_thres++;
2169 }
2170
2171 if (cs.bits.rcr_to) {
2172 rdc_stats->rcr_to++;
2173 }
2174
2175 if (cs.bits.rcr_shadow_full) {
2176 rdc_stats->rcr_shadow_full++;
2177 HXGE_FM_REPORT_ERROR(hxgep, channel,
2178 HXGE_FM_EREPORT_RDMC_RCR_SHA_FULL);
2179 rxchan_fatal = B_TRUE;
2180 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2181 "==> hxge_rx_err_evnts(channel %d): "
2182 "fatal error: rcr_shadow_full", channel));
2183 }
2184
2185 if (cs.bits.rcr_full) {
2186 rdc_stats->rcrfull++;
2187 HXGE_FM_REPORT_ERROR(hxgep, channel,
2188 HXGE_FM_EREPORT_RDMC_RCRFULL);
2189 rxchan_fatal = B_TRUE;
2190 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2191 "==> hxge_rx_err_evnts(channel %d): "
2192 "fatal error: rcrfull error", channel));
2193 }
2194
2195 if (cs.bits.rbr_empty) {
2196 rdc_stats->rbr_empty++;
2197 hxge_rx_rbr_empty_recover(hxgep, channel);
2198 }
2199
2200 if (cs.bits.rbr_full) {
2201 rdc_stats->rbrfull++;
2202 HXGE_FM_REPORT_ERROR(hxgep, channel,
2203 HXGE_FM_EREPORT_RDMC_RBRFULL);
2204 rxchan_fatal = B_TRUE;
2205 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2206 "==> hxge_rx_err_evnts(channel %d): "
2207 "fatal error: rbr_full error", channel));
2208 }
2209
2210 if (rxchan_fatal) {
2211 p_rx_rcr_ring_t rcrp;
2212 p_rx_rbr_ring_t rbrp;
2213
2214 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel];
2215 rbrp = rcrp->rx_rbr_p;
2216
2217 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2218 " hxge_rx_err_evnts: fatal error on Channel #%d\n",
2219 channel));
2220
2221 MUTEX_ENTER(&rbrp->post_lock);
2222 /* This function needs to be inside the post_lock */
2223 status = hxge_rxdma_fatal_err_recover(hxgep, channel);
2224 MUTEX_EXIT(&rbrp->post_lock);
2225 if (status == HXGE_OK) {
2226 FM_SERVICE_RESTORED(hxgep);
2227 }
2228 }
2229
2230 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_rx_err_evnts"));
2231 return (status);
2232 }
2233
2234 static hxge_status_t
hxge_map_rxdma(p_hxge_t hxgep)2235 hxge_map_rxdma(p_hxge_t hxgep)
2236 {
2237 int i, ndmas;
2238 uint16_t channel;
2239 p_rx_rbr_rings_t rx_rbr_rings;
2240 p_rx_rbr_ring_t *rbr_rings;
2241 p_rx_rcr_rings_t rx_rcr_rings;
2242 p_rx_rcr_ring_t *rcr_rings;
2243 p_rx_mbox_areas_t rx_mbox_areas_p;
2244 p_rx_mbox_t *rx_mbox_p;
2245 p_hxge_dma_pool_t dma_buf_poolp;
2246 p_hxge_dma_common_t *dma_buf_p;
2247 p_hxge_dma_pool_t dma_rbr_cntl_poolp;
2248 p_hxge_dma_common_t *dma_rbr_cntl_p;
2249 p_hxge_dma_pool_t dma_rcr_cntl_poolp;
2250 p_hxge_dma_common_t *dma_rcr_cntl_p;
2251 p_hxge_dma_pool_t dma_mbox_cntl_poolp;
2252 p_hxge_dma_common_t *dma_mbox_cntl_p;
2253 uint32_t *num_chunks;
2254 hxge_status_t status = HXGE_OK;
2255
2256 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_map_rxdma"));
2257
2258 dma_buf_poolp = hxgep->rx_buf_pool_p;
2259 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p;
2260 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p;
2261 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p;
2262
2263 if (!dma_buf_poolp->buf_allocated ||
2264 !dma_rbr_cntl_poolp->buf_allocated ||
2265 !dma_rcr_cntl_poolp->buf_allocated ||
2266 !dma_mbox_cntl_poolp->buf_allocated) {
2267 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2268 "<== hxge_map_rxdma: buf not allocated"));
2269 return (HXGE_ERROR);
2270 }
2271
2272 ndmas = dma_buf_poolp->ndmas;
2273 if (!ndmas) {
2274 HXGE_DEBUG_MSG((hxgep, RX_CTL,
2275 "<== hxge_map_rxdma: no dma allocated"));
2276 return (HXGE_ERROR);
2277 }
2278
2279 num_chunks = dma_buf_poolp->num_chunks;
2280 dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
2281 dma_rbr_cntl_p = dma_rbr_cntl_poolp->dma_buf_pool_p;
2282 dma_rcr_cntl_p = dma_rcr_cntl_poolp->dma_buf_pool_p;
2283 dma_mbox_cntl_p = dma_mbox_cntl_poolp->dma_buf_pool_p;
2284
2285 rx_rbr_rings = (p_rx_rbr_rings_t)
2286 KMEM_ZALLOC(sizeof (rx_rbr_rings_t), KM_SLEEP);
2287 rbr_rings = (p_rx_rbr_ring_t *)KMEM_ZALLOC(
2288 sizeof (p_rx_rbr_ring_t) * ndmas, KM_SLEEP);
2289
2290 rx_rcr_rings = (p_rx_rcr_rings_t)
2291 KMEM_ZALLOC(sizeof (rx_rcr_rings_t), KM_SLEEP);
2292 rcr_rings = (p_rx_rcr_ring_t *)KMEM_ZALLOC(
2293 sizeof (p_rx_rcr_ring_t) * ndmas, KM_SLEEP);
2294
2295 rx_mbox_areas_p = (p_rx_mbox_areas_t)
2296 KMEM_ZALLOC(sizeof (rx_mbox_areas_t), KM_SLEEP);
2297 rx_mbox_p = (p_rx_mbox_t *)KMEM_ZALLOC(
2298 sizeof (p_rx_mbox_t) * ndmas, KM_SLEEP);
2299
2300 /*
2301 * Timeout should be set based on the system clock divider.
2302 * The following timeout value of 1 assumes that the
2303 * granularity (1000) is 3 microseconds running at 300MHz.
2304 */
2305
2306 hxgep->intr_threshold = RXDMA_RCR_PTHRES_DEFAULT;
2307 hxgep->intr_timeout = RXDMA_RCR_TO_DEFAULT;
2308
2309 /*
2310 * Map descriptors from the buffer polls for each dam channel.
2311 */
2312 for (i = 0; i < ndmas; i++) {
2313 if (((p_hxge_dma_common_t)dma_buf_p[i]) == NULL) {
2314 status = HXGE_ERROR;
2315 goto hxge_map_rxdma_fail1;
2316 }
2317
2318 /*
2319 * Set up and prepare buffer blocks, descriptors and mailbox.
2320 */
2321 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
2322 status = hxge_map_rxdma_channel(hxgep, channel,
2323 (p_hxge_dma_common_t *)&dma_buf_p[i],
2324 (p_rx_rbr_ring_t *)&rbr_rings[i],
2325 num_chunks[i],
2326 (p_hxge_dma_common_t *)&dma_rbr_cntl_p[i],
2327 (p_hxge_dma_common_t *)&dma_rcr_cntl_p[i],
2328 (p_hxge_dma_common_t *)&dma_mbox_cntl_p[i],
2329 (p_rx_rcr_ring_t *)&rcr_rings[i],
2330 (p_rx_mbox_t *)&rx_mbox_p[i]);
2331 if (status != HXGE_OK) {
2332 goto hxge_map_rxdma_fail1;
2333 }
2334 rbr_rings[i]->index = (uint16_t)i;
2335 rcr_rings[i]->index = (uint16_t)i;
2336 rcr_rings[i]->rdc_stats = &hxgep->statsp->rdc_stats[i];
2337 }
2338
2339 rx_rbr_rings->ndmas = rx_rcr_rings->ndmas = ndmas;
2340 rx_rbr_rings->rbr_rings = rbr_rings;
2341 hxgep->rx_rbr_rings = rx_rbr_rings;
2342 rx_rcr_rings->rcr_rings = rcr_rings;
2343 hxgep->rx_rcr_rings = rx_rcr_rings;
2344
2345 rx_mbox_areas_p->rxmbox_areas = rx_mbox_p;
2346 hxgep->rx_mbox_areas_p = rx_mbox_areas_p;
2347
2348 goto hxge_map_rxdma_exit;
2349
2350 hxge_map_rxdma_fail1:
2351 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2352 "==> hxge_map_rxdma: unmap rbr,rcr (status 0x%x channel %d i %d)",
2353 status, channel, i));
2354 i--;
2355 for (; i >= 0; i--) {
2356 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
2357 hxge_unmap_rxdma_channel(hxgep, channel,
2358 rbr_rings[i], rcr_rings[i], rx_mbox_p[i]);
2359 }
2360
2361 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas);
2362 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t));
2363 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas);
2364 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t));
2365 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas);
2366 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t));
2367
2368 hxge_map_rxdma_exit:
2369 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2370 "<== hxge_map_rxdma: (status 0x%x channel %d)", status, channel));
2371
2372 return (status);
2373 }
2374
2375 static void
hxge_unmap_rxdma(p_hxge_t hxgep)2376 hxge_unmap_rxdma(p_hxge_t hxgep)
2377 {
2378 int i, ndmas;
2379 uint16_t channel;
2380 p_rx_rbr_rings_t rx_rbr_rings;
2381 p_rx_rbr_ring_t *rbr_rings;
2382 p_rx_rcr_rings_t rx_rcr_rings;
2383 p_rx_rcr_ring_t *rcr_rings;
2384 p_rx_mbox_areas_t rx_mbox_areas_p;
2385 p_rx_mbox_t *rx_mbox_p;
2386 p_hxge_dma_pool_t dma_buf_poolp;
2387 p_hxge_dma_pool_t dma_rbr_cntl_poolp;
2388 p_hxge_dma_pool_t dma_rcr_cntl_poolp;
2389 p_hxge_dma_pool_t dma_mbox_cntl_poolp;
2390 p_hxge_dma_common_t *dma_buf_p;
2391
2392 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_unmap_rxdma"));
2393
2394 dma_buf_poolp = hxgep->rx_buf_pool_p;
2395 dma_rbr_cntl_poolp = hxgep->rx_rbr_cntl_pool_p;
2396 dma_rcr_cntl_poolp = hxgep->rx_rcr_cntl_pool_p;
2397 dma_mbox_cntl_poolp = hxgep->rx_mbox_cntl_pool_p;
2398
2399 if (!dma_buf_poolp->buf_allocated ||
2400 !dma_rbr_cntl_poolp->buf_allocated ||
2401 !dma_rcr_cntl_poolp->buf_allocated ||
2402 !dma_mbox_cntl_poolp->buf_allocated) {
2403 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2404 "<== hxge_unmap_rxdma: NULL buf pointers"));
2405 return;
2406 }
2407
2408 rx_rbr_rings = hxgep->rx_rbr_rings;
2409 rx_rcr_rings = hxgep->rx_rcr_rings;
2410 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
2411 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2412 "<== hxge_unmap_rxdma: NULL pointers"));
2413 return;
2414 }
2415
2416 ndmas = rx_rbr_rings->ndmas;
2417 if (!ndmas) {
2418 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2419 "<== hxge_unmap_rxdma: no channel"));
2420 return;
2421 }
2422
2423 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2424 "==> hxge_unmap_rxdma (ndmas %d)", ndmas));
2425
2426 rbr_rings = rx_rbr_rings->rbr_rings;
2427 rcr_rings = rx_rcr_rings->rcr_rings;
2428 rx_mbox_areas_p = hxgep->rx_mbox_areas_p;
2429 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
2430 dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
2431
2432 for (i = 0; i < ndmas; i++) {
2433 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
2434 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2435 "==> hxge_unmap_rxdma (ndmas %d) channel %d",
2436 ndmas, channel));
2437 (void) hxge_unmap_rxdma_channel(hxgep, channel,
2438 (p_rx_rbr_ring_t)rbr_rings[i],
2439 (p_rx_rcr_ring_t)rcr_rings[i],
2440 (p_rx_mbox_t)rx_mbox_p[i]);
2441 }
2442
2443 KMEM_FREE(rx_rbr_rings, sizeof (rx_rbr_rings_t));
2444 KMEM_FREE(rbr_rings, sizeof (p_rx_rbr_ring_t) * ndmas);
2445 KMEM_FREE(rx_rcr_rings, sizeof (rx_rcr_rings_t));
2446 KMEM_FREE(rcr_rings, sizeof (p_rx_rcr_ring_t) * ndmas);
2447 KMEM_FREE(rx_mbox_areas_p, sizeof (rx_mbox_areas_t));
2448 KMEM_FREE(rx_mbox_p, sizeof (p_rx_mbox_t) * ndmas);
2449
2450 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma"));
2451 }
2452
2453 hxge_status_t
hxge_map_rxdma_channel(p_hxge_t hxgep,uint16_t channel,p_hxge_dma_common_t * dma_buf_p,p_rx_rbr_ring_t * rbr_p,uint32_t num_chunks,p_hxge_dma_common_t * dma_rbr_cntl_p,p_hxge_dma_common_t * dma_rcr_cntl_p,p_hxge_dma_common_t * dma_mbox_cntl_p,p_rx_rcr_ring_t * rcr_p,p_rx_mbox_t * rx_mbox_p)2454 hxge_map_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
2455 p_hxge_dma_common_t *dma_buf_p, p_rx_rbr_ring_t *rbr_p,
2456 uint32_t num_chunks, p_hxge_dma_common_t *dma_rbr_cntl_p,
2457 p_hxge_dma_common_t *dma_rcr_cntl_p, p_hxge_dma_common_t *dma_mbox_cntl_p,
2458 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p)
2459 {
2460 int status = HXGE_OK;
2461
2462 /*
2463 * Set up and prepare buffer blocks, descriptors and mailbox.
2464 */
2465 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2466 "==> hxge_map_rxdma_channel (channel %d)", channel));
2467
2468 /*
2469 * Receive buffer blocks
2470 */
2471 status = hxge_map_rxdma_channel_buf_ring(hxgep, channel,
2472 dma_buf_p, rbr_p, num_chunks);
2473 if (status != HXGE_OK) {
2474 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2475 "==> hxge_map_rxdma_channel (channel %d): "
2476 "map buffer failed 0x%x", channel, status));
2477 goto hxge_map_rxdma_channel_exit;
2478 }
2479
2480 /*
2481 * Receive block ring, completion ring and mailbox.
2482 */
2483 status = hxge_map_rxdma_channel_cfg_ring(hxgep, channel,
2484 dma_rbr_cntl_p, dma_rcr_cntl_p, dma_mbox_cntl_p,
2485 rbr_p, rcr_p, rx_mbox_p);
2486 if (status != HXGE_OK) {
2487 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2488 "==> hxge_map_rxdma_channel (channel %d): "
2489 "map config failed 0x%x", channel, status));
2490 goto hxge_map_rxdma_channel_fail2;
2491 }
2492 goto hxge_map_rxdma_channel_exit;
2493
2494 hxge_map_rxdma_channel_fail3:
2495 /* Free rbr, rcr */
2496 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2497 "==> hxge_map_rxdma_channel: free rbr/rcr (status 0x%x channel %d)",
2498 status, channel));
2499 hxge_unmap_rxdma_channel_cfg_ring(hxgep, *rcr_p, *rx_mbox_p);
2500
2501 hxge_map_rxdma_channel_fail2:
2502 /* Free buffer blocks */
2503 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2504 "==> hxge_map_rxdma_channel: free rx buffers"
2505 "(hxgep 0x%x status 0x%x channel %d)",
2506 hxgep, status, channel));
2507 hxge_unmap_rxdma_channel_buf_ring(hxgep, *rbr_p);
2508
2509 status = HXGE_ERROR;
2510
2511 hxge_map_rxdma_channel_exit:
2512 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2513 "<== hxge_map_rxdma_channel: (hxgep 0x%x status 0x%x channel %d)",
2514 hxgep, status, channel));
2515
2516 return (status);
2517 }
2518
2519 /*ARGSUSED*/
2520 static void
hxge_unmap_rxdma_channel(p_hxge_t hxgep,uint16_t channel,p_rx_rbr_ring_t rbr_p,p_rx_rcr_ring_t rcr_p,p_rx_mbox_t rx_mbox_p)2521 hxge_unmap_rxdma_channel(p_hxge_t hxgep, uint16_t channel,
2522 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
2523 {
2524 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2525 "==> hxge_unmap_rxdma_channel (channel %d)", channel));
2526
2527 /*
2528 * unmap receive block ring, completion ring and mailbox.
2529 */
2530 (void) hxge_unmap_rxdma_channel_cfg_ring(hxgep, rcr_p, rx_mbox_p);
2531
2532 /* unmap buffer blocks */
2533 (void) hxge_unmap_rxdma_channel_buf_ring(hxgep, rbr_p);
2534
2535 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_unmap_rxdma_channel"));
2536 }
2537
2538 /*ARGSUSED*/
2539 static hxge_status_t
hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep,uint16_t dma_channel,p_hxge_dma_common_t * dma_rbr_cntl_p,p_hxge_dma_common_t * dma_rcr_cntl_p,p_hxge_dma_common_t * dma_mbox_cntl_p,p_rx_rbr_ring_t * rbr_p,p_rx_rcr_ring_t * rcr_p,p_rx_mbox_t * rx_mbox_p)2540 hxge_map_rxdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel,
2541 p_hxge_dma_common_t *dma_rbr_cntl_p, p_hxge_dma_common_t *dma_rcr_cntl_p,
2542 p_hxge_dma_common_t *dma_mbox_cntl_p, p_rx_rbr_ring_t *rbr_p,
2543 p_rx_rcr_ring_t *rcr_p, p_rx_mbox_t *rx_mbox_p)
2544 {
2545 p_rx_rbr_ring_t rbrp;
2546 p_rx_rcr_ring_t rcrp;
2547 p_rx_mbox_t mboxp;
2548 p_hxge_dma_common_t cntl_dmap;
2549 p_hxge_dma_common_t dmap;
2550 p_rx_msg_t *rx_msg_ring;
2551 p_rx_msg_t rx_msg_p;
2552 rdc_rbr_cfg_a_t *rcfga_p;
2553 rdc_rbr_cfg_b_t *rcfgb_p;
2554 rdc_rcr_cfg_a_t *cfga_p;
2555 rdc_rcr_cfg_b_t *cfgb_p;
2556 rdc_rx_cfg1_t *cfig1_p;
2557 rdc_rx_cfg2_t *cfig2_p;
2558 rdc_rbr_kick_t *kick_p;
2559 uint32_t dmaaddrp;
2560 uint32_t *rbr_vaddrp;
2561 uint32_t bkaddr;
2562 hxge_status_t status = HXGE_OK;
2563 int i;
2564 uint32_t hxge_port_rcr_size;
2565
2566 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2567 "==> hxge_map_rxdma_channel_cfg_ring"));
2568
2569 cntl_dmap = *dma_rbr_cntl_p;
2570
2571 /*
2572 * Map in the receive block ring
2573 */
2574 rbrp = *rbr_p;
2575 dmap = (p_hxge_dma_common_t)&rbrp->rbr_desc;
2576 hxge_setup_dma_common(dmap, cntl_dmap, rbrp->rbb_max, 4);
2577
2578 /*
2579 * Zero out buffer block ring descriptors.
2580 */
2581 bzero((caddr_t)dmap->kaddrp, dmap->alength);
2582
2583 rcfga_p = &(rbrp->rbr_cfga);
2584 rcfgb_p = &(rbrp->rbr_cfgb);
2585 kick_p = &(rbrp->rbr_kick);
2586 rcfga_p->value = 0;
2587 rcfgb_p->value = 0;
2588 kick_p->value = 0;
2589 rbrp->rbr_addr = dmap->dma_cookie.dmac_laddress;
2590 rcfga_p->value = (rbrp->rbr_addr &
2591 (RBR_CFIG_A_STDADDR_MASK | RBR_CFIG_A_STDADDR_BASE_MASK));
2592 rcfga_p->value |= ((uint64_t)rbrp->rbb_max << RBR_CFIG_A_LEN_SHIFT);
2593
2594 /* XXXX: how to choose packet buffer sizes */
2595 rcfgb_p->bits.bufsz0 = rbrp->pkt_buf_size0;
2596 rcfgb_p->bits.vld0 = 1;
2597 rcfgb_p->bits.bufsz1 = rbrp->pkt_buf_size1;
2598 rcfgb_p->bits.vld1 = 1;
2599 rcfgb_p->bits.bufsz2 = rbrp->pkt_buf_size2;
2600 rcfgb_p->bits.vld2 = 1;
2601 rcfgb_p->bits.bksize = hxgep->rx_bksize_code;
2602
2603 /*
2604 * For each buffer block, enter receive block address to the ring.
2605 */
2606 rbr_vaddrp = (uint32_t *)dmap->kaddrp;
2607 rbrp->rbr_desc_vp = (uint32_t *)dmap->kaddrp;
2608 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2609 "==> hxge_map_rxdma_channel_cfg_ring: channel %d "
2610 "rbr_vaddrp $%p", dma_channel, rbr_vaddrp));
2611
2612 rx_msg_ring = rbrp->rx_msg_ring;
2613 for (i = 0; i < rbrp->tnblocks; i++) {
2614 rx_msg_p = rx_msg_ring[i];
2615 rx_msg_p->hxgep = hxgep;
2616 rx_msg_p->rx_rbr_p = rbrp;
2617 bkaddr = (uint32_t)
2618 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >>
2619 RBR_BKADDR_SHIFT));
2620 rx_msg_p->free = B_FALSE;
2621 rx_msg_p->max_usage_cnt = 0xbaddcafe;
2622
2623 *rbr_vaddrp++ = bkaddr;
2624 }
2625
2626 kick_p->bits.bkadd = rbrp->rbb_max;
2627 rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
2628
2629 rbrp->rbr_rd_index = 0;
2630
2631 rbrp->rbr_consumed = 0;
2632 rbrp->rbr_used = 0;
2633 rbrp->rbr_use_bcopy = B_TRUE;
2634 rbrp->rbr_bufsize_type = RCR_PKTBUFSZ_0;
2635
2636 /*
2637 * Do bcopy on packets greater than bcopy size once the lo threshold is
2638 * reached. This lo threshold should be less than the hi threshold.
2639 *
2640 * Do bcopy on every packet once the hi threshold is reached.
2641 */
2642 if (hxge_rx_threshold_lo >= hxge_rx_threshold_hi) {
2643 /* default it to use hi */
2644 hxge_rx_threshold_lo = hxge_rx_threshold_hi;
2645 }
2646 if (hxge_rx_buf_size_type > HXGE_RBR_TYPE2) {
2647 hxge_rx_buf_size_type = HXGE_RBR_TYPE2;
2648 }
2649 rbrp->rbr_bufsize_type = hxge_rx_buf_size_type;
2650
2651 switch (hxge_rx_threshold_hi) {
2652 default:
2653 case HXGE_RX_COPY_NONE:
2654 /* Do not do bcopy at all */
2655 rbrp->rbr_use_bcopy = B_FALSE;
2656 rbrp->rbr_threshold_hi = rbrp->rbb_max;
2657 break;
2658
2659 case HXGE_RX_COPY_1:
2660 case HXGE_RX_COPY_2:
2661 case HXGE_RX_COPY_3:
2662 case HXGE_RX_COPY_4:
2663 case HXGE_RX_COPY_5:
2664 case HXGE_RX_COPY_6:
2665 case HXGE_RX_COPY_7:
2666 rbrp->rbr_threshold_hi =
2667 rbrp->rbb_max * (hxge_rx_threshold_hi) /
2668 HXGE_RX_BCOPY_SCALE;
2669 break;
2670
2671 case HXGE_RX_COPY_ALL:
2672 rbrp->rbr_threshold_hi = 0;
2673 break;
2674 }
2675
2676 switch (hxge_rx_threshold_lo) {
2677 default:
2678 case HXGE_RX_COPY_NONE:
2679 /* Do not do bcopy at all */
2680 if (rbrp->rbr_use_bcopy) {
2681 rbrp->rbr_use_bcopy = B_FALSE;
2682 }
2683 rbrp->rbr_threshold_lo = rbrp->rbb_max;
2684 break;
2685
2686 case HXGE_RX_COPY_1:
2687 case HXGE_RX_COPY_2:
2688 case HXGE_RX_COPY_3:
2689 case HXGE_RX_COPY_4:
2690 case HXGE_RX_COPY_5:
2691 case HXGE_RX_COPY_6:
2692 case HXGE_RX_COPY_7:
2693 rbrp->rbr_threshold_lo =
2694 rbrp->rbb_max * (hxge_rx_threshold_lo) /
2695 HXGE_RX_BCOPY_SCALE;
2696 break;
2697
2698 case HXGE_RX_COPY_ALL:
2699 rbrp->rbr_threshold_lo = 0;
2700 break;
2701 }
2702
2703 HXGE_DEBUG_MSG((hxgep, RX_CTL,
2704 "hxge_map_rxdma_channel_cfg_ring: channel %d rbb_max %d "
2705 "rbrp->rbr_bufsize_type %d rbb_threshold_hi %d "
2706 "rbb_threshold_lo %d",
2707 dma_channel, rbrp->rbb_max, rbrp->rbr_bufsize_type,
2708 rbrp->rbr_threshold_hi, rbrp->rbr_threshold_lo));
2709
2710 /* Map in the receive completion ring */
2711 rcrp = (p_rx_rcr_ring_t)KMEM_ZALLOC(sizeof (rx_rcr_ring_t), KM_SLEEP);
2712 MUTEX_INIT(&rcrp->lock, NULL, MUTEX_DRIVER,
2713 (void *) hxgep->interrupt_cookie);
2714 rcrp->rdc = dma_channel;
2715 rcrp->hxgep = hxgep;
2716
2717 hxge_port_rcr_size = hxgep->hxge_port_rcr_size;
2718 rcrp->comp_size = hxge_port_rcr_size;
2719 rcrp->comp_wrap_mask = hxge_port_rcr_size - 1;
2720
2721 cntl_dmap = *dma_rcr_cntl_p;
2722
2723 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc;
2724 hxge_setup_dma_common(dmap, cntl_dmap, rcrp->comp_size,
2725 sizeof (rcr_entry_t));
2726 rcrp->comp_rd_index = 0;
2727 rcrp->comp_wt_index = 0;
2728 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
2729 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
2730 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
2731 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
2732 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
2733 (hxge_port_rcr_size - 1);
2734 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
2735 (hxge_port_rcr_size - 1);
2736
2737 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc);
2738 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3;
2739
2740 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2741 "==> hxge_map_rxdma_channel_cfg_ring: channel %d "
2742 "rbr_vaddrp $%p rcr_desc_rd_head_p $%p "
2743 "rcr_desc_rd_head_pp $%p rcr_desc_rd_last_p $%p "
2744 "rcr_desc_rd_last_pp $%p ",
2745 dma_channel, rbr_vaddrp, rcrp->rcr_desc_rd_head_p,
2746 rcrp->rcr_desc_rd_head_pp, rcrp->rcr_desc_last_p,
2747 rcrp->rcr_desc_last_pp));
2748
2749 /*
2750 * Zero out buffer block ring descriptors.
2751 */
2752 bzero((caddr_t)dmap->kaddrp, dmap->alength);
2753 rcrp->intr_timeout = hxgep->intr_timeout;
2754 rcrp->intr_threshold = hxgep->intr_threshold;
2755 rcrp->full_hdr_flag = B_FALSE;
2756 rcrp->sw_priv_hdr_len = 0;
2757
2758 cfga_p = &(rcrp->rcr_cfga);
2759 cfgb_p = &(rcrp->rcr_cfgb);
2760 cfga_p->value = 0;
2761 cfgb_p->value = 0;
2762 rcrp->rcr_addr = dmap->dma_cookie.dmac_laddress;
2763
2764 cfga_p->value = (rcrp->rcr_addr &
2765 (RCRCFIG_A_STADDR_MASK | RCRCFIG_A_STADDR_BASE_MASK));
2766
2767 cfga_p->value |= ((uint64_t)rcrp->comp_size << RCRCFIG_A_LEN_SHIF);
2768
2769 /*
2770 * Timeout should be set based on the system clock divider. The
2771 * following timeout value of 1 assumes that the granularity (1000) is
2772 * 3 microseconds running at 300MHz.
2773 */
2774 cfgb_p->bits.pthres = rcrp->intr_threshold;
2775 cfgb_p->bits.timeout = rcrp->intr_timeout;
2776 cfgb_p->bits.entout = 1;
2777
2778 /* Map in the mailbox */
2779 cntl_dmap = *dma_mbox_cntl_p;
2780 mboxp = (p_rx_mbox_t)KMEM_ZALLOC(sizeof (rx_mbox_t), KM_SLEEP);
2781 dmap = (p_hxge_dma_common_t)&mboxp->rx_mbox;
2782 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (rxdma_mailbox_t));
2783 cfig1_p = (rdc_rx_cfg1_t *)&mboxp->rx_cfg1;
2784 cfig2_p = (rdc_rx_cfg2_t *)&mboxp->rx_cfg2;
2785 cfig1_p->value = cfig2_p->value = 0;
2786
2787 mboxp->mbox_addr = dmap->dma_cookie.dmac_laddress;
2788 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2789 "==> hxge_map_rxdma_channel_cfg_ring: "
2790 "channel %d cfg1 0x%016llx cfig2 0x%016llx cookie 0x%016llx",
2791 dma_channel, cfig1_p->value, cfig2_p->value,
2792 mboxp->mbox_addr));
2793
2794 dmaaddrp = (uint32_t)((dmap->dma_cookie.dmac_laddress >> 32) & 0xfff);
2795 cfig1_p->bits.mbaddr_h = dmaaddrp;
2796
2797 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress & 0xffffffff);
2798 dmaaddrp = (uint32_t)(dmap->dma_cookie.dmac_laddress &
2799 RXDMA_CFIG2_MBADDR_L_MASK);
2800
2801 cfig2_p->bits.mbaddr_l = (dmaaddrp >> RXDMA_CFIG2_MBADDR_L_SHIFT);
2802
2803 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2804 "==> hxge_map_rxdma_channel_cfg_ring: channel %d damaddrp $%p "
2805 "cfg1 0x%016llx cfig2 0x%016llx",
2806 dma_channel, dmaaddrp, cfig1_p->value, cfig2_p->value));
2807
2808 cfig2_p->bits.full_hdr = rcrp->full_hdr_flag;
2809 cfig2_p->bits.offset = rcrp->sw_priv_hdr_len;
2810
2811 rbrp->rx_rcr_p = rcrp;
2812 rcrp->rx_rbr_p = rbrp;
2813 *rcr_p = rcrp;
2814 *rx_mbox_p = mboxp;
2815
2816 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2817 "<== hxge_map_rxdma_channel_cfg_ring status 0x%08x", status));
2818 return (status);
2819 }
2820
2821 /*ARGSUSED*/
2822 static void
hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep,p_rx_rcr_ring_t rcr_p,p_rx_mbox_t rx_mbox_p)2823 hxge_unmap_rxdma_channel_cfg_ring(p_hxge_t hxgep,
2824 p_rx_rcr_ring_t rcr_p, p_rx_mbox_t rx_mbox_p)
2825 {
2826 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2827 "==> hxge_unmap_rxdma_channel_cfg_ring: channel %d", rcr_p->rdc));
2828
2829 MUTEX_DESTROY(&rcr_p->lock);
2830 KMEM_FREE(rcr_p, sizeof (rx_rcr_ring_t));
2831 KMEM_FREE(rx_mbox_p, sizeof (rx_mbox_t));
2832
2833 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2834 "<== hxge_unmap_rxdma_channel_cfg_ring"));
2835 }
2836
2837 static hxge_status_t
hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep,uint16_t channel,p_hxge_dma_common_t * dma_buf_p,p_rx_rbr_ring_t * rbr_p,uint32_t num_chunks)2838 hxge_map_rxdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel,
2839 p_hxge_dma_common_t *dma_buf_p,
2840 p_rx_rbr_ring_t *rbr_p, uint32_t num_chunks)
2841 {
2842 p_rx_rbr_ring_t rbrp;
2843 p_hxge_dma_common_t dma_bufp, tmp_bufp;
2844 p_rx_msg_t *rx_msg_ring;
2845 p_rx_msg_t rx_msg_p;
2846 p_mblk_t mblk_p;
2847
2848 rxring_info_t *ring_info;
2849 hxge_status_t status = HXGE_OK;
2850 int i, j, index;
2851 uint32_t size, bsize, nblocks, nmsgs;
2852
2853 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2854 "==> hxge_map_rxdma_channel_buf_ring: channel %d", channel));
2855
2856 dma_bufp = tmp_bufp = *dma_buf_p;
2857 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2858 " hxge_map_rxdma_channel_buf_ring: channel %d to map %d "
2859 "chunks bufp 0x%016llx", channel, num_chunks, dma_bufp));
2860
2861 nmsgs = 0;
2862 for (i = 0; i < num_chunks; i++, tmp_bufp++) {
2863 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2864 "==> hxge_map_rxdma_channel_buf_ring: channel %d "
2865 "bufp 0x%016llx nblocks %d nmsgs %d",
2866 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
2867 nmsgs += tmp_bufp->nblocks;
2868 }
2869 if (!nmsgs) {
2870 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2871 "<== hxge_map_rxdma_channel_buf_ring: channel %d "
2872 "no msg blocks", channel));
2873 status = HXGE_ERROR;
2874 goto hxge_map_rxdma_channel_buf_ring_exit;
2875 }
2876 rbrp = (p_rx_rbr_ring_t)KMEM_ZALLOC(sizeof (rx_rbr_ring_t), KM_SLEEP);
2877
2878 size = nmsgs * sizeof (p_rx_msg_t);
2879 rx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
2880 ring_info = (rxring_info_t *)KMEM_ZALLOC(sizeof (rxring_info_t),
2881 KM_SLEEP);
2882
2883 MUTEX_INIT(&rbrp->lock, NULL, MUTEX_DRIVER,
2884 (void *) hxgep->interrupt_cookie);
2885 MUTEX_INIT(&rbrp->post_lock, NULL, MUTEX_DRIVER,
2886 (void *) hxgep->interrupt_cookie);
2887
2888 rbrp->rdc = channel;
2889 rbrp->num_blocks = num_chunks;
2890 rbrp->tnblocks = nmsgs;
2891 rbrp->rbb_max = nmsgs;
2892 rbrp->rbr_max_size = nmsgs;
2893 rbrp->rbr_wrap_mask = (rbrp->rbb_max - 1);
2894
2895 /*
2896 * Buffer sizes: 256, 1K, and 2K.
2897 *
2898 * Blk 0 size.
2899 */
2900 rbrp->pkt_buf_size0 = RBR_BUFSZ0_256B;
2901 rbrp->pkt_buf_size0_bytes = RBR_BUFSZ0_256_BYTES;
2902 rbrp->hpi_pkt_buf_size0 = SIZE_256B;
2903
2904 /*
2905 * Blk 1 size.
2906 */
2907 rbrp->pkt_buf_size1 = RBR_BUFSZ1_1K;
2908 rbrp->pkt_buf_size1_bytes = RBR_BUFSZ1_1K_BYTES;
2909 rbrp->hpi_pkt_buf_size1 = SIZE_1KB;
2910
2911 /*
2912 * Blk 2 size.
2913 */
2914 rbrp->pkt_buf_size2 = RBR_BUFSZ2_2K;
2915 rbrp->pkt_buf_size2_bytes = RBR_BUFSZ2_2K_BYTES;
2916 rbrp->hpi_pkt_buf_size2 = SIZE_2KB;
2917
2918 rbrp->block_size = hxgep->rx_default_block_size;
2919
2920 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2921 "==> hxge_map_rxdma_channel_buf_ring: channel %d "
2922 "actual rbr max %d rbb_max %d nmsgs %d "
2923 "rbrp->block_size %d default_block_size %d "
2924 "(config hxge_rbr_size %d hxge_rbr_spare_size %d)",
2925 channel, rbrp->rbr_max_size, rbrp->rbb_max, nmsgs,
2926 rbrp->block_size, hxgep->rx_default_block_size,
2927 hxge_rbr_size, hxge_rbr_spare_size));
2928
2929 /*
2930 * Map in buffers from the buffer pool.
2931 * Note that num_blocks is the num_chunks. For Sparc, there is likely
2932 * only one chunk. For x86, there will be many chunks.
2933 * Loop over chunks.
2934 */
2935 index = 0;
2936 for (i = 0; i < rbrp->num_blocks; i++, dma_bufp++) {
2937 bsize = dma_bufp->block_size;
2938 nblocks = dma_bufp->nblocks;
2939 ring_info->buffer[i].dvma_addr = (uint64_t)dma_bufp->ioaddr_pp;
2940 ring_info->buffer[i].buf_index = i;
2941 ring_info->buffer[i].buf_size = dma_bufp->alength;
2942 ring_info->buffer[i].start_index = index;
2943 ring_info->buffer[i].kaddr = (uint64_t)dma_bufp->kaddrp;
2944
2945 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2946 " hxge_map_rxdma_channel_buf_ring: map channel %d "
2947 "chunk %d nblocks %d chunk_size %x block_size 0x%x "
2948 "dma_bufp $%p dvma_addr $%p", channel, i,
2949 dma_bufp->nblocks,
2950 ring_info->buffer[i].buf_size, bsize, dma_bufp,
2951 ring_info->buffer[i].dvma_addr));
2952
2953 /* loop over blocks within a chunk */
2954 for (j = 0; j < nblocks; j++) {
2955 if ((rx_msg_p = hxge_allocb(bsize, BPRI_LO,
2956 dma_bufp)) == NULL) {
2957 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2958 "allocb failed (index %d i %d j %d)",
2959 index, i, j));
2960 goto hxge_map_rxdma_channel_buf_ring_fail1;
2961 }
2962 rx_msg_ring[index] = rx_msg_p;
2963 rx_msg_p->block_index = index;
2964 rx_msg_p->shifted_addr = (uint32_t)
2965 ((rx_msg_p->buf_dma.dma_cookie.dmac_laddress >>
2966 RBR_BKADDR_SHIFT));
2967 /*
2968 * Too much output
2969 * HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2970 * "index %d j %d rx_msg_p $%p mblk %p",
2971 * index, j, rx_msg_p, rx_msg_p->rx_mblk_p));
2972 */
2973 mblk_p = rx_msg_p->rx_mblk_p;
2974 mblk_p->b_wptr = mblk_p->b_rptr + bsize;
2975
2976 rbrp->rbr_ref_cnt++;
2977 index++;
2978 rx_msg_p->buf_dma.dma_channel = channel;
2979 }
2980 }
2981 if (i < rbrp->num_blocks) {
2982 goto hxge_map_rxdma_channel_buf_ring_fail1;
2983 }
2984 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
2985 "hxge_map_rxdma_channel_buf_ring: done buf init "
2986 "channel %d msg block entries %d", channel, index));
2987 ring_info->block_size_mask = bsize - 1;
2988 rbrp->rx_msg_ring = rx_msg_ring;
2989 rbrp->dma_bufp = dma_buf_p;
2990 rbrp->ring_info = ring_info;
2991
2992 status = hxge_rxbuf_index_info_init(hxgep, rbrp);
2993 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, " hxge_map_rxdma_channel_buf_ring: "
2994 "channel %d done buf info init", channel));
2995
2996 /*
2997 * Finally, permit hxge_freeb() to call hxge_post_page().
2998 */
2999 rbrp->rbr_state = RBR_POSTING;
3000
3001 *rbr_p = rbrp;
3002
3003 goto hxge_map_rxdma_channel_buf_ring_exit;
3004
3005 hxge_map_rxdma_channel_buf_ring_fail1:
3006 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3007 " hxge_map_rxdma_channel_buf_ring: failed channel (0x%x)",
3008 channel, status));
3009
3010 index--;
3011 for (; index >= 0; index--) {
3012 rx_msg_p = rx_msg_ring[index];
3013 if (rx_msg_p != NULL) {
3014 freeb(rx_msg_p->rx_mblk_p);
3015 rx_msg_ring[index] = NULL;
3016 }
3017 }
3018
3019 hxge_map_rxdma_channel_buf_ring_fail:
3020 MUTEX_DESTROY(&rbrp->post_lock);
3021 MUTEX_DESTROY(&rbrp->lock);
3022 KMEM_FREE(ring_info, sizeof (rxring_info_t));
3023 KMEM_FREE(rx_msg_ring, size);
3024 KMEM_FREE(rbrp, sizeof (rx_rbr_ring_t));
3025
3026 status = HXGE_ERROR;
3027
3028 hxge_map_rxdma_channel_buf_ring_exit:
3029 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3030 "<== hxge_map_rxdma_channel_buf_ring status 0x%08x", status));
3031
3032 return (status);
3033 }
3034
3035 /*ARGSUSED*/
3036 static void
hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep,p_rx_rbr_ring_t rbr_p)3037 hxge_unmap_rxdma_channel_buf_ring(p_hxge_t hxgep,
3038 p_rx_rbr_ring_t rbr_p)
3039 {
3040 p_rx_msg_t *rx_msg_ring;
3041 p_rx_msg_t rx_msg_p;
3042 rxring_info_t *ring_info;
3043 int i;
3044 uint32_t size;
3045
3046 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3047 "==> hxge_unmap_rxdma_channel_buf_ring"));
3048 if (rbr_p == NULL) {
3049 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3050 "<== hxge_unmap_rxdma_channel_buf_ring: NULL rbrp"));
3051 return;
3052 }
3053 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3054 "==> hxge_unmap_rxdma_channel_buf_ring: channel %d", rbr_p->rdc));
3055
3056 rx_msg_ring = rbr_p->rx_msg_ring;
3057 ring_info = rbr_p->ring_info;
3058
3059 if (rx_msg_ring == NULL || ring_info == NULL) {
3060 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3061 "<== hxge_unmap_rxdma_channel_buf_ring: "
3062 "rx_msg_ring $%p ring_info $%p", rx_msg_p, ring_info));
3063 return;
3064 }
3065
3066 size = rbr_p->tnblocks * sizeof (p_rx_msg_t);
3067 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3068 " hxge_unmap_rxdma_channel_buf_ring: channel %d chunks %d "
3069 "tnblocks %d (max %d) size ptrs %d ", rbr_p->rdc, rbr_p->num_blocks,
3070 rbr_p->tnblocks, rbr_p->rbr_max_size, size));
3071
3072 for (i = 0; i < rbr_p->tnblocks; i++) {
3073 rx_msg_p = rx_msg_ring[i];
3074 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3075 " hxge_unmap_rxdma_channel_buf_ring: "
3076 "rx_msg_p $%p", rx_msg_p));
3077 if (rx_msg_p != NULL) {
3078 freeb(rx_msg_p->rx_mblk_p);
3079 rx_msg_ring[i] = NULL;
3080 }
3081 }
3082
3083 /*
3084 * We no longer may use the mutex <post_lock>. By setting
3085 * <rbr_state> to anything but POSTING, we prevent
3086 * hxge_post_page() from accessing a dead mutex.
3087 */
3088 rbr_p->rbr_state = RBR_UNMAPPING;
3089 MUTEX_DESTROY(&rbr_p->post_lock);
3090
3091 MUTEX_DESTROY(&rbr_p->lock);
3092 KMEM_FREE(ring_info, sizeof (rxring_info_t));
3093 KMEM_FREE(rx_msg_ring, size);
3094
3095 if (rbr_p->rbr_ref_cnt == 0) {
3096 /* This is the normal state of affairs. */
3097 KMEM_FREE(rbr_p, sizeof (*rbr_p));
3098 } else {
3099 /*
3100 * Some of our buffers are still being used.
3101 * Therefore, tell hxge_freeb() this ring is
3102 * unmapped, so it may free <rbr_p> for us.
3103 */
3104 rbr_p->rbr_state = RBR_UNMAPPED;
3105 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3106 "unmap_rxdma_buf_ring: %d %s outstanding.",
3107 rbr_p->rbr_ref_cnt,
3108 rbr_p->rbr_ref_cnt == 1 ? "msg" : "msgs"));
3109 }
3110
3111 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3112 "<== hxge_unmap_rxdma_channel_buf_ring"));
3113 }
3114
3115 static hxge_status_t
hxge_rxdma_hw_start_common(p_hxge_t hxgep)3116 hxge_rxdma_hw_start_common(p_hxge_t hxgep)
3117 {
3118 hxge_status_t status = HXGE_OK;
3119
3120 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common"));
3121
3122 /*
3123 * Load the sharable parameters by writing to the function zero control
3124 * registers. These FZC registers should be initialized only once for
3125 * the entire chip.
3126 */
3127 (void) hxge_init_fzc_rx_common(hxgep);
3128
3129 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start_common"));
3130
3131 return (status);
3132 }
3133
3134 static hxge_status_t
hxge_rxdma_hw_start(p_hxge_t hxgep)3135 hxge_rxdma_hw_start(p_hxge_t hxgep)
3136 {
3137 int i, ndmas;
3138 uint16_t channel;
3139 p_rx_rbr_rings_t rx_rbr_rings;
3140 p_rx_rbr_ring_t *rbr_rings;
3141 p_rx_rcr_rings_t rx_rcr_rings;
3142 p_rx_rcr_ring_t *rcr_rings;
3143 p_rx_mbox_areas_t rx_mbox_areas_p;
3144 p_rx_mbox_t *rx_mbox_p;
3145 hxge_status_t status = HXGE_OK;
3146
3147 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start"));
3148
3149 rx_rbr_rings = hxgep->rx_rbr_rings;
3150 rx_rcr_rings = hxgep->rx_rcr_rings;
3151 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
3152 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3153 "<== hxge_rxdma_hw_start: NULL ring pointers"));
3154 return (HXGE_ERROR);
3155 }
3156
3157 ndmas = rx_rbr_rings->ndmas;
3158 if (ndmas == 0) {
3159 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3160 "<== hxge_rxdma_hw_start: no dma channel allocated"));
3161 return (HXGE_ERROR);
3162 }
3163 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3164 "==> hxge_rxdma_hw_start (ndmas %d)", ndmas));
3165
3166 /*
3167 * Scrub the RDC Rx DMA Prefetch Buffer Command.
3168 */
3169 for (i = 0; i < 128; i++) {
3170 HXGE_REG_WR64(hxgep->hpi_handle, RDC_PREF_CMD, i);
3171 }
3172
3173 /*
3174 * Scrub Rx DMA Shadow Tail Command.
3175 */
3176 for (i = 0; i < 64; i++) {
3177 HXGE_REG_WR64(hxgep->hpi_handle, RDC_SHADOW_CMD, i);
3178 }
3179
3180 /*
3181 * Scrub Rx DMA Control Fifo Command.
3182 */
3183 for (i = 0; i < 512; i++) {
3184 HXGE_REG_WR64(hxgep->hpi_handle, RDC_CTRL_FIFO_CMD, i);
3185 }
3186
3187 /*
3188 * Scrub Rx DMA Data Fifo Command.
3189 */
3190 for (i = 0; i < 1536; i++) {
3191 HXGE_REG_WR64(hxgep->hpi_handle, RDC_DATA_FIFO_CMD, i);
3192 }
3193
3194 /*
3195 * Reset the FIFO Error Stat.
3196 */
3197 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_STAT, 0xFF);
3198
3199 /* Set the error mask to receive interrupts */
3200 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0);
3201
3202 rbr_rings = rx_rbr_rings->rbr_rings;
3203 rcr_rings = rx_rcr_rings->rcr_rings;
3204 rx_mbox_areas_p = hxgep->rx_mbox_areas_p;
3205 if (rx_mbox_areas_p) {
3206 rx_mbox_p = rx_mbox_areas_p->rxmbox_areas;
3207 }
3208
3209 for (i = 0; i < ndmas; i++) {
3210 channel = rbr_rings[i]->rdc;
3211 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3212 "==> hxge_rxdma_hw_start (ndmas %d) channel %d",
3213 ndmas, channel));
3214 status = hxge_rxdma_start_channel(hxgep, channel,
3215 (p_rx_rbr_ring_t)rbr_rings[i],
3216 (p_rx_rcr_ring_t)rcr_rings[i],
3217 (p_rx_mbox_t)rx_mbox_p[i], rbr_rings[i]->rbb_max);
3218 if (status != HXGE_OK) {
3219 goto hxge_rxdma_hw_start_fail1;
3220 }
3221 }
3222
3223 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_start: "
3224 "rx_rbr_rings 0x%016llx rings 0x%016llx",
3225 rx_rbr_rings, rx_rcr_rings));
3226 goto hxge_rxdma_hw_start_exit;
3227
3228 hxge_rxdma_hw_start_fail1:
3229 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3230 "==> hxge_rxdma_hw_start: disable "
3231 "(status 0x%x channel %d i %d)", status, channel, i));
3232 for (; i >= 0; i--) {
3233 channel = rbr_rings[i]->rdc;
3234 (void) hxge_rxdma_stop_channel(hxgep, channel);
3235 }
3236
3237 hxge_rxdma_hw_start_exit:
3238 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3239 "==> hxge_rxdma_hw_start: (status 0x%x)", status));
3240 return (status);
3241 }
3242
3243 static void
hxge_rxdma_hw_stop(p_hxge_t hxgep)3244 hxge_rxdma_hw_stop(p_hxge_t hxgep)
3245 {
3246 int i, ndmas;
3247 uint16_t channel;
3248 p_rx_rbr_rings_t rx_rbr_rings;
3249 p_rx_rbr_ring_t *rbr_rings;
3250 p_rx_rcr_rings_t rx_rcr_rings;
3251
3252 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop"));
3253
3254 rx_rbr_rings = hxgep->rx_rbr_rings;
3255 rx_rcr_rings = hxgep->rx_rcr_rings;
3256
3257 if (rx_rbr_rings == NULL || rx_rcr_rings == NULL) {
3258 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3259 "<== hxge_rxdma_hw_stop: NULL ring pointers"));
3260 return;
3261 }
3262
3263 ndmas = rx_rbr_rings->ndmas;
3264 if (!ndmas) {
3265 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3266 "<== hxge_rxdma_hw_stop: no dma channel allocated"));
3267 return;
3268 }
3269
3270 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3271 "==> hxge_rxdma_hw_stop (ndmas %d)", ndmas));
3272
3273 rbr_rings = rx_rbr_rings->rbr_rings;
3274 for (i = 0; i < ndmas; i++) {
3275 channel = rbr_rings[i]->rdc;
3276 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3277 "==> hxge_rxdma_hw_stop (ndmas %d) channel %d",
3278 ndmas, channel));
3279 (void) hxge_rxdma_stop_channel(hxgep, channel);
3280 }
3281
3282 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_hw_stop: "
3283 "rx_rbr_rings 0x%016llx rings 0x%016llx",
3284 rx_rbr_rings, rx_rcr_rings));
3285
3286 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_hw_stop"));
3287 }
3288
3289 static hxge_status_t
hxge_rxdma_start_channel(p_hxge_t hxgep,uint16_t channel,p_rx_rbr_ring_t rbr_p,p_rx_rcr_ring_t rcr_p,p_rx_mbox_t mbox_p,int n_init_kick)3290 hxge_rxdma_start_channel(p_hxge_t hxgep, uint16_t channel,
3291 p_rx_rbr_ring_t rbr_p, p_rx_rcr_ring_t rcr_p, p_rx_mbox_t mbox_p,
3292 int n_init_kick)
3293 {
3294 hpi_handle_t handle;
3295 hpi_status_t rs = HPI_SUCCESS;
3296 rdc_stat_t cs;
3297 rdc_int_mask_t ent_mask;
3298 hxge_status_t status = HXGE_OK;
3299
3300 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel"));
3301
3302 handle = HXGE_DEV_HPI_HANDLE(hxgep);
3303
3304 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "hxge_rxdma_start_channel: "
3305 "hpi handle addr $%p acc $%p",
3306 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh));
3307
3308 /* Reset RXDMA channel */
3309 rs = hpi_rxdma_cfg_rdc_reset(handle, channel);
3310 if (rs != HPI_SUCCESS) {
3311 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3312 "==> hxge_rxdma_start_channel: "
3313 "reset rxdma failed (0x%08x channel %d)",
3314 status, channel));
3315 return (HXGE_ERROR | rs);
3316 }
3317 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3318 "==> hxge_rxdma_start_channel: reset done: channel %d", channel));
3319
3320 /*
3321 * Initialize the RXDMA channel specific FZC control configurations.
3322 * These FZC registers are pertaining to each RX channel (logical
3323 * pages).
3324 */
3325 status = hxge_init_fzc_rxdma_channel(hxgep,
3326 channel, rbr_p, rcr_p, mbox_p);
3327 if (status != HXGE_OK) {
3328 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3329 "==> hxge_rxdma_start_channel: "
3330 "init fzc rxdma failed (0x%08x channel %d)",
3331 status, channel));
3332 return (status);
3333 }
3334 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3335 "==> hxge_rxdma_start_channel: fzc done"));
3336
3337 /*
3338 * Zero out the shadow and prefetch ram.
3339 */
3340 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3341 "==> hxge_rxdma_start_channel: ram done"));
3342
3343 /* Set up the interrupt event masks. */
3344 ent_mask.value = 0;
3345 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
3346 if (rs != HPI_SUCCESS) {
3347 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3348 "==> hxge_rxdma_start_channel: "
3349 "init rxdma event masks failed (0x%08x channel %d)",
3350 status, channel));
3351 return (HXGE_ERROR | rs);
3352 }
3353 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3354 "event done: channel %d (mask 0x%016llx)",
3355 channel, ent_mask.value));
3356
3357 /*
3358 * Load RXDMA descriptors, buffers, mailbox, initialise the receive DMA
3359 * channels and enable each DMA channel.
3360 */
3361 status = hxge_enable_rxdma_channel(hxgep,
3362 channel, rbr_p, rcr_p, mbox_p, n_init_kick);
3363 if (status != HXGE_OK) {
3364 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3365 " hxge_rxdma_start_channel: "
3366 " init enable rxdma failed (0x%08x channel %d)",
3367 status, channel));
3368 return (status);
3369 }
3370
3371 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3372 "control done - channel %d cs 0x%016llx", channel, cs.value));
3373
3374 /*
3375 * Initialize the receive DMA control and status register
3376 * Note that rdc_stat HAS to be set after RBR and RCR rings are set
3377 */
3378 cs.value = 0;
3379 cs.bits.mex = 1;
3380 cs.bits.rcr_thres = 1;
3381 cs.bits.rcr_to = 1;
3382 cs.bits.rbr_empty = 1;
3383 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs);
3384 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3385 "channel %d rx_dma_cntl_stat 0x%0016llx", channel, cs.value));
3386 if (status != HXGE_OK) {
3387 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3388 "==> hxge_rxdma_start_channel: "
3389 "init rxdma control register failed (0x%08x channel %d",
3390 status, channel));
3391 return (status);
3392 }
3393
3394 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "==> hxge_rxdma_start_channel: "
3395 "control done - channel %d cs 0x%016llx", channel, cs.value));
3396 HXGE_DEBUG_MSG((hxgep, MEM2_CTL,
3397 "==> hxge_rxdma_start_channel: enable done"));
3398 HXGE_DEBUG_MSG((hxgep, MEM2_CTL, "<== hxge_rxdma_start_channel"));
3399 return (HXGE_OK);
3400 }
3401
3402 static hxge_status_t
hxge_rxdma_stop_channel(p_hxge_t hxgep,uint16_t channel)3403 hxge_rxdma_stop_channel(p_hxge_t hxgep, uint16_t channel)
3404 {
3405 hpi_handle_t handle;
3406 hpi_status_t rs = HPI_SUCCESS;
3407 rdc_stat_t cs;
3408 rdc_int_mask_t ent_mask;
3409 hxge_status_t status = HXGE_OK;
3410
3411 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel"));
3412
3413 handle = HXGE_DEV_HPI_HANDLE(hxgep);
3414
3415 HXGE_DEBUG_MSG((hxgep, RX_CTL, "hxge_rxdma_stop_channel: "
3416 "hpi handle addr $%p acc $%p",
3417 hxgep->hpi_handle.regp, hxgep->hpi_handle.regh));
3418
3419 /* Reset RXDMA channel */
3420 rs = hpi_rxdma_cfg_rdc_reset(handle, channel);
3421 if (rs != HPI_SUCCESS) {
3422 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3423 " hxge_rxdma_stop_channel: "
3424 " reset rxdma failed (0x%08x channel %d)",
3425 rs, channel));
3426 return (HXGE_ERROR | rs);
3427 }
3428 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3429 "==> hxge_rxdma_stop_channel: reset done"));
3430
3431 /* Set up the interrupt event masks. */
3432 ent_mask.value = RDC_INT_MASK_ALL;
3433 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
3434 if (rs != HPI_SUCCESS) {
3435 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3436 "==> hxge_rxdma_stop_channel: "
3437 "set rxdma event masks failed (0x%08x channel %d)",
3438 rs, channel));
3439 return (HXGE_ERROR | rs);
3440 }
3441 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3442 "==> hxge_rxdma_stop_channel: event done"));
3443
3444 /* Initialize the receive DMA control and status register */
3445 cs.value = 0;
3446 status = hxge_init_rxdma_channel_cntl_stat(hxgep, channel, &cs);
3447
3448 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_stop_channel: control "
3449 " to default (all 0s) 0x%08x", cs.value));
3450
3451 if (status != HXGE_OK) {
3452 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3453 " hxge_rxdma_stop_channel: init rxdma"
3454 " control register failed (0x%08x channel %d",
3455 status, channel));
3456 return (status);
3457 }
3458
3459 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3460 "==> hxge_rxdma_stop_channel: control done"));
3461
3462 /* disable dma channel */
3463 status = hxge_disable_rxdma_channel(hxgep, channel);
3464
3465 if (status != HXGE_OK) {
3466 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3467 " hxge_rxdma_stop_channel: "
3468 " init enable rxdma failed (0x%08x channel %d)",
3469 status, channel));
3470 return (status);
3471 }
3472
3473 HXGE_DEBUG_MSG((hxgep, RX_CTL,
3474 "==> hxge_rxdma_stop_channel: disable done"));
3475 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_stop_channel"));
3476
3477 return (HXGE_OK);
3478 }
3479
3480 hxge_status_t
hxge_rxdma_handle_sys_errors(p_hxge_t hxgep)3481 hxge_rxdma_handle_sys_errors(p_hxge_t hxgep)
3482 {
3483 hpi_handle_t handle;
3484 p_hxge_rdc_sys_stats_t statsp;
3485 rdc_fifo_err_stat_t stat;
3486 hxge_status_t status = HXGE_OK;
3487
3488 handle = hxgep->hpi_handle;
3489 statsp = (p_hxge_rdc_sys_stats_t)&hxgep->statsp->rdc_sys_stats;
3490
3491 /* Get the error status and clear the register */
3492 HXGE_REG_RD64(handle, RDC_FIFO_ERR_STAT, &stat.value);
3493 HXGE_REG_WR64(handle, RDC_FIFO_ERR_STAT, stat.value);
3494
3495 if (stat.bits.rx_ctrl_fifo_sec) {
3496 statsp->ctrl_fifo_sec++;
3497 if (statsp->ctrl_fifo_sec == 1)
3498 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3499 "==> hxge_rxdma_handle_sys_errors: "
3500 "rx_ctrl_fifo_sec"));
3501 }
3502
3503 if (stat.bits.rx_ctrl_fifo_ded) {
3504 /* Global fatal error encountered */
3505 statsp->ctrl_fifo_ded++;
3506 HXGE_FM_REPORT_ERROR(hxgep, 0,
3507 HXGE_FM_EREPORT_RDMC_CTRL_FIFO_DED);
3508 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3509 "==> hxge_rxdma_handle_sys_errors: "
3510 "fatal error: rx_ctrl_fifo_ded error"));
3511 }
3512
3513 if (stat.bits.rx_data_fifo_sec) {
3514 statsp->data_fifo_sec++;
3515 if (statsp->data_fifo_sec == 1)
3516 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3517 "==> hxge_rxdma_handle_sys_errors: "
3518 "rx_data_fifo_sec"));
3519 }
3520
3521 if (stat.bits.rx_data_fifo_ded) {
3522 /* Global fatal error encountered */
3523 statsp->data_fifo_ded++;
3524 HXGE_FM_REPORT_ERROR(hxgep, 0,
3525 HXGE_FM_EREPORT_RDMC_DATA_FIFO_DED);
3526 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3527 "==> hxge_rxdma_handle_sys_errors: "
3528 "fatal error: rx_data_fifo_ded error"));
3529 }
3530
3531 if (stat.bits.rx_ctrl_fifo_ded || stat.bits.rx_data_fifo_ded) {
3532 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3533 " hxge_rxdma_handle_sys_errors: fatal error\n"));
3534 status = hxge_rx_port_fatal_err_recover(hxgep);
3535 if (status == HXGE_OK) {
3536 FM_SERVICE_RESTORED(hxgep);
3537 }
3538 }
3539
3540 return (HXGE_OK);
3541 }
3542
3543 static hxge_status_t
hxge_rxdma_fatal_err_recover(p_hxge_t hxgep,uint16_t channel)3544 hxge_rxdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel)
3545 {
3546 hpi_handle_t handle;
3547 hpi_status_t rs = HPI_SUCCESS;
3548 p_rx_rbr_ring_t rbrp;
3549 p_rx_rcr_ring_t rcrp;
3550 p_rx_mbox_t mboxp;
3551 rdc_int_mask_t ent_mask;
3552 p_hxge_dma_common_t dmap;
3553 p_rx_msg_t rx_msg_p;
3554 int i;
3555 uint32_t hxge_port_rcr_size;
3556 uint64_t tmp;
3557 int n_init_kick = 0;
3558
3559 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rxdma_fatal_err_recover"));
3560
3561 /*
3562 * Stop the dma channel waits for the stop done. If the stop done bit
3563 * is not set, then create an error.
3564 */
3565
3566 handle = HXGE_DEV_HPI_HANDLE(hxgep);
3567
3568 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Rx DMA stop..."));
3569
3570 rbrp = (p_rx_rbr_ring_t)hxgep->rx_rbr_rings->rbr_rings[channel];
3571 rcrp = (p_rx_rcr_ring_t)hxgep->rx_rcr_rings->rcr_rings[channel];
3572
3573 MUTEX_ENTER(&rcrp->lock);
3574 MUTEX_ENTER(&rbrp->lock);
3575
3576 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA channel..."));
3577
3578 rs = hpi_rxdma_cfg_rdc_disable(handle, channel);
3579 if (rs != HPI_SUCCESS) {
3580 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3581 "hxge_disable_rxdma_channel:failed"));
3582 goto fail;
3583 }
3584 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxDMA interrupt..."));
3585
3586 /* Disable interrupt */
3587 ent_mask.value = RDC_INT_MASK_ALL;
3588 rs = hpi_rxdma_event_mask(handle, OP_SET, channel, &ent_mask);
3589 if (rs != HPI_SUCCESS) {
3590 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3591 "Set rxdma event masks failed (channel %d)", channel));
3592 }
3593 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel reset..."));
3594
3595 /* Reset RXDMA channel */
3596 rs = hpi_rxdma_cfg_rdc_reset(handle, channel);
3597 if (rs != HPI_SUCCESS) {
3598 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3599 "Reset rxdma failed (channel %d)", channel));
3600 goto fail;
3601 }
3602 hxge_port_rcr_size = hxgep->hxge_port_rcr_size;
3603 mboxp = (p_rx_mbox_t)hxgep->rx_mbox_areas_p->rxmbox_areas[channel];
3604
3605 rbrp->rbr_wr_index = (rbrp->rbb_max - 1);
3606 rbrp->rbr_rd_index = 0;
3607
3608 rcrp->comp_rd_index = 0;
3609 rcrp->comp_wt_index = 0;
3610 rcrp->rcr_desc_rd_head_p = rcrp->rcr_desc_first_p =
3611 (p_rcr_entry_t)DMA_COMMON_VPTR(rcrp->rcr_desc);
3612 rcrp->rcr_desc_rd_head_pp = rcrp->rcr_desc_first_pp =
3613 (p_rcr_entry_t)DMA_COMMON_IOADDR(rcrp->rcr_desc);
3614
3615 rcrp->rcr_desc_last_p = rcrp->rcr_desc_rd_head_p +
3616 (hxge_port_rcr_size - 1);
3617 rcrp->rcr_desc_last_pp = rcrp->rcr_desc_rd_head_pp +
3618 (hxge_port_rcr_size - 1);
3619
3620 rcrp->rcr_tail_begin = DMA_COMMON_IOADDR(rcrp->rcr_desc);
3621 rcrp->rcr_tail_begin = (rcrp->rcr_tail_begin & 0x7ffffULL) >> 3;
3622
3623 dmap = (p_hxge_dma_common_t)&rcrp->rcr_desc;
3624 bzero((caddr_t)dmap->kaddrp, dmap->alength);
3625
3626 HXGE_DEBUG_MSG((hxgep, RX_CTL, "rbr entries = %d\n",
3627 rbrp->rbr_max_size));
3628
3629 /* Count the number of buffers owned by the hardware at this moment */
3630 for (i = 0; i < rbrp->rbr_max_size; i++) {
3631 rx_msg_p = rbrp->rx_msg_ring[i];
3632 if (rx_msg_p->ref_cnt == 1) {
3633 n_init_kick++;
3634 }
3635 }
3636
3637 HXGE_DEBUG_MSG((hxgep, RX_CTL, "RxDMA channel re-start..."));
3638
3639 /*
3640 * This is error recover! Some buffers are owned by the hardware and
3641 * the rest are owned by the apps. We should only kick in those
3642 * owned by the hardware initially. The apps will post theirs
3643 * eventually.
3644 */
3645 (void) hxge_rxdma_start_channel(hxgep, channel, rbrp, rcrp, mboxp,
3646 n_init_kick);
3647
3648 /*
3649 * The DMA channel may disable itself automatically.
3650 * The following is a work-around.
3651 */
3652 HXGE_REG_RD64(handle, RDC_RX_CFG1, &tmp);
3653 rs = hpi_rxdma_cfg_rdc_enable(handle, channel);
3654 if (rs != HPI_SUCCESS) {
3655 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3656 "hpi_rxdma_cfg_rdc_enable (channel %d)", channel));
3657 }
3658
3659 /*
3660 * Delay a bit of time by doing reads.
3661 */
3662 for (i = 0; i < 1024; i++) {
3663 uint64_t value;
3664 RXDMA_REG_READ64(HXGE_DEV_HPI_HANDLE(hxgep),
3665 RDC_INT_MASK, i & 3, &value);
3666 }
3667
3668 MUTEX_EXIT(&rbrp->lock);
3669 MUTEX_EXIT(&rcrp->lock);
3670
3671 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rxdma_fatal_err_recover"));
3672 return (HXGE_OK);
3673
3674 fail:
3675 MUTEX_EXIT(&rbrp->lock);
3676 MUTEX_EXIT(&rcrp->lock);
3677 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3678 "Error Recovery failed for channel(%d)", channel));
3679 return (HXGE_ERROR | rs);
3680 }
3681
3682 static hxge_status_t
hxge_rx_port_fatal_err_recover(p_hxge_t hxgep)3683 hxge_rx_port_fatal_err_recover(p_hxge_t hxgep)
3684 {
3685 hxge_status_t status = HXGE_OK;
3686 p_hxge_dma_common_t *dma_buf_p;
3687 uint16_t channel;
3688 int ndmas;
3689 int i;
3690 block_reset_t reset_reg;
3691 p_rx_rcr_ring_t rcrp;
3692 p_rx_rbr_ring_t rbrp;
3693
3694 HXGE_DEBUG_MSG((hxgep, RX_CTL, "==> hxge_rx_port_fatal_err_recover"));
3695 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovering from RDC error ..."));
3696
3697 /* Disable RxMAC */
3698 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Disable RxMAC...\n"));
3699 MUTEX_ENTER(&hxgep->vmac_lock);
3700 if (hxge_rx_vmac_disable(hxgep) != HXGE_OK)
3701 goto fail;
3702
3703 HXGE_DELAY(1000);
3704
3705 /*
3706 * Reset RDC block from PEU for this fatal error
3707 */
3708 reset_reg.value = 0;
3709 reset_reg.bits.rdc_rst = 1;
3710 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
3711
3712 HXGE_DELAY(1000);
3713
3714 /* Restore any common settings after PEU reset */
3715 if (hxge_rxdma_hw_start_common(hxgep) != HXGE_OK)
3716 goto fail;
3717
3718 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Stop all RxDMA channels..."));
3719
3720 ndmas = hxgep->rx_buf_pool_p->ndmas;
3721 dma_buf_p = hxgep->rx_buf_pool_p->dma_buf_pool_p;
3722
3723 for (i = 0; i < ndmas; i++) {
3724 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
3725 rcrp = hxgep->rx_rcr_rings->rcr_rings[channel];
3726 rbrp = rcrp->rx_rbr_p;
3727
3728 MUTEX_ENTER(&rbrp->post_lock);
3729
3730 /*
3731 * This function needs to be inside the post_lock
3732 */
3733 if (hxge_rxdma_fatal_err_recover(hxgep, channel) != HXGE_OK) {
3734 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3735 "Could not recover channel %d", channel));
3736 }
3737 MUTEX_EXIT(&rbrp->post_lock);
3738 }
3739
3740 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Reset RxMAC..."));
3741
3742 /* Reset RxMAC */
3743 if (hxge_rx_vmac_reset(hxgep) != HXGE_OK) {
3744 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3745 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC"));
3746 goto fail;
3747 }
3748
3749 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-initialize RxMAC..."));
3750
3751 /* Re-Initialize RxMAC */
3752 if ((status = hxge_rx_vmac_init(hxgep)) != HXGE_OK) {
3753 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3754 "hxge_rx_port_fatal_err_recover: Failed to reset RxMAC"));
3755 goto fail;
3756 }
3757 HXGE_DEBUG_MSG((hxgep, RX_CTL, "Re-enable RxMAC..."));
3758
3759 /* Re-enable RxMAC */
3760 if ((status = hxge_rx_vmac_enable(hxgep)) != HXGE_OK) {
3761 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3762 "hxge_rx_port_fatal_err_recover: Failed to enable RxMAC"));
3763 goto fail;
3764 }
3765 MUTEX_EXIT(&hxgep->vmac_lock);
3766
3767 /* Reset the error mask since PEU reset cleared it */
3768 HXGE_REG_WR64(hxgep->hpi_handle, RDC_FIFO_ERR_INT_MASK, 0x0);
3769
3770 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3771 "Recovery Successful, RxPort Restored"));
3772 HXGE_DEBUG_MSG((hxgep, RX_CTL, "<== hxge_rx_port_fatal_err_recover"));
3773 return (HXGE_OK);
3774
3775 fail:
3776 MUTEX_EXIT(&hxgep->vmac_lock);
3777 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3778 "Error Recovery failed for hxge(%d)", hxgep->instance));
3779 return (status);
3780 }
3781
3782 static void
hxge_rbr_empty_restore(p_hxge_t hxgep,p_rx_rbr_ring_t rx_rbr_p)3783 hxge_rbr_empty_restore(p_hxge_t hxgep, p_rx_rbr_ring_t rx_rbr_p)
3784 {
3785 hpi_status_t hpi_status;
3786 hxge_status_t status;
3787 rdc_stat_t cs;
3788 p_hxge_rx_ring_stats_t rdc_stats;
3789
3790 rdc_stats = &hxgep->statsp->rdc_stats[rx_rbr_p->rdc];
3791
3792 /*
3793 * Complete the processing for the RBR Empty by:
3794 * 0) kicking back HXGE_RBR_EMPTY_THRESHOLD
3795 * packets.
3796 * 1) Disable the RX vmac.
3797 * 2) Re-enable the affected DMA channel.
3798 * 3) Re-enable the RX vmac.
3799 */
3800
3801 /*
3802 * Disable the RX VMAC, but setting the framelength
3803 * to 0, since there is a hardware bug when disabling
3804 * the vmac.
3805 */
3806 MUTEX_ENTER(&hxgep->vmac_lock);
3807 (void) hxge_rx_vmac_disable(hxgep);
3808
3809 /*
3810 * Re-arm the mex bit for interrupts to be enabled.
3811 */
3812 cs.value = 0;
3813 cs.bits.mex = 1;
3814 RXDMA_REG_WRITE64(HXGE_DEV_HPI_HANDLE(hxgep), RDC_STAT,
3815 rx_rbr_p->rdc, cs.value);
3816
3817 hpi_status = hpi_rxdma_cfg_rdc_enable(
3818 HXGE_DEV_HPI_HANDLE(hxgep), rx_rbr_p->rdc);
3819 if (hpi_status != HPI_SUCCESS) {
3820 rdc_stats->rbr_empty_fail++;
3821
3822 /* Assume we are already inside the post_lock */
3823 status = hxge_rxdma_fatal_err_recover(hxgep, rx_rbr_p->rdc);
3824 if (status != HXGE_OK) {
3825 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
3826 "hxge(%d): channel(%d) is empty.",
3827 hxgep->instance, rx_rbr_p->rdc));
3828 }
3829 }
3830
3831 /*
3832 * Re-enable the RX VMAC.
3833 */
3834 (void) hxge_rx_vmac_enable(hxgep);
3835 MUTEX_EXIT(&hxgep->vmac_lock);
3836
3837 rdc_stats->rbr_empty_restore++;
3838 rx_rbr_p->rbr_is_empty = B_FALSE;
3839 }
3840