1 /* 2 * Copyright (c) 2012-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include "efx.h" 32 #include "efx_impl.h" 33 34 35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 36 37 38 static __checkReturn efx_rc_t 39 efx_mcdi_init_rxq( 40 __in efx_nic_t *enp, 41 __in uint32_t size, 42 __in uint32_t target_evq, 43 __in uint32_t label, 44 __in uint32_t instance, 45 __in efsys_mem_t *esmp, 46 __in boolean_t disable_scatter) 47 { 48 efx_mcdi_req_t req; 49 50 uint8_t payload[ 51 MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS))]; 52 int npages = EFX_RXQ_NBUFS(size); 53 int i; 54 efx_qword_t *dma_addr; 55 uint64_t addr; 56 efx_rc_t rc; 57 58 /* If this changes, then the payload size might need to change. */ 59 EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0); 60 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS); 61 62 (void) memset(payload, 0, sizeof (payload)); 63 req.emr_cmd = MC_CMD_INIT_RXQ; 64 req.emr_in_buf = payload; 65 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages); 66 req.emr_out_buf = payload; 67 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN; 68 69 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size); 70 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq); 71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label); 72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance); 73 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS, 74 INIT_RXQ_IN_FLAG_BUFF_MODE, 0, 75 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0, 76 INIT_RXQ_IN_FLAG_TIMESTAMP, 0, 77 INIT_RXQ_IN_CRC_MODE, 0, 78 INIT_RXQ_IN_FLAG_PREFIX, 1, 79 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter); 80 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0); 81 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 82 83 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 84 addr = EFSYS_MEM_ADDR(esmp); 85 86 for (i = 0; i < npages; i++) { 87 EFX_POPULATE_QWORD_2(*dma_addr, 88 EFX_DWORD_1, (uint32_t)(addr >> 32), 89 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 90 91 dma_addr++; 92 addr += EFX_BUF_SIZE; 93 } 94 95 efx_mcdi_execute(enp, &req); 96 97 if (req.emr_rc != 0) { 98 rc = req.emr_rc; 99 goto fail1; 100 } 101 102 return (0); 103 104 fail1: 105 EFSYS_PROBE1(fail1, efx_rc_t, rc); 106 107 return (rc); 108 } 109 110 static __checkReturn efx_rc_t 111 efx_mcdi_fini_rxq( 112 __in efx_nic_t *enp, 113 __in uint32_t instance) 114 { 115 efx_mcdi_req_t req; 116 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN, 117 MC_CMD_FINI_RXQ_OUT_LEN)]; 118 efx_rc_t rc; 119 120 (void) memset(payload, 0, sizeof (payload)); 121 req.emr_cmd = MC_CMD_FINI_RXQ; 122 req.emr_in_buf = payload; 123 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 124 req.emr_out_buf = payload; 125 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 126 127 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 128 129 efx_mcdi_execute(enp, &req); 130 131 if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) { 132 rc = req.emr_rc; 133 goto fail1; 134 } 135 136 return (0); 137 138 fail1: 139 EFSYS_PROBE1(fail1, efx_rc_t, rc); 140 141 return (rc); 142 } 143 144 #if EFSYS_OPT_RX_SCALE 145 static __checkReturn efx_rc_t 146 efx_mcdi_rss_context_alloc( 147 __in efx_nic_t *enp, 148 __in efx_rx_scale_support_t scale_support, 149 __in uint32_t num_queues, 150 __out uint32_t *rss_contextp) 151 { 152 efx_mcdi_req_t req; 153 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 154 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; 155 uint32_t rss_context; 156 uint32_t context_type; 157 efx_rc_t rc; 158 159 if (num_queues > EFX_MAXRSS) { 160 rc = EINVAL; 161 goto fail1; 162 } 163 164 switch (scale_support) { 165 case EFX_RX_SCALE_EXCLUSIVE: 166 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 167 break; 168 case EFX_RX_SCALE_SHARED: 169 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 170 break; 171 default: 172 rc = EINVAL; 173 goto fail2; 174 } 175 176 (void) memset(payload, 0, sizeof (payload)); 177 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 178 req.emr_in_buf = payload; 179 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 180 req.emr_out_buf = payload; 181 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 182 183 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 184 EVB_PORT_ID_ASSIGNED); 185 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 186 /* NUM_QUEUES is only used to validate indirection table offsets */ 187 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 188 189 efx_mcdi_execute(enp, &req); 190 191 if (req.emr_rc != 0) { 192 rc = req.emr_rc; 193 goto fail3; 194 } 195 196 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 197 rc = EMSGSIZE; 198 goto fail4; 199 } 200 201 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 202 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 203 rc = ENOENT; 204 goto fail5; 205 } 206 207 *rss_contextp = rss_context; 208 209 return (0); 210 211 fail5: 212 EFSYS_PROBE(fail5); 213 fail4: 214 EFSYS_PROBE(fail4); 215 fail3: 216 EFSYS_PROBE(fail3); 217 fail2: 218 EFSYS_PROBE(fail2); 219 fail1: 220 EFSYS_PROBE1(fail1, efx_rc_t, rc); 221 222 return (rc); 223 } 224 #endif /* EFSYS_OPT_RX_SCALE */ 225 226 #if EFSYS_OPT_RX_SCALE 227 static efx_rc_t 228 efx_mcdi_rss_context_free( 229 __in efx_nic_t *enp, 230 __in uint32_t rss_context) 231 { 232 efx_mcdi_req_t req; 233 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 234 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; 235 efx_rc_t rc; 236 237 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 238 rc = EINVAL; 239 goto fail1; 240 } 241 242 (void) memset(payload, 0, sizeof (payload)); 243 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 244 req.emr_in_buf = payload; 245 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 246 req.emr_out_buf = payload; 247 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 248 249 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 250 251 efx_mcdi_execute(enp, &req); 252 253 if (req.emr_rc != 0) { 254 rc = req.emr_rc; 255 goto fail2; 256 } 257 258 return (0); 259 260 fail2: 261 EFSYS_PROBE(fail2); 262 fail1: 263 EFSYS_PROBE1(fail1, efx_rc_t, rc); 264 265 return (rc); 266 } 267 #endif /* EFSYS_OPT_RX_SCALE */ 268 269 #if EFSYS_OPT_RX_SCALE 270 static efx_rc_t 271 efx_mcdi_rss_context_set_flags( 272 __in efx_nic_t *enp, 273 __in uint32_t rss_context, 274 __in efx_rx_hash_type_t type) 275 { 276 efx_mcdi_req_t req; 277 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 278 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; 279 efx_rc_t rc; 280 281 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 282 rc = EINVAL; 283 goto fail1; 284 } 285 286 (void) memset(payload, 0, sizeof (payload)); 287 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 288 req.emr_in_buf = payload; 289 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 290 req.emr_out_buf = payload; 291 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 292 293 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 294 rss_context); 295 296 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 297 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 298 (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0, 299 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 300 (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0, 301 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 302 (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0, 303 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 304 (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0); 305 306 efx_mcdi_execute(enp, &req); 307 308 if (req.emr_rc != 0) { 309 rc = req.emr_rc; 310 goto fail2; 311 } 312 313 return (0); 314 315 fail2: 316 EFSYS_PROBE(fail2); 317 fail1: 318 EFSYS_PROBE1(fail1, efx_rc_t, rc); 319 320 return (rc); 321 } 322 #endif /* EFSYS_OPT_RX_SCALE */ 323 324 #if EFSYS_OPT_RX_SCALE 325 static efx_rc_t 326 efx_mcdi_rss_context_set_key( 327 __in efx_nic_t *enp, 328 __in uint32_t rss_context, 329 __in_ecount(n) uint8_t *key, 330 __in size_t n) 331 { 332 efx_mcdi_req_t req; 333 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 334 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; 335 efx_rc_t rc; 336 337 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 338 rc = EINVAL; 339 goto fail1; 340 } 341 342 (void) memset(payload, 0, sizeof (payload)); 343 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 344 req.emr_in_buf = payload; 345 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 346 req.emr_out_buf = payload; 347 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 348 349 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 350 rss_context); 351 352 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 353 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 354 rc = EINVAL; 355 goto fail2; 356 } 357 358 (void) memcpy(MCDI_IN2(req, uint8_t, 359 RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), key, n); 360 361 efx_mcdi_execute(enp, &req); 362 363 if (req.emr_rc != 0) { 364 rc = req.emr_rc; 365 goto fail3; 366 } 367 368 return (0); 369 370 fail3: 371 EFSYS_PROBE(fail3); 372 fail2: 373 EFSYS_PROBE(fail2); 374 fail1: 375 EFSYS_PROBE1(fail1, efx_rc_t, rc); 376 377 return (rc); 378 } 379 #endif /* EFSYS_OPT_RX_SCALE */ 380 381 #if EFSYS_OPT_RX_SCALE 382 static efx_rc_t 383 efx_mcdi_rss_context_set_table( 384 __in efx_nic_t *enp, 385 __in uint32_t rss_context, 386 __in_ecount(n) unsigned int *table, 387 __in size_t n) 388 { 389 efx_mcdi_req_t req; 390 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 391 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; 392 uint8_t *req_table; 393 int i, rc; 394 395 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 396 rc = EINVAL; 397 goto fail1; 398 } 399 400 (void) memset(payload, 0, sizeof (payload)); 401 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 402 req.emr_in_buf = payload; 403 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 404 req.emr_out_buf = payload; 405 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 406 407 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 408 rss_context); 409 410 req_table = 411 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 412 413 for (i = 0; 414 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 415 i++) { 416 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 417 } 418 419 efx_mcdi_execute(enp, &req); 420 421 if (req.emr_rc != 0) { 422 rc = req.emr_rc; 423 goto fail2; 424 } 425 426 return (0); 427 428 fail2: 429 EFSYS_PROBE(fail2); 430 fail1: 431 EFSYS_PROBE1(fail1, efx_rc_t, rc); 432 433 return (rc); 434 } 435 #endif /* EFSYS_OPT_RX_SCALE */ 436 437 438 __checkReturn efx_rc_t 439 ef10_rx_init( 440 __in efx_nic_t *enp) 441 { 442 #if EFSYS_OPT_RX_SCALE 443 444 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 445 &enp->en_rss_context) == 0) { 446 /* 447 * Allocated an exclusive RSS context, which allows both the 448 * indirection table and key to be modified. 449 */ 450 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; 451 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 452 } else { 453 /* 454 * Failed to allocate an exclusive RSS context. Continue 455 * operation without support for RSS. The pseudo-header in 456 * received packets will not contain a Toeplitz hash value. 457 */ 458 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 459 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 460 } 461 462 #endif /* EFSYS_OPT_RX_SCALE */ 463 464 return (0); 465 } 466 467 #if EFSYS_OPT_RX_SCATTER 468 __checkReturn efx_rc_t 469 ef10_rx_scatter_enable( 470 __in efx_nic_t *enp, 471 __in unsigned int buf_size) 472 { 473 _NOTE(ARGUNUSED(enp, buf_size)) 474 return (0); 475 } 476 #endif /* EFSYS_OPT_RX_SCATTER */ 477 478 #if EFSYS_OPT_RX_SCALE 479 __checkReturn efx_rc_t 480 ef10_rx_scale_mode_set( 481 __in efx_nic_t *enp, 482 __in efx_rx_hash_alg_t alg, 483 __in efx_rx_hash_type_t type, 484 __in boolean_t insert) 485 { 486 efx_rc_t rc; 487 488 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); 489 EFSYS_ASSERT3U(insert, ==, B_TRUE); 490 491 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { 492 rc = EINVAL; 493 goto fail1; 494 } 495 496 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 497 rc = ENOTSUP; 498 goto fail2; 499 } 500 501 if ((rc = efx_mcdi_rss_context_set_flags(enp, 502 enp->en_rss_context, type)) != 0) 503 goto fail3; 504 505 return (0); 506 507 fail3: 508 EFSYS_PROBE(fail3); 509 fail2: 510 EFSYS_PROBE(fail2); 511 fail1: 512 EFSYS_PROBE1(fail1, efx_rc_t, rc); 513 514 return (rc); 515 } 516 #endif /* EFSYS_OPT_RX_SCALE */ 517 518 #if EFSYS_OPT_RX_SCALE 519 __checkReturn efx_rc_t 520 ef10_rx_scale_key_set( 521 __in efx_nic_t *enp, 522 __in_ecount(n) uint8_t *key, 523 __in size_t n) 524 { 525 efx_rc_t rc; 526 527 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 528 rc = ENOTSUP; 529 goto fail1; 530 } 531 532 if ((rc = efx_mcdi_rss_context_set_key(enp, 533 enp->en_rss_context, key, n)) != 0) 534 goto fail2; 535 536 return (0); 537 538 fail2: 539 EFSYS_PROBE(fail2); 540 fail1: 541 EFSYS_PROBE1(fail1, efx_rc_t, rc); 542 543 return (rc); 544 } 545 #endif /* EFSYS_OPT_RX_SCALE */ 546 547 #if EFSYS_OPT_RX_SCALE 548 __checkReturn efx_rc_t 549 ef10_rx_scale_tbl_set( 550 __in efx_nic_t *enp, 551 __in_ecount(n) unsigned int *table, 552 __in size_t n) 553 { 554 efx_rc_t rc; 555 556 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { 557 rc = ENOTSUP; 558 goto fail1; 559 } 560 561 if ((rc = efx_mcdi_rss_context_set_table(enp, 562 enp->en_rss_context, table, n)) != 0) 563 goto fail2; 564 565 return (0); 566 567 fail2: 568 EFSYS_PROBE(fail2); 569 fail1: 570 EFSYS_PROBE1(fail1, efx_rc_t, rc); 571 572 return (rc); 573 } 574 #endif /* EFSYS_OPT_RX_SCALE */ 575 576 577 /* 578 * EF10 RX pseudo-header 579 * --------------------- 580 * 581 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 582 * 583 * +00: Toeplitz hash value. 584 * (32bit little-endian) 585 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 586 * (16bit big-endian) 587 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 588 * (16bit big-endian) 589 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 590 * (16bit little-endian) 591 * +10: MAC timestamp. Zero if timestamping is not enabled. 592 * (32bit little-endian) 593 * 594 * See "The RX Pseudo-header" in SF-109306-TC. 595 */ 596 597 __checkReturn efx_rc_t 598 ef10_rx_prefix_pktlen( 599 __in efx_nic_t *enp, 600 __in uint8_t *buffer, 601 __out uint16_t *lengthp) 602 { 603 _NOTE(ARGUNUSED(enp)) 604 /* 605 * The RX pseudo-header contains the packet length, excluding the 606 * pseudo-header. If the hardware receive datapath was operating in 607 * cut-through mode then the length in the RX pseudo-header will be 608 * zero, and the packet length must be obtained from the DMA length 609 * reported in the RX event. 610 */ 611 *lengthp = buffer[8] | (buffer[9] << 8); 612 return (0); 613 } 614 615 #if EFSYS_OPT_RX_SCALE 616 __checkReturn uint32_t 617 ef10_rx_prefix_hash( 618 __in efx_nic_t *enp, 619 __in efx_rx_hash_alg_t func, 620 __in uint8_t *buffer) 621 { 622 _NOTE(ARGUNUSED(enp)) 623 switch (func) { 624 case EFX_RX_HASHALG_TOEPLITZ: 625 return (buffer[0] | 626 (buffer[1] << 8) | 627 (buffer[2] << 16) | 628 (buffer[3] << 24)); 629 630 default: 631 EFSYS_ASSERT(0); 632 return (0); 633 } 634 } 635 #endif /* EFSYS_OPT_RX_SCALE */ 636 637 void 638 ef10_rx_qpost( 639 __in efx_rxq_t *erp, 640 __in_ecount(n) efsys_dma_addr_t *addrp, 641 __in size_t size, 642 __in unsigned int n, 643 __in unsigned int completed, 644 __in unsigned int added) 645 { 646 efx_qword_t qword; 647 unsigned int i; 648 unsigned int offset; 649 unsigned int id; 650 651 /* The client driver must not overfill the queue */ 652 EFSYS_ASSERT3U(added - completed + n, <=, 653 EFX_RXQ_LIMIT(erp->er_mask + 1)); 654 655 id = added & (erp->er_mask); 656 for (i = 0; i < n; i++) { 657 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 658 unsigned int, id, efsys_dma_addr_t, addrp[i], 659 size_t, size); 660 661 EFX_POPULATE_QWORD_3(qword, 662 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 663 ESF_DZ_RX_KER_BUF_ADDR_DW0, 664 (uint32_t)(addrp[i] & 0xffffffff), 665 ESF_DZ_RX_KER_BUF_ADDR_DW1, 666 (uint32_t)(addrp[i] >> 32)); 667 668 offset = id * sizeof (efx_qword_t); 669 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 670 671 id = (id + 1) & (erp->er_mask); 672 } 673 } 674 675 void 676 ef10_rx_qpush( 677 __in efx_rxq_t *erp, 678 __in unsigned int added, 679 __inout unsigned int *pushedp) 680 { 681 efx_nic_t *enp = erp->er_enp; 682 unsigned int pushed = *pushedp; 683 uint32_t wptr; 684 efx_dword_t dword; 685 686 /* Hardware has alignment restriction for WPTR */ 687 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN); 688 if (pushed == wptr) 689 return; 690 691 *pushedp = wptr; 692 693 /* Push the populated descriptors out */ 694 wptr &= erp->er_mask; 695 696 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 697 698 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 699 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 700 wptr, pushed & erp->er_mask); 701 EFSYS_PIO_WRITE_BARRIER(); 702 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 703 erp->er_index, &dword, B_FALSE); 704 } 705 706 __checkReturn efx_rc_t 707 ef10_rx_qflush( 708 __in efx_rxq_t *erp) 709 { 710 efx_nic_t *enp = erp->er_enp; 711 efx_rc_t rc; 712 713 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 714 goto fail1; 715 716 return (0); 717 718 fail1: 719 EFSYS_PROBE1(fail1, efx_rc_t, rc); 720 721 return (rc); 722 } 723 724 void 725 ef10_rx_qenable( 726 __in efx_rxq_t *erp) 727 { 728 /* FIXME */ 729 _NOTE(ARGUNUSED(erp)) 730 /* FIXME */ 731 } 732 733 __checkReturn efx_rc_t 734 ef10_rx_qcreate( 735 __in efx_nic_t *enp, 736 __in unsigned int index, 737 __in unsigned int label, 738 __in efx_rxq_type_t type, 739 __in efsys_mem_t *esmp, 740 __in size_t n, 741 __in uint32_t id, 742 __in efx_evq_t *eep, 743 __in efx_rxq_t *erp) 744 { 745 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 746 efx_rc_t rc; 747 boolean_t disable_scatter; 748 749 _NOTE(ARGUNUSED(erp, id)) 750 751 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 752 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 753 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 754 755 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS)); 756 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS)); 757 758 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) { 759 rc = EINVAL; 760 goto fail1; 761 } 762 if (index >= encp->enc_rxq_limit) { 763 rc = EINVAL; 764 goto fail2; 765 } 766 767 /* Scatter can only be disabled if the firmware supports doing so */ 768 if ((type != EFX_RXQ_TYPE_SCATTER) && 769 enp->en_nic_cfg.enc_rx_disable_scatter_supported) { 770 disable_scatter = B_TRUE; 771 } else { 772 disable_scatter = B_FALSE; 773 } 774 775 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index, 776 esmp, disable_scatter)) != 0) 777 goto fail3; 778 779 erp->er_eep = eep; 780 erp->er_label = label; 781 782 ef10_ev_rxlabel_init(eep, erp, label); 783 784 return (0); 785 786 fail3: 787 EFSYS_PROBE(fail3); 788 fail2: 789 EFSYS_PROBE(fail2); 790 fail1: 791 EFSYS_PROBE1(fail1, efx_rc_t, rc); 792 793 return (rc); 794 } 795 796 void 797 ef10_rx_qdestroy( 798 __in efx_rxq_t *erp) 799 { 800 efx_nic_t *enp = erp->er_enp; 801 efx_evq_t *eep = erp->er_eep; 802 unsigned int label = erp->er_label; 803 804 ef10_ev_rxlabel_fini(eep, label); 805 806 EFSYS_ASSERT(enp->en_rx_qcount != 0); 807 --enp->en_rx_qcount; 808 809 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 810 } 811 812 void 813 ef10_rx_fini( 814 __in efx_nic_t *enp) 815 { 816 #if EFSYS_OPT_RX_SCALE 817 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) { 818 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 819 } 820 enp->en_rss_context = 0; 821 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; 822 #else 823 _NOTE(ARGUNUSED(enp)) 824 #endif /* EFSYS_OPT_RX_SCALE */ 825 } 826 827 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 828