1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2018 Joyent, Inc. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/callb.h> 28 #include <sys/cpupart.h> 29 #include <sys/pool.h> 30 #include <sys/pool_pset.h> 31 #include <sys/sdt.h> 32 #include <sys/strsubr.h> 33 #include <sys/strsun.h> 34 #include <sys/vlan.h> 35 #include <inet/ipsec_impl.h> 36 #include <inet/ip_impl.h> 37 #include <inet/sadb.h> 38 #include <inet/ipsecesp.h> 39 #include <inet/ipsecah.h> 40 41 #include <sys/mac_impl.h> 42 #include <sys/mac_client_impl.h> 43 #include <sys/mac_client_priv.h> 44 #include <sys/mac_soft_ring.h> 45 #include <sys/mac_flow_impl.h> 46 #include <sys/mac_stat.h> 47 48 static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 49 static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 50 static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 51 static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 52 static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 53 54 static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 55 static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 56 static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 57 static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 58 static void mac_srs_add_glist(mac_soft_ring_set_t *); 59 static void mac_srs_remove_glist(mac_soft_ring_set_t *); 60 static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 61 static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 62 63 static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 64 static void mac_walk_srs_and_bind(int); 65 static void mac_walk_srs_and_unbind(int); 66 67 extern boolean_t mac_latency_optimize; 68 69 static kmem_cache_t *mac_srs_cache; 70 kmem_cache_t *mac_soft_ring_cache; 71 72 /* 73 * The duration in msec we wait before signalling the soft ring 74 * worker thread in case packets get queued. 75 */ 76 uint32_t mac_soft_ring_worker_wait = 0; 77 78 /* 79 * A global tunable for turning polling on/off. By default, dynamic 80 * polling is always on and is always very beneficial. It should be 81 * turned off with absolute care and for the rare workload (very 82 * low latency sensitive traffic). 83 */ 84 int mac_poll_enable = B_TRUE; 85 86 /* 87 * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 88 * Large values could end up in consuming lot of system memory and cause 89 * system hang. 90 */ 91 int mac_soft_ring_max_q_cnt = 1024; 92 int mac_soft_ring_min_q_cnt = 256; 93 int mac_soft_ring_poll_thres = 16; 94 95 boolean_t mac_tx_serialize = B_FALSE; 96 97 /* 98 * mac_tx_srs_hiwat is the queue depth threshold at which callers of 99 * mac_tx() will be notified of flow control condition. 100 * 101 * TCP does not honour flow control condition sent up by mac_tx(). 102 * Thus provision is made for TCP to allow more packets to be queued 103 * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 104 * 105 * Note that mac_tx_srs_hiwat is always be lesser than 106 * mac_tx_srs_max_q_cnt. 107 */ 108 uint32_t mac_tx_srs_max_q_cnt = 100000; 109 uint32_t mac_tx_srs_hiwat = 1000; 110 111 /* 112 * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 113 * 114 * Global tunables that determines the number of soft rings to be used for 115 * fanning out incoming traffic on a link. These count will be used only 116 * when no explicit set of CPUs was assigned to the data-links. 117 * 118 * mac_rx_soft_ring_count tunable will come into effect only if 119 * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 120 * default only for sun4v platforms. 121 * 122 * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 123 * 10Gbps link and is not dependent upon mac_soft_ring_enable. 124 * 125 * The number of soft rings for fanout for a link or a flow is determined 126 * by mac_compute_soft_ring_count() routine. This routine will take into 127 * account mac_soft_ring_enable, mac_rx_soft_ring_count and 128 * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 129 * 130 * If a bandwidth is specified, the determination of the number of soft 131 * rings is based on specified bandwidth, CPU speed and number of CPUs in 132 * the system. 133 */ 134 uint_t mac_rx_soft_ring_count = 8; 135 uint_t mac_rx_soft_ring_10gig_count = 8; 136 137 /* 138 * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 139 * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 140 * list is used to walk the list of all MAC threads when a CPU is 141 * coming online or going offline. 142 */ 143 static mac_soft_ring_set_t *mac_srs_g_list = NULL; 144 static krwlock_t mac_srs_g_lock; 145 146 /* 147 * Whether the SRS threads should be bound, or not. 148 */ 149 boolean_t mac_srs_thread_bind = B_TRUE; 150 151 /* 152 * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 153 * dladm command would override this. 154 */ 155 boolean_t mac_tx_intr_retarget = B_FALSE; 156 boolean_t mac_rx_intr_retarget = B_FALSE; 157 158 /* 159 * If cpu bindings are specified by user, then Tx SRS and its soft 160 * rings should also be bound to the CPUs specified by user. The 161 * CPUs for Tx bindings are at the end of the cpu list provided by 162 * the user. If enough CPUs are not available (for Tx and Rx 163 * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 164 */ 165 #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 166 processorid_t cpuid; \ 167 int i; \ 168 mac_soft_ring_t *softring; \ 169 mac_cpus_t *srs_cpu; \ 170 \ 171 srs_cpu = &mac_tx_srs->srs_cpu; \ 172 cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 173 mac_srs_worker_bind(mac_tx_srs, cpuid); \ 174 if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 175 for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 176 cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 177 softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 178 if (cpuid != -1) { \ 179 (void) mac_soft_ring_bind(softring, \ 180 cpuid); \ 181 } \ 182 } \ 183 } \ 184 } 185 186 /* 187 * Re-targeting is allowed only for exclusive group or for primary. 188 */ 189 #define RETARGETABLE_CLIENT(group, mcip) \ 190 ((((group) != NULL) && \ 191 ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 192 mac_is_primary_client(mcip)) 193 194 #define MAC_RING_RETARGETABLE(ring) \ 195 (((ring) != NULL) && \ 196 ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 197 !((ring)->mr_info.mri_intr.mi_ddi_shared)) 198 199 200 /* INIT and FINI ROUTINES */ 201 202 void 203 mac_soft_ring_init(void) 204 { 205 mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 206 sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 207 208 mac_srs_cache = kmem_cache_create("mac_srs_cache", 209 sizeof (mac_soft_ring_set_t), 210 64, NULL, NULL, NULL, NULL, NULL, 0); 211 212 rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 213 mutex_enter(&cpu_lock); 214 register_cpu_setup_func(mac_srs_cpu_setup, NULL); 215 mutex_exit(&cpu_lock); 216 } 217 218 void 219 mac_soft_ring_finish(void) 220 { 221 mutex_enter(&cpu_lock); 222 unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 223 mutex_exit(&cpu_lock); 224 rw_destroy(&mac_srs_g_lock); 225 kmem_cache_destroy(mac_soft_ring_cache); 226 kmem_cache_destroy(mac_srs_cache); 227 } 228 229 static void 230 mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 231 { 232 mac_soft_ring_t *softring, *next, *head; 233 234 /* 235 * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 236 * DR. The callbacks from DR are called with cpu_lock held, and hence 237 * can't wait to grab the mac perimeter. The soft ring list is hence 238 * protected for read access by srs_lock. Changing the soft ring list 239 * needs the mac perimeter and the srs_lock. 240 */ 241 mutex_enter(&mac_srs->srs_lock); 242 243 head = mac_srs->srs_soft_ring_head; 244 mac_srs->srs_soft_ring_head = NULL; 245 mac_srs->srs_soft_ring_tail = NULL; 246 mac_srs->srs_soft_ring_count = 0; 247 248 mutex_exit(&mac_srs->srs_lock); 249 250 for (softring = head; softring != NULL; softring = next) { 251 next = softring->s_ring_next; 252 mac_soft_ring_free(softring); 253 } 254 } 255 256 static void 257 mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 258 { 259 ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 260 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 261 262 rw_enter(&mac_srs_g_lock, RW_WRITER); 263 mutex_enter(&mac_srs->srs_lock); 264 265 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 266 267 if (mac_srs_g_list == NULL) { 268 mac_srs_g_list = mac_srs; 269 } else { 270 mac_srs->srs_next = mac_srs_g_list; 271 mac_srs_g_list->srs_prev = mac_srs; 272 mac_srs->srs_prev = NULL; 273 mac_srs_g_list = mac_srs; 274 } 275 mac_srs->srs_state |= SRS_IN_GLIST; 276 277 mutex_exit(&mac_srs->srs_lock); 278 rw_exit(&mac_srs_g_lock); 279 } 280 281 static void 282 mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 283 { 284 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 285 286 rw_enter(&mac_srs_g_lock, RW_WRITER); 287 mutex_enter(&mac_srs->srs_lock); 288 289 ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 290 291 if (mac_srs == mac_srs_g_list) { 292 mac_srs_g_list = mac_srs->srs_next; 293 if (mac_srs_g_list != NULL) 294 mac_srs_g_list->srs_prev = NULL; 295 } else { 296 mac_srs->srs_prev->srs_next = mac_srs->srs_next; 297 if (mac_srs->srs_next != NULL) 298 mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 299 } 300 mac_srs->srs_state &= ~SRS_IN_GLIST; 301 302 mutex_exit(&mac_srs->srs_lock); 303 rw_exit(&mac_srs_g_lock); 304 } 305 306 /* POLLING SETUP AND TEAR DOWN ROUTINES */ 307 308 /* 309 * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 310 * 311 * These routines are used to call back into the upper layer 312 * (primarily TCP squeue) to stop polling the soft rings or 313 * restart polling. 314 */ 315 void 316 mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 317 mac_soft_ring_set_t *mac_srs) 318 { 319 mac_soft_ring_t *softring; 320 321 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 322 323 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 324 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 325 return; 326 } 327 328 for (softring = mac_srs->srs_soft_ring_head; 329 softring != NULL; softring = softring->s_ring_next) { 330 if ((softring->s_ring_type & ST_RING_TCP) && 331 (softring->s_ring_rx_arg2 != NULL)) { 332 mcip->mci_resource_quiesce(mcip->mci_resource_arg, 333 softring->s_ring_rx_arg2); 334 } 335 } 336 } 337 338 void 339 mac_srs_client_poll_restart(mac_client_impl_t *mcip, 340 mac_soft_ring_set_t *mac_srs) 341 { 342 mac_soft_ring_t *softring; 343 344 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 345 346 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 347 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 348 return; 349 } 350 351 for (softring = mac_srs->srs_soft_ring_head; 352 softring != NULL; softring = softring->s_ring_next) { 353 if ((softring->s_ring_type & ST_RING_TCP) && 354 (softring->s_ring_rx_arg2 != NULL)) { 355 mcip->mci_resource_restart(mcip->mci_resource_arg, 356 softring->s_ring_rx_arg2); 357 } 358 } 359 } 360 361 /* 362 * Register the given SRS and associated soft rings with the consumer and 363 * enable the polling interface used by the consumer.(i.e IP) over this 364 * SRS and associated soft rings. 365 */ 366 void 367 mac_srs_client_poll_enable(mac_client_impl_t *mcip, 368 mac_soft_ring_set_t *mac_srs) 369 { 370 mac_rx_fifo_t mrf; 371 mac_soft_ring_t *softring; 372 373 ASSERT(mac_srs->srs_mcip == mcip); 374 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 375 376 if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 377 return; 378 379 bzero(&mrf, sizeof (mac_rx_fifo_t)); 380 mrf.mrf_type = MAC_RX_FIFO; 381 382 /* 383 * A SRS is capable of acting as a soft ring for cases 384 * where no fanout is needed. This is the case for userland 385 * flows. 386 */ 387 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 388 return; 389 390 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 391 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 392 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 393 mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 394 395 softring = mac_srs->srs_soft_ring_head; 396 while (softring != NULL) { 397 if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 398 /* 399 * TCP and UDP support DLS bypass. Squeue polling 400 * support implies DLS bypass since the squeue poll 401 * path does not have DLS processing. 402 */ 403 mac_soft_ring_dls_bypass(softring, 404 mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 405 } 406 /* 407 * Non-TCP protocols don't support squeues. Hence we don't 408 * make any ring addition callbacks for non-TCP rings 409 */ 410 if (!(softring->s_ring_type & ST_RING_TCP)) { 411 softring->s_ring_rx_arg2 = NULL; 412 softring = softring->s_ring_next; 413 continue; 414 } 415 mrf.mrf_rx_arg = softring; 416 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 417 mrf.mrf_cpu_id = softring->s_ring_cpuid; 418 mrf.mrf_flow_priority = mac_srs->srs_pri; 419 420 softring->s_ring_rx_arg2 = mcip->mci_resource_add( 421 mcip->mci_resource_arg, (mac_resource_t *)&mrf); 422 423 softring = softring->s_ring_next; 424 } 425 } 426 427 /* 428 * Unregister the given SRS and associated soft rings with the consumer and 429 * disable the polling interface used by the consumer.(i.e IP) over this 430 * SRS and associated soft rings. 431 */ 432 void 433 mac_srs_client_poll_disable(mac_client_impl_t *mcip, 434 mac_soft_ring_set_t *mac_srs) 435 { 436 mac_soft_ring_t *softring; 437 438 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 439 440 /* 441 * A SRS is capable of acting as a soft ring for cases 442 * where no protocol fanout is needed. This is the case 443 * for userland flows. Nothing to do here. 444 */ 445 if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 446 return; 447 448 mutex_enter(&mac_srs->srs_lock); 449 if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 450 ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 451 mutex_exit(&mac_srs->srs_lock); 452 return; 453 } 454 mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 455 mutex_exit(&mac_srs->srs_lock); 456 457 /* 458 * DLS bypass is now disabled in the case of both TCP and UDP. 459 * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 460 * callback. In addition, in the case of TCP, invoke IP's callback 461 * for ring removal. 462 */ 463 for (softring = mac_srs->srs_soft_ring_head; 464 softring != NULL; softring = softring->s_ring_next) { 465 if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 466 continue; 467 468 if ((softring->s_ring_type & ST_RING_TCP) && 469 softring->s_ring_rx_arg2 != NULL) { 470 mcip->mci_resource_remove(mcip->mci_resource_arg, 471 softring->s_ring_rx_arg2); 472 } 473 474 mutex_enter(&softring->s_ring_lock); 475 while (softring->s_ring_state & S_RING_PROC) { 476 softring->s_ring_state |= S_RING_CLIENT_WAIT; 477 cv_wait(&softring->s_ring_client_cv, 478 &softring->s_ring_lock); 479 } 480 softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 481 softring->s_ring_rx_arg2 = NULL; 482 softring->s_ring_rx_func = mac_rx_deliver; 483 softring->s_ring_rx_arg1 = mcip; 484 mutex_exit(&softring->s_ring_lock); 485 } 486 } 487 488 /* 489 * Enable or disable poll capability of the SRS on the underlying Rx ring. 490 * 491 * There is a need to enable or disable the poll capability of an SRS over an 492 * Rx ring depending on the number of mac clients sharing the ring and also 493 * whether user flows are configured on it. However the poll state is actively 494 * manipulated by the SRS worker and poll threads and uncoordinated changes by 495 * yet another thread to the underlying capability can surprise them leading 496 * to assert failures. Instead we quiesce the SRS, make the changes and then 497 * restart the SRS. 498 */ 499 static void 500 mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 501 boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 502 { 503 boolean_t need_restart = B_FALSE; 504 mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 505 mac_ring_t *ring; 506 507 if (!SRS_QUIESCED(mac_srs)) { 508 mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 509 need_restart = B_TRUE; 510 } 511 512 ring = mac_srs->srs_ring; 513 if ((ring != NULL) && 514 (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 515 if (turn_off_poll_capab) 516 mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 517 else if (mac_poll_enable) 518 mac_srs->srs_state |= SRS_POLLING_CAPAB; 519 } 520 srs_rx->sr_lower_proc = rx_func; 521 522 if (need_restart) 523 mac_rx_srs_restart(mac_srs); 524 } 525 526 /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 527 528 /* 529 * Return the next CPU to be used to bind a MAC kernel thread. 530 * If a cpupart is specified, the cpu chosen must be from that 531 * cpu partition. 532 */ 533 static processorid_t 534 mac_next_bind_cpu(cpupart_t *cpupart) 535 { 536 static cpu_t *cp = NULL; 537 cpu_t *cp_start; 538 539 ASSERT(MUTEX_HELD(&cpu_lock)); 540 541 if (cp == NULL) 542 cp = cpu_list; 543 544 cp = cp->cpu_next_onln; 545 cp_start = cp; 546 547 do { 548 if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 549 return (cp->cpu_id); 550 551 } while ((cp = cp->cpu_next_onln) != cp_start); 552 553 return (-1); /* No matching CPU found online */ 554 } 555 556 /* ARGSUSED */ 557 static int 558 mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 559 { 560 ASSERT(MUTEX_HELD(&cpu_lock)); 561 switch (what) { 562 case CPU_CONFIG: 563 case CPU_ON: 564 case CPU_CPUPART_IN: 565 mac_walk_srs_and_bind(id); 566 break; 567 568 case CPU_UNCONFIG: 569 case CPU_OFF: 570 case CPU_CPUPART_OUT: 571 mac_walk_srs_and_unbind(id); 572 break; 573 574 default: 575 break; 576 } 577 return (0); 578 } 579 580 /* 581 * mac_compute_soft_ring_count(): 582 * 583 * This routine computes the number of soft rings needed to handle incoming 584 * load given a flow_entry. 585 * 586 * The routine does the following: 587 * 1) soft rings will be created if mac_soft_ring_enable is set. 588 * 2) If the underlying link is a 10Gbps link, then soft rings will be 589 * created even if mac_soft_ring_enable is not set. The number of soft 590 * rings, so created, will equal mac_rx_soft_ring_10gig_count. 591 * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 592 * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 593 * 10Gbps link. 594 * 595 * If a bandwidth limit is specified, the number that gets computed is 596 * dependent upon CPU speed, the number of Rx rings configured, and 597 * the bandwidth limit. 598 * If more Rx rings are available, less number of soft rings is needed. 599 * 600 * mac_use_bw_heuristic is another "hidden" variable that can be used to 601 * override the default use of soft ring count computation. Depending upon 602 * the usefulness of it, mac_use_bw_heuristic can later be made into a 603 * data-link property or removed altogether. 604 * 605 * TODO: Cleanup and tighten some of the assumptions. 606 */ 607 boolean_t mac_use_bw_heuristic = B_TRUE; 608 static int 609 mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 610 { 611 uint64_t cpu_speed, bw = 0; 612 int srings = 0; 613 boolean_t bw_enabled = B_FALSE; 614 615 ASSERT(!(flent->fe_type & FLOW_USER)); 616 if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 617 mac_use_bw_heuristic) { 618 /* bandwidth enabled */ 619 bw_enabled = B_TRUE; 620 bw = flent->fe_resource_props.mrp_maxbw; 621 } 622 if (!bw_enabled) { 623 /* No bandwidth enabled */ 624 if (mac_soft_ring_enable) 625 srings = mac_rx_soft_ring_count; 626 627 /* Is this a 10Gig link? */ 628 flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 629 MAC_STAT_IFSPEED); 630 /* convert to Mbps */ 631 if (((flent->fe_nic_speed)/1000000) > 1000 && 632 mac_rx_soft_ring_10gig_count > 0) { 633 /* This is a 10Gig link */ 634 srings = mac_rx_soft_ring_10gig_count; 635 /* 636 * Use 2 times mac_rx_soft_ring_10gig_count for 637 * sun4v systems. 638 */ 639 if (mac_soft_ring_enable) 640 srings = srings * 2; 641 } 642 } else { 643 /* 644 * Soft ring computation using CPU speed and specified 645 * bandwidth limit. 646 */ 647 /* Assumption: all CPUs have the same frequency */ 648 cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 649 650 /* cpu_speed is in MHz; make bw in units of Mbps. */ 651 bw = bw/1000000; 652 653 if (bw >= 1000) { 654 /* 655 * bw is greater than or equal to 1Gbps. 656 * The number of soft rings required is a function 657 * of bandwidth and CPU speed. To keep this simple, 658 * let's use this rule: 1GHz CPU can handle 1Gbps. 659 * If bw is less than 1 Gbps, then there is no need 660 * for soft rings. Assumption is that CPU speeds 661 * (on modern systems) are at least 1GHz. 662 */ 663 srings = bw/cpu_speed; 664 if (srings <= 1 && mac_soft_ring_enable) { 665 /* 666 * Give at least 2 soft rings 667 * for sun4v systems 668 */ 669 srings = 2; 670 } 671 } 672 } 673 /* 674 * If the flent has multiple Rx SRSs, then each SRS need not 675 * have that many soft rings on top of it. The number of 676 * soft rings for each Rx SRS is found by dividing srings by 677 * rx_srs_cnt. 678 */ 679 if (rx_srs_cnt > 1) { 680 int remainder; 681 682 remainder = srings%rx_srs_cnt; 683 srings = srings/rx_srs_cnt; 684 if (remainder != 0) 685 srings++; 686 /* 687 * Fanning out to 1 soft ring is not very useful. 688 * Set it as well to 0 and mac_srs_fanout_init() 689 * will take care of creating a single soft ring 690 * for proto fanout. 691 */ 692 if (srings == 1) 693 srings = 0; 694 } 695 /* Do some more massaging */ 696 srings = min(srings, maxcpus); 697 srings = min(srings, MAX_SR_FANOUT); 698 return (srings); 699 } 700 701 /* 702 * mac_tx_cpu_init: 703 * set up CPUs for Tx interrupt re-targeting and Tx worker 704 * thread binding 705 */ 706 static void 707 mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 708 cpupart_t *cpupart) 709 { 710 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 711 mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 712 mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 713 mac_soft_ring_t *sringp; 714 mac_ring_t *ring; 715 processorid_t worker_cpuid; 716 boolean_t retargetable_client = B_FALSE; 717 int i, j; 718 719 if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 720 flent->fe_mcip)) { 721 retargetable_client = B_TRUE; 722 } 723 724 if (MAC_TX_SOFT_RINGS(tx_srs)) { 725 if (mrp != NULL) 726 j = mrp->mrp_ncpus - 1; 727 for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 728 if (mrp != NULL) { 729 if (j < 0) 730 j = mrp->mrp_ncpus - 1; 731 worker_cpuid = mrp->mrp_cpu[j]; 732 } else { 733 /* 734 * Bind interrupt to the next CPU available 735 * and leave the worker unbound. 736 */ 737 worker_cpuid = -1; 738 } 739 sringp = tx_srs->srs_tx_soft_rings[i]; 740 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 741 srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 742 if (MAC_RING_RETARGETABLE(ring) && 743 retargetable_client) { 744 mutex_enter(&cpu_lock); 745 srs_cpu->mc_tx_intr_cpu[i] = 746 (mrp != NULL) ? mrp->mrp_cpu[j] : 747 (mac_tx_intr_retarget ? 748 mac_next_bind_cpu(cpupart) : -1); 749 mutex_exit(&cpu_lock); 750 } else { 751 srs_cpu->mc_tx_intr_cpu[i] = -1; 752 } 753 if (mrp != NULL) 754 j--; 755 } 756 } else { 757 /* Tx mac_ring_handle_t is stored in st_arg2 */ 758 srs_cpu->mc_tx_fanout_cpus[0] = 759 (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 760 ring = (mac_ring_t *)srs_tx->st_arg2; 761 if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 762 mutex_enter(&cpu_lock); 763 srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 764 mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 765 (mac_tx_intr_retarget ? 766 mac_next_bind_cpu(cpupart) : -1); 767 mutex_exit(&cpu_lock); 768 } else { 769 srs_cpu->mc_tx_intr_cpu[0] = -1; 770 } 771 } 772 } 773 774 /* 775 * Assignment of user specified CPUs to a link. 776 * 777 * Minimum CPUs required to get an optimal assignmet: 778 * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 779 * flag is set -- one for polling, one for fanout soft ring. 780 * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 781 * for polling, one for SRS worker thread and one for fanout soft ring. 782 * 783 * The CPUs needed for Tx side is equal to the number of Tx rings 784 * the link is using. 785 * 786 * mac_flow_user_cpu_init() categorizes the CPU assignment depending 787 * upon the number of CPUs in 3 different buckets. 788 * 789 * In the first bucket, the most optimal case is handled. The user has 790 * passed enough number of CPUs and every thread gets its own CPU. 791 * 792 * The second and third are the sub-optimal cases. Enough CPUs are not 793 * available. 794 * 795 * The second bucket handles the case where atleast one distinct CPU is 796 * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 797 * SRS or soft rings). 798 * 799 * In the third case (worst case scenario), specified CPU count is less 800 * than the Rx rings configured for the link. In this case, we round 801 * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 802 */ 803 static void 804 mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 805 { 806 mac_soft_ring_set_t *rx_srs, *tx_srs; 807 int i, srs_cnt; 808 mac_cpus_t *srs_cpu; 809 int no_of_cpus, cpu_cnt; 810 int rx_srs_cnt, reqd_rx_cpu_cnt; 811 int fanout_cpu_cnt, reqd_tx_cpu_cnt; 812 int reqd_poll_worker_cnt, fanout_cnt_per_srs; 813 mac_resource_props_t *emrp = &flent->fe_effective_props; 814 815 ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 816 /* 817 * The check for nbc_ncpus to be within limits for 818 * the user specified case was done earlier and if 819 * not within limits, an error would have been 820 * returned to the user. 821 */ 822 ASSERT(mrp->mrp_ncpus > 0); 823 824 no_of_cpus = mrp->mrp_ncpus; 825 826 if (mrp->mrp_rx_intr_cpu != -1) { 827 /* 828 * interrupt has been re-targetted. Poll 829 * thread needs to be bound to interrupt 830 * CPU. 831 * 832 * Find where in the list is the intr 833 * CPU and swap it with the first one. 834 * We will be using the first CPU in the 835 * list for poll. 836 */ 837 for (i = 0; i < no_of_cpus; i++) { 838 if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 839 break; 840 } 841 mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 842 mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 843 } 844 845 /* 846 * Requirements: 847 * The number of CPUs that each Rx ring needs is dependent 848 * upon mac_latency_optimize flag. 849 * 1) If set, atleast 2 CPUs are needed -- one for 850 * polling, one for fanout soft ring. 851 * 2) If not set, then atleast 3 CPUs are needed -- one 852 * for polling, one for srs worker thread, and one for 853 * fanout soft ring. 854 */ 855 rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 856 (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 857 reqd_rx_cpu_cnt = mac_latency_optimize ? 858 (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 859 860 /* How many CPUs are needed for Tx side? */ 861 tx_srs = flent->fe_tx_srs; 862 reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 863 tx_srs->srs_tx_ring_count : 1; 864 865 /* CPUs needed for Rx SRSes poll and worker threads */ 866 reqd_poll_worker_cnt = mac_latency_optimize ? 867 rx_srs_cnt : rx_srs_cnt * 2; 868 869 /* Has the user provided enough CPUs? */ 870 if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 871 /* 872 * Best case scenario. There is enough CPUs. All 873 * Rx rings will get their own set of CPUs plus 874 * Tx soft rings will get their own. 875 */ 876 /* 877 * fanout_cpu_cnt is the number of CPUs available 878 * for Rx side fanout soft rings. 879 */ 880 fanout_cpu_cnt = no_of_cpus - 881 reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 882 883 /* 884 * Divide fanout_cpu_cnt by rx_srs_cnt to find 885 * out how many fanout soft rings each Rx SRS 886 * can have. 887 */ 888 fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 889 890 /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 891 fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 892 893 /* Do the assignment for the default Rx ring */ 894 cpu_cnt = 0; 895 rx_srs = flent->fe_rx_srs[0]; 896 ASSERT(rx_srs->srs_ring == NULL); 897 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 898 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 899 srs_cpu = &rx_srs->srs_cpu; 900 srs_cpu->mc_ncpus = no_of_cpus; 901 bcopy(mrp->mrp_cpu, 902 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 903 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 904 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 905 /* Retarget the interrupt to the same CPU as the poll */ 906 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 907 srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 908 srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 909 for (i = 0; i < fanout_cnt_per_srs; i++) 910 srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 911 912 /* Do the assignment for h/w Rx SRSes */ 913 if (flent->fe_rx_srs_cnt > 1) { 914 cpu_cnt = 0; 915 for (srs_cnt = 1; 916 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 917 rx_srs = flent->fe_rx_srs[srs_cnt]; 918 ASSERT(rx_srs->srs_ring != NULL); 919 if (rx_srs->srs_fanout_state == 920 SRS_FANOUT_INIT) { 921 rx_srs->srs_fanout_state = 922 SRS_FANOUT_REINIT; 923 } 924 srs_cpu = &rx_srs->srs_cpu; 925 srs_cpu->mc_ncpus = no_of_cpus; 926 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 927 sizeof (srs_cpu->mc_cpus)); 928 srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 929 /* The first CPU in the list is the intr CPU */ 930 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 931 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 932 srs_cpu->mc_rx_workerid = 933 (mac_latency_optimize ? 934 srs_cpu->mc_rx_pollid : 935 mrp->mrp_cpu[cpu_cnt++]); 936 for (i = 0; i < fanout_cnt_per_srs; i++) { 937 srs_cpu->mc_rx_fanout_cpus[i] = 938 mrp->mrp_cpu[cpu_cnt++]; 939 } 940 ASSERT(cpu_cnt <= no_of_cpus); 941 } 942 } 943 goto tx_cpu_init; 944 } 945 946 /* 947 * Sub-optimal case. 948 * We have the following information: 949 * no_of_cpus - no. of cpus that user passed. 950 * rx_srs_cnt - no. of rx rings. 951 * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 952 * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 953 * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 954 */ 955 /* 956 * If we bind the Rx fanout soft rings to the same CPUs 957 * as poll/worker, would that be enough? 958 */ 959 if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 960 boolean_t worker_assign = B_FALSE; 961 962 /* 963 * If mac_latency_optimize is not set, are there 964 * enough CPUs to assign a CPU for worker also? 965 */ 966 if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 967 worker_assign = B_TRUE; 968 /* 969 * Zero'th Rx SRS is the default Rx ring. It is not 970 * associated with h/w Rx ring. 971 */ 972 rx_srs = flent->fe_rx_srs[0]; 973 ASSERT(rx_srs->srs_ring == NULL); 974 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 975 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 976 cpu_cnt = 0; 977 srs_cpu = &rx_srs->srs_cpu; 978 srs_cpu->mc_ncpus = no_of_cpus; 979 bcopy(mrp->mrp_cpu, 980 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 981 srs_cpu->mc_rx_fanout_cnt = 1; 982 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 983 /* Retarget the interrupt to the same CPU as the poll */ 984 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 985 srs_cpu->mc_rx_workerid = 986 ((!mac_latency_optimize && worker_assign) ? 987 mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 988 989 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 990 991 /* Do CPU bindings for SRSes having h/w Rx rings */ 992 if (flent->fe_rx_srs_cnt > 1) { 993 cpu_cnt = 0; 994 for (srs_cnt = 1; 995 srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 996 rx_srs = flent->fe_rx_srs[srs_cnt]; 997 ASSERT(rx_srs->srs_ring != NULL); 998 if (rx_srs->srs_fanout_state == 999 SRS_FANOUT_INIT) { 1000 rx_srs->srs_fanout_state = 1001 SRS_FANOUT_REINIT; 1002 } 1003 srs_cpu = &rx_srs->srs_cpu; 1004 srs_cpu->mc_ncpus = no_of_cpus; 1005 bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1006 sizeof (srs_cpu->mc_cpus)); 1007 srs_cpu->mc_rx_pollid = 1008 mrp->mrp_cpu[cpu_cnt]; 1009 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1010 srs_cpu->mc_rx_workerid = 1011 ((!mac_latency_optimize && worker_assign) ? 1012 mrp->mrp_cpu[++cpu_cnt] : 1013 srs_cpu->mc_rx_pollid); 1014 srs_cpu->mc_rx_fanout_cnt = 1; 1015 srs_cpu->mc_rx_fanout_cpus[0] = 1016 mrp->mrp_cpu[cpu_cnt]; 1017 cpu_cnt++; 1018 ASSERT(cpu_cnt <= no_of_cpus); 1019 } 1020 } 1021 goto tx_cpu_init; 1022 } 1023 1024 /* 1025 * Real sub-optimal case. Not enough CPUs for poll and 1026 * Tx soft rings. Do a round robin assignment where 1027 * each Rx SRS will get the same CPU for poll, worker 1028 * and fanout soft ring. 1029 */ 1030 cpu_cnt = 0; 1031 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1032 rx_srs = flent->fe_rx_srs[srs_cnt]; 1033 srs_cpu = &rx_srs->srs_cpu; 1034 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1035 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1036 srs_cpu->mc_ncpus = no_of_cpus; 1037 bcopy(mrp->mrp_cpu, 1038 srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 1039 srs_cpu->mc_rx_fanout_cnt = 1; 1040 srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 1041 /* Retarget the interrupt to the same CPU as the poll */ 1042 srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 1043 srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 1044 srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1045 if (++cpu_cnt >= no_of_cpus) 1046 cpu_cnt = 0; 1047 } 1048 1049 tx_cpu_init: 1050 mac_tx_cpu_init(flent, mrp, NULL); 1051 1052 /* 1053 * Copy the user specified CPUs to the effective CPUs 1054 */ 1055 for (i = 0; i < mrp->mrp_ncpus; i++) { 1056 emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 1057 } 1058 emrp->mrp_ncpus = mrp->mrp_ncpus; 1059 emrp->mrp_mask = mrp->mrp_mask; 1060 bzero(emrp->mrp_pool, MAXPATHLEN); 1061 } 1062 1063 /* 1064 * mac_flow_cpu_init(): 1065 * 1066 * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1067 * the CPU binding information in srs_cpu for all Rx SRSes associated 1068 * with a flent. 1069 */ 1070 static void 1071 mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1072 { 1073 mac_soft_ring_set_t *rx_srs; 1074 processorid_t cpuid; 1075 int i, j, k, srs_cnt, nscpus, maxcpus, soft_ring_cnt = 0; 1076 mac_cpus_t *srs_cpu; 1077 mac_resource_props_t *emrp = &flent->fe_effective_props; 1078 uint32_t cpus[MRP_NCPUS]; 1079 1080 /* 1081 * The maximum number of CPUs available can either be 1082 * the number of CPUs in the pool or the number of CPUs 1083 * in the system. 1084 */ 1085 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 1086 1087 /* 1088 * Compute the number of soft rings needed on top for each Rx 1089 * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1090 * associated with h/w Rx rings. Soft ring count needed for 1091 * each h/w Rx SRS is computed and the same is applied to 1092 * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1093 * is the software classified Rx SRS. 1094 */ 1095 soft_ring_cnt = mac_compute_soft_ring_count(flent, 1096 flent->fe_rx_srs_cnt - 1, maxcpus); 1097 if (soft_ring_cnt == 0) { 1098 /* 1099 * Even when soft_ring_cnt is 0, we still need 1100 * to create a soft ring for TCP, UDP and 1101 * OTHER. So set it to 1. 1102 */ 1103 soft_ring_cnt = 1; 1104 } 1105 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1106 rx_srs = flent->fe_rx_srs[srs_cnt]; 1107 srs_cpu = &rx_srs->srs_cpu; 1108 if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1109 rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1110 srs_cpu->mc_ncpus = soft_ring_cnt; 1111 srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1112 mutex_enter(&cpu_lock); 1113 for (j = 0; j < soft_ring_cnt; j++) { 1114 cpuid = mac_next_bind_cpu(cpupart); 1115 srs_cpu->mc_cpus[j] = cpuid; 1116 srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1117 } 1118 cpuid = mac_next_bind_cpu(cpupart); 1119 srs_cpu->mc_rx_pollid = cpuid; 1120 srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 1121 srs_cpu->mc_rx_pollid : -1); 1122 /* increment ncpus to account for polling cpu */ 1123 srs_cpu->mc_ncpus++; 1124 srs_cpu->mc_cpus[j++] = cpuid; 1125 if (!mac_latency_optimize) { 1126 cpuid = mac_next_bind_cpu(cpupart); 1127 srs_cpu->mc_ncpus++; 1128 srs_cpu->mc_cpus[j++] = cpuid; 1129 } 1130 srs_cpu->mc_rx_workerid = cpuid; 1131 mutex_exit(&cpu_lock); 1132 } 1133 1134 nscpus = 0; 1135 for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1136 rx_srs = flent->fe_rx_srs[srs_cnt]; 1137 srs_cpu = &rx_srs->srs_cpu; 1138 for (j = 0; j < srs_cpu->mc_ncpus; j++) { 1139 cpus[nscpus++] = srs_cpu->mc_cpus[j]; 1140 } 1141 } 1142 1143 1144 /* 1145 * Copy cpu list to fe_effective_props 1146 * without duplicates. 1147 */ 1148 k = 0; 1149 for (i = 0; i < nscpus; i++) { 1150 for (j = 0; j < k; j++) { 1151 if (emrp->mrp_cpu[j] == cpus[i]) 1152 break; 1153 } 1154 if (j == k) 1155 emrp->mrp_cpu[k++] = cpus[i]; 1156 } 1157 emrp->mrp_ncpus = k; 1158 1159 mac_tx_cpu_init(flent, NULL, cpupart); 1160 } 1161 1162 /* 1163 * DATAPATH SETUP ROUTINES 1164 * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1165 */ 1166 1167 /* 1168 * mac_srs_fanout_list_alloc: 1169 * 1170 * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 1171 * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 1172 * array space is needed to store Tx soft rings. Thus we allocate so 1173 * much array space for srs_tx_soft_rings. 1174 * 1175 * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 1176 * of space to st_soft_rings. This array is used for quick access to 1177 * soft ring associated with a pseudo Tx ring based on the pseudo 1178 * ring's index (mr_index). 1179 */ 1180 static void 1181 mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1182 { 1183 mac_client_impl_t *mcip = mac_srs->srs_mcip; 1184 1185 if (mac_srs->srs_type & SRST_TX) { 1186 mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 1187 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1188 MAX_RINGS_PER_GROUP, KM_SLEEP); 1189 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 1190 mac_srs_tx_t *tx = &mac_srs->srs_tx; 1191 1192 tx->st_soft_rings = (mac_soft_ring_t **) 1193 kmem_zalloc(sizeof (mac_soft_ring_t *) * 1194 MAX_RINGS_PER_GROUP, KM_SLEEP); 1195 } 1196 } else { 1197 mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 1198 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1199 KM_SLEEP); 1200 mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 1201 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1202 KM_SLEEP); 1203 mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 1204 kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 1205 KM_SLEEP); 1206 } 1207 } 1208 1209 static void 1210 mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1211 { 1212 cpu_t *cp; 1213 boolean_t clear = B_FALSE; 1214 1215 ASSERT(MUTEX_HELD(&cpu_lock)); 1216 1217 if (!mac_srs_thread_bind) 1218 return; 1219 1220 cp = cpu_get(cpuid); 1221 if (cp == NULL || !cpu_is_online(cp)) 1222 return; 1223 1224 mutex_enter(&mac_srs->srs_lock); 1225 mac_srs->srs_state |= SRS_WORKER_BOUND; 1226 if (mac_srs->srs_worker_cpuid != -1) 1227 clear = B_TRUE; 1228 mac_srs->srs_worker_cpuid = cpuid; 1229 mutex_exit(&mac_srs->srs_lock); 1230 1231 if (clear) 1232 thread_affinity_clear(mac_srs->srs_worker); 1233 1234 thread_affinity_set(mac_srs->srs_worker, cpuid); 1235 DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1236 } 1237 1238 static void 1239 mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1240 { 1241 cpu_t *cp; 1242 boolean_t clear = B_FALSE; 1243 1244 ASSERT(MUTEX_HELD(&cpu_lock)); 1245 1246 if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1247 return; 1248 1249 cp = cpu_get(cpuid); 1250 if (cp == NULL || !cpu_is_online(cp)) 1251 return; 1252 1253 mutex_enter(&mac_srs->srs_lock); 1254 mac_srs->srs_state |= SRS_POLL_BOUND; 1255 if (mac_srs->srs_poll_cpuid != -1) 1256 clear = B_TRUE; 1257 mac_srs->srs_poll_cpuid = cpuid; 1258 mutex_exit(&mac_srs->srs_lock); 1259 1260 if (clear) 1261 thread_affinity_clear(mac_srs->srs_poll_thr); 1262 1263 thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1264 DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1265 } 1266 1267 /* 1268 * Re-target interrupt to the passed CPU. If re-target is successful, 1269 * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 1270 */ 1271 void 1272 mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1273 { 1274 cpu_t *cp; 1275 mac_ring_t *ring = mac_srs->srs_ring; 1276 mac_intr_t *mintr = &ring->mr_info.mri_intr; 1277 flow_entry_t *flent = mac_srs->srs_flent; 1278 boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 1279 1280 ASSERT(MUTEX_HELD(&cpu_lock)); 1281 1282 /* 1283 * Don't re-target the interrupt for these cases: 1284 * 1) ring is NULL 1285 * 2) the interrupt is shared (mi_ddi_shared) 1286 * 3) ddi_handle is NULL and !primary 1287 * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 1288 * Case 3 & 4 are because of mac_client_intr_cpu() routine. 1289 * This routine will re-target fixed interrupt for primary 1290 * mac client if the client has only one ring. In that 1291 * case, mc_rx_intr_cpu will already have the correct value. 1292 */ 1293 if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 1294 (mintr->mi_ddi_handle == NULL && !primary) || (primary && 1295 mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 1296 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1297 return; 1298 } 1299 1300 if (mintr->mi_ddi_handle == NULL) 1301 return; 1302 1303 cp = cpu_get(cpuid); 1304 if (cp == NULL || !cpu_is_online(cp)) 1305 return; 1306 1307 /* Drop the cpu_lock as set_intr_affinity() holds it */ 1308 mutex_exit(&cpu_lock); 1309 if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 1310 mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 1311 else 1312 mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 1313 mutex_enter(&cpu_lock); 1314 } 1315 1316 /* 1317 * Re-target Tx interrupts 1318 */ 1319 void 1320 mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 1321 { 1322 cpu_t *cp; 1323 mac_ring_t *ring; 1324 mac_intr_t *mintr; 1325 mac_soft_ring_t *sringp; 1326 mac_srs_tx_t *srs_tx; 1327 mac_cpus_t *srs_cpu; 1328 processorid_t cpuid; 1329 int i; 1330 1331 ASSERT(MUTEX_HELD(&cpu_lock)); 1332 1333 srs_cpu = &mac_srs->srs_cpu; 1334 if (MAC_TX_SOFT_RINGS(mac_srs)) { 1335 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 1336 sringp = mac_srs->srs_tx_soft_rings[i]; 1337 ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 1338 cpuid = srs_cpu->mc_tx_intr_cpu[i]; 1339 cp = cpu_get(cpuid); 1340 if (cp == NULL || !cpu_is_online(cp) || 1341 !MAC_RING_RETARGETABLE(ring)) { 1342 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1343 continue; 1344 } 1345 mintr = &ring->mr_info.mri_intr; 1346 /* 1347 * Drop the cpu_lock as set_intr_affinity() 1348 * holds it 1349 */ 1350 mutex_exit(&cpu_lock); 1351 if (set_intr_affinity(mintr->mi_ddi_handle, 1352 cpuid) == DDI_SUCCESS) { 1353 srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 1354 } else { 1355 srs_cpu->mc_tx_retargeted_cpu[i] = -1; 1356 } 1357 mutex_enter(&cpu_lock); 1358 } 1359 } else { 1360 cpuid = srs_cpu->mc_tx_intr_cpu[0]; 1361 cp = cpu_get(cpuid); 1362 if (cp == NULL || !cpu_is_online(cp)) { 1363 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1364 return; 1365 } 1366 srs_tx = &mac_srs->srs_tx; 1367 ring = (mac_ring_t *)srs_tx->st_arg2; 1368 if (MAC_RING_RETARGETABLE(ring)) { 1369 mintr = &ring->mr_info.mri_intr; 1370 mutex_exit(&cpu_lock); 1371 if ((set_intr_affinity(mintr->mi_ddi_handle, 1372 cpuid) == DDI_SUCCESS)) { 1373 srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 1374 } else { 1375 srs_cpu->mc_tx_retargeted_cpu[0] = -1; 1376 } 1377 mutex_enter(&cpu_lock); 1378 } 1379 } 1380 } 1381 1382 /* 1383 * When a CPU comes back online, bind the MAC kernel threads which 1384 * were previously bound to that CPU, and had to be unbound because 1385 * the CPU was going away. 1386 * 1387 * These functions are called with cpu_lock held and hence we can't 1388 * cv_wait to grab the mac perimeter. Since these functions walk the soft 1389 * ring list of an SRS without being in the perimeter, the list itself 1390 * is protected by the SRS lock. 1391 */ 1392 static void 1393 mac_walk_srs_and_bind(int cpuid) 1394 { 1395 mac_soft_ring_set_t *mac_srs; 1396 mac_soft_ring_t *soft_ring; 1397 1398 rw_enter(&mac_srs_g_lock, RW_READER); 1399 1400 if ((mac_srs = mac_srs_g_list) == NULL) 1401 goto done; 1402 1403 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1404 if (mac_srs->srs_worker_cpuid == -1 && 1405 mac_srs->srs_worker_cpuid_save == cpuid) { 1406 mac_srs->srs_worker_cpuid_save = -1; 1407 mac_srs_worker_bind(mac_srs, cpuid); 1408 } 1409 1410 if (!(mac_srs->srs_type & SRST_TX)) { 1411 if (mac_srs->srs_poll_cpuid == -1 && 1412 mac_srs->srs_poll_cpuid_save == cpuid) { 1413 mac_srs->srs_poll_cpuid_save = -1; 1414 mac_srs_poll_bind(mac_srs, cpuid); 1415 } 1416 } 1417 1418 /* Next tackle the soft rings associated with the srs */ 1419 mutex_enter(&mac_srs->srs_lock); 1420 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1421 soft_ring = soft_ring->s_ring_next) { 1422 if (soft_ring->s_ring_cpuid == -1 && 1423 soft_ring->s_ring_cpuid_save == cpuid) { 1424 soft_ring->s_ring_cpuid_save = -1; 1425 (void) mac_soft_ring_bind(soft_ring, cpuid); 1426 } 1427 } 1428 mutex_exit(&mac_srs->srs_lock); 1429 } 1430 done: 1431 rw_exit(&mac_srs_g_lock); 1432 } 1433 1434 /* 1435 * Change the priority of the SRS's poll and worker thread. Additionally, 1436 * update the priority of the worker threads for the SRS's soft rings. 1437 * Need to modify any associated squeue threads. 1438 */ 1439 void 1440 mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1441 { 1442 mac_soft_ring_t *ringp; 1443 1444 mac_srs->srs_pri = prival; 1445 thread_lock(mac_srs->srs_worker); 1446 (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1447 thread_unlock(mac_srs->srs_worker); 1448 if (mac_srs->srs_poll_thr != NULL) { 1449 thread_lock(mac_srs->srs_poll_thr); 1450 (void) thread_change_pri(mac_srs->srs_poll_thr, 1451 mac_srs->srs_pri, 0); 1452 thread_unlock(mac_srs->srs_poll_thr); 1453 } 1454 if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1455 return; 1456 while (ringp != mac_srs->srs_soft_ring_tail) { 1457 thread_lock(ringp->s_ring_worker); 1458 (void) thread_change_pri(ringp->s_ring_worker, 1459 mac_srs->srs_pri, 0); 1460 thread_unlock(ringp->s_ring_worker); 1461 ringp = ringp->s_ring_next; 1462 } 1463 ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1464 thread_lock(ringp->s_ring_worker); 1465 (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1466 thread_unlock(ringp->s_ring_worker); 1467 } 1468 1469 /* 1470 * Change the receive bandwidth limit. 1471 */ 1472 static void 1473 mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1474 { 1475 mac_soft_ring_t *softring; 1476 1477 mutex_enter(&srs->srs_lock); 1478 mutex_enter(&srs->srs_bw->mac_bw_lock); 1479 1480 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1481 /* Reset bandwidth limit */ 1482 if (srs->srs_type & SRST_BW_CONTROL) { 1483 softring = srs->srs_soft_ring_head; 1484 while (softring != NULL) { 1485 softring->s_ring_type &= ~ST_RING_BW_CTL; 1486 softring = softring->s_ring_next; 1487 } 1488 srs->srs_type &= ~SRST_BW_CONTROL; 1489 srs->srs_drain_func = mac_rx_srs_drain; 1490 } 1491 } else { 1492 /* Set/Modify bandwidth limit */ 1493 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1494 /* 1495 * Give twice the queuing capability before 1496 * dropping packets. The unit is bytes/tick. 1497 */ 1498 srs->srs_bw->mac_bw_drop_threshold = 1499 srs->srs_bw->mac_bw_limit << 1; 1500 if (!(srs->srs_type & SRST_BW_CONTROL)) { 1501 softring = srs->srs_soft_ring_head; 1502 while (softring != NULL) { 1503 softring->s_ring_type |= ST_RING_BW_CTL; 1504 softring = softring->s_ring_next; 1505 } 1506 srs->srs_type |= SRST_BW_CONTROL; 1507 srs->srs_drain_func = mac_rx_srs_drain_bw; 1508 } 1509 } 1510 done: 1511 mutex_exit(&srs->srs_bw->mac_bw_lock); 1512 mutex_exit(&srs->srs_lock); 1513 } 1514 1515 /* Change the transmit bandwidth limit */ 1516 static void 1517 mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1518 { 1519 uint32_t tx_mode, ring_info = 0; 1520 mac_srs_tx_t *srs_tx = &srs->srs_tx; 1521 mac_client_impl_t *mcip = srs->srs_mcip; 1522 1523 /* 1524 * We need to quiesce/restart the client here because mac_tx() and 1525 * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1526 * st_mode and related fields, which are modified by the code below. 1527 */ 1528 mac_tx_client_quiesce((mac_client_handle_t)mcip); 1529 1530 mutex_enter(&srs->srs_lock); 1531 mutex_enter(&srs->srs_bw->mac_bw_lock); 1532 1533 tx_mode = srs_tx->st_mode; 1534 if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1535 /* Reset bandwidth limit */ 1536 if (tx_mode == SRS_TX_BW) { 1537 if (srs_tx->st_arg2 != NULL) 1538 ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1539 if (mac_tx_serialize || 1540 (ring_info & MAC_RING_TX_SERIALIZE)) { 1541 srs_tx->st_mode = SRS_TX_SERIALIZE; 1542 } else { 1543 srs_tx->st_mode = SRS_TX_DEFAULT; 1544 } 1545 } else if (tx_mode == SRS_TX_BW_FANOUT) { 1546 srs_tx->st_mode = SRS_TX_FANOUT; 1547 } else if (tx_mode == SRS_TX_BW_AGGR) { 1548 srs_tx->st_mode = SRS_TX_AGGR; 1549 } 1550 srs->srs_type &= ~SRST_BW_CONTROL; 1551 } else { 1552 /* Set/Modify bandwidth limit */ 1553 srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1554 /* 1555 * Give twice the queuing capability before 1556 * dropping packets. The unit is bytes/tick. 1557 */ 1558 srs->srs_bw->mac_bw_drop_threshold = 1559 srs->srs_bw->mac_bw_limit << 1; 1560 srs->srs_type |= SRST_BW_CONTROL; 1561 if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 1562 tx_mode != SRS_TX_BW_AGGR) { 1563 if (tx_mode == SRS_TX_SERIALIZE || 1564 tx_mode == SRS_TX_DEFAULT) { 1565 srs_tx->st_mode = SRS_TX_BW; 1566 } else if (tx_mode == SRS_TX_FANOUT) { 1567 srs_tx->st_mode = SRS_TX_BW_FANOUT; 1568 } else if (tx_mode == SRS_TX_AGGR) { 1569 srs_tx->st_mode = SRS_TX_BW_AGGR; 1570 } else { 1571 ASSERT(0); 1572 } 1573 } 1574 } 1575 done: 1576 srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1577 mutex_exit(&srs->srs_bw->mac_bw_lock); 1578 mutex_exit(&srs->srs_lock); 1579 1580 mac_tx_client_restart((mac_client_handle_t)mcip); 1581 } 1582 1583 /* 1584 * The uber function that deals with any update to bandwidth limits. 1585 */ 1586 void 1587 mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1588 { 1589 int count; 1590 1591 for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1592 mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1593 mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1594 } 1595 1596 /* 1597 * When the first sub-flow is added to a link, we disable polling on the 1598 * link and also modify the entry point to mac_rx_srs_subflow_process(). 1599 * (polling is disabled because with the subflow added, accounting 1600 * for polling needs additional logic, it is assumed that when a subflow is 1601 * added, we can take some hit as a result of disabling polling rather than 1602 * adding more complexity - if this becomes a perf. issue we need to 1603 * re-rvaluate this logic). When the last subflow is removed, we turn back 1604 * polling and also reset the entry point to mac_rx_srs_process(). 1605 * 1606 * In the future if there are multiple SRS, we can simply 1607 * take one and give it to the flow rather than disabling polling and 1608 * resetting the entry point. 1609 */ 1610 void 1611 mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1612 { 1613 flow_entry_t *flent = mcip->mci_flent; 1614 int i; 1615 mac_impl_t *mip = mcip->mci_mip; 1616 mac_rx_func_t rx_func; 1617 uint_t rx_srs_cnt; 1618 boolean_t enable_classifier; 1619 1620 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1621 1622 enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1623 1624 rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1625 mac_rx_srs_process; 1626 1627 /* Tell mac_srs_poll_state_change to disable polling if necessary */ 1628 if (mip->mi_state_flags & MIS_POLL_DISABLE) 1629 enable_classifier = B_TRUE; 1630 1631 /* 1632 * If receive function has already been configured correctly for 1633 * current subflow configuration, do nothing. 1634 */ 1635 if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1636 return; 1637 1638 rx_srs_cnt = flent->fe_rx_srs_cnt; 1639 for (i = 0; i < rx_srs_cnt; i++) { 1640 ASSERT(flent->fe_rx_srs[i] != NULL); 1641 mac_srs_poll_state_change(flent->fe_rx_srs[i], 1642 enable_classifier, rx_func); 1643 } 1644 1645 /* 1646 * Change the S/W classifier so that we can land in the 1647 * correct processing function with correct argument. 1648 * If all subflows have been removed we can revert to 1649 * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process(). 1650 */ 1651 mutex_enter(&flent->fe_lock); 1652 flent->fe_cb_fn = (flow_fn_t)rx_func; 1653 flent->fe_cb_arg1 = (void *)mip; 1654 flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1655 mutex_exit(&flent->fe_lock); 1656 } 1657 1658 static void 1659 mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1660 { 1661 int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0; 1662 mac_soft_ring_t *softring; 1663 1664 softring = mac_srs->srs_soft_ring_head; 1665 if (softring == NULL) { 1666 ASSERT(mac_srs->srs_soft_ring_count == 0); 1667 mac_srs->srs_tcp_ring_count = 0; 1668 mac_srs->srs_udp_ring_count = 0; 1669 mac_srs->srs_oth_ring_count = 0; 1670 mac_srs->srs_tx_ring_count = 0; 1671 return; 1672 } 1673 1674 while (softring != NULL) { 1675 if (softring->s_ring_type & ST_RING_TCP) { 1676 mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 1677 } else if (softring->s_ring_type & ST_RING_UDP) { 1678 mac_srs->srs_udp_soft_rings[udp_count++] = softring; 1679 } else if (softring->s_ring_type & ST_RING_OTH) { 1680 mac_srs->srs_oth_soft_rings[oth_count++] = softring; 1681 } else { 1682 ASSERT(softring->s_ring_type & ST_RING_TX); 1683 mac_srs->srs_tx_soft_rings[tx_count++] = softring; 1684 } 1685 softring = softring->s_ring_next; 1686 } 1687 1688 ASSERT(mac_srs->srs_soft_ring_count == 1689 (tcp_count + udp_count + oth_count + tx_count)); 1690 mac_srs->srs_tcp_ring_count = tcp_count; 1691 mac_srs->srs_udp_ring_count = udp_count; 1692 mac_srs->srs_oth_ring_count = oth_count; 1693 mac_srs->srs_tx_ring_count = tx_count; 1694 } 1695 1696 void 1697 mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 1698 mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1699 processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1700 mac_resource_handle_t x_arg2, boolean_t set_bypass) 1701 { 1702 mac_soft_ring_t *softring; 1703 mac_rx_fifo_t mrf; 1704 1705 bzero(&mrf, sizeof (mac_rx_fifo_t)); 1706 mrf.mrf_type = MAC_RX_FIFO; 1707 mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1708 mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 1709 mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 1710 mrf.mrf_flow_priority = pri; 1711 1712 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1713 (type|ST_RING_TCP), pri, mcip, mac_srs, 1714 cpuid, rx_func, x_arg1, x_arg2); 1715 softring->s_ring_rx_arg2 = NULL; 1716 1717 /* 1718 * TCP and UDP support DLS bypass. In addition TCP 1719 * squeue can also poll their corresponding soft rings. 1720 */ 1721 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1722 mac_soft_ring_dls_bypass(softring, 1723 mcip->mci_direct_rx_fn, 1724 mcip->mci_direct_rx_arg); 1725 1726 mrf.mrf_rx_arg = softring; 1727 mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1728 1729 /* 1730 * Make a call in IP to get a TCP squeue assigned to 1731 * this softring to maintain full CPU locality through 1732 * the stack and allow the squeue to be able to poll 1733 * the softring so the flow control can be pushed 1734 * all the way to H/W. 1735 */ 1736 softring->s_ring_rx_arg2 = 1737 mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1738 (mac_resource_t *)&mrf); 1739 } 1740 1741 /* 1742 * Non-TCP protocols don't support squeues. Hence we 1743 * don't make any ring addition callbacks for non-TCP 1744 * rings. Now create the UDP softring and allow it to 1745 * bypass the DLS layer. 1746 */ 1747 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1748 (type|ST_RING_UDP), pri, mcip, mac_srs, 1749 cpuid, rx_func, x_arg1, x_arg2); 1750 softring->s_ring_rx_arg2 = NULL; 1751 1752 if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1753 mac_soft_ring_dls_bypass(softring, 1754 mcip->mci_direct_rx_fn, 1755 mcip->mci_direct_rx_arg); 1756 } 1757 1758 /* Create the Oth softrings which has to go through the DLS */ 1759 softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1760 (type|ST_RING_OTH), pri, mcip, mac_srs, 1761 cpuid, rx_func, x_arg1, x_arg2); 1762 softring->s_ring_rx_arg2 = NULL; 1763 } 1764 1765 /* 1766 * This routine associates a CPU or a set of CPU to process incoming 1767 * traffic from a mac client. If multiple CPUs are specified, then 1768 * so many soft rings are created with each soft ring worker thread 1769 * bound to a CPU in the set. Each soft ring in turn will be 1770 * associated with an squeue and the squeue will be moved to the 1771 * same CPU as that of the soft ring's. 1772 */ 1773 static void 1774 mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 1775 void *x_arg1, mac_resource_handle_t x_arg2, 1776 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1777 { 1778 mac_soft_ring_t *softring; 1779 uint32_t soft_ring_flag = 0; 1780 processorid_t cpuid = -1; 1781 int i, srings_present, new_fanout_cnt; 1782 mac_cpus_t *srs_cpu; 1783 1784 /* fanout state is REINIT. Set it back to INIT */ 1785 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1786 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1787 1788 /* how many are present right now */ 1789 srings_present = mac_rx_srs->srs_tcp_ring_count; 1790 /* new request */ 1791 srs_cpu = &mac_rx_srs->srs_cpu; 1792 new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1793 1794 mutex_enter(&mac_rx_srs->srs_lock); 1795 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1796 soft_ring_flag |= ST_RING_BW_CTL; 1797 mutex_exit(&mac_rx_srs->srs_lock); 1798 1799 if (new_fanout_cnt > srings_present) { 1800 /* soft rings increased */ 1801 mutex_enter(&mac_rx_srs->srs_lock); 1802 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1803 mutex_exit(&mac_rx_srs->srs_lock); 1804 1805 for (i = mac_rx_srs->srs_tcp_ring_count; 1806 i < new_fanout_cnt; i++) { 1807 /* 1808 * Create the protocol softrings and set the 1809 * DLS bypass where possible. 1810 */ 1811 mac_srs_create_proto_softrings(i, soft_ring_flag, 1812 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1813 rx_func, x_arg1, x_arg2, B_TRUE); 1814 } 1815 mac_srs_update_fanout_list(mac_rx_srs); 1816 } else if (new_fanout_cnt < srings_present) { 1817 /* soft rings decreased */ 1818 if (new_fanout_cnt == 1) { 1819 mutex_enter(&mac_rx_srs->srs_lock); 1820 mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1821 ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1822 mutex_exit(&mac_rx_srs->srs_lock); 1823 } 1824 /* Get rid of extra soft rings */ 1825 for (i = new_fanout_cnt; 1826 i < mac_rx_srs->srs_tcp_ring_count; i++) { 1827 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1828 if (softring->s_ring_rx_arg2 != NULL) { 1829 mcip->mci_resource_remove( 1830 (void *)mcip->mci_resource_arg, 1831 softring->s_ring_rx_arg2); 1832 } 1833 mac_soft_ring_remove(mac_rx_srs, 1834 mac_rx_srs->srs_tcp_soft_rings[i]); 1835 mac_soft_ring_remove(mac_rx_srs, 1836 mac_rx_srs->srs_udp_soft_rings[i]); 1837 mac_soft_ring_remove(mac_rx_srs, 1838 mac_rx_srs->srs_oth_soft_rings[i]); 1839 } 1840 mac_srs_update_fanout_list(mac_rx_srs); 1841 } 1842 1843 ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1844 mutex_enter(&cpu_lock); 1845 for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 1846 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1847 (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1848 cpuid); 1849 (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1850 cpuid); 1851 (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1852 cpuid); 1853 softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1854 if (softring->s_ring_rx_arg2 != NULL) { 1855 mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1856 softring->s_ring_rx_arg2, cpuid); 1857 } 1858 } 1859 1860 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1861 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1862 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1863 /* 1864 * Bind Tx srs and soft ring threads too. Let's bind tx 1865 * srs to the last cpu in mrp list. 1866 */ 1867 if (mac_tx_srs != NULL) { 1868 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1869 mac_tx_srs_retarget_intr(mac_tx_srs); 1870 } 1871 mutex_exit(&cpu_lock); 1872 } 1873 1874 /* 1875 * Bind SRS threads and soft rings to CPUs/create fanout list. 1876 */ 1877 void 1878 mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 1879 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 1880 mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 1881 cpupart_t *cpupart) 1882 { 1883 int i; 1884 processorid_t cpuid; 1885 uint32_t soft_ring_flag = 0; 1886 int soft_ring_cnt; 1887 mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1888 1889 /* 1890 * Remove the no soft ring flag and we will adjust it 1891 * appropriately further down. 1892 */ 1893 mutex_enter(&mac_rx_srs->srs_lock); 1894 mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1895 mutex_exit(&mac_rx_srs->srs_lock); 1896 1897 ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1898 1899 if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1900 soft_ring_flag |= ST_RING_BW_CTL; 1901 1902 ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1903 mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1904 /* 1905 * Ring count can be 0 if no fanout is required and no cpu 1906 * were specified. Leave the SRS worker and poll thread 1907 * unbound 1908 */ 1909 ASSERT(mrp != NULL); 1910 soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 1911 1912 /* Step 1: bind cpu contains cpu list where threads need to bind */ 1913 if (soft_ring_cnt > 0) { 1914 mutex_enter(&cpu_lock); 1915 for (i = 0; i < soft_ring_cnt; i++) { 1916 cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1917 /* Create the protocol softrings */ 1918 mac_srs_create_proto_softrings(i, soft_ring_flag, 1919 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1920 rx_func, x_arg1, x_arg2, B_FALSE); 1921 } 1922 mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 1923 mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 1924 mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1925 /* 1926 * Bind Tx srs and soft ring threads too. 1927 * Let's bind tx srs to the last cpu in 1928 * mrp list. 1929 */ 1930 if (mac_tx_srs == NULL) { 1931 mutex_exit(&cpu_lock); 1932 goto alldone; 1933 } 1934 1935 BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1936 mac_tx_srs_retarget_intr(mac_tx_srs); 1937 mutex_exit(&cpu_lock); 1938 } else { 1939 mutex_enter(&cpu_lock); 1940 /* 1941 * For a subflow, mrp_workerid and mrp_pollid 1942 * is not set. 1943 */ 1944 mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 1945 mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 1946 mutex_exit(&cpu_lock); 1947 goto no_softrings; 1948 } 1949 1950 alldone: 1951 if (soft_ring_cnt > 1) 1952 mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1953 mac_srs_update_fanout_list(mac_rx_srs); 1954 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1955 return; 1956 1957 no_softrings: 1958 if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1959 mutex_enter(&cpu_lock); 1960 cpuid = mac_next_bind_cpu(cpupart); 1961 /* Create the protocol softrings */ 1962 mac_srs_create_proto_softrings(0, soft_ring_flag, 1963 mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1964 rx_func, x_arg1, x_arg2, B_FALSE); 1965 mutex_exit(&cpu_lock); 1966 } else { 1967 /* 1968 * This is the case when there is no fanout which is 1969 * true for subflows. 1970 */ 1971 mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1972 } 1973 mac_srs_update_fanout_list(mac_rx_srs); 1974 mac_srs_client_poll_enable(mcip, mac_rx_srs); 1975 } 1976 1977 /* 1978 * Calls mac_srs_fanout_init() or modify() depending upon whether 1979 * the SRS is getting initialized or re-initialized. 1980 */ 1981 void 1982 mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1983 mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1984 mac_resource_handle_t x_arg2, cpupart_t *cpupart) 1985 { 1986 mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1987 int i, rx_srs_cnt; 1988 1989 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 1990 1991 /* 1992 * Aggr ports do not have SRSes. This function should never be 1993 * called on an aggr port. 1994 */ 1995 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 1996 mac_rx_srs = flent->fe_rx_srs[0]; 1997 1998 /* 1999 * Set up the fanout on the tx side only once, with the 2000 * first rx SRS. The CPU binding, fanout, and bandwidth 2001 * criteria are common to both RX and TX, so 2002 * initializing them along side avoids redundant code. 2003 */ 2004 mac_tx_srs = flent->fe_tx_srs; 2005 rx_srs_cnt = flent->fe_rx_srs_cnt; 2006 2007 /* No fanout for subflows */ 2008 if (flent->fe_type & FLOW_USER) { 2009 mac_srs_fanout_init(mcip, mrp, rx_func, 2010 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2011 cpupart); 2012 return; 2013 } 2014 2015 if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 2016 mac_flow_user_cpu_init(flent, mrp); 2017 else 2018 mac_flow_cpu_init(flent, cpupart); 2019 2020 mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2021 2022 /* 2023 * Set up fanout for both SW (0th SRS) and HW classified 2024 * SRS (the rest of Rx SRSs in flent). 2025 */ 2026 for (i = 0; i < rx_srs_cnt; i++) { 2027 mac_rx_srs = flent->fe_rx_srs[i]; 2028 if (i != 0) 2029 mac_tx_srs = NULL; 2030 switch (mac_rx_srs->srs_fanout_state) { 2031 case SRS_FANOUT_UNINIT: 2032 mac_srs_fanout_init(mcip, mrp, rx_func, 2033 x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 2034 cpupart); 2035 break; 2036 case SRS_FANOUT_INIT: 2037 break; 2038 case SRS_FANOUT_REINIT: 2039 mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 2040 mac_srs_fanout_modify(mcip, rx_func, x_arg1, 2041 x_arg2, mac_rx_srs, mac_tx_srs); 2042 mac_rx_srs_restart(mac_rx_srs); 2043 break; 2044 default: 2045 VERIFY(mac_rx_srs->srs_fanout_state <= 2046 SRS_FANOUT_REINIT); 2047 break; 2048 } 2049 } 2050 } 2051 2052 /* 2053 * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2054 * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2055 * processing is created. 2056 * 2057 * Details on Rx SRS: 2058 * Create a SRS and also add the necessary soft rings for TCP and 2059 * non-TCP based on fanout type and count specified. 2060 * 2061 * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2062 * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2063 * to be heavily modified. 2064 * 2065 * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2066 */ 2067 mac_soft_ring_set_t * 2068 mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2069 mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2070 mac_ring_t *ring) 2071 { 2072 mac_soft_ring_set_t *mac_srs; 2073 mac_srs_rx_t *srs_rx; 2074 mac_srs_tx_t *srs_tx; 2075 mac_bw_ctl_t *mac_bw; 2076 mac_resource_props_t *mrp; 2077 boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2078 2079 mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2080 bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2081 srs_rx = &mac_srs->srs_rx; 2082 srs_tx = &mac_srs->srs_tx; 2083 2084 mutex_enter(&flent->fe_lock); 2085 2086 /* 2087 * Get the bandwidth control structure from the flent. Get 2088 * rid of any residual values in the control structure for 2089 * the tx bw struct and also for the rx, if the rx srs is 2090 * the 1st one being brought up (the rx bw ctl struct may 2091 * be shared by multiple SRSs) 2092 */ 2093 if (is_tx_srs) { 2094 mac_srs->srs_bw = &flent->fe_tx_bw; 2095 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2096 flent->fe_tx_srs = mac_srs; 2097 } else { 2098 /* 2099 * The bw counter (stored in the flent) is shared 2100 * by SRS's within an rx group. 2101 */ 2102 mac_srs->srs_bw = &flent->fe_rx_bw; 2103 /* First rx SRS, clear the bw structure */ 2104 if (flent->fe_rx_srs_cnt == 0) 2105 bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2106 2107 /* 2108 * It is better to panic here rather than just assert because 2109 * on a non-debug kernel we might end up courrupting memory 2110 * and making it difficult to debug. 2111 */ 2112 if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 2113 panic("Array Overrun detected due to MAC client %p " 2114 " having more rings than %d", (void *)mcip, 2115 MAX_RINGS_PER_GROUP); 2116 } 2117 flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2118 flent->fe_rx_srs_cnt++; 2119 } 2120 mac_srs->srs_flent = flent; 2121 mutex_exit(&flent->fe_lock); 2122 2123 mac_srs->srs_state = 0; 2124 mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2125 mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2126 mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 2127 mac_srs->srs_mcip = mcip; 2128 mac_srs_fanout_list_alloc(mac_srs); 2129 2130 /* 2131 * For a flow we use the underlying MAC client's priority range with 2132 * the priority value to find an absolute priority value. For a MAC 2133 * client we use the MAC client's maximum priority as the value. 2134 */ 2135 mrp = &flent->fe_effective_props; 2136 if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2137 mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2138 mcip->mci_max_pri, mrp->mrp_priority); 2139 } else { 2140 mac_srs->srs_pri = mcip->mci_max_pri; 2141 } 2142 /* 2143 * We need to insert the SRS in the global list before 2144 * binding the SRS and SR threads. Otherwise there is a 2145 * is a small window where the cpu reconfig callbacks 2146 * may miss the SRS in the list walk and DR could fail 2147 * as there are bound threads. 2148 */ 2149 mac_srs_add_glist(mac_srs); 2150 2151 /* Initialize bw limit */ 2152 if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2153 mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2154 2155 mac_bw = mac_srs->srs_bw; 2156 mutex_enter(&mac_bw->mac_bw_lock); 2157 mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2158 2159 /* 2160 * Give twice the queuing capability before 2161 * dropping packets. The unit is bytes/tick. 2162 */ 2163 mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2164 mutex_exit(&mac_bw->mac_bw_lock); 2165 mac_srs->srs_type |= SRST_BW_CONTROL; 2166 } else { 2167 mac_srs->srs_drain_func = mac_rx_srs_drain; 2168 } 2169 2170 /* 2171 * We use the following policy to control Receive 2172 * Side Dynamic Polling: 2173 * 1) We switch to poll mode anytime the processing thread causes 2174 * a backlog to build up in SRS and its associated Soft Rings 2175 * (sr_poll_pkt_cnt > 0). 2176 * 2) As long as the backlog stays under the low water mark 2177 * (sr_lowat), we poll the H/W for more packets. 2178 * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2179 * stay in poll mode but don't poll the H/W for more packets. 2180 * 4) Anytime in polling mode, if we poll the H/W for packets and 2181 * find nothing plus we have an existing backlog 2182 * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2183 * the H/W for packets anymore (let the polling thread go to sleep). 2184 * 5) Once the backlog is relieved (packets are processed) we reenable 2185 * polling (by signalling the poll thread) only when the backlog 2186 * dips below sr_poll_thres. 2187 * 6) sr_hiwat is used exclusively when we are not polling capable 2188 * and is used to decide when to drop packets so the SRS queue 2189 * length doesn't grow infinitely. 2190 */ 2191 if (!is_tx_srs) { 2192 srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2193 /* Low water mark needs to be less than high water mark */ 2194 srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2195 mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2196 (mac_soft_ring_max_q_cnt >> 2); 2197 /* Poll threshold need to be half of low water mark or less */ 2198 srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2199 (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2200 (srs_rx->sr_lowat >> 1); 2201 if (mac_latency_optimize) 2202 mac_srs->srs_state |= SRS_LATENCY_OPT; 2203 else 2204 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2205 } 2206 2207 mac_srs->srs_worker = thread_create(NULL, 0, 2208 mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2209 2210 if (is_tx_srs) { 2211 /* Handle everything about Tx SRS and return */ 2212 mac_srs->srs_drain_func = mac_tx_srs_drain; 2213 srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2214 srs_tx->st_hiwat = 2215 (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2216 mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2217 srs_tx->st_arg1 = x_arg1; 2218 srs_tx->st_arg2 = x_arg2; 2219 goto done; 2220 } 2221 2222 if ((srs_type & SRST_FLOW) != 0 || 2223 FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2224 srs_rx->sr_lower_proc = mac_rx_srs_process; 2225 else 2226 srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2227 2228 srs_rx->sr_func = rx_func; 2229 srs_rx->sr_arg1 = x_arg1; 2230 srs_rx->sr_arg2 = x_arg2; 2231 2232 if (ring != NULL) { 2233 uint_t ring_info; 2234 2235 /* Is the mac_srs created over the RX default group? */ 2236 if (ring->mr_gh == (mac_group_handle_t) 2237 MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2238 mac_srs->srs_type |= SRST_DEFAULT_GRP; 2239 } 2240 mac_srs->srs_ring = ring; 2241 ring->mr_srs = mac_srs; 2242 ring->mr_classify_type = MAC_HW_CLASSIFIER; 2243 ring->mr_flag |= MR_INCIPIENT; 2244 2245 if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 2246 FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2247 mac_srs->srs_state |= SRS_POLLING_CAPAB; 2248 2249 mac_srs->srs_poll_thr = thread_create(NULL, 0, 2250 mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2251 mac_srs->srs_pri); 2252 /* 2253 * Some drivers require serialization and don't send 2254 * packet chains in interrupt context. For such 2255 * drivers, we should always queue in the soft ring 2256 * so that we get a chance to switch into polling 2257 * mode under backlog. 2258 */ 2259 ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 2260 if (ring_info & MAC_RING_RX_ENQUEUE) 2261 mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2262 } 2263 done: 2264 mac_srs_stat_create(mac_srs); 2265 return (mac_srs); 2266 } 2267 2268 /* 2269 * Figure out the number of soft rings required. Its dependant on 2270 * if protocol fanout is required (for LINKs), global settings 2271 * require us to do fanout for performance (based on mac_soft_ring_enable), 2272 * or user has specifically requested fanout. 2273 */ 2274 static uint32_t 2275 mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2276 { 2277 uint32_t fanout_type; 2278 mac_resource_props_t *mrp = &flent->fe_effective_props; 2279 2280 /* no fanout for subflows */ 2281 switch (link_type) { 2282 case SRST_FLOW: 2283 fanout_type = SRST_NO_SOFT_RINGS; 2284 break; 2285 case SRST_LINK: 2286 fanout_type = SRST_FANOUT_PROTO; 2287 break; 2288 } 2289 2290 /* A primary NIC/link is being plumbed */ 2291 if (flent->fe_type & FLOW_PRIMARY_MAC) { 2292 if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2293 fanout_type |= SRST_FANOUT_SRC_IP; 2294 } 2295 } else if (flent->fe_type & FLOW_VNIC) { 2296 /* A VNIC is being created */ 2297 if (mrp != NULL && mrp->mrp_ncpus > 0) { 2298 fanout_type |= SRST_FANOUT_SRC_IP; 2299 } 2300 } 2301 2302 return (fanout_type); 2303 } 2304 2305 /* 2306 * Change a group from h/w to s/w classification. 2307 */ 2308 void 2309 mac_rx_switch_grp_to_sw(mac_group_t *group) 2310 { 2311 mac_ring_t *ring; 2312 mac_soft_ring_set_t *mac_srs; 2313 2314 for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2315 if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2316 /* 2317 * Remove the SRS associated with the HW ring. 2318 * As a result, polling will be disabled. 2319 */ 2320 mac_srs = ring->mr_srs; 2321 ASSERT(mac_srs != NULL); 2322 mac_rx_srs_remove(mac_srs); 2323 ring->mr_srs = NULL; 2324 } 2325 2326 if (ring->mr_state != MR_INUSE) 2327 (void) mac_start_ring(ring); 2328 2329 /* 2330 * We need to perform SW classification 2331 * for packets landing in these rings 2332 */ 2333 ring->mr_flag = 0; 2334 ring->mr_classify_type = MAC_SW_CLASSIFIER; 2335 } 2336 } 2337 2338 /* 2339 * Create the Rx SRS for S/W classifier and for each ring in the 2340 * group (if exclusive group). Also create the Tx SRS. 2341 */ 2342 void 2343 mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2344 uint32_t link_type) 2345 { 2346 cpupart_t *cpupart; 2347 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2348 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2349 boolean_t use_default = B_FALSE; 2350 2351 mac_rx_srs_group_setup(mcip, flent, link_type); 2352 mac_tx_srs_group_setup(mcip, flent, link_type); 2353 2354 /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */ 2355 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 2356 return; 2357 2358 pool_lock(); 2359 cpupart = mac_pset_find(mrp, &use_default); 2360 mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 2361 mac_rx_deliver, mcip, NULL, cpupart); 2362 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 2363 pool_unlock(); 2364 } 2365 2366 /* 2367 * Set up the Rx SRSes. If there is no group associated with the 2368 * client, then only setup SW classification. If the client has 2369 * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an 2370 * SRS for each HW ring. If the client is sharing a group, then make 2371 * sure to teardown the HW SRSes. 2372 */ 2373 void 2374 mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2375 uint32_t link_type) 2376 { 2377 mac_impl_t *mip = mcip->mci_mip; 2378 mac_soft_ring_set_t *mac_srs; 2379 mac_ring_t *ring; 2380 uint32_t fanout_type; 2381 mac_group_t *rx_group = flent->fe_rx_ring_group; 2382 boolean_t no_unicast; 2383 2384 /* 2385 * If this is an an aggr port, then don't setup Rx SRS and Rx 2386 * soft rings as they won't be used. However, we still need to 2387 * start the rings to receive data on them. 2388 */ 2389 if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) { 2390 if (rx_group == NULL) 2391 return; 2392 2393 for (ring = rx_group->mrg_rings; ring != NULL; 2394 ring = ring->mr_next) { 2395 if (ring->mr_state != MR_INUSE) 2396 (void) mac_start_ring(ring); 2397 } 2398 2399 return; 2400 } 2401 2402 /* 2403 * Aggr ports should never have SRSes. 2404 */ 2405 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2406 2407 fanout_type = mac_find_fanout(flent, link_type); 2408 no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0; 2409 2410 /* Create the SRS for SW classification if none exists */ 2411 if (flent->fe_rx_srs[0] == NULL) { 2412 ASSERT(flent->fe_rx_srs_cnt == 0); 2413 mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2414 mac_rx_deliver, mcip, NULL, NULL); 2415 mutex_enter(&flent->fe_lock); 2416 flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2417 flent->fe_cb_arg1 = (void *)mip; 2418 flent->fe_cb_arg2 = (void *)mac_srs; 2419 mutex_exit(&flent->fe_lock); 2420 } 2421 2422 if (rx_group == NULL) 2423 return; 2424 2425 /* 2426 * If the group is marked RESERVED then setup an SRS and 2427 * fanout for each HW ring. 2428 */ 2429 switch (rx_group->mrg_state) { 2430 case MAC_GROUP_STATE_RESERVED: 2431 for (ring = rx_group->mrg_rings; ring != NULL; 2432 ring = ring->mr_next) { 2433 uint16_t vid = i_mac_flow_vid(mcip->mci_flent); 2434 2435 switch (ring->mr_state) { 2436 case MR_INUSE: 2437 case MR_FREE: 2438 if (ring->mr_srs != NULL) 2439 break; 2440 if (ring->mr_state != MR_INUSE) 2441 (void) mac_start_ring(ring); 2442 2443 /* 2444 * If a client requires SW VLAN 2445 * filtering or has no unicast address 2446 * then we don't create any HW ring 2447 * SRSes. 2448 */ 2449 if ((!MAC_GROUP_HW_VLAN(rx_group) && 2450 vid != VLAN_ID_NONE) || no_unicast) 2451 break; 2452 2453 /* 2454 * When a client has exclusive use of 2455 * a group, and that group's traffic 2456 * is fully HW classified, we create 2457 * an SRS for each HW ring in order to 2458 * make use of dynamic polling of said 2459 * HW rings. 2460 */ 2461 mac_srs = mac_srs_create(mcip, flent, 2462 fanout_type | link_type, 2463 mac_rx_deliver, mcip, NULL, ring); 2464 break; 2465 default: 2466 cmn_err(CE_PANIC, 2467 "srs_setup: mcip = %p " 2468 "trying to add UNKNOWN ring = %p\n", 2469 (void *)mcip, (void *)ring); 2470 break; 2471 } 2472 } 2473 break; 2474 case MAC_GROUP_STATE_SHARED: 2475 /* 2476 * When a group is shared by multiple clients, we must 2477 * use SW classifiction to ensure packets are 2478 * delivered to the correct client. 2479 */ 2480 mac_rx_switch_grp_to_sw(rx_group); 2481 break; 2482 default: 2483 ASSERT(B_FALSE); 2484 break; 2485 } 2486 } 2487 2488 /* 2489 * Set up the TX SRS. 2490 */ 2491 void 2492 mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2493 uint32_t link_type) 2494 { 2495 /* 2496 * If this is an exclusive client (e.g. an aggr port), then 2497 * don't setup Tx SRS and Tx soft rings as they won't be used. 2498 * However, we still need to start the rings to send data 2499 * across them. 2500 */ 2501 if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 2502 mac_ring_t *ring; 2503 mac_group_t *grp; 2504 2505 grp = (mac_group_t *)flent->fe_tx_ring_group; 2506 2507 if (grp == NULL) 2508 return; 2509 2510 for (ring = grp->mrg_rings; ring != NULL; 2511 ring = ring->mr_next) { 2512 if (ring->mr_state != MR_INUSE) 2513 (void) mac_start_ring(ring); 2514 } 2515 2516 return; 2517 } 2518 2519 /* 2520 * Aggr ports should never have SRSes. 2521 */ 2522 ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 2523 2524 if (flent->fe_tx_srs == NULL) { 2525 (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 2526 NULL, mcip, NULL, NULL); 2527 } 2528 2529 mac_tx_srs_setup(mcip, flent); 2530 } 2531 2532 /* 2533 * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown 2534 * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set 2535 * when we wish to move a MAC client from one group to another. In 2536 * that case, we need to release the current HW SRSes but keep the SW 2537 * SRS for continued traffic classifiction. 2538 */ 2539 void 2540 mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 2541 { 2542 mac_soft_ring_set_t *mac_srs; 2543 int i; 2544 int count = flent->fe_rx_srs_cnt; 2545 2546 for (i = 0; i < count; i++) { 2547 if (i == 0 && hwonly) 2548 continue; 2549 mac_srs = flent->fe_rx_srs[i]; 2550 mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2551 mac_srs_free(mac_srs); 2552 flent->fe_rx_srs[i] = NULL; 2553 flent->fe_rx_srs_cnt--; 2554 } 2555 2556 /* 2557 * If we are only tearing down the HW SRSes then there must be 2558 * one SRS left for SW classification. Otherwise we are tearing 2559 * down both HW and SW and there should be no SRSes left. 2560 */ 2561 if (hwonly) 2562 VERIFY3S(flent->fe_rx_srs_cnt, ==, 1); 2563 else 2564 VERIFY3S(flent->fe_rx_srs_cnt, ==, 0); 2565 } 2566 2567 /* 2568 * Remove the TX SRS. 2569 */ 2570 void 2571 mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2572 uint32_t link_type) 2573 { 2574 mac_soft_ring_set_t *tx_srs; 2575 mac_srs_tx_t *tx; 2576 2577 if ((tx_srs = flent->fe_tx_srs) == NULL) 2578 return; 2579 2580 tx = &tx_srs->srs_tx; 2581 switch (link_type) { 2582 case SRST_FLOW: 2583 /* 2584 * For flows, we need to work with passed 2585 * flent to find the Rx/Tx SRS. 2586 */ 2587 mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2588 break; 2589 case SRST_LINK: 2590 mac_tx_client_condemn((mac_client_handle_t)mcip); 2591 if (tx->st_arg2 != NULL) { 2592 ASSERT(tx_srs->srs_type & SRST_TX); 2593 /* 2594 * The ring itself will be stopped when 2595 * we release the group or in the 2596 * mac_datapath_teardown (for the default 2597 * group) 2598 */ 2599 tx->st_arg2 = NULL; 2600 } 2601 break; 2602 default: 2603 ASSERT(B_FALSE); 2604 break; 2605 } 2606 mac_srs_free(tx_srs); 2607 flent->fe_tx_srs = NULL; 2608 } 2609 2610 /* 2611 * This is the group state machine. 2612 * 2613 * The state of an Rx group is given by 2614 * the following table. The default group and its rings are started in 2615 * mac_start itself and the default group stays in SHARED state until 2616 * mac_stop at which time the group and rings are stopped and and it 2617 * reverts to the Registered state. 2618 * 2619 * Typically this function is called on a group after adding or removing a 2620 * client from it, to find out what should be the new state of the group. 2621 * If the new state is RESERVED, then the client that owns this group 2622 * exclusively is also returned. Note that adding or removing a client from 2623 * a group could also impact the default group and the caller needs to 2624 * evaluate the effect on the default group. 2625 * 2626 * Group type # of clients mi_nactiveclients Group State 2627 * in the group 2628 * 2629 * Non-default 0 N.A. REGISTERED 2630 * Non-default 1 N.A. RESERVED 2631 * 2632 * Default 0 N.A. SHARED 2633 * Default 1 1 RESERVED 2634 * Default 1 > 1 SHARED 2635 * Default > 1 N.A. SHARED 2636 * 2637 * For a TX group, the following is the state table. 2638 * 2639 * Group type # of clients Group State 2640 * in the group 2641 * 2642 * Non-default 0 REGISTERED 2643 * Non-default 1 RESERVED 2644 * 2645 * Default 0 REGISTERED 2646 * Default 1 RESERVED 2647 * Default > 1 SHARED 2648 */ 2649 mac_group_state_t 2650 mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 2651 mac_group_t *defgrp, boolean_t rx_group) 2652 { 2653 mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2654 2655 *group_only_mcip = NULL; 2656 2657 /* Non-default group */ 2658 2659 if (grp != defgrp) { 2660 if (MAC_GROUP_NO_CLIENT(grp)) 2661 return (MAC_GROUP_STATE_REGISTERED); 2662 2663 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2664 if (*group_only_mcip != NULL) 2665 return (MAC_GROUP_STATE_RESERVED); 2666 2667 return (MAC_GROUP_STATE_SHARED); 2668 } 2669 2670 /* Default group */ 2671 2672 if (MAC_GROUP_NO_CLIENT(grp)) { 2673 if (rx_group) 2674 return (MAC_GROUP_STATE_SHARED); 2675 else 2676 return (MAC_GROUP_STATE_REGISTERED); 2677 } 2678 *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2679 if (*group_only_mcip == NULL) 2680 return (MAC_GROUP_STATE_SHARED); 2681 2682 if (rx_group && mip->mi_nactiveclients != 1) 2683 return (MAC_GROUP_STATE_SHARED); 2684 2685 ASSERT(*group_only_mcip != NULL); 2686 return (MAC_GROUP_STATE_RESERVED); 2687 } 2688 2689 /* 2690 * OVERVIEW NOTES FOR DATAPATH 2691 * =========================== 2692 * 2693 * Create an SRS and setup the corresponding flow function and args. 2694 * Add a classification rule for the flow specified by 'flent' and program 2695 * the hardware classifier when applicable. 2696 * 2697 * Rx ring assignment, SRS, polling and B/W enforcement 2698 * ---------------------------------------------------- 2699 * 2700 * We try to use H/W classification on NIC and assign traffic to a 2701 * MAC address to a particular Rx ring. There is a 1-1 mapping 2702 * between a SRS and a Rx ring. The SRS (short for soft ring set) 2703 * dynamically switches the underlying Rx ring between interrupt 2704 * and polling mode and enforces any specified B/W control. 2705 * 2706 * There is always a SRS created and tied to each H/W and S/W rule. 2707 * Whenever we create a H/W rule, we always add the the same rule to 2708 * S/W classifier and tie a SRS to it. 2709 * 2710 * In case a B/W control is specified, its broken into bytes 2711 * per ticks and as soon as the quota for a tick is exhausted, 2712 * the underlying Rx ring is forced into poll mode for remianing 2713 * tick. The SRS poll thread only polls for bytes that are 2714 * allowed to come in the SRS. We typically let 4x the configured 2715 * B/W worth of packets to come in the SRS (to prevent unnecessary 2716 * drops due to bursts) but only process the specified amount. 2717 * 2718 * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2719 * Rx rings (and corresponding SRSs) assigned to it. The SRS 2720 * in turn can have softrings to do protocol level fanout or 2721 * softrings to do S/W based fanout or both. In case the NIC 2722 * has no Rx rings, we do S/W classification to respective SRS. 2723 * The S/W classification rule is always setup and ready. This 2724 * allows the MAC layer to reassign Rx rings whenever needed 2725 * but packets still continue to flow via the default path and 2726 * getting S/W classified to correct SRS. 2727 * 2728 * In other cases where a NIC or VNIC is plumbed, our goal is use 2729 * H/W classifier and get two Rx ring assigned for the Link. One 2730 * for TCP and one for UDP|SCTP. The respective SRS still do the 2731 * polling on the Rx ring. For Link that is plumbed for IP, there 2732 * is a TCP squeue which also does polling and can control the 2733 * the Rx ring directly (where SRS is just pass through). For 2734 * the following cases, the SRS does the polling underneath. 2735 * 1) non IP based Links (Links which are not plumbed via ifconfig) 2736 * and paths which have no IP squeues (UDP & SCTP) 2737 * 2) If B/W control is specified on the Link 2738 * 3) If S/W fanout is secified 2739 * 2740 * Note1: As of current implementation, we try to assign only 1 Rx 2741 * ring per Link and more than 1 Rx ring for primary Link for 2742 * H/W based fanout. We always create following softrings per SRS: 2743 * 1) TCP softring which is polled by TCP squeue where possible 2744 * (and also bypasses DLS) 2745 * 2) UDP/SCTP based which bypasses DLS 2746 * 3) OTH softring which goes via DLS (currently deal with IPv6 2747 * and non TCP/UDP/SCTP for IPv4 packets). 2748 * 2749 * It is necessary to create 3 softrings since SRS has to poll 2750 * the single Rx ring underneath and enforce any link level B/W 2751 * control (we can't switch the Rx ring in poll mode just based 2752 * on TCP squeue if the same Rx ring is sharing UDP and other 2753 * traffic as well). Once polling is done and any Link level B/W 2754 * control is specified, the packets are assigned to respective 2755 * softring based on protocol. Since TCP has IP based squeue 2756 * which benefits by polling, we separate TCP packets into 2757 * its own softring which can be polled by IP squeue. We need 2758 * to separate out UDP/SCTP to UDP softring since it can bypass 2759 * the DLS layer which has heavy performance advanatges and we 2760 * need a softring (OTH) for rest. 2761 * 2762 * ToDo: The 3 softrings for protocol are needed only till we can 2763 * get rid of DLS from datapath, make IPv4 and IPv6 paths 2764 * symmetric (deal with mac_header_info for v6 and polling for 2765 * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2766 * are generic), and bring SAP based classification to MAC layer 2767 * 2768 * H/W and S/W based fanout and multiple Rx rings per Link 2769 * ------------------------------------------------------- 2770 * 2771 * In case, fanout is requested (or determined automatically based 2772 * on Link speed and processor speed), we try to assign multiple 2773 * Rx rings per Link with their respective SRS. In this case 2774 * the NIC should be capable of fanning out incoming packets between 2775 * the assigned Rx rings (H/W based fanout). All the SRS 2776 * individually switch their Rx ring between interrupt and polling 2777 * mode but share a common B/W control counter in case of Link 2778 * level B/W is specified. 2779 * 2780 * If S/W based fanout is specified in lieu of H/W based fanout, 2781 * the Link SRS creates the specified number of softrings for 2782 * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2783 * out to the correct softring based on their protocol and 2784 * protocol specific hash function. 2785 * 2786 * Primary and non primary MAC clients 2787 * ----------------------------------- 2788 * 2789 * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2790 * and are a Layer 2 construct. 2791 * 2792 * Primary NIC: 2793 * The Link that owns the primary MAC address and typically 2794 * is used as the data NIC in non virtualized cases. As such 2795 * H/W resources are preferntially given to primary NIC. As 2796 * far as code is concerned, there is no difference in the 2797 * primary NIC vs VNICs. They are all treated as Links. 2798 * At the very first call to mac_unicast_add() we program the S/W 2799 * classifier for the primary MAC address, get a soft ring set 2800 * (and soft rings based on 'ip_soft_ring_cnt') 2801 * and a Rx ring assigned for polling to get enabled. 2802 * When IP get plumbed and negotiates polling, we can 2803 * let squeue do the polling on TCP softring. 2804 * 2805 * VNICs: 2806 * Same as any other Link. As long as the H/W resource assignments 2807 * are equal, the data path and setup for all Links is same. 2808 * 2809 * Flows: 2810 * Can be configured on Links. They have their own SRS and the 2811 * S/W classifier is programmed appropriately based on the flow. 2812 * The flows typically deal with layer 3 and above and 2813 * creates a soft ring set specific to the flow. The receive 2814 * side function is switched from mac_rx_srs_process to 2815 * mac_rx_srs_subflow_process which first tries to assign the 2816 * packet to appropriate flow SRS and failing which assigns it 2817 * to link SRS. This allows us to avoid the layered approach 2818 * which gets complex. 2819 * 2820 * By the time mac_datapath_setup() completes, we already have the 2821 * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2822 * and S/W classifiers programmed. IP is not plumbed yet (and might 2823 * never be for Virtual Machines guest OS path). When IP is plumbed 2824 * (for both NIC and VNIC), we do a capability negotiation for polling 2825 * and upcall functions etc. 2826 * 2827 * Rx ring Assignement NOTES 2828 * ------------------------- 2829 * 2830 * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2831 * as NIC with a single default ring), we assign the only ring to 2832 * primary Link. The primary Link SRS can do polling on it as long as 2833 * it is the only link in use and we compare the MAC address for unicast 2834 * packets before accepting an incoming packet (there is no need for S/W 2835 * classification in this case). We disable polling on the only ring the 2836 * moment 2nd link gets created (the polling remains enabled even though 2837 * there are broadcast and * multicast flows created). 2838 * 2839 * If the NIC has more than 1 Rx ring, we assign the default ring (the 2840 * 1st ring) to deal with broadcast, multicast and traffic for other 2841 * NICs which needs S/W classification. We assign the primary mac 2842 * addresses to another ring by specifiying a classification rule for 2843 * primary unicast MAC address to the selected ring. The primary Link 2844 * (and its SRS) can continue to poll the assigned Rx ring at all times 2845 * independantly. 2846 * 2847 * Note: In future, if no fanout is specified, we try to assign 2 Rx 2848 * rings for the primary Link with the primary MAC address + TCP going 2849 * to one ring and primary MAC address + UDP|SCTP going to other ring. 2850 * Any remaining traffic for primary MAC address can go to the default 2851 * Rx ring and get S/W classified. This way the respective SRSs don't 2852 * need to do proto fanout and don't need to have softrings at all and 2853 * can poll their respective Rx rings. 2854 * 2855 * As an optimization, when a new NIC or VNIC is created, we can get 2856 * only one Rx ring and make it a TCP specific Rx ring and use the 2857 * H/W default Rx ring for the rest (this Rx ring is never polled). 2858 * 2859 * For clients that don't have MAC address, but want to receive and 2860 * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 2861 * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 2862 * always give the default group and use software classification (i.e. 2863 * even if this is the only client in the default group, we will 2864 * leave group as shared). 2865 */ 2866 2867 int 2868 mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2869 uint32_t link_type) 2870 { 2871 mac_impl_t *mip = mcip->mci_mip; 2872 mac_group_t *rgroup = NULL; 2873 mac_group_t *tgroup = NULL; 2874 mac_group_t *default_rgroup; 2875 mac_group_t *default_tgroup; 2876 int err; 2877 uint16_t vid; 2878 uint8_t *mac_addr; 2879 mac_group_state_t next_state; 2880 mac_client_impl_t *group_only_mcip; 2881 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 2882 mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 2883 boolean_t rxhw; 2884 boolean_t txhw; 2885 boolean_t use_default = B_FALSE; 2886 cpupart_t *cpupart; 2887 boolean_t no_unicast; 2888 boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 2889 mac_client_impl_t *reloc_pmcip = NULL; 2890 boolean_t use_hw; 2891 2892 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2893 2894 switch (link_type) { 2895 case SRST_FLOW: 2896 mac_srs_group_setup(mcip, flent, link_type); 2897 return (0); 2898 2899 case SRST_LINK: 2900 no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 2901 mac_addr = flent->fe_flow_desc.fd_dst_mac; 2902 2903 /* Default RX group */ 2904 default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 2905 2906 /* Default TX group */ 2907 default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 2908 2909 if (no_unicast) { 2910 rgroup = default_rgroup; 2911 tgroup = default_tgroup; 2912 goto grp_found; 2913 } 2914 rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 2915 (mrp->mrp_nrxrings > 0 || 2916 (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 2917 txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 2918 (mrp->mrp_ntxrings > 0 || 2919 (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 2920 2921 /* 2922 * All the rings initially belong to the default group 2923 * under dynamic grouping. The primary client uses the 2924 * default group when it is the only client. The 2925 * default group is also used as the destination for 2926 * all multicast and broadcast traffic of all clients. 2927 * Therefore, the primary client loses its ability to 2928 * poll the softrings on addition of a second client. 2929 * To avoid a performance penalty, MAC will move the 2930 * primary client to a dedicated group when it can. 2931 * 2932 * When using static grouping, the primary client 2933 * begins life on a non-default group. There is 2934 * no moving needed upon addition of a second client. 2935 */ 2936 if (!isprimary && mip->mi_nactiveclients == 2 && 2937 (group_only_mcip = mac_primary_client_handle(mip)) != 2938 NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 2939 reloc_pmcip = mac_check_primary_relocation( 2940 group_only_mcip, rxhw); 2941 } 2942 2943 /* 2944 * Check to see if we can get an exclusive group for 2945 * this mac address or if there already exists a 2946 * group that has this mac address (case of VLANs). 2947 * If no groups are available, use the default group. 2948 */ 2949 rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 2950 if (rgroup == NULL && rxhw) { 2951 err = ENOSPC; 2952 goto setup_failed; 2953 } else if (rgroup == NULL) { 2954 rgroup = default_rgroup; 2955 } 2956 2957 /* 2958 * If we are adding a second client to a 2959 * non-default group then we need to move the 2960 * existing client to the default group and 2961 * add the new client to the default group as 2962 * well. 2963 */ 2964 if (rgroup != default_rgroup && 2965 rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) { 2966 group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup); 2967 err = mac_rx_switch_group(group_only_mcip, rgroup, 2968 default_rgroup); 2969 2970 if (err != 0) 2971 goto setup_failed; 2972 2973 rgroup = default_rgroup; 2974 } 2975 2976 /* 2977 * Check to see if we can get an exclusive group for 2978 * this mac client. If no groups are available, use 2979 * the default group. 2980 */ 2981 tgroup = mac_reserve_tx_group(mcip, B_FALSE); 2982 if (tgroup == NULL && txhw) { 2983 if (rgroup != NULL && rgroup != default_rgroup) 2984 mac_release_rx_group(mcip, rgroup); 2985 err = ENOSPC; 2986 goto setup_failed; 2987 } else if (tgroup == NULL) { 2988 tgroup = default_tgroup; 2989 } 2990 2991 /* 2992 * Some NICs don't support any Rx rings, so there may not 2993 * even be a default group. 2994 */ 2995 grp_found: 2996 if (rgroup != NULL) { 2997 if (rgroup != default_rgroup && 2998 MAC_GROUP_NO_CLIENT(rgroup) && 2999 (rxhw || mcip->mci_share != 0)) { 3000 MAC_RX_GRP_RESERVED(mip); 3001 if (mip->mi_rx_group_type == 3002 MAC_GROUP_TYPE_DYNAMIC) { 3003 MAC_RX_RING_RESERVED(mip, 3004 rgroup->mrg_cur_count); 3005 } 3006 } 3007 3008 flent->fe_rx_ring_group = rgroup; 3009 /* 3010 * Add the client to the group and update the 3011 * group's state. If rgroup != default_group 3012 * then the rgroup should only ever have one 3013 * client and be in the RESERVED state. But no 3014 * matter what, the default_rgroup will enter 3015 * the SHARED state since it has to receive 3016 * all broadcast and multicast traffic. This 3017 * case is handled later in the function. 3018 */ 3019 mac_group_add_client(rgroup, mcip); 3020 next_state = mac_group_next_state(rgroup, 3021 &group_only_mcip, default_rgroup, B_TRUE); 3022 mac_set_group_state(rgroup, next_state); 3023 } 3024 3025 if (tgroup != NULL) { 3026 if (tgroup != default_tgroup && 3027 MAC_GROUP_NO_CLIENT(tgroup) && 3028 (txhw || mcip->mci_share != 0)) { 3029 MAC_TX_GRP_RESERVED(mip); 3030 if (mip->mi_tx_group_type == 3031 MAC_GROUP_TYPE_DYNAMIC) { 3032 MAC_TX_RING_RESERVED(mip, 3033 tgroup->mrg_cur_count); 3034 } 3035 } 3036 flent->fe_tx_ring_group = tgroup; 3037 mac_group_add_client(tgroup, mcip); 3038 next_state = mac_group_next_state(tgroup, 3039 &group_only_mcip, default_tgroup, B_FALSE); 3040 tgroup->mrg_state = next_state; 3041 } 3042 3043 /* We are setting up minimal datapath only */ 3044 if (no_unicast) { 3045 mac_srs_group_setup(mcip, flent, link_type); 3046 break; 3047 } 3048 3049 /* Program software classification. */ 3050 if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 3051 goto setup_failed; 3052 3053 /* Program hardware classification. */ 3054 vid = i_mac_flow_vid(flent); 3055 use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0; 3056 err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw); 3057 3058 if (err != 0) 3059 goto setup_failed; 3060 3061 mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 3062 VERIFY3P(mcip->mci_unicast, !=, NULL); 3063 3064 /* 3065 * Setup the Rx and Tx SRSes. If the client has a 3066 * reserved group, then mac_srs_group_setup() creates 3067 * the required SRSes for the HW rings. If we have a 3068 * shared group, mac_srs_group_setup() dismantles the 3069 * HW SRSes of the previously exclusive group. 3070 */ 3071 mac_srs_group_setup(mcip, flent, link_type); 3072 3073 /* (Re)init the v6 token & local addr used by link protection */ 3074 mac_protect_update_mac_token(mcip); 3075 break; 3076 3077 default: 3078 ASSERT(B_FALSE); 3079 break; 3080 } 3081 3082 /* 3083 * All broadcast and multicast traffic is received only on the default 3084 * group. If we have setup the datapath for a non-default group above 3085 * then move the default group to shared state to allow distribution of 3086 * incoming broadcast traffic to the other groups and dismantle the 3087 * SRSes over the default group. 3088 */ 3089 if (rgroup != NULL) { 3090 if (rgroup != default_rgroup) { 3091 if (default_rgroup->mrg_state == 3092 MAC_GROUP_STATE_RESERVED) { 3093 group_only_mcip = MAC_GROUP_ONLY_CLIENT( 3094 default_rgroup); 3095 ASSERT(group_only_mcip != NULL && 3096 mip->mi_nactiveclients > 1); 3097 3098 mac_set_group_state(default_rgroup, 3099 MAC_GROUP_STATE_SHARED); 3100 mac_rx_srs_group_setup(group_only_mcip, 3101 group_only_mcip->mci_flent, SRST_LINK); 3102 pool_lock(); 3103 cpupart = mac_pset_find(mrp, &use_default); 3104 mac_fanout_setup(group_only_mcip, 3105 group_only_mcip->mci_flent, 3106 MCIP_RESOURCE_PROPS(group_only_mcip), 3107 mac_rx_deliver, group_only_mcip, NULL, 3108 cpupart); 3109 mac_set_pool_effective(use_default, cpupart, 3110 mrp, emrp); 3111 pool_unlock(); 3112 } 3113 ASSERT(default_rgroup->mrg_state == 3114 MAC_GROUP_STATE_SHARED); 3115 } 3116 3117 /* 3118 * A VLAN MAC client on a reserved group still 3119 * requires SW classification if the MAC doesn't 3120 * provide VLAN HW filtering. 3121 * 3122 * Clients with no unicast address also require SW 3123 * classification. 3124 */ 3125 if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 3126 ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) || 3127 no_unicast)) { 3128 mac_rx_switch_grp_to_sw(rgroup); 3129 } 3130 3131 } 3132 3133 mac_set_rings_effective(mcip); 3134 return (0); 3135 3136 setup_failed: 3137 /* Switch the primary back to default group */ 3138 if (reloc_pmcip != NULL) { 3139 (void) mac_rx_switch_group(reloc_pmcip, 3140 reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 3141 } 3142 mac_datapath_teardown(mcip, flent, link_type); 3143 return (err); 3144 } 3145 3146 void 3147 mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3148 uint32_t link_type) 3149 { 3150 mac_impl_t *mip = mcip->mci_mip; 3151 mac_group_t *group = NULL; 3152 mac_client_impl_t *grp_only_mcip; 3153 flow_entry_t *group_only_flent; 3154 mac_group_t *default_group; 3155 boolean_t check_default_group = B_FALSE; 3156 mac_group_state_t next_state; 3157 mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 3158 uint16_t vid; 3159 3160 ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3161 3162 switch (link_type) { 3163 case SRST_FLOW: 3164 mac_rx_srs_group_teardown(flent, B_FALSE); 3165 mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3166 return; 3167 3168 case SRST_LINK: 3169 /* Stop sending packets */ 3170 mac_tx_client_block(mcip); 3171 group = flent->fe_rx_ring_group; 3172 vid = i_mac_flow_vid(flent); 3173 3174 /* 3175 * Stop the packet flow from the hardware by disabling 3176 * any hardware filters assigned to this client. 3177 */ 3178 if (mcip->mci_unicast != NULL) { 3179 int err; 3180 3181 err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid); 3182 3183 if (err != 0) { 3184 cmn_err(CE_WARN, "%s: failed to remove a MAC HW" 3185 " filters because of error 0x%x", 3186 mip->mi_name, err); 3187 } 3188 3189 mcip->mci_unicast = NULL; 3190 } 3191 3192 /* Stop the packets coming from the S/W classifier */ 3193 mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3194 mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3195 3196 /* Quiesce and destroy all the SRSes. */ 3197 mac_rx_srs_group_teardown(flent, B_FALSE); 3198 mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 3199 3200 ASSERT3P(mcip->mci_flent, ==, flent); 3201 ASSERT3P(flent->fe_next, ==, NULL); 3202 3203 /* 3204 * Release our hold on the group as well. We need 3205 * to check if the shared group has only one client 3206 * left who can use it exclusively. Also, if we 3207 * were the last client, release the group. 3208 */ 3209 default_group = MAC_DEFAULT_RX_GROUP(mip); 3210 if (group != NULL) { 3211 mac_group_remove_client(group, mcip); 3212 next_state = mac_group_next_state(group, 3213 &grp_only_mcip, default_group, B_TRUE); 3214 3215 if (next_state == MAC_GROUP_STATE_RESERVED) { 3216 /* 3217 * Only one client left on this RX group. 3218 */ 3219 VERIFY3P(grp_only_mcip, !=, NULL); 3220 mac_set_group_state(group, 3221 MAC_GROUP_STATE_RESERVED); 3222 group_only_flent = grp_only_mcip->mci_flent; 3223 3224 /* 3225 * The only remaining client has exclusive 3226 * access on the group. Allow it to 3227 * dynamically poll the H/W rings etc. 3228 */ 3229 mac_rx_srs_group_setup(grp_only_mcip, 3230 group_only_flent, SRST_LINK); 3231 mac_fanout_setup(grp_only_mcip, 3232 group_only_flent, 3233 MCIP_RESOURCE_PROPS(grp_only_mcip), 3234 mac_rx_deliver, grp_only_mcip, NULL, NULL); 3235 mac_rx_group_unmark(group, MR_INCIPIENT); 3236 mac_set_rings_effective(grp_only_mcip); 3237 } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3238 /* 3239 * This is a non-default group being freed up. 3240 * We need to reevaluate the default group 3241 * to see if the primary client can get 3242 * exclusive access to the default group. 3243 */ 3244 VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip)); 3245 if (mrp->mrp_mask & MRP_RX_RINGS) { 3246 MAC_RX_GRP_RELEASED(mip); 3247 if (mip->mi_rx_group_type == 3248 MAC_GROUP_TYPE_DYNAMIC) { 3249 MAC_RX_RING_RELEASED(mip, 3250 group->mrg_cur_count); 3251 } 3252 } 3253 mac_release_rx_group(mcip, group); 3254 mac_set_group_state(group, 3255 MAC_GROUP_STATE_REGISTERED); 3256 check_default_group = B_TRUE; 3257 } else { 3258 VERIFY3S(next_state, ==, 3259 MAC_GROUP_STATE_SHARED); 3260 mac_set_group_state(group, 3261 MAC_GROUP_STATE_SHARED); 3262 mac_rx_group_unmark(group, MR_CONDEMNED); 3263 } 3264 flent->fe_rx_ring_group = NULL; 3265 } 3266 /* 3267 * Remove the client from the TX group. Additionally, if 3268 * this a non-default group, then we also need to release 3269 * the group. 3270 */ 3271 group = flent->fe_tx_ring_group; 3272 default_group = MAC_DEFAULT_TX_GROUP(mip); 3273 if (group != NULL) { 3274 mac_group_remove_client(group, mcip); 3275 next_state = mac_group_next_state(group, 3276 &grp_only_mcip, default_group, B_FALSE); 3277 if (next_state == MAC_GROUP_STATE_REGISTERED) { 3278 if (group != default_group) { 3279 if (mrp->mrp_mask & MRP_TX_RINGS) { 3280 MAC_TX_GRP_RELEASED(mip); 3281 if (mip->mi_tx_group_type == 3282 MAC_GROUP_TYPE_DYNAMIC) { 3283 MAC_TX_RING_RELEASED( 3284 mip, group-> 3285 mrg_cur_count); 3286 } 3287 } 3288 mac_release_tx_group(mcip, group); 3289 /* 3290 * If the default group is reserved, 3291 * then we need to set the effective 3292 * rings as we would have given 3293 * back some rings when the group 3294 * was released 3295 */ 3296 if (mip->mi_tx_group_type == 3297 MAC_GROUP_TYPE_DYNAMIC && 3298 default_group->mrg_state == 3299 MAC_GROUP_STATE_RESERVED) { 3300 grp_only_mcip = 3301 MAC_GROUP_ONLY_CLIENT 3302 (default_group); 3303 mac_set_rings_effective( 3304 grp_only_mcip); 3305 } 3306 } else { 3307 mac_ring_t *ring; 3308 int cnt; 3309 int ringcnt; 3310 3311 /* 3312 * Stop all the rings except the 3313 * default ring. 3314 */ 3315 ringcnt = group->mrg_cur_count; 3316 ring = group->mrg_rings; 3317 for (cnt = 0; cnt < ringcnt; cnt++) { 3318 if (ring->mr_state == 3319 MR_INUSE && ring != 3320 (mac_ring_t *) 3321 mip->mi_default_tx_ring) { 3322 mac_stop_ring(ring); 3323 ring->mr_flag = 0; 3324 } 3325 ring = ring->mr_next; 3326 } 3327 } 3328 } else if (next_state == MAC_GROUP_STATE_RESERVED) { 3329 mac_set_rings_effective(grp_only_mcip); 3330 } 3331 flent->fe_tx_ring_group = NULL; 3332 group->mrg_state = next_state; 3333 } 3334 break; 3335 default: 3336 ASSERT(B_FALSE); 3337 break; 3338 } 3339 3340 /* 3341 * The mac client using the default group gets exclusive access to the 3342 * default group if and only if it is the sole client on the entire 3343 * mip. If so set the group state to reserved, and set up the SRSes 3344 * over the default group. 3345 */ 3346 if (check_default_group) { 3347 default_group = MAC_DEFAULT_RX_GROUP(mip); 3348 VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED); 3349 next_state = mac_group_next_state(default_group, 3350 &grp_only_mcip, default_group, B_TRUE); 3351 if (next_state == MAC_GROUP_STATE_RESERVED) { 3352 VERIFY3P(grp_only_mcip, !=, NULL); 3353 VERIFY3U(mip->mi_nactiveclients, ==, 1); 3354 mac_set_group_state(default_group, 3355 MAC_GROUP_STATE_RESERVED); 3356 mac_rx_srs_group_setup(grp_only_mcip, 3357 grp_only_mcip->mci_flent, SRST_LINK); 3358 mac_fanout_setup(grp_only_mcip, 3359 grp_only_mcip->mci_flent, 3360 MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 3361 grp_only_mcip, NULL, NULL); 3362 mac_rx_group_unmark(default_group, MR_INCIPIENT); 3363 mac_set_rings_effective(grp_only_mcip); 3364 } 3365 } 3366 3367 /* 3368 * If the primary is the only one left and the MAC supports 3369 * dynamic grouping, we need to see if the primary needs to 3370 * be moved to the default group so that it can use all the 3371 * H/W rings. 3372 */ 3373 if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 3374 mip->mi_nactiveclients == 1 && 3375 mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 3376 default_group = MAC_DEFAULT_RX_GROUP(mip); 3377 grp_only_mcip = mac_primary_client_handle(mip); 3378 if (grp_only_mcip == NULL) 3379 return; 3380 group_only_flent = grp_only_mcip->mci_flent; 3381 mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 3382 /* 3383 * If the primary has an explicit property set, leave it 3384 * alone. 3385 */ 3386 if (mrp->mrp_mask & MRP_RX_RINGS) 3387 return; 3388 /* 3389 * Switch the primary to the default group. 3390 */ 3391 (void) mac_rx_switch_group(grp_only_mcip, 3392 group_only_flent->fe_rx_ring_group, default_group); 3393 } 3394 } 3395 3396 /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3397 3398 static void 3399 mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3400 { 3401 if (mac_srs->srs_type & SRST_TX) { 3402 mac_srs_tx_t *tx; 3403 3404 ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 3405 ASSERT(mac_srs->srs_udp_soft_rings == NULL); 3406 ASSERT(mac_srs->srs_oth_soft_rings == NULL); 3407 ASSERT(mac_srs->srs_tx_soft_rings != NULL); 3408 kmem_free(mac_srs->srs_tx_soft_rings, 3409 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3410 mac_srs->srs_tx_soft_rings = NULL; 3411 tx = &mac_srs->srs_tx; 3412 if (tx->st_soft_rings != NULL) { 3413 kmem_free(tx->st_soft_rings, 3414 sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 3415 } 3416 } else { 3417 ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3418 ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3419 kmem_free(mac_srs->srs_tcp_soft_rings, 3420 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3421 mac_srs->srs_tcp_soft_rings = NULL; 3422 ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3423 kmem_free(mac_srs->srs_udp_soft_rings, 3424 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3425 mac_srs->srs_udp_soft_rings = NULL; 3426 ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3427 kmem_free(mac_srs->srs_oth_soft_rings, 3428 sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3429 mac_srs->srs_oth_soft_rings = NULL; 3430 } 3431 } 3432 3433 /* 3434 * An RX SRS is attached to at most one mac_ring. 3435 * A TX SRS has no rings. 3436 */ 3437 static void 3438 mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3439 { 3440 mac_client_impl_t *mcip; 3441 mac_ring_t *ring; 3442 flow_entry_t *flent; 3443 3444 ring = mac_srs->srs_ring; 3445 if (mac_srs->srs_type & SRST_TX) { 3446 ASSERT(ring == NULL); 3447 return; 3448 } 3449 3450 if (ring == NULL) 3451 return; 3452 3453 /* 3454 * Broadcast flows don't have a client impl association, but they 3455 * use only soft rings. 3456 */ 3457 flent = mac_srs->srs_flent; 3458 mcip = flent->fe_mcip; 3459 ASSERT(mcip != NULL); 3460 3461 ring->mr_classify_type = MAC_NO_CLASSIFIER; 3462 ring->mr_srs = NULL; 3463 } 3464 3465 /* 3466 * Physical unlink and free of the data structures happen below. This is 3467 * driven from mac_flow_destroy(), on the last refrele of a flow. 3468 * 3469 * Assumes Rx srs is 1-1 mapped with an ring. 3470 */ 3471 void 3472 mac_srs_free(mac_soft_ring_set_t *mac_srs) 3473 { 3474 ASSERT(mac_srs->srs_mcip == NULL || 3475 MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3476 ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3477 SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3478 3479 mac_pkt_drop(NULL, NULL, mac_srs->srs_first, B_FALSE); 3480 mac_srs_ring_free(mac_srs); 3481 mac_srs_soft_rings_free(mac_srs); 3482 mac_srs_fanout_list_free(mac_srs); 3483 3484 mac_srs->srs_bw = NULL; 3485 mac_srs_stat_delete(mac_srs); 3486 kmem_cache_free(mac_srs_cache, mac_srs); 3487 } 3488 3489 static void 3490 mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3491 { 3492 mac_soft_ring_t *softring; 3493 3494 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3495 3496 mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3497 if (s_ring_flag == S_RING_CONDEMNED) { 3498 while (mac_srs->srs_soft_ring_condemned_count != 3499 mac_srs->srs_soft_ring_count) 3500 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3501 } else { 3502 while (mac_srs->srs_soft_ring_quiesced_count != 3503 mac_srs->srs_soft_ring_count) 3504 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3505 } 3506 mutex_exit(&mac_srs->srs_lock); 3507 3508 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3509 softring = softring->s_ring_next) { 3510 (void) untimeout(softring->s_ring_tid); 3511 softring->s_ring_tid = NULL; 3512 } 3513 3514 (void) untimeout(mac_srs->srs_tid); 3515 mac_srs->srs_tid = NULL; 3516 3517 mutex_enter(&mac_srs->srs_lock); 3518 } 3519 3520 /* 3521 * The block comment above mac_rx_classify_flow_state_change explains the 3522 * background. At this point upcalls from the driver (both hardware classified 3523 * and software classified) have been cut off. We now need to quiesce the 3524 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3525 * the master controller. The steps involved are described below in the function 3526 */ 3527 void 3528 mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3529 { 3530 uint_t s_ring_flag; 3531 uint_t srs_poll_wait_flag; 3532 3533 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3534 ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3535 3536 if (mac_srs->srs_state & SRS_CONDEMNED) { 3537 s_ring_flag = S_RING_CONDEMNED; 3538 srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3539 } else { 3540 s_ring_flag = S_RING_QUIESCE; 3541 srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3542 } 3543 3544 /* 3545 * In the case of Rx SRS wait till the poll thread is done. 3546 */ 3547 if ((mac_srs->srs_type & SRST_TX) == 0 && 3548 mac_srs->srs_poll_thr != NULL) { 3549 while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3550 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3551 3552 /* 3553 * Turn off polling as part of the quiesce operation. 3554 */ 3555 MAC_SRS_POLLING_OFF(mac_srs); 3556 mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3557 } 3558 3559 /* 3560 * Then signal the soft ring worker threads to quiesce or quit 3561 * as needed and then wait till that happens. 3562 */ 3563 mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3564 3565 if (mac_srs->srs_state & SRS_CONDEMNED) 3566 mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3567 else 3568 mac_srs->srs_state |= SRS_QUIESCE_DONE; 3569 cv_signal(&mac_srs->srs_quiesce_done_cv); 3570 } 3571 3572 /* 3573 * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3574 * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3575 * or poll thread. This function is internal to the quiescing logic and is 3576 * called internally from the SRS quiesce or flow quiesce or client quiesce 3577 * higher level functions. 3578 */ 3579 void 3580 mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3581 { 3582 mac_ring_t *ring; 3583 3584 ring = mac_srs->srs_ring; 3585 ASSERT(ring == NULL || ring->mr_refcnt == 0); 3586 3587 if (srs_flag == SRS_CONDEMNED) { 3588 /* 3589 * The SRS is going away. We need to unbind the SRS and SR 3590 * threads before removing from the global SRS list. Otherwise 3591 * there is a small window where the cpu reconfig callbacks 3592 * may miss the SRS in the list walk and DR could fail since 3593 * there are still bound threads. 3594 */ 3595 mac_srs_threads_unbind(mac_srs); 3596 mac_srs_remove_glist(mac_srs); 3597 } 3598 /* 3599 * Wakeup the SRS worker and poll threads. 3600 */ 3601 mutex_enter(&mac_srs->srs_lock); 3602 mac_srs->srs_state |= srs_flag; 3603 cv_signal(&mac_srs->srs_async); 3604 cv_signal(&mac_srs->srs_cv); 3605 mutex_exit(&mac_srs->srs_lock); 3606 } 3607 3608 /* 3609 * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3610 * from the driver are done, then the Rx SRS is quiesced and only then can 3611 * we signal the soft rings. Thus this function can't be called arbitrarily 3612 * without satisfying the prerequisites. On the Tx side, the threads from 3613 * top need to quiesced, then the Tx SRS and only then can we signal the 3614 * Tx soft rings. 3615 */ 3616 static void 3617 mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3618 { 3619 mac_soft_ring_t *softring; 3620 3621 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3622 softring = softring->s_ring_next) 3623 mac_soft_ring_signal(softring, sr_flag); 3624 } 3625 3626 /* 3627 * The block comment above mac_rx_classify_flow_state_change explains the 3628 * background. At this point the SRS is quiesced and we need to restart the 3629 * SRS worker, poll, and softring threads. The SRS worker thread serves as 3630 * the master controller. The steps involved are described below in the function 3631 */ 3632 void 3633 mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3634 { 3635 boolean_t iam_rx_srs; 3636 mac_soft_ring_t *softring; 3637 3638 ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3639 if ((mac_srs->srs_type & SRST_TX) != 0) { 3640 iam_rx_srs = B_FALSE; 3641 ASSERT((mac_srs->srs_state & 3642 (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3643 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3644 } else { 3645 iam_rx_srs = B_TRUE; 3646 ASSERT((mac_srs->srs_state & 3647 (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3648 (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3649 if (mac_srs->srs_poll_thr != NULL) { 3650 ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3651 SRS_POLL_THR_QUIESCED); 3652 } 3653 } 3654 3655 /* 3656 * Signal any quiesced soft ring workers to restart and wait for the 3657 * soft ring down count to come down to zero. 3658 */ 3659 if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3660 for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3661 softring = softring->s_ring_next) { 3662 if (!(softring->s_ring_state & S_RING_QUIESCE)) 3663 continue; 3664 mac_soft_ring_signal(softring, S_RING_RESTART); 3665 } 3666 while (mac_srs->srs_soft_ring_quiesced_count != 0) 3667 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3668 } 3669 3670 mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3671 if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3672 /* 3673 * Signal the poll thread and ask it to restart. Wait till it 3674 * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3675 * cleared. 3676 */ 3677 mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3678 cv_signal(&mac_srs->srs_cv); 3679 while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3680 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3681 ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3682 } 3683 /* Wake up any waiter waiting for the restart to complete */ 3684 mac_srs->srs_state |= SRS_RESTART_DONE; 3685 cv_signal(&mac_srs->srs_quiesce_done_cv); 3686 } 3687 3688 static void 3689 mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3690 { 3691 mutex_enter(&mac_srs->srs_lock); 3692 if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3693 ASSERT(mac_srs->srs_worker_cpuid == -1); 3694 mutex_exit(&mac_srs->srs_lock); 3695 return; 3696 } 3697 3698 mac_srs->srs_worker_cpuid = -1; 3699 mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3700 thread_affinity_clear(mac_srs->srs_worker); 3701 mutex_exit(&mac_srs->srs_lock); 3702 } 3703 3704 static void 3705 mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3706 { 3707 mutex_enter(&mac_srs->srs_lock); 3708 if (mac_srs->srs_poll_thr == NULL || 3709 (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3710 ASSERT(mac_srs->srs_poll_cpuid == -1); 3711 mutex_exit(&mac_srs->srs_lock); 3712 return; 3713 } 3714 3715 mac_srs->srs_poll_cpuid = -1; 3716 mac_srs->srs_state &= ~SRS_POLL_BOUND; 3717 thread_affinity_clear(mac_srs->srs_poll_thr); 3718 mutex_exit(&mac_srs->srs_lock); 3719 } 3720 3721 static void 3722 mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3723 { 3724 mac_soft_ring_t *soft_ring; 3725 3726 ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3727 3728 mutex_enter(&cpu_lock); 3729 mac_srs_worker_unbind(mac_srs); 3730 if (!(mac_srs->srs_type & SRST_TX)) 3731 mac_srs_poll_unbind(mac_srs); 3732 3733 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3734 soft_ring = soft_ring->s_ring_next) { 3735 mac_soft_ring_unbind(soft_ring); 3736 } 3737 mutex_exit(&cpu_lock); 3738 } 3739 3740 /* 3741 * When a CPU is going away, unbind all MAC threads which are bound 3742 * to that CPU. The affinity of the thread to the CPU is saved to allow 3743 * the thread to be rebound to the CPU if it comes back online. 3744 */ 3745 static void 3746 mac_walk_srs_and_unbind(int cpuid) 3747 { 3748 mac_soft_ring_set_t *mac_srs; 3749 mac_soft_ring_t *soft_ring; 3750 3751 rw_enter(&mac_srs_g_lock, RW_READER); 3752 3753 if ((mac_srs = mac_srs_g_list) == NULL) 3754 goto done; 3755 3756 for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3757 if (mac_srs->srs_worker_cpuid == cpuid) { 3758 mac_srs->srs_worker_cpuid_save = cpuid; 3759 mac_srs_worker_unbind(mac_srs); 3760 } 3761 3762 if (!(mac_srs->srs_type & SRST_TX)) { 3763 if (mac_srs->srs_poll_cpuid == cpuid) { 3764 mac_srs->srs_poll_cpuid_save = cpuid; 3765 mac_srs_poll_unbind(mac_srs); 3766 } 3767 } 3768 3769 /* Next tackle the soft rings associated with the srs */ 3770 mutex_enter(&mac_srs->srs_lock); 3771 for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3772 soft_ring = soft_ring->s_ring_next) { 3773 if (soft_ring->s_ring_cpuid == cpuid) { 3774 soft_ring->s_ring_cpuid_save = cpuid; 3775 mac_soft_ring_unbind(soft_ring); 3776 } 3777 } 3778 mutex_exit(&mac_srs->srs_lock); 3779 } 3780 done: 3781 rw_exit(&mac_srs_g_lock); 3782 } 3783 3784 /* TX SETUP and TEARDOWN ROUTINES */ 3785 3786 /* 3787 * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3788 * handle the case where the number of rings is one. I.e. there is 3789 * a ring pointed to by mac_srs->srs_tx_arg2. 3790 */ 3791 void 3792 mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3793 { 3794 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3795 mac_soft_ring_t *soft_ring; 3796 int count = mac_srs->srs_tx_ring_count; 3797 uint32_t soft_ring_type = ST_RING_TX; 3798 uint_t ring_info; 3799 3800 ASSERT(mac_srs->srs_state & SRS_QUIESCE); 3801 ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 3802 if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 3803 soft_ring_type |= ST_RING_WORKER_ONLY; 3804 soft_ring = mac_soft_ring_create(count, 0, 3805 soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3806 NULL, mcip, (mac_resource_handle_t)tx_ring); 3807 mac_srs->srs_tx_ring_count++; 3808 mac_srs_update_fanout_list(mac_srs); 3809 /* 3810 * put this soft ring in quiesce mode too so when we restart 3811 * all soft rings in the srs are in the same state. 3812 */ 3813 mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3814 } 3815 3816 static void 3817 mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3818 { 3819 int sringcnt; 3820 3821 mutex_enter(&mac_srs->srs_lock); 3822 sringcnt = mac_srs->srs_soft_ring_count; 3823 ASSERT(sringcnt > 0); 3824 mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3825 3826 ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3827 while (mac_srs->srs_soft_ring_condemned_count != 1) 3828 cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3829 3830 if (softring == mac_srs->srs_soft_ring_head) { 3831 mac_srs->srs_soft_ring_head = softring->s_ring_next; 3832 if (mac_srs->srs_soft_ring_head != NULL) { 3833 mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3834 } else { 3835 mac_srs->srs_soft_ring_tail = NULL; 3836 } 3837 } else { 3838 softring->s_ring_prev->s_ring_next = 3839 softring->s_ring_next; 3840 if (softring->s_ring_next != NULL) { 3841 softring->s_ring_next->s_ring_prev = 3842 softring->s_ring_prev; 3843 } else { 3844 mac_srs->srs_soft_ring_tail = 3845 softring->s_ring_prev; 3846 } 3847 } 3848 mac_srs->srs_soft_ring_count--; 3849 3850 mac_srs->srs_soft_ring_condemned_count--; 3851 mutex_exit(&mac_srs->srs_lock); 3852 3853 mac_soft_ring_free(softring); 3854 } 3855 3856 void 3857 mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3858 { 3859 int i; 3860 mac_soft_ring_t *soft_ring, *remove_sring; 3861 mac_client_impl_t *mcip = mac_srs->srs_mcip; 3862 3863 mutex_enter(&mac_srs->srs_lock); 3864 for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 3865 soft_ring = mac_srs->srs_tx_soft_rings[i]; 3866 if (soft_ring->s_ring_tx_arg2 == tx_ring) 3867 break; 3868 } 3869 mutex_exit(&mac_srs->srs_lock); 3870 ASSERT(i < mac_srs->srs_tx_ring_count); 3871 remove_sring = soft_ring; 3872 /* 3873 * In the case of aggr, the soft ring associated with a Tx ring 3874 * is also stored in st_soft_rings[] array. That entry should 3875 * be removed. 3876 */ 3877 if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 3878 mac_srs_tx_t *tx = &mac_srs->srs_tx; 3879 3880 ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 3881 tx->st_soft_rings[tx_ring->mr_index] = NULL; 3882 } 3883 mac_soft_ring_remove(mac_srs, remove_sring); 3884 mac_srs_update_fanout_list(mac_srs); 3885 } 3886 3887 /* 3888 * mac_tx_srs_setup(): 3889 * Used to setup Tx rings. If no free Tx ring is available, then default 3890 * Tx ring is used. 3891 */ 3892 void 3893 mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 3894 { 3895 mac_impl_t *mip = mcip->mci_mip; 3896 mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 3897 int i; 3898 int tx_ring_count = 0; 3899 uint32_t soft_ring_type; 3900 mac_group_t *grp = NULL; 3901 mac_ring_t *ring; 3902 mac_srs_tx_t *tx = &tx_srs->srs_tx; 3903 boolean_t is_aggr; 3904 uint_t ring_info = 0; 3905 3906 is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0; 3907 grp = flent->fe_tx_ring_group; 3908 if (grp == NULL) { 3909 ring = (mac_ring_t *)mip->mi_default_tx_ring; 3910 goto no_group; 3911 } 3912 tx_ring_count = grp->mrg_cur_count; 3913 ring = grp->mrg_rings; 3914 /* 3915 * An attempt is made to reserve 'tx_ring_count' number 3916 * of Tx rings. If tx_ring_count is 0, default Tx ring 3917 * is used. If it is 1, an attempt is made to reserve one 3918 * Tx ring. In both the cases, the ring information is 3919 * stored in Tx SRS. If multiple Tx rings are specified, 3920 * then each Tx ring will have a Tx-side soft ring. All 3921 * these soft rings will be hang off Tx SRS. 3922 */ 3923 switch (grp->mrg_state) { 3924 case MAC_GROUP_STATE_SHARED: 3925 case MAC_GROUP_STATE_RESERVED: 3926 if (tx_ring_count <= 1 && !is_aggr) { 3927 no_group: 3928 if (ring != NULL && 3929 ring->mr_state != MR_INUSE) { 3930 (void) mac_start_ring(ring); 3931 ring_info = mac_hwring_getinfo( 3932 (mac_ring_handle_t)ring); 3933 } 3934 tx->st_arg2 = (void *)ring; 3935 mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 3936 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3937 tx->st_mode = SRS_TX_BW; 3938 } else if (mac_tx_serialize || 3939 (ring_info & MAC_RING_TX_SERIALIZE)) { 3940 tx->st_mode = SRS_TX_SERIALIZE; 3941 } else { 3942 tx->st_mode = SRS_TX_DEFAULT; 3943 } 3944 break; 3945 } 3946 soft_ring_type = ST_RING_TX; 3947 if (tx_srs->srs_type & SRST_BW_CONTROL) { 3948 tx->st_mode = is_aggr ? 3949 SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 3950 } else { 3951 tx->st_mode = is_aggr ? SRS_TX_AGGR : 3952 SRS_TX_FANOUT; 3953 } 3954 for (i = 0; i < tx_ring_count; i++) { 3955 ASSERT(ring != NULL); 3956 switch (ring->mr_state) { 3957 case MR_INUSE: 3958 case MR_FREE: 3959 ASSERT(ring->mr_srs == NULL); 3960 3961 if (ring->mr_state != MR_INUSE) 3962 (void) mac_start_ring(ring); 3963 ring_info = mac_hwring_getinfo( 3964 (mac_ring_handle_t)ring); 3965 if (mac_tx_serialize || (ring_info & 3966 MAC_RING_TX_SERIALIZE)) { 3967 soft_ring_type |= 3968 ST_RING_WORKER_ONLY; 3969 } 3970 (void) mac_soft_ring_create(i, 0, 3971 soft_ring_type, maxclsyspri, 3972 mcip, tx_srs, -1, NULL, mcip, 3973 (mac_resource_handle_t)ring); 3974 break; 3975 default: 3976 cmn_err(CE_PANIC, 3977 "srs_setup: mcip = %p " 3978 "trying to add UNKNOWN ring = %p\n", 3979 (void *)mcip, (void *)ring); 3980 break; 3981 } 3982 ring = ring->mr_next; 3983 } 3984 mac_srs_update_fanout_list(tx_srs); 3985 break; 3986 default: 3987 ASSERT(B_FALSE); 3988 break; 3989 } 3990 tx->st_func = mac_tx_get_func(tx->st_mode); 3991 if (is_aggr) { 3992 VERIFY(i_mac_capab_get((mac_handle_t)mip, 3993 MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 3994 } 3995 DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 3996 int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 3997 } 3998 3999 /* 4000 * Update the fanout of a client if its recorded link speed doesn't match 4001 * its current link speed. 4002 */ 4003 void 4004 mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 4005 { 4006 uint64_t link_speed; 4007 mac_resource_props_t *mcip_mrp; 4008 flow_entry_t *flent = mcip->mci_flent; 4009 mac_soft_ring_set_t *rx_srs; 4010 mac_cpus_t *srs_cpu; 4011 int soft_ring_count, maxcpus; 4012 4013 ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 4014 4015 link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 4016 MAC_STAT_IFSPEED); 4017 4018 if ((link_speed != 0) && 4019 (link_speed != mcip->mci_flent->fe_nic_speed)) { 4020 mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 4021 /* 4022 * Before calling mac_fanout_setup(), check to see if 4023 * the SRSes already have the right number of soft 4024 * rings. mac_fanout_setup() is a heavy duty operation 4025 * where new cpu bindings are done for SRS and soft 4026 * ring threads and interrupts re-targeted. 4027 */ 4028 maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 4029 soft_ring_count = mac_compute_soft_ring_count(flent, 4030 flent->fe_rx_srs_cnt - 1, maxcpus); 4031 /* 4032 * If soft_ring_count returned by 4033 * mac_compute_soft_ring_count() is 0, bump it 4034 * up by 1 because we always have atleast one 4035 * TCP, UDP, and OTH soft ring associated with 4036 * an SRS. 4037 */ 4038 soft_ring_count = (soft_ring_count == 0) ? 4039 1 : soft_ring_count; 4040 rx_srs = flent->fe_rx_srs[0]; 4041 srs_cpu = &rx_srs->srs_cpu; 4042 if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 4043 mac_fanout_setup(mcip, flent, mcip_mrp, 4044 mac_rx_deliver, mcip, NULL, cpupart); 4045 } 4046 } 4047 } 4048 4049 /* 4050 * Walk through the list of MAC clients for the MAC. 4051 * For each active MAC client, recompute the number of soft rings 4052 * associated with every client, only if current speed is different 4053 * from the speed that was previously used for soft ring computation. 4054 * If the cable is disconnected whlie the NIC is started, we would get 4055 * notification with speed set to 0. We do not recompute in that case. 4056 */ 4057 void 4058 mac_fanout_recompute(mac_impl_t *mip) 4059 { 4060 mac_client_impl_t *mcip; 4061 cpupart_t *cpupart; 4062 boolean_t use_default; 4063 mac_resource_props_t *mrp, *emrp; 4064 4065 i_mac_perim_enter(mip); 4066 if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 4067 mip->mi_linkstate != LINK_STATE_UP) { 4068 i_mac_perim_exit(mip); 4069 return; 4070 } 4071 4072 for (mcip = mip->mi_clients_list; mcip != NULL; 4073 mcip = mcip->mci_client_next) { 4074 /* Aggr port clients don't have SRSes. */ 4075 if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 4076 continue; 4077 4078 if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 4079 !MCIP_DATAPATH_SETUP(mcip)) 4080 continue; 4081 mrp = MCIP_RESOURCE_PROPS(mcip); 4082 emrp = MCIP_EFFECTIVE_PROPS(mcip); 4083 use_default = B_FALSE; 4084 pool_lock(); 4085 cpupart = mac_pset_find(mrp, &use_default); 4086 mac_fanout_recompute_client(mcip, cpupart); 4087 mac_set_pool_effective(use_default, cpupart, mrp, emrp); 4088 pool_unlock(); 4089 } 4090 4091 i_mac_perim_exit(mip); 4092 } 4093 4094 /* 4095 * Given a MAC, change the polling state for all its MAC clients. 'enable' is 4096 * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 4097 * default. 4098 */ 4099 void 4100 mac_poll_state_change(mac_handle_t mh, boolean_t enable) 4101 { 4102 mac_impl_t *mip = (mac_impl_t *)mh; 4103 mac_client_impl_t *mcip; 4104 4105 i_mac_perim_enter(mip); 4106 if (enable) 4107 mip->mi_state_flags &= ~MIS_POLL_DISABLE; 4108 else 4109 mip->mi_state_flags |= MIS_POLL_DISABLE; 4110 for (mcip = mip->mi_clients_list; mcip != NULL; 4111 mcip = mcip->mci_client_next) 4112 mac_client_update_classifier(mcip, B_TRUE); 4113 i_mac_perim_exit(mip); 4114 } 4115