1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 */ 26 27 /* 28 * SunOS 5.x Multithreaded STREAMS DLPI FCIP Module 29 * This is a pseudo driver module to handle encapsulation of IP and ARP 30 * datagrams over FibreChannel interfaces. FCIP is a cloneable STREAMS 31 * driver module which interfaces with IP/ARP using DLPI. This module 32 * is a Style-2 DLS provider. 33 * 34 * The implementation of this module is based on RFC 2625 which gives 35 * details on the encapsulation of IP/ARP data over FibreChannel. 36 * The fcip module needs to resolve an IP address to a port address before 37 * sending data to a destination port. A FC device port has 2 addresses 38 * associated with it: A 8 byte World Wide unique Port Name and a 3 byte 39 * volatile Port number or Port_ID. 40 * 41 * The mapping between a IP address and the World Wide Port Name is handled 42 * by the ARP layer since the IP over FC draft requires the MAC address to 43 * be the least significant six bytes of the WorldWide Port Names. The 44 * fcip module however needs to identify the destination port uniquely when 45 * the destination FC device has multiple FC ports. 46 * 47 * The FC layer mapping between the World Wide Port Name and the Port_ID 48 * will be handled through the use of a fabric name server or through the 49 * use of the FARP ELS command as described in the draft. Since the Port_IDs 50 * are volatile, the mapping between the World Wide Port Name and Port_IDs 51 * must be maintained and validated before use each time a datagram 52 * needs to be sent to the destination ports. The FC transport module 53 * informs the fcip module of all changes to states of ports on the 54 * fabric through registered callbacks. This enables the fcip module 55 * to maintain the WW_PN to Port_ID mappings current. 56 * 57 * For details on how this module interfaces with the FibreChannel Transport 58 * modules, refer to PSARC/1997/385. Chapter 3 of the FibreChannel Transport 59 * Programming guide details the APIs between ULPs and the Transport. 60 * 61 * Now for some Caveats: 62 * 63 * RFC 2625 requires that a FibreChannel Port name (the Port WWN) have 64 * the NAA bits set to '0001' indicating a IEEE 48bit address which 65 * corresponds to a ULA (Universal LAN MAC address). But with FibreChannel 66 * adapters containing 2 or more ports, IEEE naming cannot identify the 67 * ports on an adapter uniquely so we will in the first implementation 68 * be operating only on Port 0 of each adapter. 69 */ 70 71 #include <sys/types.h> 72 #include <sys/errno.h> 73 #include <sys/debug.h> 74 #include <sys/time.h> 75 #include <sys/sysmacros.h> 76 #include <sys/systm.h> 77 #include <sys/user.h> 78 #include <sys/stropts.h> 79 #include <sys/stream.h> 80 #include <sys/strlog.h> 81 #include <sys/strsubr.h> 82 #include <sys/cmn_err.h> 83 #include <sys/cpu.h> 84 #include <sys/kmem.h> 85 #include <sys/conf.h> 86 #include <sys/ddi.h> 87 #include <sys/sunddi.h> 88 #include <sys/ksynch.h> 89 #include <sys/stat.h> 90 #include <sys/kstat.h> 91 #include <sys/vtrace.h> 92 #include <sys/strsun.h> 93 #include <sys/varargs.h> 94 #include <sys/modctl.h> 95 #include <sys/thread.h> 96 #include <sys/var.h> 97 #include <sys/proc.h> 98 #include <inet/common.h> 99 #include <netinet/ip6.h> 100 #include <inet/ip.h> 101 #include <inet/arp.h> 102 #include <inet/mi.h> 103 #include <inet/nd.h> 104 #include <sys/dlpi.h> 105 #include <sys/ethernet.h> 106 #include <sys/file.h> 107 #include <sys/syslog.h> 108 #include <sys/disp.h> 109 #include <sys/taskq.h> 110 111 /* 112 * Leadville includes 113 */ 114 115 #include <sys/fibre-channel/fc.h> 116 #include <sys/fibre-channel/impl/fc_ulpif.h> 117 #include <sys/fibre-channel/ulp/fcip.h> 118 119 /* 120 * TNF Probe/trace facility include 121 */ 122 #if defined(lint) || defined(FCIP_TNF_ENABLED) 123 #include <sys/tnf_probe.h> 124 #endif 125 126 #define FCIP_ESBALLOC 127 128 /* 129 * Function prototypes 130 */ 131 132 /* standard loadable modules entry points */ 133 static int fcip_attach(dev_info_t *, ddi_attach_cmd_t); 134 static int fcip_detach(dev_info_t *, ddi_detach_cmd_t); 135 static void fcip_dodetach(struct fcipstr *slp); 136 static int fcip_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, 137 void *arg, void **result); 138 139 140 /* streams specific */ 141 static void fcip_setipq(struct fcip *fptr); 142 static int fcip_wput(queue_t *, mblk_t *); 143 static int fcip_wsrv(queue_t *); 144 static void fcip_proto(queue_t *, mblk_t *); 145 static void fcip_ioctl(queue_t *, mblk_t *); 146 static int fcip_open(queue_t *wq, dev_t *devp, int flag, 147 int sflag, cred_t *credp); 148 static int fcip_close(queue_t *rq, int flag, cred_t *credp); 149 static int fcip_start(queue_t *wq, mblk_t *mp, struct fcip *fptr, 150 struct fcip_dest *fdestp, int flags); 151 static void fcip_sendup(struct fcip *fptr, mblk_t *mp, 152 struct fcipstr *(*acceptfunc)()); 153 static struct fcipstr *fcip_accept(struct fcipstr *slp, struct fcip *fptr, 154 int type, la_wwn_t *dhostp); 155 static mblk_t *fcip_addudind(struct fcip *fptr, mblk_t *mp, 156 fcph_network_hdr_t *nhdr, int type); 157 static int fcip_setup_mac_addr(struct fcip *fptr); 158 static void fcip_kstat_init(struct fcip *fptr); 159 static int fcip_stat_update(kstat_t *, int); 160 161 162 /* dlpi specific */ 163 static void fcip_spareq(queue_t *wq, mblk_t *mp); 164 static void fcip_pareq(queue_t *wq, mblk_t *mp); 165 static void fcip_ubreq(queue_t *wq, mblk_t *mp); 166 static void fcip_breq(queue_t *wq, mblk_t *mp); 167 static void fcip_dreq(queue_t *wq, mblk_t *mp); 168 static void fcip_areq(queue_t *wq, mblk_t *mp); 169 static void fcip_udreq(queue_t *wq, mblk_t *mp); 170 static void fcip_ireq(queue_t *wq, mblk_t *mp); 171 static void fcip_dl_ioc_hdr_info(queue_t *wq, mblk_t *mp); 172 173 174 /* solaris sundry, DR/CPR etc */ 175 static int fcip_cache_constructor(void *buf, void *arg, int size); 176 static void fcip_cache_destructor(void *buf, void *size); 177 static int fcip_handle_suspend(fcip_port_info_t *fport, fc_detach_cmd_t cmd); 178 static int fcip_handle_resume(fcip_port_info_t *fport, 179 fc_ulp_port_info_t *port_info, fc_attach_cmd_t cmd); 180 static fcip_port_info_t *fcip_softstate_free(fcip_port_info_t *fport); 181 static int fcip_port_attach_handler(struct fcip *fptr); 182 183 184 /* 185 * ulp - transport interface function prototypes 186 */ 187 static int fcip_port_attach(opaque_t ulp_handle, fc_ulp_port_info_t *, 188 fc_attach_cmd_t cmd, uint32_t sid); 189 static int fcip_port_detach(opaque_t ulp_handle, fc_ulp_port_info_t *, 190 fc_detach_cmd_t cmd); 191 static int fcip_port_ioctl(opaque_t ulp_handle, opaque_t port_handle, 192 dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp, int *rval, 193 uint32_t claimed); 194 static void fcip_statec_cb(opaque_t ulp_handle, opaque_t phandle, 195 uint32_t port_state, uint32_t port_top, fc_portmap_t changelist[], 196 uint32_t listlen, uint32_t sid); 197 static int fcip_els_cb(opaque_t ulp_handle, opaque_t phandle, 198 fc_unsol_buf_t *buf, uint32_t claimed); 199 static int fcip_data_cb(opaque_t ulp_handle, opaque_t phandle, 200 fc_unsol_buf_t *payload, uint32_t claimed); 201 202 203 /* Routing table specific */ 204 static void fcip_handle_topology(struct fcip *fptr); 205 static int fcip_init_port(struct fcip *fptr); 206 struct fcip_routing_table *fcip_lookup_rtable(struct fcip *fptr, 207 la_wwn_t *pwwn, int matchflag); 208 static void fcip_rt_update(struct fcip *fptr, fc_portmap_t *devlist, 209 uint32_t listlen); 210 static void fcip_rt_flush(struct fcip *fptr); 211 static void fcip_rte_remove_deferred(void *arg); 212 static int fcip_do_plogi(struct fcip *fptr, struct fcip_routing_table *frp); 213 214 215 /* dest table specific */ 216 static struct fcip_dest *fcip_get_dest(struct fcip *fptr, 217 la_wwn_t *dlphys); 218 static struct fcip_dest *fcip_add_dest(struct fcip *fptr, 219 struct fcip_routing_table *frp); 220 static int fcip_dest_add_broadcast_entry(struct fcip *fptr, int new_flag); 221 static uint32_t fcip_get_broadcast_did(struct fcip *fptr); 222 static void fcip_cleanup_dest(struct fcip *fptr); 223 224 225 /* helper functions */ 226 static fcip_port_info_t *fcip_get_port(opaque_t phandle); 227 static int fcip_wwn_compare(la_wwn_t *wwn1, la_wwn_t *wwn2, int flag); 228 static void fcip_ether_to_str(struct ether_addr *e, caddr_t s); 229 static int fcip_port_get_num_pkts(struct fcip *fptr); 230 static int fcip_check_port_busy(struct fcip *fptr); 231 static void fcip_check_remove_minor_node(void); 232 static int fcip_set_wwn(la_wwn_t *pwwn); 233 static int fcip_plogi_in_progress(struct fcip *fptr); 234 static int fcip_check_port_exists(struct fcip *fptr); 235 static int fcip_is_supported_fc_topology(int fc_topology); 236 237 238 /* pkt specific */ 239 static fcip_pkt_t *fcip_pkt_alloc(struct fcip *fptr, mblk_t *bp, 240 int flags, int datalen); 241 static void fcip_pkt_free(struct fcip_pkt *fcip_pkt, int flags); 242 static fcip_pkt_t *fcip_ipkt_alloc(struct fcip *fptr, int cmdlen, 243 int resplen, opaque_t pd, int flags); 244 static void fcip_ipkt_free(fcip_pkt_t *fcip_pkt); 245 static void fcip_ipkt_callback(fc_packet_t *fc_pkt); 246 static void fcip_free_pkt_dma(fcip_pkt_t *fcip_pkt); 247 static void fcip_pkt_callback(fc_packet_t *fc_pkt); 248 static void fcip_init_unicast_pkt(fcip_pkt_t *fcip_pkt, fc_portid_t sid, 249 fc_portid_t did, void (*comp) ()); 250 static int fcip_transport(fcip_pkt_t *fcip_pkt); 251 static void fcip_pkt_timeout(void *arg); 252 static void fcip_timeout(void *arg); 253 static void fcip_fdestp_enqueue_pkt(struct fcip_dest *fdestp, 254 fcip_pkt_t *fcip_pkt); 255 static int fcip_fdestp_dequeue_pkt(struct fcip_dest *fdestp, 256 fcip_pkt_t *fcip_pkt); 257 static int fcip_sendup_constructor(void *buf, void *arg, int flags); 258 static void fcip_sendup_thr(void *arg); 259 static int fcip_sendup_alloc_enque(struct fcip *ftpr, mblk_t *mp, 260 struct fcipstr *(*f)()); 261 262 /* 263 * zero copy inbound data handling 264 */ 265 #ifdef FCIP_ESBALLOC 266 static void fcip_ubfree(char *arg); 267 #endif /* FCIP_ESBALLOC */ 268 269 #if !defined(FCIP_ESBALLOC) 270 static void *fcip_allocb(size_t size, uint_t pri); 271 #endif 272 273 274 /* FCIP FARP support functions */ 275 static struct fcip_dest *fcip_do_farp(struct fcip *fptr, la_wwn_t *pwwn, 276 char *ip_addr, size_t ip_addr_len, int flags); 277 static void fcip_init_broadcast_pkt(fcip_pkt_t *fcip_pkt, void (*comp) (), 278 int is_els); 279 static int fcip_handle_farp_request(struct fcip *fptr, la_els_farp_t *fcmd); 280 static int fcip_handle_farp_response(struct fcip *fptr, la_els_farp_t *fcmd); 281 static void fcip_cache_arp_broadcast(struct fcip *ftpr, fc_unsol_buf_t *buf); 282 static void fcip_port_ns(void *arg); 283 284 #ifdef DEBUG 285 286 #include <sys/debug.h> 287 288 #define FCIP_DEBUG_DEFAULT 0x1 289 #define FCIP_DEBUG_ATTACH 0x2 290 #define FCIP_DEBUG_INIT 0x4 291 #define FCIP_DEBUG_DETACH 0x8 292 #define FCIP_DEBUG_DLPI 0x10 293 #define FCIP_DEBUG_ELS 0x20 294 #define FCIP_DEBUG_DOWNSTREAM 0x40 295 #define FCIP_DEBUG_UPSTREAM 0x80 296 #define FCIP_DEBUG_MISC 0x100 297 298 #define FCIP_DEBUG_STARTUP (FCIP_DEBUG_ATTACH|FCIP_DEBUG_INIT) 299 #define FCIP_DEBUG_DATAOUT (FCIP_DEBUG_DLPI|FCIP_DEBUG_DOWNSTREAM) 300 #define FCIP_DEBUG_DATAIN (FCIP_DEBUG_ELS|FCIP_DEBUG_UPSTREAM) 301 302 static int fcip_debug = FCIP_DEBUG_DEFAULT; 303 304 #define FCIP_DEBUG(level, args) \ 305 if (fcip_debug & (level)) cmn_err args; 306 307 #else /* DEBUG */ 308 309 #define FCIP_DEBUG(level, args) /* do nothing */ 310 311 #endif /* DEBUG */ 312 313 #define KIOIP KSTAT_INTR_PTR(fcip->fcip_intrstats) 314 315 /* 316 * Endian independent ethernet to WWN copy 317 */ 318 #define ether_to_wwn(E, W) \ 319 bzero((void *)(W), sizeof (la_wwn_t)); \ 320 bcopy((void *)(E), (void *)&((W)->raw_wwn[2]), ETHERADDRL); \ 321 (W)->raw_wwn[0] |= 0x10 322 323 /* 324 * wwn_to_ether : Endian independent, copies a WWN to struct ether_addr. 325 * The args to the macro are pointers to WWN and ether_addr structures 326 */ 327 #define wwn_to_ether(W, E) \ 328 bcopy((void *)&((W)->raw_wwn[2]), (void *)E, ETHERADDRL) 329 330 /* 331 * The module_info structure contains identification and limit values. 332 * All queues associated with a certain driver share the same module_info 333 * structures. This structure defines the characteristics of that driver/ 334 * module's queues. The module name must be unique. The max and min packet 335 * sizes limit the no. of characters in M_DATA messages. The Hi and Lo 336 * water marks are for flow control when a module has a service procedure. 337 */ 338 static struct module_info fcipminfo = { 339 FCIPIDNUM, /* mi_idnum : Module ID num */ 340 FCIPNAME, /* mi_idname: Module Name */ 341 FCIPMINPSZ, /* mi_minpsz: Min packet size */ 342 FCIPMAXPSZ, /* mi_maxpsz: Max packet size */ 343 FCIPHIWAT, /* mi_hiwat : High water mark */ 344 FCIPLOWAT /* mi_lowat : Low water mark */ 345 }; 346 347 /* 348 * The qinit structres contain the module put, service. open and close 349 * procedure pointers. All modules and drivers with the same streamtab 350 * file (i.e same fmodsw or cdevsw entry points) point to the same 351 * upstream (read) and downstream (write) qinit structs. 352 */ 353 static struct qinit fcip_rinit = { 354 NULL, /* qi_putp */ 355 NULL, /* qi_srvp */ 356 fcip_open, /* qi_qopen */ 357 fcip_close, /* qi_qclose */ 358 NULL, /* qi_qadmin */ 359 &fcipminfo, /* qi_minfo */ 360 NULL /* qi_mstat */ 361 }; 362 363 static struct qinit fcip_winit = { 364 fcip_wput, /* qi_putp */ 365 fcip_wsrv, /* qi_srvp */ 366 NULL, /* qi_qopen */ 367 NULL, /* qi_qclose */ 368 NULL, /* qi_qadmin */ 369 &fcipminfo, /* qi_minfo */ 370 NULL /* qi_mstat */ 371 }; 372 373 /* 374 * streamtab contains pointers to the read and write qinit structures 375 */ 376 377 static struct streamtab fcip_info = { 378 &fcip_rinit, /* st_rdinit */ 379 &fcip_winit, /* st_wrinit */ 380 NULL, /* st_muxrinit */ 381 NULL, /* st_muxwrinit */ 382 }; 383 384 static struct cb_ops fcip_cb_ops = { 385 nodev, /* open */ 386 nodev, /* close */ 387 nodev, /* strategy */ 388 nodev, /* print */ 389 nodev, /* dump */ 390 nodev, /* read */ 391 nodev, /* write */ 392 nodev, /* ioctl */ 393 nodev, /* devmap */ 394 nodev, /* mmap */ 395 nodev, /* segmap */ 396 nochpoll, /* poll */ 397 ddi_prop_op, /* cb_prop_op */ 398 &fcip_info, /* streamtab */ 399 D_MP | D_HOTPLUG, /* Driver compatibility flag */ 400 CB_REV, /* rev */ 401 nodev, /* int (*cb_aread)() */ 402 nodev /* int (*cb_awrite)() */ 403 }; 404 405 /* 406 * autoconfiguration routines. 407 */ 408 static struct dev_ops fcip_ops = { 409 DEVO_REV, /* devo_rev, */ 410 0, /* refcnt */ 411 fcip_getinfo, /* info */ 412 nulldev, /* identify */ 413 nulldev, /* probe */ 414 fcip_attach, /* attach */ 415 fcip_detach, /* detach */ 416 nodev, /* RESET */ 417 &fcip_cb_ops, /* driver operations */ 418 NULL, /* bus operations */ 419 ddi_power /* power management */ 420 }; 421 422 #define FCIP_VERSION "1.61" 423 #define FCIP_NAME "SunFC FCIP v" FCIP_VERSION 424 425 #define PORT_DRIVER "fp" 426 427 #define GETSTRUCT(struct, number) \ 428 ((struct *)kmem_zalloc((size_t)(sizeof (struct) * (number)), \ 429 KM_SLEEP)) 430 431 static struct modldrv modldrv = { 432 &mod_driverops, /* Type of module - driver */ 433 FCIP_NAME, /* Name of module */ 434 &fcip_ops, /* driver ops */ 435 }; 436 437 static struct modlinkage modlinkage = { 438 MODREV_1, (void *)&modldrv, NULL 439 }; 440 441 442 /* 443 * Now for some global statics 444 */ 445 static uint32_t fcip_ub_nbufs = FCIP_UB_NBUFS; 446 static uint32_t fcip_ub_size = FCIP_UB_SIZE; 447 static int fcip_pkt_ttl_ticks = FCIP_PKT_TTL; 448 static int fcip_tick_incr = 1; 449 static int fcip_wait_cmds = FCIP_WAIT_CMDS; 450 static int fcip_num_attaching = 0; 451 static int fcip_port_attach_pending = 0; 452 static int fcip_create_nodes_on_demand = 1; /* keep it similar to fcp */ 453 static int fcip_cache_on_arp_broadcast = 0; 454 static int fcip_farp_supported = 0; 455 static int fcip_minor_node_created = 0; 456 457 /* 458 * Supported FCAs 459 */ 460 #define QLC_PORT_1_ID_BITS 0x100 461 #define QLC_PORT_2_ID_BITS 0x101 462 #define QLC_PORT_NAA 0x2 463 #define QLC_MODULE_NAME "qlc" 464 #define IS_QLC_PORT(port_dip) \ 465 (strcmp(ddi_driver_name(ddi_get_parent((port_dip))),\ 466 QLC_MODULE_NAME) == 0) 467 468 469 /* 470 * fcip softstate structures head. 471 */ 472 473 static void *fcip_softp = NULL; 474 475 /* 476 * linked list of active (inuse) driver streams 477 */ 478 479 static int fcip_num_instances = 0; 480 static dev_info_t *fcip_module_dip = (dev_info_t *)0; 481 482 483 /* 484 * Ethernet broadcast address: Broadcast addressing in IP over fibre 485 * channel should be the IEEE ULA (also the low 6 bytes of the Port WWN). 486 * 487 * The broadcast addressing varies for differing topologies a node may be in: 488 * - On a private loop the ARP broadcast is a class 3 sequence sent 489 * using OPNfr (Open Broadcast Replicate primitive) followed by 490 * the ARP frame to D_ID 0xFFFFFF 491 * 492 * - On a public Loop the broadcast sequence is sent to AL_PA 0x00 493 * (no OPNfr primitive). 494 * 495 * - For direct attach and point to point topologies we just send 496 * the frame to D_ID 0xFFFFFF 497 * 498 * For public loop the handling would probably be different - for now 499 * I'll just declare this struct - It can be deleted if not necessary. 500 * 501 */ 502 503 504 /* 505 * DL_INFO_ACK template for the fcip module. The dl_info_ack_t structure is 506 * returned as a part of an DL_INFO_ACK message which is a M_PCPROTO message 507 * returned in response to a DL_INFO_REQ message sent to us from a DLS user 508 * Let us fake an ether header as much as possible. 509 * 510 * dl_addr_length is the Provider's DLSAP addr which is SAP addr + 511 * Physical addr of the provider. We set this to 512 * ushort_t + sizeof (la_wwn_t) for Fibre Channel ports. 513 * dl_mac_type Lets just use DL_ETHER - we can try using DL_IPFC, a new 514 * dlpi.h define later. 515 * dl_sap_length -2 indicating the SAP address follows the Physical addr 516 * component in the DLSAP addr. 517 * dl_service_mode: DLCLDS - connectionless data link service. 518 * 519 */ 520 521 static dl_info_ack_t fcip_infoack = { 522 DL_INFO_ACK, /* dl_primitive */ 523 FCIPMTU, /* dl_max_sdu */ 524 0, /* dl_min_sdu */ 525 FCIPADDRL, /* dl_addr_length */ 526 DL_ETHER, /* dl_mac_type */ 527 0, /* dl_reserved */ 528 0, /* dl_current_state */ 529 -2, /* dl_sap_length */ 530 DL_CLDLS, /* dl_service_mode */ 531 0, /* dl_qos_length */ 532 0, /* dl_qos_offset */ 533 0, /* dl_range_length */ 534 0, /* dl_range_offset */ 535 DL_STYLE2, /* dl_provider_style */ 536 sizeof (dl_info_ack_t), /* dl_addr_offset */ 537 DL_VERSION_2, /* dl_version */ 538 ETHERADDRL, /* dl_brdcst_addr_length */ 539 sizeof (dl_info_ack_t) + FCIPADDRL, /* dl_brdcst_addr_offset */ 540 0 /* dl_growth */ 541 }; 542 543 /* 544 * FCIP broadcast address definition. 545 */ 546 static struct ether_addr fcipnhbroadcastaddr = { 547 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 548 }; 549 550 /* 551 * RFC2625 requires the broadcast ARP address in the ARP data payload to 552 * be set to 0x00 00 00 00 00 00 for ARP broadcast packets 553 */ 554 static struct ether_addr fcip_arpbroadcast_addr = { 555 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 556 }; 557 558 559 #define ether_bcopy(src, dest) bcopy((src), (dest), ETHERADDRL); 560 561 /* 562 * global kernel locks 563 */ 564 static kcondvar_t fcip_global_cv; 565 static kmutex_t fcip_global_mutex; 566 567 /* 568 * fctl external defines 569 */ 570 extern int fc_ulp_add(fc_ulp_modinfo_t *); 571 572 /* 573 * fctl data structures 574 */ 575 576 #define FCIP_REV 0x07 577 578 /* linked list of port info structures */ 579 static fcip_port_info_t *fcip_port_head = NULL; 580 581 /* linked list of fcip structures */ 582 static struct fcipstr *fcipstrup = NULL; 583 static krwlock_t fcipstruplock; 584 585 586 /* 587 * Module information structure. This structure gives the FC Transport modules 588 * information about an ULP that registers with it. 589 */ 590 static fc_ulp_modinfo_t fcip_modinfo = { 591 0, /* for xref checks? */ 592 FCTL_ULP_MODREV_4, /* FCIP revision */ 593 FC_TYPE_IS8802_SNAP, /* type 5 for SNAP encapsulated datagrams */ 594 FCIP_NAME, /* module name as in the modldrv struct */ 595 0x0, /* get all statec callbacks for now */ 596 fcip_port_attach, /* port attach callback */ 597 fcip_port_detach, /* port detach callback */ 598 fcip_port_ioctl, /* port ioctl callback */ 599 fcip_els_cb, /* els callback */ 600 fcip_data_cb, /* data callback */ 601 fcip_statec_cb /* state change callback */ 602 }; 603 604 605 /* 606 * Solaris 9 and up, the /kernel/drv/fp.conf file will have the following entry 607 * 608 * ddi-forceattach=1; 609 * 610 * This will ensure that fp is loaded at bootup. No additional checks are needed 611 */ 612 int 613 _init(void) 614 { 615 int rval; 616 617 FCIP_TNF_LOAD(); 618 619 /* 620 * Initialize the mutexs used by port attach and other callbacks. 621 * The transport can call back into our port_attach_callback 622 * routine even before _init() completes and bad things can happen. 623 */ 624 mutex_init(&fcip_global_mutex, NULL, MUTEX_DRIVER, NULL); 625 cv_init(&fcip_global_cv, NULL, CV_DRIVER, NULL); 626 rw_init(&fcipstruplock, NULL, RW_DRIVER, NULL); 627 628 mutex_enter(&fcip_global_mutex); 629 fcip_port_attach_pending = 1; 630 mutex_exit(&fcip_global_mutex); 631 632 /* 633 * Now attempt to register fcip with the transport. 634 * If fc_ulp_add fails, fcip module will not be loaded. 635 */ 636 rval = fc_ulp_add(&fcip_modinfo); 637 if (rval != FC_SUCCESS) { 638 mutex_destroy(&fcip_global_mutex); 639 cv_destroy(&fcip_global_cv); 640 rw_destroy(&fcipstruplock); 641 switch (rval) { 642 case FC_ULP_SAMEMODULE: 643 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN, 644 "!fcip: module is already registered with" 645 " transport")); 646 rval = EEXIST; 647 break; 648 case FC_ULP_SAMETYPE: 649 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN, 650 "!fcip: Another module of the same ULP type 0x%x" 651 " is already registered with the transport", 652 fcip_modinfo.ulp_type)); 653 rval = EEXIST; 654 break; 655 case FC_BADULP: 656 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN, 657 "!fcip: Current fcip version 0x%x does not match" 658 " fctl version", 659 fcip_modinfo.ulp_rev)); 660 rval = ENODEV; 661 break; 662 default: 663 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN, 664 "!fcip: fc_ulp_add failed with status 0x%x", rval)); 665 rval = ENODEV; 666 break; 667 } 668 FCIP_TNF_UNLOAD(&modlinkage); 669 return (rval); 670 } 671 672 if ((rval = ddi_soft_state_init(&fcip_softp, sizeof (struct fcip), 673 FCIP_NUM_INSTANCES)) != 0) { 674 mutex_destroy(&fcip_global_mutex); 675 cv_destroy(&fcip_global_cv); 676 rw_destroy(&fcipstruplock); 677 (void) fc_ulp_remove(&fcip_modinfo); 678 FCIP_TNF_UNLOAD(&modlinkage); 679 return (rval); 680 } 681 682 if ((rval = mod_install(&modlinkage)) != 0) { 683 FCIP_TNF_UNLOAD(&modlinkage); 684 (void) fc_ulp_remove(&fcip_modinfo); 685 mutex_destroy(&fcip_global_mutex); 686 cv_destroy(&fcip_global_cv); 687 rw_destroy(&fcipstruplock); 688 ddi_soft_state_fini(&fcip_softp); 689 } 690 return (rval); 691 } 692 693 /* 694 * Unload the port driver if this was the only ULP loaded and then 695 * deregister with the transport. 696 */ 697 int 698 _fini(void) 699 { 700 int rval; 701 int rval1; 702 703 /* 704 * Do not permit the module to be unloaded before a port 705 * attach callback has happened. 706 */ 707 mutex_enter(&fcip_global_mutex); 708 if (fcip_num_attaching || fcip_port_attach_pending) { 709 mutex_exit(&fcip_global_mutex); 710 return (EBUSY); 711 } 712 mutex_exit(&fcip_global_mutex); 713 714 if ((rval = mod_remove(&modlinkage)) != 0) { 715 return (rval); 716 } 717 718 /* 719 * unregister with the transport layer 720 */ 721 rval1 = fc_ulp_remove(&fcip_modinfo); 722 723 /* 724 * If the ULP was not registered with the transport, init should 725 * have failed. If transport has no knowledge of our existence 726 * we should simply bail out and succeed 727 */ 728 #ifdef DEBUG 729 if (rval1 == FC_BADULP) { 730 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN, 731 "fcip: ULP was never registered with the transport")); 732 rval = ENODEV; 733 } else if (rval1 == FC_BADTYPE) { 734 FCIP_DEBUG(FCIP_DEBUG_DEFAULT, (CE_WARN, 735 "fcip: No ULP of this type 0x%x was registered with " 736 "transport", fcip_modinfo.ulp_type)); 737 rval = ENODEV; 738 } 739 #endif /* DEBUG */ 740 741 mutex_destroy(&fcip_global_mutex); 742 rw_destroy(&fcipstruplock); 743 cv_destroy(&fcip_global_cv); 744 ddi_soft_state_fini(&fcip_softp); 745 746 FCIP_TNF_UNLOAD(&modlinkage); 747 748 return (rval); 749 } 750 751 /* 752 * Info about this loadable module 753 */ 754 int 755 _info(struct modinfo *modinfop) 756 { 757 return (mod_info(&modlinkage, modinfop)); 758 } 759 760 /* 761 * The port attach callback is invoked by the port driver when a FCA 762 * port comes online and binds with the transport layer. The transport 763 * then callsback into all ULP modules registered with it. The Port attach 764 * call back will also provide the ULP module with the Port's WWN and S_ID 765 */ 766 /* ARGSUSED */ 767 static int 768 fcip_port_attach(opaque_t ulp_handle, fc_ulp_port_info_t *port_info, 769 fc_attach_cmd_t cmd, uint32_t sid) 770 { 771 int rval = FC_FAILURE; 772 int instance; 773 struct fcip *fptr; 774 fcip_port_info_t *fport = NULL; 775 fcip_port_info_t *cur_fport; 776 fc_portid_t src_id; 777 778 switch (cmd) { 779 case FC_CMD_ATTACH: { 780 la_wwn_t *ww_pn = NULL; 781 /* 782 * It was determined that, as per spec, the lower 48 bits of 783 * the port-WWN will always be unique. This will make the MAC 784 * address (i.e the lower 48 bits of the WWN), that IP/ARP 785 * depend on, unique too. Hence we should be able to remove the 786 * restriction of attaching to only one of the ports of 787 * multi port FCAs. 788 * 789 * Earlier, fcip used to attach only to qlc module and fail 790 * silently for attach failures resulting from unknown FCAs or 791 * unsupported FCA ports. Now, we'll do no such checks. 792 */ 793 ww_pn = &port_info->port_pwwn; 794 795 FCIP_TNF_PROBE_2((fcip_port_attach, "fcip io", /* CSTYLED */, 796 tnf_string, msg, "port id bits", 797 tnf_opaque, nport_id, ww_pn->w.nport_id)); 798 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE, 799 "port id bits: 0x%x", ww_pn->w.nport_id)); 800 /* 801 * A port has come online 802 */ 803 mutex_enter(&fcip_global_mutex); 804 fcip_num_instances++; 805 fcip_num_attaching++; 806 807 if (fcip_port_head == NULL) { 808 /* OK to sleep here ? */ 809 fport = kmem_zalloc(sizeof (fcip_port_info_t), 810 KM_NOSLEEP); 811 if (fport == NULL) { 812 fcip_num_instances--; 813 fcip_num_attaching--; 814 ASSERT(fcip_num_attaching >= 0); 815 mutex_exit(&fcip_global_mutex); 816 rval = FC_FAILURE; 817 cmn_err(CE_WARN, "!fcip(%d): port attach " 818 "failed: alloc failed", 819 ddi_get_instance(port_info->port_dip)); 820 goto done; 821 } 822 fcip_port_head = fport; 823 } else { 824 /* 825 * traverse the port list and also check for 826 * duplicate port attaches - Nothing wrong in being 827 * paranoid Heh Heh. 828 */ 829 cur_fport = fcip_port_head; 830 while (cur_fport != NULL) { 831 if (cur_fport->fcipp_handle == 832 port_info->port_handle) { 833 fcip_num_instances--; 834 fcip_num_attaching--; 835 ASSERT(fcip_num_attaching >= 0); 836 mutex_exit(&fcip_global_mutex); 837 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN, 838 "!fcip(%d): port already " 839 "attached!!", ddi_get_instance( 840 port_info->port_dip))); 841 rval = FC_FAILURE; 842 goto done; 843 } 844 cur_fport = cur_fport->fcipp_next; 845 } 846 fport = kmem_zalloc(sizeof (fcip_port_info_t), 847 KM_NOSLEEP); 848 if (fport == NULL) { 849 rval = FC_FAILURE; 850 fcip_num_instances--; 851 fcip_num_attaching--; 852 ASSERT(fcip_num_attaching >= 0); 853 mutex_exit(&fcip_global_mutex); 854 cmn_err(CE_WARN, "!fcip(%d): port attach " 855 "failed: alloc failed", 856 ddi_get_instance(port_info->port_dip)); 857 goto done; 858 } 859 fport->fcipp_next = fcip_port_head; 860 fcip_port_head = fport; 861 } 862 863 mutex_exit(&fcip_global_mutex); 864 865 /* 866 * now fill in the details about the port itself 867 */ 868 fport->fcipp_linkage = *port_info->port_linkage; 869 fport->fcipp_handle = port_info->port_handle; 870 fport->fcipp_dip = port_info->port_dip; 871 fport->fcipp_topology = port_info->port_flags; 872 fport->fcipp_pstate = port_info->port_state; 873 fport->fcipp_naa = port_info->port_pwwn.w.naa_id; 874 bcopy(&port_info->port_pwwn, &fport->fcipp_pwwn, 875 sizeof (la_wwn_t)); 876 bcopy(&port_info->port_nwwn, &fport->fcipp_nwwn, 877 sizeof (la_wwn_t)); 878 fport->fcipp_fca_pkt_size = port_info->port_fca_pkt_size; 879 fport->fcipp_cmd_dma_attr = *port_info->port_cmd_dma_attr; 880 fport->fcipp_resp_dma_attr = *port_info->port_resp_dma_attr; 881 fport->fcipp_fca_acc_attr = *port_info->port_acc_attr; 882 src_id.port_id = sid; 883 src_id.priv_lilp_posit = 0; 884 fport->fcipp_sid = src_id; 885 886 /* 887 * allocate soft state for this instance 888 */ 889 instance = ddi_get_instance(fport->fcipp_dip); 890 if (ddi_soft_state_zalloc(fcip_softp, 891 instance) != DDI_SUCCESS) { 892 rval = FC_FAILURE; 893 cmn_err(CE_WARN, "!fcip(%d): port attach failed: " 894 "soft state alloc failed", instance); 895 goto failure; 896 } 897 898 fptr = ddi_get_soft_state(fcip_softp, instance); 899 900 if (fptr == NULL) { 901 rval = FC_FAILURE; 902 cmn_err(CE_WARN, "!fcip(%d): port attach failed: " 903 "failure to get soft state", instance); 904 goto failure; 905 } 906 907 /* 908 * initialize all mutexes and locks required for this module 909 */ 910 mutex_init(&fptr->fcip_mutex, NULL, MUTEX_DRIVER, NULL); 911 mutex_init(&fptr->fcip_ub_mutex, NULL, MUTEX_DRIVER, NULL); 912 mutex_init(&fptr->fcip_rt_mutex, NULL, MUTEX_DRIVER, NULL); 913 mutex_init(&fptr->fcip_dest_mutex, NULL, MUTEX_DRIVER, NULL); 914 mutex_init(&fptr->fcip_sendup_mutex, NULL, MUTEX_DRIVER, NULL); 915 cv_init(&fptr->fcip_farp_cv, NULL, CV_DRIVER, NULL); 916 cv_init(&fptr->fcip_sendup_cv, NULL, CV_DRIVER, NULL); 917 cv_init(&fptr->fcip_ub_cv, NULL, CV_DRIVER, NULL); 918 919 mutex_enter(&fptr->fcip_mutex); 920 921 fptr->fcip_dip = fport->fcipp_dip; /* parent's dip */ 922 fptr->fcip_instance = instance; 923 fptr->fcip_ub_upstream = 0; 924 925 if (FC_PORT_STATE_MASK(port_info->port_state) == 926 FC_STATE_ONLINE) { 927 fptr->fcip_port_state = FCIP_PORT_ONLINE; 928 if (fptr->fcip_flags & FCIP_LINK_DOWN) { 929 fptr->fcip_flags &= ~FCIP_LINK_DOWN; 930 } 931 } else { 932 fptr->fcip_port_state = FCIP_PORT_OFFLINE; 933 } 934 935 fptr->fcip_flags |= FCIP_ATTACHING; 936 fptr->fcip_port_info = fport; 937 938 /* 939 * Extract our MAC addr from our port's WWN. The lower 48 940 * bits will be our MAC address 941 */ 942 wwn_to_ether(&fport->fcipp_nwwn, &fptr->fcip_macaddr); 943 944 fport->fcipp_fcip = fptr; 945 946 FCIP_DEBUG(FCIP_DEBUG_ATTACH, 947 (CE_NOTE, "fcipdest : 0x%lx, rtable : 0x%lx", 948 (long)(sizeof (fptr->fcip_dest)), 949 (long)(sizeof (fptr->fcip_rtable)))); 950 951 bzero(fptr->fcip_dest, sizeof (fptr->fcip_dest)); 952 bzero(fptr->fcip_rtable, sizeof (fptr->fcip_rtable)); 953 954 /* 955 * create a taskq to handle sundry jobs for the driver 956 * This way we can have jobs run in parallel 957 */ 958 fptr->fcip_tq = taskq_create("fcip_tasks", 959 FCIP_NUM_THREADS, MINCLSYSPRI, FCIP_MIN_TASKS, 960 FCIP_MAX_TASKS, TASKQ_PREPOPULATE); 961 962 mutex_exit(&fptr->fcip_mutex); 963 964 /* 965 * create a separate thread to handle all unsolicited 966 * callback handling. This is because unsolicited_callback 967 * can happen from an interrupt context and the upstream 968 * modules can put new messages right back in the same 969 * thread context. This usually works fine, but sometimes 970 * we may have to block to obtain the dest struct entries 971 * for some remote ports. 972 */ 973 mutex_enter(&fptr->fcip_sendup_mutex); 974 if (thread_create(NULL, DEFAULTSTKSZ, 975 (void (*)())fcip_sendup_thr, (caddr_t)fptr, 0, &p0, 976 TS_RUN, minclsyspri) == NULL) { 977 mutex_exit(&fptr->fcip_sendup_mutex); 978 cmn_err(CE_WARN, 979 "!unable to create fcip sendup thread for " 980 " instance: 0x%x", instance); 981 rval = FC_FAILURE; 982 goto done; 983 } 984 fptr->fcip_sendup_thr_initted = 1; 985 fptr->fcip_sendup_head = fptr->fcip_sendup_tail = NULL; 986 mutex_exit(&fptr->fcip_sendup_mutex); 987 988 989 /* Let the attach handler do the rest */ 990 if (fcip_port_attach_handler(fptr) != FC_SUCCESS) { 991 /* 992 * We have already cleaned up so return 993 */ 994 rval = FC_FAILURE; 995 cmn_err(CE_WARN, "!fcip(%d): port attach failed", 996 instance); 997 goto done; 998 } 999 1000 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_CONT, 1001 "!fcip attach for port instance (0x%x) successful", 1002 instance)); 1003 1004 rval = FC_SUCCESS; 1005 goto done; 1006 } 1007 case FC_CMD_POWER_UP: 1008 /* FALLTHROUGH */ 1009 case FC_CMD_RESUME: 1010 mutex_enter(&fcip_global_mutex); 1011 fport = fcip_port_head; 1012 while (fport != NULL) { 1013 if (fport->fcipp_handle == port_info->port_handle) { 1014 break; 1015 } 1016 fport = fport->fcipp_next; 1017 } 1018 if (fport == NULL) { 1019 rval = FC_SUCCESS; 1020 mutex_exit(&fcip_global_mutex); 1021 goto done; 1022 } 1023 rval = fcip_handle_resume(fport, port_info, cmd); 1024 mutex_exit(&fcip_global_mutex); 1025 goto done; 1026 1027 default: 1028 FCIP_TNF_PROBE_2((fcip_port_attach, "fcip io", /* CSTYLED */, 1029 tnf_string, msg, "unknown command type", 1030 tnf_uint, cmd, cmd)); 1031 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN, 1032 "unknown cmd type 0x%x in port_attach", cmd)); 1033 rval = FC_FAILURE; 1034 goto done; 1035 } 1036 1037 failure: 1038 if (fport) { 1039 mutex_enter(&fcip_global_mutex); 1040 fcip_num_attaching--; 1041 ASSERT(fcip_num_attaching >= 0); 1042 (void) fcip_softstate_free(fport); 1043 fcip_port_attach_pending = 0; 1044 mutex_exit(&fcip_global_mutex); 1045 } 1046 return (rval); 1047 1048 done: 1049 mutex_enter(&fcip_global_mutex); 1050 fcip_port_attach_pending = 0; 1051 mutex_exit(&fcip_global_mutex); 1052 return (rval); 1053 } 1054 1055 /* 1056 * fcip_port_attach_handler : Completes the port attach operation after 1057 * the ulp_port_attach routine has completed its ground work. The job 1058 * of this function among other things is to obtain and handle topology 1059 * specifics, initialize a port, setup broadcast address entries in 1060 * the fcip tables etc. This routine cleans up behind itself on failures. 1061 * Returns FC_SUCCESS or FC_FAILURE. 1062 */ 1063 static int 1064 fcip_port_attach_handler(struct fcip *fptr) 1065 { 1066 fcip_port_info_t *fport = fptr->fcip_port_info; 1067 int rval = FC_FAILURE; 1068 1069 ASSERT(fport != NULL); 1070 1071 mutex_enter(&fcip_global_mutex); 1072 1073 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE, 1074 "fcip module dip: %p instance: %d", 1075 (void *)fcip_module_dip, ddi_get_instance(fptr->fcip_dip))); 1076 1077 if (fcip_module_dip == NULL) { 1078 clock_t fcip_lbolt; 1079 1080 fcip_lbolt = ddi_get_lbolt(); 1081 /* 1082 * we need to use the fcip devinfo for creating 1083 * the clone device node, but the fcip attach 1084 * (from its conf file entry claiming to be a 1085 * child of pseudo) may not have happened yet. 1086 * wait here for 10 seconds and fail port attach 1087 * if the fcip devinfo is not attached yet 1088 */ 1089 fcip_lbolt += drv_usectohz(FCIP_INIT_DELAY); 1090 1091 FCIP_DEBUG(FCIP_DEBUG_ATTACH, 1092 (CE_WARN, "cv_timedwait lbolt %lx", fcip_lbolt)); 1093 1094 (void) cv_timedwait(&fcip_global_cv, &fcip_global_mutex, 1095 fcip_lbolt); 1096 1097 if (fcip_module_dip == NULL) { 1098 mutex_exit(&fcip_global_mutex); 1099 1100 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN, 1101 "fcip attach did not happen")); 1102 goto port_attach_cleanup; 1103 } 1104 } 1105 1106 if ((!fcip_minor_node_created) && 1107 fcip_is_supported_fc_topology(fport->fcipp_topology)) { 1108 /* 1109 * Checking for same topologies which are considered valid 1110 * by fcip_handle_topology(). Dont create a minor node if 1111 * nothing is hanging off the FC port. 1112 */ 1113 if (ddi_create_minor_node(fcip_module_dip, "fcip", S_IFCHR, 1114 ddi_get_instance(fptr->fcip_dip), DDI_PSEUDO, 1115 CLONE_DEV) == DDI_FAILURE) { 1116 mutex_exit(&fcip_global_mutex); 1117 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_WARN, 1118 "failed to create minor node for fcip(%d)", 1119 ddi_get_instance(fptr->fcip_dip))); 1120 goto port_attach_cleanup; 1121 } 1122 fcip_minor_node_created++; 1123 } 1124 mutex_exit(&fcip_global_mutex); 1125 1126 /* 1127 * initialize port for traffic 1128 */ 1129 if (fcip_init_port(fptr) != FC_SUCCESS) { 1130 /* fcip_init_port has already cleaned up its stuff */ 1131 1132 mutex_enter(&fcip_global_mutex); 1133 1134 if ((fcip_num_instances == 1) && 1135 (fcip_minor_node_created == 1)) { 1136 /* Remove minor node iff this is the last instance */ 1137 ddi_remove_minor_node(fcip_module_dip, NULL); 1138 } 1139 1140 mutex_exit(&fcip_global_mutex); 1141 1142 goto port_attach_cleanup; 1143 } 1144 1145 mutex_enter(&fptr->fcip_mutex); 1146 fptr->fcip_flags &= ~FCIP_ATTACHING; 1147 fptr->fcip_flags |= FCIP_INITED; 1148 fptr->fcip_timeout_ticks = 0; 1149 1150 /* 1151 * start the timeout threads 1152 */ 1153 fptr->fcip_timeout_id = timeout(fcip_timeout, fptr, 1154 drv_usectohz(1000000)); 1155 1156 mutex_exit(&fptr->fcip_mutex); 1157 mutex_enter(&fcip_global_mutex); 1158 fcip_num_attaching--; 1159 ASSERT(fcip_num_attaching >= 0); 1160 mutex_exit(&fcip_global_mutex); 1161 rval = FC_SUCCESS; 1162 return (rval); 1163 1164 port_attach_cleanup: 1165 mutex_enter(&fcip_global_mutex); 1166 (void) fcip_softstate_free(fport); 1167 fcip_num_attaching--; 1168 ASSERT(fcip_num_attaching >= 0); 1169 mutex_exit(&fcip_global_mutex); 1170 rval = FC_FAILURE; 1171 return (rval); 1172 } 1173 1174 1175 /* 1176 * Handler for DDI_RESUME operations. Port must be ready to restart IP 1177 * traffic on resume 1178 */ 1179 static int 1180 fcip_handle_resume(fcip_port_info_t *fport, fc_ulp_port_info_t *port_info, 1181 fc_attach_cmd_t cmd) 1182 { 1183 int rval = FC_SUCCESS; 1184 struct fcip *fptr = fport->fcipp_fcip; 1185 struct fcipstr *tslp; 1186 int index; 1187 1188 1189 ASSERT(fptr != NULL); 1190 1191 mutex_enter(&fptr->fcip_mutex); 1192 1193 if (cmd == FC_CMD_POWER_UP) { 1194 fptr->fcip_flags &= ~(FCIP_POWER_DOWN); 1195 if (fptr->fcip_flags & FCIP_SUSPENDED) { 1196 mutex_exit(&fptr->fcip_mutex); 1197 return (FC_SUCCESS); 1198 } 1199 } else if (cmd == FC_CMD_RESUME) { 1200 fptr->fcip_flags &= ~(FCIP_SUSPENDED); 1201 } else { 1202 mutex_exit(&fptr->fcip_mutex); 1203 return (FC_FAILURE); 1204 } 1205 1206 /* 1207 * set the current port state and topology 1208 */ 1209 fport->fcipp_topology = port_info->port_flags; 1210 fport->fcipp_pstate = port_info->port_state; 1211 1212 rw_enter(&fcipstruplock, RW_READER); 1213 for (tslp = fcipstrup; tslp; tslp = tslp->sl_nextp) { 1214 if (tslp->sl_fcip == fptr) { 1215 break; 1216 } 1217 } 1218 rw_exit(&fcipstruplock); 1219 1220 /* 1221 * No active streams on this port 1222 */ 1223 if (tslp == NULL) { 1224 rval = FC_SUCCESS; 1225 goto done; 1226 } 1227 1228 mutex_enter(&fptr->fcip_rt_mutex); 1229 for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) { 1230 struct fcip_routing_table *frp; 1231 1232 frp = fptr->fcip_rtable[index]; 1233 while (frp) { 1234 uint32_t did; 1235 /* 1236 * Mark the broadcast RTE available again. It 1237 * was marked SUSPENDED during SUSPEND. 1238 */ 1239 did = fcip_get_broadcast_did(fptr); 1240 if (frp->fcipr_d_id.port_id == did) { 1241 frp->fcipr_state = 0; 1242 index = FCIP_RT_HASH_ELEMS; 1243 break; 1244 } 1245 frp = frp->fcipr_next; 1246 } 1247 } 1248 mutex_exit(&fptr->fcip_rt_mutex); 1249 1250 /* 1251 * fcip_handle_topology will update the port entries in the 1252 * routing table. 1253 * fcip_handle_topology also takes care of resetting the 1254 * fcipr_state field in the routing table structure. The entries 1255 * were set to RT_INVALID during suspend. 1256 */ 1257 fcip_handle_topology(fptr); 1258 1259 done: 1260 /* 1261 * Restart the timeout thread 1262 */ 1263 fptr->fcip_timeout_id = timeout(fcip_timeout, fptr, 1264 drv_usectohz(1000000)); 1265 mutex_exit(&fptr->fcip_mutex); 1266 return (rval); 1267 } 1268 1269 1270 /* 1271 * Insert a destination port entry into the routing table for 1272 * this port 1273 */ 1274 static void 1275 fcip_rt_update(struct fcip *fptr, fc_portmap_t *devlist, uint32_t listlen) 1276 { 1277 struct fcip_routing_table *frp; 1278 fcip_port_info_t *fport = fptr->fcip_port_info; 1279 int hash_bucket, i; 1280 fc_portmap_t *pmap; 1281 char wwn_buf[20]; 1282 1283 FCIP_TNF_PROBE_2((fcip_rt_update, "fcip io", /* CSTYLED */, 1284 tnf_string, msg, "enter", 1285 tnf_int, listlen, listlen)); 1286 1287 ASSERT(!mutex_owned(&fptr->fcip_mutex)); 1288 mutex_enter(&fptr->fcip_rt_mutex); 1289 1290 for (i = 0; i < listlen; i++) { 1291 pmap = &(devlist[i]); 1292 1293 frp = fcip_lookup_rtable(fptr, &(pmap->map_pwwn), 1294 FCIP_COMPARE_PWWN); 1295 /* 1296 * If an entry for a port in the devlist exists in the 1297 * in the per port routing table, make sure the data 1298 * is current. We need to do this irrespective of the 1299 * underlying port topology. 1300 */ 1301 switch (pmap->map_type) { 1302 /* FALLTHROUGH */ 1303 case PORT_DEVICE_NOCHANGE: 1304 /* FALLTHROUGH */ 1305 case PORT_DEVICE_USER_LOGIN: 1306 /* FALLTHROUGH */ 1307 case PORT_DEVICE_CHANGED: 1308 /* FALLTHROUGH */ 1309 case PORT_DEVICE_NEW: 1310 if (frp == NULL) { 1311 goto add_new_entry; 1312 } else if (frp) { 1313 goto update_entry; 1314 } else { 1315 continue; 1316 } 1317 1318 case PORT_DEVICE_OLD: 1319 /* FALLTHROUGH */ 1320 case PORT_DEVICE_USER_LOGOUT: 1321 /* 1322 * Mark entry for removal from Routing Table if 1323 * one exists. Let the timeout thread actually 1324 * remove the entry after we've given up hopes 1325 * of the port ever showing up. 1326 */ 1327 if (frp) { 1328 uint32_t did; 1329 1330 /* 1331 * Mark the routing table as invalid to bail 1332 * the packets early that are in transit 1333 */ 1334 did = fptr->fcip_broadcast_did; 1335 if (frp->fcipr_d_id.port_id != did) { 1336 frp->fcipr_pd = NULL; 1337 frp->fcipr_state = FCIP_RT_INVALID; 1338 frp->fcipr_invalid_timeout = 1339 fptr->fcip_timeout_ticks + 1340 FCIP_RTE_TIMEOUT; 1341 } 1342 } 1343 continue; 1344 1345 default: 1346 FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN, 1347 "unknown map flags in rt_update")); 1348 continue; 1349 } 1350 add_new_entry: 1351 ASSERT(frp == NULL); 1352 hash_bucket = FCIP_RT_HASH(pmap->map_pwwn.raw_wwn); 1353 1354 ASSERT(hash_bucket < FCIP_RT_HASH_ELEMS); 1355 1356 FCIP_TNF_PROBE_2((fcip_rt_update, "cfip io", /* CSTYLED */, 1357 tnf_string, msg, 1358 "add new entry", 1359 tnf_int, hashbucket, hash_bucket)); 1360 1361 frp = (struct fcip_routing_table *) 1362 kmem_zalloc(sizeof (struct fcip_routing_table), KM_SLEEP); 1363 /* insert at beginning of hash bucket */ 1364 frp->fcipr_next = fptr->fcip_rtable[hash_bucket]; 1365 fptr->fcip_rtable[hash_bucket] = frp; 1366 fc_wwn_to_str(&pmap->map_pwwn, wwn_buf); 1367 FCIP_DEBUG(FCIP_DEBUG_ATTACH, (CE_NOTE, 1368 "added entry for pwwn %s and d_id 0x%x", 1369 wwn_buf, pmap->map_did.port_id)); 1370 update_entry: 1371 bcopy((void *)&pmap->map_pwwn, 1372 (void *)&frp->fcipr_pwwn, sizeof (la_wwn_t)); 1373 bcopy((void *)&pmap->map_nwwn, (void *)&frp->fcipr_nwwn, 1374 sizeof (la_wwn_t)); 1375 frp->fcipr_d_id = pmap->map_did; 1376 frp->fcipr_state = pmap->map_state; 1377 frp->fcipr_pd = pmap->map_pd; 1378 1379 /* 1380 * If there is no pd for a destination port that is not 1381 * a broadcast entry, the port is pretty much unusable - so 1382 * mark the port for removal so we can try adding back the 1383 * entry again. 1384 */ 1385 if ((frp->fcipr_pd == NULL) && 1386 (frp->fcipr_d_id.port_id != fptr->fcip_broadcast_did)) { 1387 frp->fcipr_state = PORT_DEVICE_INVALID; 1388 frp->fcipr_invalid_timeout = fptr->fcip_timeout_ticks + 1389 (FCIP_RTE_TIMEOUT / 2); 1390 } 1391 frp->fcipr_fca_dev = 1392 fc_ulp_get_fca_device(fport->fcipp_handle, pmap->map_did); 1393 1394 /* 1395 * login to the remote port. Don't worry about 1396 * plogi failures for now 1397 */ 1398 if (pmap->map_pd != NULL) { 1399 (void) fcip_do_plogi(fptr, frp); 1400 } else if (FC_TOP_EXTERNAL(fport->fcipp_topology)) { 1401 fc_wwn_to_str(&frp->fcipr_pwwn, wwn_buf); 1402 FCIP_DEBUG(FCIP_DEBUG_MISC, (CE_NOTE, 1403 "logging into pwwn %s, d_id 0x%x", 1404 wwn_buf, frp->fcipr_d_id.port_id)); 1405 (void) fcip_do_plogi(fptr, frp); 1406 } 1407 1408 FCIP_TNF_BYTE_ARRAY(fcip_rt_update, "fcip io", "detail", 1409 "new wwn in rt", pwwn, 1410 &frp->fcipr_pwwn, sizeof (la_wwn_t)); 1411 } 1412 mutex_exit(&fptr->fcip_rt_mutex); 1413 } 1414 1415 1416 /* 1417 * return a matching routing table entry for a given fcip instance 1418 */ 1419 struct fcip_routing_table * 1420 fcip_lookup_rtable(struct fcip *fptr, la_wwn_t *wwn, int matchflag) 1421 { 1422 struct fcip_routing_table *frp = NULL; 1423 int hash_bucket; 1424 1425 1426 FCIP_TNF_PROBE_1((fcip_lookup_rtable, "fcip io", /* CSTYLED */, 1427 tnf_string, msg, "enter")); 1428 FCIP_TNF_BYTE_ARRAY(fcip_lookup_rtable, "fcip io", "detail", 1429 "rtable lookup for", wwn, 1430 &wwn->raw_wwn, sizeof (la_wwn_t)); 1431 FCIP_TNF_PROBE_2((fcip_lookup_rtable, "fcip io", /* CSTYLED */, 1432 tnf_string, msg, "match by", 1433 tnf_int, matchflag, matchflag)); 1434 1435 ASSERT(mutex_owned(&fptr->fcip_rt_mutex)); 1436 1437 hash_bucket = FCIP_RT_HASH(wwn->raw_wwn); 1438 frp = fptr->fcip_rtable[hash_bucket]; 1439 while (frp != NULL) { 1440 1441 FCIP_TNF_BYTE_ARRAY(fcip_lookup_rtable, "fcip io", "detail", 1442 "rtable entry", nwwn, 1443 &(frp->fcipr_nwwn.raw_wwn), sizeof (la_wwn_t)); 1444 1445 if (fcip_wwn_compare(&frp->fcipr_pwwn, wwn, matchflag) == 0) { 1446 break; 1447 } 1448 1449 frp = frp->fcipr_next; 1450 } 1451 FCIP_TNF_PROBE_2((fcip_lookup_rtable, "fcip io", /* CSTYLED */, 1452 tnf_string, msg, "lookup result", 1453 tnf_opaque, frp, frp)); 1454 return (frp); 1455 } 1456 1457 /* 1458 * Attach of fcip under pseudo. The actual setup of the interface 1459 * actually happens in fcip_port_attach on a callback from the 1460 * transport. The port_attach callback however can proceed only 1461 * after the devinfo for fcip has been created under pseudo 1462 */ 1463 static int 1464 fcip_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 1465 { 1466 switch ((int)cmd) { 1467 1468 case DDI_ATTACH: { 1469 ASSERT(fcip_module_dip == NULL); 1470 fcip_module_dip = dip; 1471 1472 /* 1473 * this call originates as a result of fcip's conf 1474 * file entry and will result in a fcip instance being 1475 * a child of pseudo. We should ensure here that the port 1476 * driver (fp) has been loaded and initted since we would 1477 * never get a port attach callback without fp being loaded. 1478 * If we are unable to succesfully load and initalize fp - 1479 * just fail this attach. 1480 */ 1481 mutex_enter(&fcip_global_mutex); 1482 1483 FCIP_DEBUG(FCIP_DEBUG_ATTACH, 1484 (CE_WARN, "global cv - signaling")); 1485 1486 cv_signal(&fcip_global_cv); 1487 1488 FCIP_DEBUG(FCIP_DEBUG_ATTACH, 1489 (CE_WARN, "global cv - signaled")); 1490 mutex_exit(&fcip_global_mutex); 1491 return (DDI_SUCCESS); 1492 } 1493 case DDI_RESUME: 1494 /* 1495 * Resume appears trickier 1496 */ 1497 return (DDI_SUCCESS); 1498 default: 1499 return (DDI_FAILURE); 1500 } 1501 } 1502 1503 1504 /* 1505 * The detach entry point to permit unloading fcip. We make sure 1506 * there are no active streams before we proceed with the detach 1507 */ 1508 /* ARGSUSED */ 1509 static int 1510 fcip_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 1511 { 1512 struct fcip *fptr; 1513 fcip_port_info_t *fport; 1514 int detached; 1515 1516 switch (cmd) { 1517 case DDI_DETACH: { 1518 /* 1519 * If we got here, any active streams should have been 1520 * unplumbed but check anyway 1521 */ 1522 mutex_enter(&fcip_global_mutex); 1523 if (fcipstrup != NULL) { 1524 mutex_exit(&fcip_global_mutex); 1525 return (DDI_FAILURE); 1526 } 1527 1528 if (fcip_port_head != NULL) { 1529 /* 1530 * Check to see if we have unattached/unbound 1531 * ports. If all the ports are unattached/unbound go 1532 * ahead and unregister with the transport 1533 */ 1534 fport = fcip_port_head; 1535 while (fport != NULL) { 1536 fptr = fport->fcipp_fcip; 1537 if (fptr == NULL) { 1538 continue; 1539 } 1540 mutex_enter(&fptr->fcip_mutex); 1541 fptr->fcip_flags |= FCIP_DETACHING; 1542 if (fptr->fcip_ipq || 1543 fptr->fcip_flags & (FCIP_IN_TIMEOUT | 1544 FCIP_IN_CALLBACK | FCIP_ATTACHING | 1545 FCIP_SUSPENDED | FCIP_POWER_DOWN | 1546 FCIP_REG_INPROGRESS)) { 1547 FCIP_TNF_PROBE_1((fcip_detach, 1548 "fcip io", /* CSTYLED */, 1549 tnf_string, msg, 1550 "fcip instance busy")); 1551 1552 mutex_exit(&fptr->fcip_mutex); 1553 FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN, 1554 "fcip instance busy")); 1555 break; 1556 } 1557 /* 1558 * Check for any outstanding pkts. If yes 1559 * fail the detach 1560 */ 1561 mutex_enter(&fptr->fcip_dest_mutex); 1562 if (fcip_port_get_num_pkts(fptr) > 0) { 1563 mutex_exit(&fptr->fcip_dest_mutex); 1564 mutex_exit(&fptr->fcip_mutex); 1565 FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN, 1566 "fcip instance busy - pkts " 1567 "pending")); 1568 break; 1569 } 1570 mutex_exit(&fptr->fcip_dest_mutex); 1571 1572 mutex_enter(&fptr->fcip_rt_mutex); 1573 if (fcip_plogi_in_progress(fptr)) { 1574 mutex_exit(&fptr->fcip_rt_mutex); 1575 mutex_exit(&fptr->fcip_mutex); 1576 FCIP_DEBUG(FCIP_DEBUG_DETACH, (CE_WARN, 1577 "fcip instance busy - plogi in " 1578 "progress")); 1579 break; 1580 } 1581 mutex_exit(&fptr->fcip_rt_mutex); 1582 1583 mutex_exit(&fptr->fcip_mutex); 1584 fport = fport->fcipp_next; 1585 } 1586 /* 1587 * if fport is non NULL - we have active ports 1588 */ 1589 if (fport != NULL) { 1590 /* 1591 * Remove the DETACHING flags on the ports 1592 */ 1593 fport = fcip_port_head; 1594 while (fport != NULL) { 1595 fptr = fport->fcipp_fcip; 1596 mutex_enter(&fptr->fcip_mutex); 1597 fptr->fcip_flags &= ~(FCIP_DETACHING); 1598 mutex_exit(&fptr->fcip_mutex); 1599 fport = fport->fcipp_next; 1600 } 1601 mutex_exit(&fcip_global_mutex); 1602 return (DDI_FAILURE); 1603 } 1604 } 1605 1606 /* 1607 * free up all softstate structures 1608 */ 1609 fport = fcip_port_head; 1610 while (fport != NULL) { 1611 detached = 1; 1612 1613 fptr = fport->fcipp_fcip; 1614 if (fptr) { 1615 mutex_enter(&fptr->fcip_mutex); 1616 /* 1617 * Check to see if somebody beat us to the 1618 * punch 1619 */ 1620 detached = fptr->fcip_flags & FCIP_DETACHED; 1621 fptr->fcip_flags &= ~(FCIP_DETACHING); 1622 fptr->fcip_flags |= FCIP_DETACHED; 1623 mutex_exit(&fptr->fcip_mutex); 1624 } 1625 1626 if (!detached) { 1627 fport = fcip_softstate_free(fport); 1628 } else { 1629 /* 1630 * If the port was marked as detached 1631 * but it was still in the list, that 1632 * means another thread has marked it 1633 * but we got in while it released the 1634 * fcip_global_mutex in softstate_free. 1635 * Given that, we're still safe to use 1636 * fport->fcipp_next to find out what 1637 * the next port on the list is. 1638 */ 1639 fport = fport->fcipp_next; 1640 } 1641 1642 FCIP_DEBUG(FCIP_DEBUG_DETACH, 1643 (CE_NOTE, "detaching port")); 1644 1645 FCIP_TNF_PROBE_1((fcip_detach, 1646 "fcip io", /* CSTYLED */, tnf_string, 1647 msg, "detaching port")); 1648 } 1649 1650 /* 1651 * If we haven't removed all the port structures, we 1652 * aren't yet ready to be detached. 1653 */ 1654 if (fcip_port_head != NULL) { 1655 mutex_exit(&fcip_global_mutex); 1656 return (DDI_FAILURE); 1657 } 1658 1659 fcip_num_instances = 0; 1660 mutex_exit(&fcip_global_mutex); 1661 fcip_module_dip = NULL; 1662 return (DDI_SUCCESS); 1663 } 1664 case DDI_SUSPEND: 1665 return (DDI_SUCCESS); 1666 default: 1667 return (DDI_FAILURE); 1668 } 1669 } 1670 1671 /* 1672 * The port_detach callback is called from the transport when a 1673 * FC port is being removed from the transport's control. This routine 1674 * provides fcip with an opportunity to cleanup all activities and 1675 * structures on the port marked for removal. 1676 */ 1677 /* ARGSUSED */ 1678 static int 1679 fcip_port_detach(opaque_t ulp_handle, fc_ulp_port_info_t *port_info, 1680 fc_detach_cmd_t cmd) 1681 { 1682 int rval = FC_FAILURE; 1683 fcip_port_info_t *fport; 1684 struct fcip *fptr; 1685 struct fcipstr *strp; 1686 1687 switch (cmd) { 1688 case FC_CMD_DETACH: { 1689 mutex_enter(&fcip_global_mutex); 1690 1691 if (fcip_port_head == NULL) { 1692 /* 1693 * we are all done but our fini has not been 1694 * called yet!! Let's hope we have no active 1695 * fcip instances here. - strange secnario but 1696 * no harm in having this return a success. 1697 */ 1698 fcip_check_remove_minor_node(); 1699 1700 mutex_exit(&fcip_global_mutex); 1701 return (FC_SUCCESS); 1702 } else { 1703 /* 1704 * traverse the port list 1705 */ 1706 fport = fcip_port_head; 1707 while (fport != NULL) { 1708 if (fport->fcipp_handle == 1709 port_info->port_handle) { 1710 fptr = fport->fcipp_fcip; 1711 1712 /* 1713 * Fail the port detach if there is 1714 * still an attached, bound stream on 1715 * this interface. 1716 */ 1717 1718 rw_enter(&fcipstruplock, RW_READER); 1719 1720 for (strp = fcipstrup; strp != NULL; 1721 strp = strp->sl_nextp) { 1722 if (strp->sl_fcip == fptr) { 1723 rw_exit(&fcipstruplock); 1724 mutex_exit( 1725 &fcip_global_mutex); 1726 return (FC_FAILURE); 1727 } 1728 } 1729 1730 rw_exit(&fcipstruplock); 1731 1732 /* 1733 * fail port detach if we are in 1734 * the middle of a deferred port attach 1735 * or if the port has outstanding pkts 1736 */ 1737 if (fptr != NULL) { 1738 mutex_enter(&fptr->fcip_mutex); 1739 if (fcip_check_port_busy 1740 (fptr) || 1741 (fptr->fcip_flags & 1742 FCIP_DETACHED)) { 1743 mutex_exit( 1744 &fptr->fcip_mutex); 1745 mutex_exit( 1746 &fcip_global_mutex); 1747 return (FC_FAILURE); 1748 } 1749 1750 fptr->fcip_flags |= 1751 FCIP_DETACHED; 1752 mutex_exit(&fptr->fcip_mutex); 1753 } 1754 (void) fcip_softstate_free(fport); 1755 1756 fcip_check_remove_minor_node(); 1757 mutex_exit(&fcip_global_mutex); 1758 return (FC_SUCCESS); 1759 } 1760 fport = fport->fcipp_next; 1761 } 1762 ASSERT(fport == NULL); 1763 } 1764 mutex_exit(&fcip_global_mutex); 1765 break; 1766 } 1767 case FC_CMD_POWER_DOWN: 1768 /* FALLTHROUGH */ 1769 case FC_CMD_SUSPEND: 1770 mutex_enter(&fcip_global_mutex); 1771 fport = fcip_port_head; 1772 while (fport != NULL) { 1773 if (fport->fcipp_handle == port_info->port_handle) { 1774 break; 1775 } 1776 fport = fport->fcipp_next; 1777 } 1778 if (fport == NULL) { 1779 mutex_exit(&fcip_global_mutex); 1780 break; 1781 } 1782 rval = fcip_handle_suspend(fport, cmd); 1783 mutex_exit(&fcip_global_mutex); 1784 break; 1785 default: 1786 FCIP_DEBUG(FCIP_DEBUG_DETACH, 1787 (CE_WARN, "unknown port detach command!!")); 1788 break; 1789 } 1790 return (rval); 1791 } 1792 1793 1794 /* 1795 * Returns 0 if the port is not busy, else returns non zero. 1796 */ 1797 static int 1798 fcip_check_port_busy(struct fcip *fptr) 1799 { 1800 int rval = 0, num_pkts = 0; 1801 1802 ASSERT(fptr != NULL); 1803 ASSERT(MUTEX_HELD(&fptr->fcip_mutex)); 1804 1805 mutex_enter(&fptr->fcip_dest_mutex); 1806 1807 if (fptr->fcip_flags & FCIP_PORT_BUSY || 1808 ((num_pkts = fcip_port_get_num_pkts(fptr)) > 0) || 1809 fptr->fcip_num_ipkts_pending) { 1810 rval = 1; 1811 FCIP_DEBUG(FCIP_DEBUG_DETACH, 1812 (CE_NOTE, "!fcip_check_port_busy: port is busy " 1813 "fcip_flags: 0x%x, num_pkts: 0x%x, ipkts_pending: 0x%lx!", 1814 fptr->fcip_flags, num_pkts, fptr->fcip_num_ipkts_pending)); 1815 } 1816 1817 mutex_exit(&fptr->fcip_dest_mutex); 1818 return (rval); 1819 } 1820 1821 /* 1822 * Helper routine to remove fcip's minor node 1823 * There is one minor node per system and it should be removed if there are no 1824 * other fcip instances (which has a 1:1 mapping for fp instances) present 1825 */ 1826 static void 1827 fcip_check_remove_minor_node(void) 1828 { 1829 ASSERT(MUTEX_HELD(&fcip_global_mutex)); 1830 1831 /* 1832 * If there are no more fcip (fp) instances, remove the 1833 * minor node for fcip. 1834 * Reset fcip_minor_node_created to invalidate it. 1835 */ 1836 if (fcip_num_instances == 0 && (fcip_module_dip != NULL)) { 1837 ddi_remove_minor_node(fcip_module_dip, NULL); 1838 fcip_minor_node_created = 0; 1839 } 1840 } 1841 1842 /* 1843 * This routine permits the suspend operation during a CPR/System 1844 * power management operation. The routine basically quiesces I/Os 1845 * on all active interfaces 1846 */ 1847 static int 1848 fcip_handle_suspend(fcip_port_info_t *fport, fc_detach_cmd_t cmd) 1849 { 1850 struct fcip *fptr = fport->fcipp_fcip; 1851 timeout_id_t tid; 1852 int index; 1853 int tryagain = 0; 1854 int count; 1855 struct fcipstr *tslp; 1856 1857 1858 ASSERT(fptr != NULL); 1859 mutex_enter(&fptr->fcip_mutex); 1860 1861 /* 1862 * Fail if we are in the middle of a callback. Don't use delay during 1863 * suspend since clock intrs are not available so busy wait 1864 */ 1865 count = 0; 1866 while (count++ < 15 && 1867 ((fptr->fcip_flags & FCIP_IN_CALLBACK) || 1868 (fptr->fcip_flags & FCIP_IN_TIMEOUT))) { 1869 mutex_exit(&fptr->fcip_mutex); 1870 drv_usecwait(1000000); 1871 mutex_enter(&fptr->fcip_mutex); 1872 } 1873 1874 if (fptr->fcip_flags & FCIP_IN_CALLBACK || 1875 fptr->fcip_flags & FCIP_IN_TIMEOUT) { 1876 mutex_exit(&fptr->fcip_mutex); 1877 return (FC_FAILURE); 1878 } 1879 1880 if (cmd == FC_CMD_POWER_DOWN) { 1881 if (fptr->fcip_flags & FCIP_SUSPENDED) { 1882 fptr->fcip_flags |= FCIP_POWER_DOWN; 1883 mutex_exit(&fptr->fcip_mutex); 1884 goto success; 1885 } else { 1886 fptr->fcip_flags |= FCIP_POWER_DOWN; 1887 } 1888 } else if (cmd == FC_CMD_SUSPEND) { 1889 fptr->fcip_flags |= FCIP_SUSPENDED; 1890 } else { 1891 mutex_exit(&fptr->fcip_mutex); 1892 return (FC_FAILURE); 1893 } 1894 1895 mutex_exit(&fptr->fcip_mutex); 1896 /* 1897 * If no streams are plumbed - its the easiest case - Just 1898 * bail out without having to do much 1899 */ 1900 1901 rw_enter(&fcipstruplock, RW_READER); 1902 for (tslp = fcipstrup; tslp; tslp = tslp->sl_nextp) { 1903 if (tslp->sl_fcip == fptr) { 1904 break; 1905 } 1906 } 1907 rw_exit(&fcipstruplock); 1908 1909 /* 1910 * No active streams on this port 1911 */ 1912 if (tslp == NULL) { 1913 goto success; 1914 } 1915 1916 /* 1917 * Walk through each Routing table structure and check if 1918 * the destination table has any outstanding commands. If yes 1919 * wait for the commands to drain. Since we go through each 1920 * routing table entry in succession, it may be wise to wait 1921 * only a few seconds for each entry. 1922 */ 1923 mutex_enter(&fptr->fcip_rt_mutex); 1924 while (!tryagain) { 1925 1926 tryagain = 0; 1927 for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) { 1928 struct fcip_routing_table *frp; 1929 struct fcip_dest *fdestp; 1930 la_wwn_t *pwwn; 1931 int hash_bucket; 1932 1933 frp = fptr->fcip_rtable[index]; 1934 while (frp) { 1935 /* 1936 * Mark the routing table as SUSPENDED. Even 1937 * mark the broadcast entry SUSPENDED to 1938 * prevent any ARP or other broadcasts. We 1939 * can reset the state of the broadcast 1940 * RTE when we resume. 1941 */ 1942 frp->fcipr_state = FCIP_RT_SUSPENDED; 1943 pwwn = &frp->fcipr_pwwn; 1944 1945 /* 1946 * Get hold of destination pointer 1947 */ 1948 mutex_enter(&fptr->fcip_dest_mutex); 1949 1950 hash_bucket = FCIP_DEST_HASH(pwwn->raw_wwn); 1951 ASSERT(hash_bucket < FCIP_DEST_HASH_ELEMS); 1952 1953 fdestp = fptr->fcip_dest[hash_bucket]; 1954 while (fdestp != NULL) { 1955 mutex_enter(&fdestp->fcipd_mutex); 1956 if (fdestp->fcipd_rtable) { 1957 if (fcip_wwn_compare(pwwn, 1958 &fdestp->fcipd_pwwn, 1959 FCIP_COMPARE_PWWN) == 0) { 1960 mutex_exit( 1961 &fdestp->fcipd_mutex); 1962 break; 1963 } 1964 } 1965 mutex_exit(&fdestp->fcipd_mutex); 1966 fdestp = fdestp->fcipd_next; 1967 } 1968 1969 mutex_exit(&fptr->fcip_dest_mutex); 1970 if (fdestp == NULL) { 1971 frp = frp->fcipr_next; 1972 continue; 1973 } 1974 1975 /* 1976 * Wait for fcip_wait_cmds seconds for 1977 * the commands to drain. 1978 */ 1979 count = 0; 1980 mutex_enter(&fdestp->fcipd_mutex); 1981 while (fdestp->fcipd_ncmds && 1982 count < fcip_wait_cmds) { 1983 mutex_exit(&fdestp->fcipd_mutex); 1984 mutex_exit(&fptr->fcip_rt_mutex); 1985 drv_usecwait(1000000); 1986 mutex_enter(&fptr->fcip_rt_mutex); 1987 mutex_enter(&fdestp->fcipd_mutex); 1988 count++; 1989 } 1990 /* 1991 * Check if we were able to drain all cmds 1992 * successfully. Else continue with other 1993 * ports and try during the second pass 1994 */ 1995 if (fdestp->fcipd_ncmds) { 1996 tryagain++; 1997 } 1998 mutex_exit(&fdestp->fcipd_mutex); 1999 2000 frp = frp->fcipr_next; 2001 } 2002 } 2003 if (tryagain == 0) { 2004 break; 2005 } 2006 } 2007 mutex_exit(&fptr->fcip_rt_mutex); 2008 2009 if (tryagain) { 2010 mutex_enter(&fptr->fcip_mutex); 2011 fptr->fcip_flags &= ~(FCIP_SUSPENDED | FCIP_POWER_DOWN); 2012 mutex_exit(&fptr->fcip_mutex); 2013 return (FC_FAILURE); 2014 } 2015 2016 success: 2017 mutex_enter(&fptr->fcip_mutex); 2018 tid = fptr->fcip_timeout_id; 2019 fptr->fcip_timeout_id = NULL; 2020 mutex_exit(&fptr->fcip_mutex); 2021 2022 (void) untimeout(tid); 2023 2024 return (FC_SUCCESS); 2025 } 2026 2027 /* 2028 * the getinfo(9E) entry point 2029 */ 2030 /* ARGSUSED */ 2031 static int 2032 fcip_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 2033 { 2034 int rval = DDI_FAILURE; 2035 2036 switch (cmd) { 2037 case DDI_INFO_DEVT2DEVINFO: 2038 *result = fcip_module_dip; 2039 if (*result) 2040 rval = DDI_SUCCESS; 2041 break; 2042 2043 case DDI_INFO_DEVT2INSTANCE: 2044 *result = (void *)0; 2045 rval = DDI_SUCCESS; 2046 break; 2047 default: 2048 break; 2049 } 2050 2051 return (rval); 2052 } 2053 2054 /* 2055 * called from fcip_attach to initialize kstats for the link 2056 */ 2057 /* ARGSUSED */ 2058 static void 2059 fcip_kstat_init(struct fcip *fptr) 2060 { 2061 int instance; 2062 char buf[16]; 2063 struct fcipstat *fcipstatp; 2064 2065 ASSERT(mutex_owned(&fptr->fcip_mutex)); 2066 2067 instance = ddi_get_instance(fptr->fcip_dip); 2068 (void) sprintf(buf, "fcip%d", instance); 2069 2070 #ifdef kstat 2071 fptr->fcip_kstatp = kstat_create("fcip", instance, buf, "net", 2072 KSTAT_TYPE_NAMED, 2073 (sizeof (struct fcipstat)/ sizeof (kstat_named_t)), 2074 KSTAT_FLAG_PERSISTENT); 2075 #else 2076 fptr->fcip_kstatp = kstat_create("fcip", instance, buf, "net", 2077 KSTAT_TYPE_NAMED, 2078 (sizeof (struct fcipstat)/ sizeof (kstat_named_t)), 0); 2079 #endif 2080 if (fptr->fcip_kstatp == NULL) { 2081 FCIP_DEBUG(FCIP_DEBUG_INIT, (CE_WARN, "kstat created failed")); 2082 return; 2083 } 2084 2085 fcipstatp = (struct fcipstat *)fptr->fcip_kstatp->ks_data; 2086 kstat_named_init(&fcipstatp->fcips_ipackets, "ipackets", 2087 KSTAT_DATA_ULONG); 2088 kstat_named_init(&fcipstatp->fcips_ierrors, "ierrors", 2089 KSTAT_DATA_ULONG); 2090 kstat_named_init(&fcipstatp->fcips_opackets, "opackets", 2091 KSTAT_DATA_ULONG); 2092 kstat_named_init(&fcipstatp->fcips_oerrors, "oerrors", 2093 KSTAT_DATA_ULONG); 2094 kstat_named_init(&fcipstatp->fcips_collisions, "collisions", 2095 KSTAT_DATA_ULONG); 2096 kstat_named_init(&fcipstatp->fcips_nocanput, "nocanput", 2097 KSTAT_DATA_ULONG); 2098 kstat_named_init(&fcipstatp->fcips_allocbfail, "allocbfail", 2099 KSTAT_DATA_ULONG); 2100 2101 kstat_named_init(&fcipstatp->fcips_defer, "defer", 2102 KSTAT_DATA_ULONG); 2103 kstat_named_init(&fcipstatp->fcips_fram, "fram", 2104 KSTAT_DATA_ULONG); 2105 kstat_named_init(&fcipstatp->fcips_crc, "crc", 2106 KSTAT_DATA_ULONG); 2107 kstat_named_init(&fcipstatp->fcips_oflo, "oflo", 2108 KSTAT_DATA_ULONG); 2109 kstat_named_init(&fcipstatp->fcips_uflo, "uflo", 2110 KSTAT_DATA_ULONG); 2111 kstat_named_init(&fcipstatp->fcips_missed, "missed", 2112 KSTAT_DATA_ULONG); 2113 kstat_named_init(&fcipstatp->fcips_tlcol, "tlcol", 2114 KSTAT_DATA_ULONG); 2115 kstat_named_init(&fcipstatp->fcips_trtry, "trtry", 2116 KSTAT_DATA_ULONG); 2117 kstat_named_init(&fcipstatp->fcips_tnocar, "tnocar", 2118 KSTAT_DATA_ULONG); 2119 kstat_named_init(&fcipstatp->fcips_inits, "inits", 2120 KSTAT_DATA_ULONG); 2121 kstat_named_init(&fcipstatp->fcips_notbufs, "notbufs", 2122 KSTAT_DATA_ULONG); 2123 kstat_named_init(&fcipstatp->fcips_norbufs, "norbufs", 2124 KSTAT_DATA_ULONG); 2125 kstat_named_init(&fcipstatp->fcips_allocbfail, "allocbfail", 2126 KSTAT_DATA_ULONG); 2127 2128 /* 2129 * required by kstat for MIB II objects(RFC 1213) 2130 */ 2131 kstat_named_init(&fcipstatp->fcips_rcvbytes, "fcips_rcvbytes", 2132 KSTAT_DATA_ULONG); /* # octets received */ 2133 /* MIB - ifInOctets */ 2134 kstat_named_init(&fcipstatp->fcips_xmtbytes, "fcips_xmtbytes", 2135 KSTAT_DATA_ULONG); /* # octets xmitted */ 2136 /* MIB - ifOutOctets */ 2137 kstat_named_init(&fcipstatp->fcips_multircv, "fcips_multircv", 2138 KSTAT_DATA_ULONG); /* # multicast packets */ 2139 /* delivered to upper layer */ 2140 /* MIB - ifInNUcastPkts */ 2141 kstat_named_init(&fcipstatp->fcips_multixmt, "fcips_multixmt", 2142 KSTAT_DATA_ULONG); /* # multicast packets */ 2143 /* requested to be sent */ 2144 /* MIB - ifOutNUcastPkts */ 2145 kstat_named_init(&fcipstatp->fcips_brdcstrcv, "fcips_brdcstrcv", 2146 KSTAT_DATA_ULONG); /* # broadcast packets */ 2147 /* delivered to upper layer */ 2148 /* MIB - ifInNUcastPkts */ 2149 kstat_named_init(&fcipstatp->fcips_brdcstxmt, "fcips_brdcstxmt", 2150 KSTAT_DATA_ULONG); /* # broadcast packets */ 2151 /* requested to be sent */ 2152 /* MIB - ifOutNUcastPkts */ 2153 kstat_named_init(&fcipstatp->fcips_norcvbuf, "fcips_norcvbuf", 2154 KSTAT_DATA_ULONG); /* # rcv packets discarded */ 2155 /* MIB - ifInDiscards */ 2156 kstat_named_init(&fcipstatp->fcips_noxmtbuf, "fcips_noxmtbuf", 2157 KSTAT_DATA_ULONG); /* # xmt packets discarded */ 2158 2159 fptr->fcip_kstatp->ks_update = fcip_stat_update; 2160 fptr->fcip_kstatp->ks_private = (void *) fptr; 2161 kstat_install(fptr->fcip_kstatp); 2162 } 2163 2164 /* 2165 * Update the defined kstats for netstat et al to use 2166 */ 2167 /* ARGSUSED */ 2168 static int 2169 fcip_stat_update(kstat_t *fcip_statp, int val) 2170 { 2171 struct fcipstat *fcipstatp; 2172 struct fcip *fptr; 2173 2174 fptr = (struct fcip *)fcip_statp->ks_private; 2175 fcipstatp = (struct fcipstat *)fcip_statp->ks_data; 2176 2177 if (val == KSTAT_WRITE) { 2178 fptr->fcip_ipackets = fcipstatp->fcips_ipackets.value.ul; 2179 fptr->fcip_ierrors = fcipstatp->fcips_ierrors.value.ul; 2180 fptr->fcip_opackets = fcipstatp->fcips_opackets.value.ul; 2181 fptr->fcip_oerrors = fcipstatp->fcips_oerrors.value.ul; 2182 fptr->fcip_collisions = fcipstatp->fcips_collisions.value.ul; 2183 fptr->fcip_defer = fcipstatp->fcips_defer.value.ul; 2184 fptr->fcip_fram = fcipstatp->fcips_fram.value.ul; 2185 fptr->fcip_crc = fcipstatp->fcips_crc.value.ul; 2186 fptr->fcip_oflo = fcipstatp->fcips_oflo.value.ul; 2187 fptr->fcip_uflo = fcipstatp->fcips_uflo.value.ul; 2188 fptr->fcip_missed = fcipstatp->fcips_missed.value.ul; 2189 fptr->fcip_tlcol = fcipstatp->fcips_tlcol.value.ul; 2190 fptr->fcip_trtry = fcipstatp->fcips_trtry.value.ul; 2191 fptr->fcip_tnocar = fcipstatp->fcips_tnocar.value.ul; 2192 fptr->fcip_inits = fcipstatp->fcips_inits.value.ul; 2193 fptr->fcip_notbufs = fcipstatp->fcips_notbufs.value.ul; 2194 fptr->fcip_norbufs = fcipstatp->fcips_norbufs.value.ul; 2195 fptr->fcip_nocanput = fcipstatp->fcips_nocanput.value.ul; 2196 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2197 fptr->fcip_rcvbytes = fcipstatp->fcips_rcvbytes.value.ul; 2198 fptr->fcip_xmtbytes = fcipstatp->fcips_xmtbytes.value.ul; 2199 fptr->fcip_multircv = fcipstatp->fcips_multircv.value.ul; 2200 fptr->fcip_multixmt = fcipstatp->fcips_multixmt.value.ul; 2201 fptr->fcip_brdcstrcv = fcipstatp->fcips_brdcstrcv.value.ul; 2202 fptr->fcip_norcvbuf = fcipstatp->fcips_norcvbuf.value.ul; 2203 fptr->fcip_noxmtbuf = fcipstatp->fcips_noxmtbuf.value.ul; 2204 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2205 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2206 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2207 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2208 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2209 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2210 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2211 fptr->fcip_allocbfail = fcipstatp->fcips_allocbfail.value.ul; 2212 2213 } else { 2214 fcipstatp->fcips_ipackets.value.ul = fptr->fcip_ipackets; 2215 fcipstatp->fcips_ierrors.value.ul = fptr->fcip_ierrors; 2216 fcipstatp->fcips_opackets.value.ul = fptr->fcip_opackets; 2217 fcipstatp->fcips_oerrors.value.ul = fptr->fcip_oerrors; 2218 fcipstatp->fcips_collisions.value.ul = fptr->fcip_collisions; 2219 fcipstatp->fcips_nocanput.value.ul = fptr->fcip_nocanput; 2220 fcipstatp->fcips_allocbfail.value.ul = fptr->fcip_allocbfail; 2221 fcipstatp->fcips_defer.value.ul = fptr->fcip_defer; 2222 fcipstatp->fcips_fram.value.ul = fptr->fcip_fram; 2223 fcipstatp->fcips_crc.value.ul = fptr->fcip_crc; 2224 fcipstatp->fcips_oflo.value.ul = fptr->fcip_oflo; 2225 fcipstatp->fcips_uflo.value.ul = fptr->fcip_uflo; 2226 fcipstatp->fcips_missed.value.ul = fptr->fcip_missed; 2227 fcipstatp->fcips_tlcol.value.ul = fptr->fcip_tlcol; 2228 fcipstatp->fcips_trtry.value.ul = fptr->fcip_trtry; 2229 fcipstatp->fcips_tnocar.value.ul = fptr->fcip_tnocar; 2230 fcipstatp->fcips_inits.value.ul = fptr->fcip_inits; 2231 fcipstatp->fcips_norbufs.value.ul = fptr->fcip_norbufs; 2232 fcipstatp->fcips_notbufs.value.ul = fptr->fcip_notbufs; 2233 fcipstatp->fcips_rcvbytes.value.ul = fptr->fcip_rcvbytes; 2234 fcipstatp->fcips_xmtbytes.value.ul = fptr->fcip_xmtbytes; 2235 fcipstatp->fcips_multircv.value.ul = fptr->fcip_multircv; 2236 fcipstatp->fcips_multixmt.value.ul = fptr->fcip_multixmt; 2237 fcipstatp->fcips_brdcstrcv.value.ul = fptr->fcip_brdcstrcv; 2238 fcipstatp->fcips_brdcstxmt.value.ul = fptr->fcip_brdcstxmt; 2239 fcipstatp->fcips_norcvbuf.value.ul = fptr->fcip_norcvbuf; 2240 fcipstatp->fcips_noxmtbuf.value.ul = fptr->fcip_noxmtbuf; 2241 2242 } 2243 return (0); 2244 } 2245 2246 2247 /* 2248 * fcip_statec_cb: handles all required state change callback notifications 2249 * it receives from the transport 2250 */ 2251 /* ARGSUSED */ 2252 static void 2253 fcip_statec_cb(opaque_t ulp_handle, opaque_t phandle, 2254 uint32_t port_state, uint32_t port_top, fc_portmap_t changelist[], 2255 uint32_t listlen, uint32_t sid) 2256 { 2257 fcip_port_info_t *fport; 2258 struct fcip *fptr; 2259 struct fcipstr *slp; 2260 queue_t *wrq; 2261 int instance; 2262 int index; 2263 struct fcip_routing_table *frtp; 2264 2265 fport = fcip_get_port(phandle); 2266 2267 if (fport == NULL) { 2268 return; 2269 } 2270 2271 fptr = fport->fcipp_fcip; 2272 ASSERT(fptr != NULL); 2273 2274 if (fptr == NULL) { 2275 return; 2276 } 2277 2278 instance = ddi_get_instance(fport->fcipp_dip); 2279 2280 FCIP_TNF_PROBE_4((fcip_statec_cb, "fcip io", /* CSTYLED */, 2281 tnf_string, msg, "state change callback", 2282 tnf_uint, instance, instance, 2283 tnf_uint, S_ID, sid, 2284 tnf_int, count, listlen)); 2285 FCIP_DEBUG(FCIP_DEBUG_ELS, 2286 (CE_NOTE, "fcip%d, state change callback: state:0x%x, " 2287 "S_ID:0x%x, count:0x%x", instance, port_state, sid, listlen)); 2288 2289 mutex_enter(&fptr->fcip_mutex); 2290 2291 if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) || 2292 (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) { 2293 mutex_exit(&fptr->fcip_mutex); 2294 return; 2295 } 2296 2297 /* 2298 * set fcip flags to indicate we are in the middle of a 2299 * state change callback so we can wait till the statechange 2300 * is handled before succeeding/failing the SUSPEND/POWER DOWN. 2301 */ 2302 fptr->fcip_flags |= FCIP_IN_SC_CB; 2303 2304 fport->fcipp_pstate = port_state; 2305 2306 /* 2307 * Check if topology changed. If Yes - Modify the broadcast 2308 * RTE entries to understand the new broadcast D_IDs 2309 */ 2310 if (fport->fcipp_topology != port_top && 2311 (port_top != FC_TOP_UNKNOWN)) { 2312 /* REMOVE later */ 2313 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE, 2314 "topology changed: Old topology: 0x%x New topology 0x%x", 2315 fport->fcipp_topology, port_top)); 2316 /* 2317 * If topology changed - attempt a rediscovery of 2318 * devices. Helps specially in Fabric/Public loops 2319 * and if on_demand_node_creation is disabled 2320 */ 2321 fport->fcipp_topology = port_top; 2322 fcip_handle_topology(fptr); 2323 } 2324 2325 mutex_exit(&fptr->fcip_mutex); 2326 2327 switch (FC_PORT_STATE_MASK(port_state)) { 2328 case FC_STATE_ONLINE: 2329 /* FALLTHROUGH */ 2330 case FC_STATE_LIP: 2331 /* FALLTHROUGH */ 2332 case FC_STATE_LIP_LBIT_SET: 2333 2334 /* 2335 * nothing to do here actually other than if we 2336 * were actually logged onto a port in the devlist 2337 * (which indicates active communication between 2338 * the host port and the port in the changelist). 2339 * If however we are in a private loop or point to 2340 * point mode, we need to check for any IP capable 2341 * ports and update our routing table. 2342 */ 2343 switch (port_top) { 2344 case FC_TOP_FABRIC: 2345 /* 2346 * This indicates a fabric port with a NameServer. 2347 * Check the devlist to see if we are in active 2348 * communication with a port on the devlist. 2349 */ 2350 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE, 2351 "Statec_cb: fabric topology")); 2352 fcip_rt_update(fptr, changelist, listlen); 2353 break; 2354 case FC_TOP_NO_NS: 2355 /* 2356 * No nameserver - so treat it like a Private loop 2357 * or point to point topology and get a map of 2358 * devices on the link and get IP capable ports to 2359 * to update the routing table. 2360 */ 2361 FCIP_DEBUG(FCIP_DEBUG_ELS, 2362 (CE_NOTE, "Statec_cb: NO_NS topology")); 2363 /* FALLTHROUGH */ 2364 case FC_TOP_PRIVATE_LOOP: 2365 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE, 2366 "Statec_cb: Pvt_Loop topology")); 2367 /* FALLTHROUGH */ 2368 case FC_TOP_PT_PT: 2369 /* 2370 * call get_port_map() and update routing table 2371 */ 2372 fcip_rt_update(fptr, changelist, listlen); 2373 break; 2374 default: 2375 FCIP_DEBUG(FCIP_DEBUG_ELS, 2376 (CE_NOTE, "Statec_cb: Unknown topology")); 2377 } 2378 2379 /* 2380 * We should now enable the Queues and permit I/Os 2381 * to flow through downstream. The update of routing 2382 * table should have flushed out any port entries that 2383 * don't exist or are not available after the state change 2384 */ 2385 mutex_enter(&fptr->fcip_mutex); 2386 fptr->fcip_port_state = FCIP_PORT_ONLINE; 2387 if (fptr->fcip_flags & FCIP_LINK_DOWN) { 2388 fptr->fcip_flags &= ~FCIP_LINK_DOWN; 2389 } 2390 mutex_exit(&fptr->fcip_mutex); 2391 2392 /* 2393 * Enable write queues 2394 */ 2395 rw_enter(&fcipstruplock, RW_READER); 2396 for (slp = fcipstrup; slp != NULL; slp = slp->sl_nextp) { 2397 if (slp && slp->sl_fcip == fptr) { 2398 wrq = WR(slp->sl_rq); 2399 if (wrq->q_flag & QFULL) { 2400 qenable(wrq); 2401 } 2402 } 2403 } 2404 rw_exit(&fcipstruplock); 2405 break; 2406 case FC_STATE_OFFLINE: 2407 /* 2408 * mark the port_state OFFLINE and wait for it to 2409 * become online. Any new messages in this state will 2410 * simply be queued back up. If the port does not 2411 * come online in a short while, we can begin failing 2412 * messages and flush the routing table 2413 */ 2414 mutex_enter(&fptr->fcip_mutex); 2415 fptr->fcip_mark_offline = fptr->fcip_timeout_ticks + 2416 FCIP_OFFLINE_TIMEOUT; 2417 fptr->fcip_port_state = FCIP_PORT_OFFLINE; 2418 mutex_exit(&fptr->fcip_mutex); 2419 2420 /* 2421 * Mark all Routing table entries as invalid to prevent 2422 * any commands from trickling through to ports that 2423 * have disappeared from under us 2424 */ 2425 mutex_enter(&fptr->fcip_rt_mutex); 2426 for (index = 0; index < FCIP_RT_HASH_ELEMS; index++) { 2427 frtp = fptr->fcip_rtable[index]; 2428 while (frtp) { 2429 frtp->fcipr_state = PORT_DEVICE_INVALID; 2430 frtp = frtp->fcipr_next; 2431 } 2432 } 2433 mutex_exit(&fptr->fcip_rt_mutex); 2434 2435 break; 2436 2437 case FC_STATE_RESET_REQUESTED: 2438 /* 2439 * Release all Unsolicited buffers back to transport/FCA. 2440 * This also means the port state is marked offline - so 2441 * we may have to do what OFFLINE state requires us to do. 2442 * Care must be taken to wait for any active unsolicited 2443 * buffer with the other Streams modules - so wait for 2444 * a freeb if the unsolicited buffer is passed back all 2445 * the way upstream. 2446 */ 2447 mutex_enter(&fptr->fcip_mutex); 2448 2449 #ifdef FCIP_ESBALLOC 2450 while (fptr->fcip_ub_upstream) { 2451 cv_wait(&fptr->fcip_ub_cv, &fptr->fcip_mutex); 2452 } 2453 #endif /* FCIP_ESBALLOC */ 2454 2455 fptr->fcip_mark_offline = fptr->fcip_timeout_ticks + 2456 FCIP_OFFLINE_TIMEOUT; 2457 fptr->fcip_port_state = FCIP_PORT_OFFLINE; 2458 mutex_exit(&fptr->fcip_mutex); 2459 break; 2460 2461 case FC_STATE_DEVICE_CHANGE: 2462 if (listlen) { 2463 fcip_rt_update(fptr, changelist, listlen); 2464 } 2465 break; 2466 case FC_STATE_RESET: 2467 /* 2468 * Not much to do I guess - wait for port to become 2469 * ONLINE. If the port doesn't become online in a short 2470 * while, the upper layers abort any request themselves. 2471 * We can just putback the messages in the streams queues 2472 * if the link is offline 2473 */ 2474 break; 2475 } 2476 mutex_enter(&fptr->fcip_mutex); 2477 fptr->fcip_flags &= ~(FCIP_IN_SC_CB); 2478 mutex_exit(&fptr->fcip_mutex); 2479 } 2480 2481 /* 2482 * Given a port handle, return the fcip_port_info structure corresponding 2483 * to that port handle. The transport allocates and communicates with 2484 * ULPs using port handles 2485 */ 2486 static fcip_port_info_t * 2487 fcip_get_port(opaque_t phandle) 2488 { 2489 fcip_port_info_t *fport; 2490 2491 ASSERT(phandle != NULL); 2492 2493 mutex_enter(&fcip_global_mutex); 2494 fport = fcip_port_head; 2495 2496 while (fport != NULL) { 2497 if (fport->fcipp_handle == phandle) { 2498 /* found */ 2499 break; 2500 } 2501 fport = fport->fcipp_next; 2502 } 2503 2504 mutex_exit(&fcip_global_mutex); 2505 2506 return (fport); 2507 } 2508 2509 /* 2510 * Handle inbound ELS requests received by the transport. We are only 2511 * intereseted in FARP/InARP mostly. 2512 */ 2513 /* ARGSUSED */ 2514 static int 2515 fcip_els_cb(opaque_t ulp_handle, opaque_t phandle, 2516 fc_unsol_buf_t *buf, uint32_t claimed) 2517 { 2518 fcip_port_info_t *fport; 2519 struct fcip *fptr; 2520 int instance; 2521 uchar_t r_ctl; 2522 uchar_t ls_code; 2523 la_els_farp_t farp_cmd; 2524 la_els_farp_t *fcmd; 2525 int rval = FC_UNCLAIMED; 2526 2527 fport = fcip_get_port(phandle); 2528 if (fport == NULL) { 2529 return (FC_UNCLAIMED); 2530 } 2531 2532 fptr = fport->fcipp_fcip; 2533 ASSERT(fptr != NULL); 2534 if (fptr == NULL) { 2535 return (FC_UNCLAIMED); 2536 } 2537 2538 instance = ddi_get_instance(fport->fcipp_dip); 2539 2540 mutex_enter(&fptr->fcip_mutex); 2541 if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) || 2542 (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) { 2543 mutex_exit(&fptr->fcip_mutex); 2544 return (FC_UNCLAIMED); 2545 } 2546 2547 /* 2548 * set fcip flags to indicate we are in the middle of a 2549 * ELS callback so we can wait till the statechange 2550 * is handled before succeeding/failing the SUSPEND/POWER DOWN. 2551 */ 2552 fptr->fcip_flags |= FCIP_IN_ELS_CB; 2553 mutex_exit(&fptr->fcip_mutex); 2554 2555 FCIP_TNF_PROBE_2((fcip_els_cb, "fcip io", /* CSTYLED */, 2556 tnf_string, msg, "ELS callback", 2557 tnf_uint, instance, instance)); 2558 2559 FCIP_DEBUG(FCIP_DEBUG_ELS, 2560 (CE_NOTE, "fcip%d, ELS callback , ", instance)); 2561 2562 r_ctl = buf->ub_frame.r_ctl; 2563 switch (r_ctl & R_CTL_ROUTING) { 2564 case R_CTL_EXTENDED_SVC: 2565 if (r_ctl == R_CTL_ELS_REQ) { 2566 ls_code = buf->ub_buffer[0]; 2567 if (ls_code == LA_ELS_FARP_REQ) { 2568 /* 2569 * Inbound FARP broadcast request 2570 */ 2571 if (buf->ub_bufsize != sizeof (la_els_farp_t)) { 2572 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN, 2573 "Invalid FARP req buffer size " 2574 "expected 0x%lx, got 0x%x", 2575 (long)(sizeof (la_els_farp_t)), 2576 buf->ub_bufsize)); 2577 rval = FC_UNCLAIMED; 2578 goto els_cb_done; 2579 } 2580 fcmd = (la_els_farp_t *)buf; 2581 if (fcip_wwn_compare(&fcmd->resp_nwwn, 2582 &fport->fcipp_nwwn, 2583 FCIP_COMPARE_NWWN) != 0) { 2584 rval = FC_UNCLAIMED; 2585 goto els_cb_done; 2586 } 2587 /* 2588 * copy the FARP request and release the 2589 * unsolicited buffer 2590 */ 2591 fcmd = &farp_cmd; 2592 bcopy((void *)buf, (void *)fcmd, 2593 sizeof (la_els_farp_t)); 2594 (void) fc_ulp_ubrelease(fport->fcipp_handle, 1, 2595 &buf->ub_token); 2596 2597 if (fcip_farp_supported && 2598 fcip_handle_farp_request(fptr, fcmd) == 2599 FC_SUCCESS) { 2600 /* 2601 * We successfully sent out a FARP 2602 * reply to the requesting port 2603 */ 2604 rval = FC_SUCCESS; 2605 goto els_cb_done; 2606 } else { 2607 rval = FC_UNCLAIMED; 2608 goto els_cb_done; 2609 } 2610 } 2611 } else if (r_ctl == R_CTL_ELS_RSP) { 2612 ls_code = buf->ub_buffer[0]; 2613 if (ls_code == LA_ELS_FARP_REPLY) { 2614 /* 2615 * We received a REPLY to our FARP request 2616 */ 2617 if (buf->ub_bufsize != sizeof (la_els_farp_t)) { 2618 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN, 2619 "Invalid FARP req buffer size " 2620 "expected 0x%lx, got 0x%x", 2621 (long)(sizeof (la_els_farp_t)), 2622 buf->ub_bufsize)); 2623 rval = FC_UNCLAIMED; 2624 goto els_cb_done; 2625 } 2626 fcmd = &farp_cmd; 2627 bcopy((void *)buf, (void *)fcmd, 2628 sizeof (la_els_farp_t)); 2629 (void) fc_ulp_ubrelease(fport->fcipp_handle, 1, 2630 &buf->ub_token); 2631 if (fcip_farp_supported && 2632 fcip_handle_farp_response(fptr, fcmd) == 2633 FC_SUCCESS) { 2634 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_NOTE, 2635 "Successfully recevied a FARP " 2636 "response")); 2637 mutex_enter(&fptr->fcip_mutex); 2638 fptr->fcip_farp_rsp_flag = 1; 2639 cv_signal(&fptr->fcip_farp_cv); 2640 mutex_exit(&fptr->fcip_mutex); 2641 rval = FC_SUCCESS; 2642 goto els_cb_done; 2643 } else { 2644 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN, 2645 "Unable to handle a FARP response " 2646 "receive")); 2647 rval = FC_UNCLAIMED; 2648 goto els_cb_done; 2649 } 2650 } 2651 } 2652 break; 2653 default: 2654 break; 2655 } 2656 els_cb_done: 2657 mutex_enter(&fptr->fcip_mutex); 2658 fptr->fcip_flags &= ~(FCIP_IN_ELS_CB); 2659 mutex_exit(&fptr->fcip_mutex); 2660 return (rval); 2661 } 2662 2663 2664 /* 2665 * Handle inbound FARP requests 2666 */ 2667 static int 2668 fcip_handle_farp_request(struct fcip *fptr, la_els_farp_t *fcmd) 2669 { 2670 fcip_pkt_t *fcip_pkt; 2671 fc_packet_t *fc_pkt; 2672 fcip_port_info_t *fport = fptr->fcip_port_info; 2673 int rval = FC_FAILURE; 2674 opaque_t fca_dev; 2675 fc_portmap_t map; 2676 struct fcip_routing_table *frp; 2677 struct fcip_dest *fdestp; 2678 2679 /* 2680 * Add an entry for the remote port into our routing and destination 2681 * tables. 2682 */ 2683 map.map_did = fcmd->req_id; 2684 map.map_hard_addr.hard_addr = fcmd->req_id.port_id; 2685 map.map_state = PORT_DEVICE_VALID; 2686 map.map_type = PORT_DEVICE_NEW; 2687 map.map_flags = 0; 2688 map.map_pd = NULL; 2689 bcopy((void *)&fcmd->req_pwwn, (void *)&map.map_pwwn, 2690 sizeof (la_wwn_t)); 2691 bcopy((void *)&fcmd->req_nwwn, (void *)&map.map_nwwn, 2692 sizeof (la_wwn_t)); 2693 fcip_rt_update(fptr, &map, 1); 2694 mutex_enter(&fptr->fcip_rt_mutex); 2695 frp = fcip_lookup_rtable(fptr, &fcmd->req_pwwn, FCIP_COMPARE_NWWN); 2696 mutex_exit(&fptr->fcip_rt_mutex); 2697 2698 fdestp = fcip_add_dest(fptr, frp); 2699 2700 fcip_pkt = fcip_ipkt_alloc(fptr, sizeof (la_els_farp_t), 2701 sizeof (la_els_farp_t), NULL, KM_SLEEP); 2702 if (fcip_pkt == NULL) { 2703 rval = FC_FAILURE; 2704 goto farp_done; 2705 } 2706 /* 2707 * Fill in our port's PWWN and NWWN 2708 */ 2709 fcmd->resp_pwwn = fport->fcipp_pwwn; 2710 fcmd->resp_nwwn = fport->fcipp_nwwn; 2711 2712 fcip_init_unicast_pkt(fcip_pkt, fport->fcipp_sid, 2713 fcmd->req_id, NULL); 2714 2715 fca_dev = 2716 fc_ulp_get_fca_device(fport->fcipp_handle, fcmd->req_id); 2717 fc_pkt = FCIP_PKT_TO_FC_PKT(fcip_pkt); 2718 fc_pkt->pkt_cmd_fhdr.r_ctl = R_CTL_ELS_RSP; 2719 fc_pkt->pkt_fca_device = fca_dev; 2720 fcip_pkt->fcip_pkt_dest = fdestp; 2721 2722 /* 2723 * Attempt a PLOGI again 2724 */ 2725 if (fcmd->resp_flags & FARP_INIT_P_LOGI) { 2726 if (fcip_do_plogi(fptr, frp) != FC_SUCCESS) { 2727 /* 2728 * Login to the remote port failed. There is no 2729 * point continuing with the FARP request further 2730 * so bail out here. 2731 */ 2732 frp->fcipr_state = PORT_DEVICE_INVALID; 2733 rval = FC_FAILURE; 2734 goto farp_done; 2735 } 2736 } 2737 2738 FCIP_CP_OUT(fcmd, fc_pkt->pkt_cmd, fc_pkt->pkt_cmd_acc, 2739 sizeof (la_els_farp_t)); 2740 2741 rval = fc_ulp_issue_els(fport->fcipp_handle, fc_pkt); 2742 if (rval != FC_SUCCESS) { 2743 FCIP_TNF_PROBE_2((fcip_handle_farp_request, "fcip io", 2744 /* CSTYLED */, tnf_string, msg, 2745 "fcip_transport of farp reply failed", 2746 tnf_uint, rval, rval)); 2747 FCIP_DEBUG(FCIP_DEBUG_ELS, (CE_WARN, 2748 "fcip_transport of farp reply failed 0x%x", rval)); 2749 } 2750 2751 farp_done: 2752 return (rval); 2753 } 2754 2755 2756 /* 2757 * Handle FARP responses to our FARP requests. When we receive a FARP 2758 * reply, we need to add the entry for the Port that replied into our 2759 * routing and destination hash tables. It is possible that the remote 2760 * port did not login into us (FARP responses can be received without 2761 * a PLOGI) 2762 */ 2763 static int 2764 fcip_handle_farp_response(struct fcip *fptr, la_els_farp_t *fcmd) 2765 { 2766 int rval = FC_FAILURE; 2767 fc_portmap_t map; 2768 struct fcip_routing_table *frp; 2769 struct fcip_dest *fdestp; 2770 2771 /* 2772 * Add an entry for the remote port into our routing and destination 2773 * tables. 2774 */ 2775 map.map_did = fcmd->dest_id; 2776 map.map_hard_addr.hard_addr = fcmd->dest_id.port_id; 2777 map.map_state = PORT_DEVICE_VALID; 2778 map.map_type = PORT_DEVICE_NEW; 2779 map.map_flags = 0; 2780 map.map_pd = NULL; 2781 bcopy((void *)&fcmd->resp_pwwn, (void *)&map.map_pwwn, 2782 sizeof (la_wwn_t)); 2783 bcopy((void *)&fcmd->resp_nwwn, (void *)&map.map_nwwn, 2784 sizeof (la_wwn_t)); 2785 fcip_rt_update(fptr, &map, 1); 2786 mutex_enter(&fptr->fcip_rt_mutex); 2787 frp = fcip_lookup_rtable(fptr, &fcmd->resp_pwwn, FCIP_COMPARE_NWWN); 2788 mutex_exit(&fptr->fcip_rt_mutex); 2789 2790 fdestp = fcip_add_dest(fptr, frp); 2791 2792 if (fdestp != NULL) { 2793 rval = FC_SUCCESS; 2794 } 2795 return (rval); 2796 } 2797 2798 2799 #define FCIP_HDRS_LENGTH \ 2800 sizeof (fcph_network_hdr_t)+sizeof (llc_snap_hdr_t)+sizeof (ipha_t) 2801 2802 /* 2803 * fcip_data_cb is the heart of most IP operations. This routine is called 2804 * by the transport when any unsolicited IP data arrives at a port (which 2805 * is almost all IP data). This routine then strips off the Network header 2806 * from the payload (after authenticating the received payload ofcourse), 2807 * creates a message blk and sends the data upstream. You will see ugly 2808 * #defines because of problems with using esballoc() as opposed to 2809 * allocb to prevent an extra copy of data. We should probably move to 2810 * esballoc entirely when the MTU eventually will be larger than 1500 bytes 2811 * since copies will get more expensive then. At 1500 byte MTUs, there is 2812 * no noticable difference between using allocb and esballoc. The other 2813 * caveat is that the qlc firmware still cannot tell us accurately the 2814 * no. of valid bytes in the unsol buffer it DMA'ed so we have to resort 2815 * to looking into the IP header and hoping that the no. of bytes speficified 2816 * in the header was actually received. 2817 */ 2818 /* ARGSUSED */ 2819 static int 2820 fcip_data_cb(opaque_t ulp_handle, opaque_t phandle, 2821 fc_unsol_buf_t *buf, uint32_t claimed) 2822 { 2823 fcip_port_info_t *fport; 2824 struct fcip *fptr; 2825 fcph_network_hdr_t *nhdr; 2826 llc_snap_hdr_t *snaphdr; 2827 mblk_t *bp; 2828 uint32_t len; 2829 uint32_t hdrlen; 2830 ushort_t type; 2831 ipha_t *iphdr; 2832 int rval; 2833 2834 #ifdef FCIP_ESBALLOC 2835 frtn_t *free_ubuf; 2836 struct fcip_esballoc_arg *fesb_argp; 2837 #endif /* FCIP_ESBALLOC */ 2838 2839 fport = fcip_get_port(phandle); 2840 if (fport == NULL) { 2841 return (FC_UNCLAIMED); 2842 } 2843 2844 fptr = fport->fcipp_fcip; 2845 ASSERT(fptr != NULL); 2846 2847 if (fptr == NULL) { 2848 return (FC_UNCLAIMED); 2849 } 2850 2851 mutex_enter(&fptr->fcip_mutex); 2852 if ((fptr->fcip_flags & (FCIP_DETACHING | FCIP_DETACHED)) || 2853 (fptr->fcip_flags & (FCIP_SUSPENDED | FCIP_POWER_DOWN))) { 2854 mutex_exit(&fptr->fcip_mutex); 2855 rval = FC_UNCLAIMED; 2856 goto data_cb_done; 2857 } 2858 2859 /* 2860 * set fcip flags to indicate we are in the middle of a 2861 * data callback so we can wait till the statechange 2862 * is handled before succeeding/failing the SUSPEND/POWER DOWN. 2863 */ 2864 fptr->fcip_flags |= FCIP_IN_DATA_CB; 2865 mutex_exit(&fptr->fcip_mutex); 2866 2867 FCIP_TNF_PROBE_2((fcip_data_cb, "fcip io", /* CSTYLED */, 2868 tnf_string, msg, "data callback", 2869 tnf_int, instance, ddi_get_instance(fport->fcipp_dip))); 2870 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2871 (CE_NOTE, "fcip%d, data callback", 2872 ddi_get_instance(fport->fcipp_dip))); 2873 2874 /* 2875 * get to the network and snap headers in the payload 2876 */ 2877 nhdr = (fcph_network_hdr_t *)buf->ub_buffer; 2878 snaphdr = (llc_snap_hdr_t *)(buf->ub_buffer + 2879 sizeof (fcph_network_hdr_t)); 2880 2881 hdrlen = sizeof (fcph_network_hdr_t) + sizeof (llc_snap_hdr_t); 2882 2883 /* 2884 * get the IP header to obtain the no. of bytes we need to read 2885 * off from the unsol buffer. This obviously is because not all 2886 * data fills up the unsol buffer completely and the firmware 2887 * doesn't tell us how many valid bytes are in there as well 2888 */ 2889 iphdr = (ipha_t *)(buf->ub_buffer + hdrlen); 2890 snaphdr->pid = BE_16(snaphdr->pid); 2891 type = snaphdr->pid; 2892 2893 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2894 (CE_CONT, "SNAPHDR: dsap %x, ssap %x, ctrl %x\n", 2895 snaphdr->dsap, snaphdr->ssap, snaphdr->ctrl)); 2896 2897 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2898 (CE_CONT, "oui[0] 0x%x oui[1] 0x%x oui[2] 0x%x pid 0x%x\n", 2899 snaphdr->oui[0], snaphdr->oui[1], snaphdr->oui[2], snaphdr->pid)); 2900 2901 /* Authneticate, Authenticate */ 2902 if (type == ETHERTYPE_IP) { 2903 len = hdrlen + BE_16(iphdr->ipha_length); 2904 } else if (type == ETHERTYPE_ARP) { 2905 len = hdrlen + 28; 2906 } else { 2907 len = buf->ub_bufsize; 2908 } 2909 2910 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2911 (CE_CONT, "effective packet length is %d bytes.\n", len)); 2912 2913 if (len < hdrlen || len > FCIP_UB_SIZE) { 2914 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2915 (CE_NOTE, "Incorrect buffer size %d bytes", len)); 2916 rval = FC_UNCLAIMED; 2917 goto data_cb_done; 2918 } 2919 2920 if (buf->ub_frame.type != FC_TYPE_IS8802_SNAP) { 2921 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_NOTE, "Not IP/ARP data")); 2922 rval = FC_UNCLAIMED; 2923 goto data_cb_done; 2924 } 2925 2926 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_NOTE, "checking wwn")); 2927 2928 if ((fcip_wwn_compare(&nhdr->net_dest_addr, &fport->fcipp_pwwn, 2929 FCIP_COMPARE_NWWN) != 0) && 2930 (!IS_BROADCAST_ADDR(&nhdr->net_dest_addr))) { 2931 rval = FC_UNCLAIMED; 2932 goto data_cb_done; 2933 } else if (fcip_cache_on_arp_broadcast && 2934 IS_BROADCAST_ADDR(&nhdr->net_dest_addr)) { 2935 fcip_cache_arp_broadcast(fptr, buf); 2936 } 2937 2938 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_NOTE, "Allocate streams block")); 2939 2940 /* 2941 * Using esballoc instead of allocb should be faster, atleast at 2942 * larger MTUs than 1500 bytes. Someday we'll get there :) 2943 */ 2944 #if defined(FCIP_ESBALLOC) 2945 /* 2946 * allocate memory for the frtn function arg. The Function 2947 * (fcip_ubfree) arg is a struct fcip_esballoc_arg type 2948 * which contains pointers to the unsol buffer and the 2949 * opaque port handle for releasing the unsol buffer back to 2950 * the FCA for reuse 2951 */ 2952 fesb_argp = (struct fcip_esballoc_arg *) 2953 kmem_zalloc(sizeof (struct fcip_esballoc_arg), KM_NOSLEEP); 2954 2955 if (fesb_argp == NULL) { 2956 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2957 (CE_WARN, "esballoc of mblk failed in data_cb")); 2958 rval = FC_UNCLAIMED; 2959 goto data_cb_done; 2960 } 2961 /* 2962 * Check with KM_NOSLEEP 2963 */ 2964 free_ubuf = (frtn_t *)kmem_zalloc(sizeof (frtn_t), KM_NOSLEEP); 2965 if (free_ubuf == NULL) { 2966 kmem_free(fesb_argp, sizeof (struct fcip_esballoc_arg)); 2967 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2968 (CE_WARN, "esballoc of mblk failed in data_cb")); 2969 rval = FC_UNCLAIMED; 2970 goto data_cb_done; 2971 } 2972 2973 fesb_argp->frtnp = free_ubuf; 2974 fesb_argp->buf = buf; 2975 fesb_argp->phandle = phandle; 2976 free_ubuf->free_func = fcip_ubfree; 2977 free_ubuf->free_arg = (char *)fesb_argp; 2978 if ((bp = (mblk_t *)esballoc((unsigned char *)buf->ub_buffer, 2979 len, BPRI_MED, free_ubuf)) == NULL) { 2980 kmem_free(fesb_argp, sizeof (struct fcip_esballoc_arg)); 2981 kmem_free(free_ubuf, sizeof (frtn_t)); 2982 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2983 (CE_WARN, "esballoc of mblk failed in data_cb")); 2984 rval = FC_UNCLAIMED; 2985 goto data_cb_done; 2986 } 2987 #elif !defined(FCIP_ESBALLOC) 2988 /* 2989 * allocate streams mblk and copy the contents of the 2990 * unsolicited buffer into this newly alloc'ed mblk 2991 */ 2992 if ((bp = (mblk_t *)fcip_allocb((size_t)len, BPRI_LO)) == NULL) { 2993 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 2994 (CE_WARN, "alloc of mblk failed in data_cb")); 2995 rval = FC_UNCLAIMED; 2996 goto data_cb_done; 2997 } 2998 2999 /* 3000 * Unsolicited buffers handed up to us from the FCA must be 3001 * endian clean so just bcopy the data into our mblk. Else 3002 * we may have to either copy the data byte by byte or 3003 * use the ddi_rep_get* routines to do the copy for us. 3004 */ 3005 bcopy(buf->ub_buffer, bp->b_rptr, len); 3006 3007 /* 3008 * for esballoc'ed mblks - free the UB in the frtn function 3009 * along with the memory allocated for the function arg. 3010 * for allocb'ed mblk - release the unsolicited buffer here 3011 */ 3012 (void) fc_ulp_ubrelease(phandle, 1, &buf->ub_token); 3013 3014 #endif /* FCIP_ESBALLOC */ 3015 3016 bp->b_wptr = bp->b_rptr + len; 3017 fptr->fcip_ipackets++; 3018 3019 if (type == ETHERTYPE_IP) { 3020 mutex_enter(&fptr->fcip_mutex); 3021 fptr->fcip_ub_upstream++; 3022 mutex_exit(&fptr->fcip_mutex); 3023 bp->b_rptr += hdrlen; 3024 3025 /* 3026 * Check if ipq is valid in the sendup thread 3027 */ 3028 if (fcip_sendup_alloc_enque(fptr, bp, NULL) != FC_SUCCESS) { 3029 freemsg(bp); 3030 } 3031 } else { 3032 /* 3033 * We won't get ethernet 802.3 packets in FCIP but we may get 3034 * types other than ETHERTYPE_IP, such as ETHERTYPE_ARP. Let 3035 * fcip_sendup() do the matching. 3036 */ 3037 mutex_enter(&fptr->fcip_mutex); 3038 fptr->fcip_ub_upstream++; 3039 mutex_exit(&fptr->fcip_mutex); 3040 if (fcip_sendup_alloc_enque(fptr, bp, 3041 fcip_accept) != FC_SUCCESS) { 3042 freemsg(bp); 3043 } 3044 } 3045 3046 rval = FC_SUCCESS; 3047 3048 /* 3049 * Unset fcip_flags to indicate we are out of callback and return 3050 */ 3051 data_cb_done: 3052 mutex_enter(&fptr->fcip_mutex); 3053 fptr->fcip_flags &= ~(FCIP_IN_DATA_CB); 3054 mutex_exit(&fptr->fcip_mutex); 3055 return (rval); 3056 } 3057 3058 #if !defined(FCIP_ESBALLOC) 3059 /* 3060 * Allocate a message block for the inbound data to be sent upstream. 3061 */ 3062 static void * 3063 fcip_allocb(size_t size, uint_t pri) 3064 { 3065 mblk_t *mp; 3066 3067 if ((mp = allocb(size, pri)) == NULL) { 3068 return (NULL); 3069 } 3070 return (mp); 3071 } 3072 3073 #endif 3074 3075 /* 3076 * This helper routine kmem cache alloc's a sendup element for enquing 3077 * into the sendup list for callbacks upstream from the dedicated sendup 3078 * thread. We enque the msg buf into the sendup list and cv_signal the 3079 * sendup thread to finish the callback for us. 3080 */ 3081 static int 3082 fcip_sendup_alloc_enque(struct fcip *fptr, mblk_t *mp, struct fcipstr *(*f)()) 3083 { 3084 struct fcip_sendup_elem *msg_elem; 3085 int rval = FC_FAILURE; 3086 3087 FCIP_TNF_PROBE_1((fcip_sendup_alloc_enque, "fcip io", /* CSTYLED */, 3088 tnf_string, msg, "sendup msg enque")); 3089 msg_elem = kmem_cache_alloc(fptr->fcip_sendup_cache, KM_NOSLEEP); 3090 if (msg_elem == NULL) { 3091 /* drop pkt to floor - update stats */ 3092 rval = FC_FAILURE; 3093 goto sendup_alloc_done; 3094 } 3095 msg_elem->fcipsu_mp = mp; 3096 msg_elem->fcipsu_func = f; 3097 3098 mutex_enter(&fptr->fcip_sendup_mutex); 3099 if (fptr->fcip_sendup_head == NULL) { 3100 fptr->fcip_sendup_head = fptr->fcip_sendup_tail = msg_elem; 3101 } else { 3102 fptr->fcip_sendup_tail->fcipsu_next = msg_elem; 3103 fptr->fcip_sendup_tail = msg_elem; 3104 } 3105 fptr->fcip_sendup_cnt++; 3106 cv_signal(&fptr->fcip_sendup_cv); 3107 mutex_exit(&fptr->fcip_sendup_mutex); 3108 rval = FC_SUCCESS; 3109 3110 sendup_alloc_done: 3111 return (rval); 3112 } 3113 3114 /* 3115 * One of the ways of performing the WWN to D_ID mapping required for 3116 * IPFC data is to cache the unsolicited ARP broadcast messages received 3117 * and update the routing table to add entry for the destination port 3118 * if we are the intended recipient of the ARP broadcast message. This is 3119 * one of the methods recommended in the rfc to obtain the WWN to D_ID mapping 3120 * but is not typically used unless enabled. The driver prefers to use the 3121 * nameserver/lilp map to obtain this mapping. 3122 */ 3123 static void 3124 fcip_cache_arp_broadcast(struct fcip *fptr, fc_unsol_buf_t *buf) 3125 { 3126 fcip_port_info_t *fport; 3127 fcph_network_hdr_t *nhdr; 3128 struct fcip_routing_table *frp; 3129 fc_portmap_t map; 3130 3131 fport = fptr->fcip_port_info; 3132 if (fport == NULL) { 3133 return; 3134 } 3135 ASSERT(fport != NULL); 3136 3137 nhdr = (fcph_network_hdr_t *)buf->ub_buffer; 3138 3139 mutex_enter(&fptr->fcip_rt_mutex); 3140 frp = fcip_lookup_rtable(fptr, &nhdr->net_src_addr, FCIP_COMPARE_NWWN); 3141 mutex_exit(&fptr->fcip_rt_mutex); 3142 if (frp == NULL) { 3143 map.map_did.port_id = buf->ub_frame.s_id; 3144 map.map_hard_addr.hard_addr = buf->ub_frame.s_id; 3145 map.map_state = PORT_DEVICE_VALID; 3146 map.map_type = PORT_DEVICE_NEW; 3147 map.map_flags = 0; 3148 map.map_pd = NULL; 3149 bcopy((void *)&nhdr->net_src_addr, (void *)&map.map_pwwn, 3150 sizeof (la_wwn_t)); 3151 bcopy((void *)&nhdr->net_src_addr, (void *)&map.map_nwwn, 3152 sizeof (la_wwn_t)); 3153 fcip_rt_update(fptr, &map, 1); 3154 mutex_enter(&fptr->fcip_rt_mutex); 3155 frp = fcip_lookup_rtable(fptr, &nhdr->net_src_addr, 3156 FCIP_COMPARE_NWWN); 3157 mutex_exit(&fptr->fcip_rt_mutex); 3158 3159 (void) fcip_add_dest(fptr, frp); 3160 } 3161 3162 } 3163 3164 /* 3165 * This is a dedicated thread to do callbacks from fcip's data callback 3166 * routines into the modules upstream. The reason for this thread is 3167 * the data callback function can be called from an interrupt context and 3168 * the upstream modules *can* make calls downstream in the same thread 3169 * context. If the call is to a fabric port which is not yet in our 3170 * routing tables, we may have to query the nameserver/fabric for the 3171 * MAC addr to Port_ID mapping which may be blocking calls. 3172 */ 3173 static void 3174 fcip_sendup_thr(void *arg) 3175 { 3176 struct fcip *fptr = (struct fcip *)arg; 3177 struct fcip_sendup_elem *msg_elem; 3178 queue_t *ip4q = NULL; 3179 3180 CALLB_CPR_INIT(&fptr->fcip_cpr_info, &fptr->fcip_sendup_mutex, 3181 callb_generic_cpr, "fcip_sendup_thr"); 3182 3183 mutex_enter(&fptr->fcip_sendup_mutex); 3184 for (;;) { 3185 3186 while (fptr->fcip_sendup_thr_initted && 3187 fptr->fcip_sendup_head == NULL) { 3188 CALLB_CPR_SAFE_BEGIN(&fptr->fcip_cpr_info); 3189 cv_wait(&fptr->fcip_sendup_cv, 3190 &fptr->fcip_sendup_mutex); 3191 CALLB_CPR_SAFE_END(&fptr->fcip_cpr_info, 3192 &fptr->fcip_sendup_mutex); 3193 } 3194 3195 if (fptr->fcip_sendup_thr_initted == 0) { 3196 break; 3197 } 3198 3199 FCIP_TNF_PROBE_1((fcip_sendup_thr, "fcip io", /* CSTYLED */, 3200 tnf_string, msg, "fcip sendup thr - new msg")); 3201 3202 msg_elem = fptr->fcip_sendup_head; 3203 fptr->fcip_sendup_head = msg_elem->fcipsu_next; 3204 msg_elem->fcipsu_next = NULL; 3205 mutex_exit(&fptr->fcip_sendup_mutex); 3206 3207 if (msg_elem->fcipsu_func == NULL) { 3208 /* 3209 * Message for ipq. Check to see if the ipq is 3210 * is still valid. Since the thread is asynchronous, 3211 * there could have been a close on the stream 3212 */ 3213 mutex_enter(&fptr->fcip_mutex); 3214 if (fptr->fcip_ipq && canputnext(fptr->fcip_ipq)) { 3215 ip4q = fptr->fcip_ipq; 3216 mutex_exit(&fptr->fcip_mutex); 3217 putnext(ip4q, msg_elem->fcipsu_mp); 3218 } else { 3219 mutex_exit(&fptr->fcip_mutex); 3220 freemsg(msg_elem->fcipsu_mp); 3221 } 3222 } else { 3223 fcip_sendup(fptr, msg_elem->fcipsu_mp, 3224 msg_elem->fcipsu_func); 3225 } 3226 3227 #if !defined(FCIP_ESBALLOC) 3228 /* 3229 * for allocb'ed mblk - decrement upstream count here 3230 */ 3231 mutex_enter(&fptr->fcip_mutex); 3232 ASSERT(fptr->fcip_ub_upstream > 0); 3233 fptr->fcip_ub_upstream--; 3234 mutex_exit(&fptr->fcip_mutex); 3235 #endif /* FCIP_ESBALLOC */ 3236 3237 kmem_cache_free(fptr->fcip_sendup_cache, (void *)msg_elem); 3238 mutex_enter(&fptr->fcip_sendup_mutex); 3239 fptr->fcip_sendup_cnt--; 3240 } 3241 3242 3243 #ifndef __lock_lint 3244 CALLB_CPR_EXIT(&fptr->fcip_cpr_info); 3245 #else 3246 mutex_exit(&fptr->fcip_sendup_mutex); 3247 #endif /* __lock_lint */ 3248 3249 /* Wake up fcip detach thread by the end */ 3250 cv_signal(&fptr->fcip_sendup_cv); 3251 3252 thread_exit(); 3253 } 3254 3255 #ifdef FCIP_ESBALLOC 3256 3257 /* 3258 * called from the stream head when it is done using an unsolicited buffer. 3259 * We release this buffer then to the FCA for reuse. 3260 */ 3261 static void 3262 fcip_ubfree(char *arg) 3263 { 3264 struct fcip_esballoc_arg *fesb_argp = (struct fcip_esballoc_arg *)arg; 3265 fc_unsol_buf_t *ubuf; 3266 frtn_t *frtnp; 3267 fcip_port_info_t *fport; 3268 struct fcip *fptr; 3269 3270 3271 fport = fcip_get_port(fesb_argp->phandle); 3272 fptr = fport->fcipp_fcip; 3273 3274 ASSERT(fesb_argp != NULL); 3275 ubuf = fesb_argp->buf; 3276 frtnp = fesb_argp->frtnp; 3277 3278 3279 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, 3280 (CE_WARN, "freeing ubuf after esballoc in fcip_ubfree")); 3281 (void) fc_ulp_ubrelease(fesb_argp->phandle, 1, &ubuf->ub_token); 3282 3283 mutex_enter(&fptr->fcip_mutex); 3284 ASSERT(fptr->fcip_ub_upstream > 0); 3285 fptr->fcip_ub_upstream--; 3286 cv_signal(&fptr->fcip_ub_cv); 3287 mutex_exit(&fptr->fcip_mutex); 3288 3289 kmem_free(frtnp, sizeof (frtn_t)); 3290 kmem_free(fesb_argp, sizeof (struct fcip_esballoc_arg)); 3291 } 3292 3293 #endif /* FCIP_ESBALLOC */ 3294 3295 /* 3296 * handle data other than that of type ETHERTYPE_IP and send it on its 3297 * way upstream to the right streams module to handle 3298 */ 3299 static void 3300 fcip_sendup(struct fcip *fptr, mblk_t *mp, struct fcipstr *(*acceptfunc)()) 3301 { 3302 struct fcipstr *slp, *nslp; 3303 la_wwn_t *dhostp; 3304 mblk_t *nmp; 3305 uint32_t isgroupaddr; 3306 int type; 3307 uint32_t hdrlen; 3308 fcph_network_hdr_t *nhdr; 3309 llc_snap_hdr_t *snaphdr; 3310 3311 FCIP_TNF_PROBE_1((fcip_sendup, "fcip io", /* CSTYLED */, 3312 tnf_string, msg, "fcip sendup")); 3313 nhdr = (fcph_network_hdr_t *)mp->b_rptr; 3314 snaphdr = 3315 (llc_snap_hdr_t *)(mp->b_rptr + sizeof (fcph_network_hdr_t)); 3316 dhostp = &nhdr->net_dest_addr; 3317 type = snaphdr->pid; 3318 hdrlen = sizeof (fcph_network_hdr_t) + sizeof (llc_snap_hdr_t); 3319 3320 /* No group address with fibre channel */ 3321 isgroupaddr = 0; 3322 3323 /* 3324 * While holding a reader lock on the linked list of streams structures, 3325 * attempt to match the address criteria for each stream 3326 * and pass up the raw M_DATA ("fastpath") or a DL_UNITDATA_IND. 3327 */ 3328 3329 rw_enter(&fcipstruplock, RW_READER); 3330 3331 if ((slp = (*acceptfunc)(fcipstrup, fptr, type, dhostp)) == NULL) { 3332 rw_exit(&fcipstruplock); 3333 FCIP_TNF_PROBE_1((fcip_sendup, "fcip io", /* CSTYLED */, 3334 tnf_string, msg, "fcip sendup - no slp")); 3335 freemsg(mp); 3336 return; 3337 } 3338 3339 /* 3340 * Loop on matching open streams until (*acceptfunc)() returns NULL. 3341 */ 3342 for (; nslp = (*acceptfunc)(slp->sl_nextp, fptr, type, dhostp); 3343 slp = nslp) { 3344 if (canputnext(slp->sl_rq)) { 3345 if (nmp = dupmsg(mp)) { 3346 if ((slp->sl_flags & FCIP_SLFAST) && 3347 !isgroupaddr) { 3348 nmp->b_rptr += hdrlen; 3349 putnext(slp->sl_rq, nmp); 3350 } else if (slp->sl_flags & FCIP_SLRAW) { 3351 /* No headers when FCIP_SLRAW is set */ 3352 putnext(slp->sl_rq, nmp); 3353 } else if ((nmp = fcip_addudind(fptr, nmp, 3354 nhdr, type))) { 3355 putnext(slp->sl_rq, nmp); 3356 } 3357 } 3358 } 3359 } 3360 3361 /* 3362 * Do the last one. 3363 */ 3364 if (canputnext(slp->sl_rq)) { 3365 if (slp->sl_flags & FCIP_SLFAST) { 3366 mp->b_rptr += hdrlen; 3367 putnext(slp->sl_rq, mp); 3368 } else if (slp->sl_flags & FCIP_SLRAW) { 3369 putnext(slp->sl_rq, mp); 3370 } else if ((mp = fcip_addudind(fptr, mp, nhdr, type))) { 3371 putnext(slp->sl_rq, mp); 3372 } 3373 } else { 3374 freemsg(mp); 3375 } 3376 FCIP_TNF_PROBE_1((fcip_sendup, "fcip io", /* CSTYLED */, 3377 tnf_string, msg, "fcip sendup done")); 3378 3379 rw_exit(&fcipstruplock); 3380 } 3381 3382 /* 3383 * Match the stream based on type and wwn if necessary. 3384 * Destination wwn dhostp is passed to this routine is reserved 3385 * for future usage. We don't need to use it right now since port 3386 * to fcip instance mapping is unique and wwn is already validated when 3387 * packet comes to fcip. 3388 */ 3389 /* ARGSUSED */ 3390 static struct fcipstr * 3391 fcip_accept(struct fcipstr *slp, struct fcip *fptr, int type, la_wwn_t *dhostp) 3392 { 3393 t_uscalar_t sap; 3394 3395 FCIP_TNF_PROBE_1((fcip_accept, "fcip io", /* CSTYLED */, 3396 tnf_string, msg, "fcip accept")); 3397 3398 for (; slp; slp = slp->sl_nextp) { 3399 sap = slp->sl_sap; 3400 FCIP_DEBUG(FCIP_DEBUG_UPSTREAM, (CE_CONT, 3401 "fcip_accept: checking next sap = %x, type = %x", 3402 sap, type)); 3403 3404 if ((slp->sl_fcip == fptr) && (type == sap)) { 3405 return (slp); 3406 } 3407 } 3408 return (NULL); 3409 } 3410 3411 /* 3412 * Handle DL_UNITDATA_IND messages 3413 */ 3414 static mblk_t * 3415 fcip_addudind(struct fcip *fptr, mblk_t *mp, fcph_network_hdr_t *nhdr, 3416 int type) 3417 { 3418 dl_unitdata_ind_t *dludindp; 3419 struct fcipdladdr *dlap; 3420 mblk_t *nmp; 3421 int size; 3422 uint32_t hdrlen; 3423 struct ether_addr src_addr; 3424 struct ether_addr dest_addr; 3425 3426 3427 hdrlen = (sizeof (llc_snap_hdr_t) + sizeof (fcph_network_hdr_t)); 3428 mp->b_rptr += hdrlen; 3429 3430 FCIP_TNF_PROBE_1((fcip_addudind, "fcip io", /* CSTYLED */, 3431 tnf_string, msg, "fcip addudind")); 3432 3433 /* 3434 * Allocate an M_PROTO mblk for the DL_UNITDATA_IND. 3435 */ 3436 size = sizeof (dl_unitdata_ind_t) + FCIPADDRL + FCIPADDRL; 3437 if ((nmp = allocb(size, BPRI_LO)) == NULL) { 3438 fptr->fcip_allocbfail++; 3439 freemsg(mp); 3440 return (NULL); 3441 } 3442 DB_TYPE(nmp) = M_PROTO; 3443 nmp->b_wptr = nmp->b_datap->db_lim; 3444 nmp->b_rptr = nmp->b_wptr - size; 3445 3446 /* 3447 * Construct a DL_UNITDATA_IND primitive. 3448 */ 3449 dludindp = (dl_unitdata_ind_t *)nmp->