xref: /illumos-gate/usr/src/uts/common/io/fibre-channel/impl/fctl.c (revision d1580181)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
 * Copyright (c) 2015 Joyent, Inc.  All rights reserved.
 */
/*
 * Fibre channel Transport Library (fctl)
 *
 * Function naming conventions:
 *		Functions called from ULPs begin with fc_ulp_
 *		Functions called from FCAs begin with fc_fca_
 *		Internal functions begin with fctl_
 *
 * Fibre channel packet layout:
 *	  +---------------------+<--------+
 *	  |			|	  |
 *	  | ULP Packet private	|	  |
 *	  |			|	  |
 *	  +---------------------+	  |
 *	  |			|---------+
 *	  |  struct  fc_packet	|---------+
 *	  |			|	  |
 *	  +---------------------+<--------+
 *	  |			|
 *	  | FCA Packet private	|
 *	  |			|
 *	  +---------------------+
 *
 * So you  loved  the  ascii  art ?  It's  strongly  desirable	to  cache
 * allocate the entire packet in one common  place.  So we define a set a
 * of rules.  In a  contiguous	block of memory,  the top  portion of the
 * block points to ulp packet  private	area, next follows the	fc_packet
 * structure used  extensively by all the consumers and what follows this
 * is the FCA packet private.  Note that given a packet	 structure, it is
 * possible  to get to the  ULP	 and  FCA  Packet  private  fields  using
 * ulp_private and fca_private fields (which hold pointers) respectively.
 *
 * It should be noted with a grain of salt that ULP Packet  private  size
 * varies  between two different  ULP types, So this poses a challenge to
 * compute the correct	size of the whole block on a per port basis.  The
 * transport  layer  doesn't have a problem in dealing with  FCA   packet
 * private  sizes as it is the sole  manager of ports  underneath.  Since
 * it's not a good idea to cache allocate  different  sizes of memory for
 * different ULPs and have the ability to choose from one of these caches
 * based on ULP type during every packet  allocation,  the transport some
 * what	 wisely (?)  hands off this job of cache  allocation  to the ULPs
 * themselves.
 *
 * That means FCAs need to make their  packet  private size  known to the
 * transport   to  pass	 it  up	 to  the   ULPs.  This	is  done   during
 * fc_fca_attach().  And the transport passes this size up to ULPs during
 * fc_ulp_port_attach() of each ULP.
 *
 * This	 leaves	 us with  another  possible  question;	How  are  packets
 * allocated for ELS's started by the transport	 itself ?  Well, the port
 * driver  during  attach  time, cache	allocates  on a per port basis to
 * handle ELSs too.
 */

#include <sys/note.h>
#include <sys/types.h>
#include <sys/varargs.h>
#include <sys/param.h>
#include <sys/errno.h>
#include <sys/uio.h>
#include <sys/buf.h>
#include <sys/modctl.h>
#include <sys/open.h>
#include <sys/kmem.h>
#include <sys/poll.h>
#include <sys/conf.h>
#include <sys/cmn_err.h>
#include <sys/stat.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/promif.h>
#include <sys/byteorder.h>
#include <sys/fibre-channel/fc.h>
#include <sys/fibre-channel/impl/fc_ulpif.h>
#include <sys/fibre-channel/impl/fc_fcaif.h>
#include <sys/fibre-channel/impl/fctl_private.h>
#include <sys/fibre-channel/impl/fc_portif.h>

/* These are referenced by fp.c!  */
int did_table_size = D_ID_HASH_TABLE_SIZE;
int pwwn_table_size = PWWN_HASH_TABLE_SIZE;

static fc_ulp_module_t	*fctl_ulp_modules;
static fc_fca_port_t	*fctl_fca_portlist;
static fc_ulp_list_t	*fctl_ulp_list;

static char fctl_greeting[] =
	"fctl: %s ULP same type (0x%x) as existing module.\n";

static char *fctl_undefined = "Undefined";

/*
 * This lock protects the fc_ulp_module_t linked list (i.e. mod_next field)
 */

static krwlock_t fctl_ulp_lock;

/*
 * The fctl_mod_ports_lock protects the mod_ports element in the
 * fc_ulp_ports_t structure
 */

static krwlock_t fctl_mod_ports_lock;

/*
 * fctl_port_lock protects the linked list of local port structures
 * (fctl_fca_portlist).	 When walking the list, this lock must be obtained
 * prior to any local port locks.
 */

static kmutex_t fctl_port_lock;
static kmutex_t	fctl_ulp_list_mutex;

static fctl_nwwn_list_t		*fctl_nwwn_hash_table;
static kmutex_t			fctl_nwwn_hash_mutex;
int fctl_nwwn_table_size = NWWN_HASH_TABLE_SIZE;

#if	!defined(lint)
_NOTE(MUTEX_PROTECTS_DATA(fctl_nwwn_hash_mutex, fctl_nwwn_hash_table))
_NOTE(MUTEX_PROTECTS_DATA(fctl_ulp_list_mutex, fctl_ulp_list))
_NOTE(RWLOCK_PROTECTS_DATA(fctl_ulp_lock, ulp_module::mod_next))
_NOTE(RWLOCK_PROTECTS_DATA(fctl_mod_ports_lock, ulp_module::mod_ports
    ulp_ports::port_handle))
_NOTE(DATA_READABLE_WITHOUT_LOCK(ulp_module::mod_info))
_NOTE(MUTEX_PROTECTS_DATA(ulp_ports::port_mutex, ulp_ports::port_statec
    ulp_ports::port_dstate))
#endif /* lint */

#define	FCTL_VERSION		"20090729-1.70"
#define	FCTL_NAME_VERSION	"SunFC Transport v" FCTL_VERSION

char *fctl_version = FCTL_NAME_VERSION;

extern struct mod_ops mod_miscops;

static struct modlmisc modlmisc = {
	&mod_miscops,			/* type of module */
	FCTL_NAME_VERSION		/* Module name */
};

static struct modlinkage modlinkage = {
	MODREV_1, (void *)&modlmisc, NULL
};

static struct bus_ops fctl_fca_busops = {
	BUSO_REV,
	nullbusmap,			/* bus_map */
	NULL,				/* bus_get_intrspec */
	NULL,				/* bus_add_intrspec */
	NULL,				/* bus_remove_intrspec */
	i_ddi_map_fault,		/* bus_map_fault */
	NULL,				/* bus_dma_map */
	ddi_dma_allochdl,		/* bus_dma_allochdl */
	ddi_dma_freehdl,		/* bus_dma_freehdl */
	ddi_dma_bindhdl,		/* bus_dma_bindhdl */
	ddi_dma_unbindhdl,		/* bus_unbindhdl */
	ddi_dma_flush,			/* bus_dma_flush */
	ddi_dma_win,			/* bus_dma_win */
	ddi_dma_mctl,			/* bus_dma_ctl */
	fctl_fca_bus_ctl,		/* bus_ctl */
	ddi_bus_prop_op,		/* bus_prop_op */
	NULL,				/* bus_get_eventcookie */
	NULL,				/* bus_add_eventcall */
	NULL,				/* bus_remove_event */
	NULL,				/* bus_post_event */
	NULL,				/* bus_intr_ctl */
	NULL,				/* bus_config */
	NULL,				/* bus_unconfig */
	NULL,				/* bus_fm_init */
	NULL,				/* bus_fm_fini */
	NULL,				/* bus_fm_access_enter */
	NULL,				/* bus_fm_access_exit */
	NULL,				/* bus_power */
	NULL
};

struct kmem_cache *fctl_job_cache;

static fc_errmap_t fc_errlist [] = {
	{ FC_FAILURE,		"Operation failed"			},
	{ FC_SUCCESS,		"Operation success"			},
	{ FC_CAP_ERROR,		"Capability error"			},
	{ FC_CAP_FOUND,		"Capability found"			},
	{ FC_CAP_SETTABLE,	"Capability settable"			},
	{ FC_UNBOUND,		"Port not bound"			},
	{ FC_NOMEM,		"No memory"				},
	{ FC_BADPACKET,		"Bad packet"				},
	{ FC_OFFLINE,		"Port offline"				},
	{ FC_OLDPORT,		"Old Port"				},
	{ FC_NO_MAP,		"No map available"			},
	{ FC_TRANSPORT_ERROR,	"Transport error"			},
	{ FC_ELS_FREJECT,	"ELS Frejected"				},
	{ FC_ELS_PREJECT,	"ELS PRejected"				},
	{ FC_ELS_BAD,		"Bad ELS request"			},
	{ FC_ELS_MALFORMED,	"Malformed ELS request"			},
	{ FC_TOOMANY,		"Too many commands"			},
	{ FC_UB_BADTOKEN,	"Bad Unsolicited buffer token"		},
	{ FC_UB_ERROR,		"Unsolicited buffer error"		},
	{ FC_UB_BUSY,		"Unsolicited buffer busy"		},
	{ FC_BADULP,		"Bad ULP"				},
	{ FC_BADTYPE,		"Bad Type"				},
	{ FC_UNCLAIMED,		"Not Claimed"				},
	{ FC_ULP_SAMEMODULE,	"Same ULP Module"			},
	{ FC_ULP_SAMETYPE,	"Same ULP Type"				},
	{ FC_ABORTED,		"Command Aborted"			},
	{ FC_ABORT_FAILED,	"Abort Failed"				},
	{ FC_BADEXCHANGE,	"Bad Exchange"				},
	{ FC_BADWWN,		"Bad World Wide Name"			},
	{ FC_BADDEV,		"Bad Device"				},
	{ FC_BADCMD,		"Bad Command"				},
	{ FC_BADOBJECT,		"Bad Object"				},
	{ FC_BADPORT,		"Bad Port"				},
	{ FC_NOTTHISPORT,	"Not on this Port"			},
	{ FC_PREJECT,		"Operation Prejected"			},
	{ FC_FREJECT,		"Operation Frejected"			},
	{ FC_PBUSY,		"Operation Pbusyed"			},
	{ FC_FBUSY,		"Operation Fbusyed"			},
	{ FC_ALREADY,		"Already done"				},
	{ FC_LOGINREQ,		"PLOGI Required"			},
	{ FC_RESETFAIL,		"Reset operation failed"		},
	{ FC_INVALID_REQUEST,	"Invalid Request"			},
	{ FC_OUTOFBOUNDS,	"Out of Bounds"				},
	{ FC_TRAN_BUSY,		"Command transport Busy"		},
	{ FC_STATEC_BUSY,	"State change Busy"			},
	{ FC_DEVICE_BUSY,	"Port driver is working on this device"	}
};

fc_pkt_reason_t remote_stop_reasons [] = {
	{ FC_REASON_ABTS,	"Abort Sequence"	},
	{ FC_REASON_ABTX,	"Abort Exchange"	},
	{ FC_REASON_INVALID,	NULL			}
};

fc_pkt_reason_t general_reasons [] = {
	{ FC_REASON_HW_ERROR,		"Hardware Error"		},
	{ FC_REASON_SEQ_TIMEOUT,	"Sequence Timeout"		},
	{ FC_REASON_ABORTED,		"Aborted"			},
	{ FC_REASON_ABORT_FAILED,	"Abort Failed"			},
	{ FC_REASON_NO_CONNECTION,	"No Connection"			},
	{ FC_REASON_XCHG_DROPPED,	"Exchange Dropped"		},
	{ FC_REASON_ILLEGAL_FRAME,	"Illegal Frame"			},
	{ FC_REASON_ILLEGAL_LENGTH,	"Illegal Length"		},
	{ FC_REASON_UNSUPPORTED,	"Unsuported"			},
	{ FC_REASON_RX_BUF_TIMEOUT,	"Receive Buffer Timeout"	},
	{ FC_REASON_FCAL_OPN_FAIL,	"FC AL Open Failed"		},
	{ FC_REASON_OVERRUN,		"Over run"			},
	{ FC_REASON_QFULL,		"Queue Full"			},
	{ FC_REASON_ILLEGAL_REQ,	"Illegal Request",		},
	{ FC_REASON_PKT_BUSY,		"Busy"				},
	{ FC_REASON_OFFLINE,		"Offline"			},
	{ FC_REASON_BAD_XID,		"Bad Exchange Id"		},
	{ FC_REASON_XCHG_BSY,		"Exchange Busy"			},
	{ FC_REASON_NOMEM,		"No Memory"			},
	{ FC_REASON_BAD_SID,		"Bad S_ID"			},
	{ FC_REASON_NO_SEQ_INIT,	"No Sequence Initiative"	},
	{ FC_REASON_DIAG_BUSY,		"Diagnostic Busy"		},
	{ FC_REASON_DMA_ERROR,		"DMA Error"			},
	{ FC_REASON_CRC_ERROR,		"CRC Error"			},
	{ FC_REASON_ABORT_TIMEOUT,	"Abort Timeout"			},
	{ FC_REASON_FCA_UNIQUE,		"FCA Unique"			},
	{ FC_REASON_INVALID,		NULL				}
};

fc_pkt_reason_t rjt_reasons [] = {
	{ FC_REASON_INVALID_D_ID,	"Invalid D_ID"			},
	{ FC_REASON_INVALID_S_ID,	"Invalid S_ID"			},
	{ FC_REASON_TEMP_UNAVAILABLE,	"Temporarily Unavailable"	},
	{ FC_REASON_PERM_UNAVAILABLE,	"Permamnently Unavailable"	},
	{ FC_REASON_CLASS_NOT_SUPP,	"Class Not Supported",		},
	{ FC_REASON_DELIMTER_USAGE_ERROR,
	    "Delimeter Usage Error"		},
	{ FC_REASON_TYPE_NOT_SUPP,	"Type Not Supported"		},
	{ FC_REASON_INVALID_LINK_CTRL,	"Invalid Link Control"		},
	{ FC_REASON_INVALID_R_CTL,	"Invalid R_CTL"			},
	{ FC_REASON_INVALID_F_CTL,	"Invalid F_CTL"			},
	{ FC_REASON_INVALID_OX_ID,	"Invalid OX_ID"			},
	{ FC_REASON_INVALID_RX_ID,	"Invalid RX_ID"			},
	{ FC_REASON_INVALID_SEQ_ID,	"Invalid Sequence ID"		},
	{ FC_REASON_INVALID_DF_CTL,	"Invalid DF_CTL"		},
	{ FC_REASON_INVALID_SEQ_CNT,	"Invalid Sequence count"	},
	{ FC_REASON_INVALID_PARAM,	"Invalid Parameter"		},
	{ FC_REASON_EXCH_ERROR,		"Exchange Error"		},
	{ FC_REASON_PROTOCOL_ERROR,	"Protocol Error"		},
	{ FC_REASON_INCORRECT_LENGTH,	"Incorrect Length"		},
	{ FC_REASON_UNEXPECTED_ACK,	"Unexpected Ack"		},
	{ FC_REASON_UNEXPECTED_LR,	"Unexpected Link reset"		},
	{ FC_REASON_LOGIN_REQUIRED,	"Login Required"		},
	{ FC_REASON_EXCESSIVE_SEQS,	"Excessive Sequences"
	    " Attempted"			},
	{ FC_REASON_EXCH_UNABLE,	"Exchange incapable"		},
	{ FC_REASON_ESH_NOT_SUPP,	"Expiration Security Header "
	    "Not Supported"			},
	{ FC_REASON_NO_FABRIC_PATH,	"No Fabric Path"		},
	{ FC_REASON_VENDOR_UNIQUE,	"Vendor Unique"			},
	{ FC_REASON_INVALID,		NULL				}
};

fc_pkt_reason_t n_port_busy_reasons [] = {
	{ FC_REASON_PHYSICAL_BUSY,		"Physical Busy"		},
	{ FC_REASON_N_PORT_RESOURCE_BSY,	"Resource Busy"		},
	{ FC_REASON_N_PORT_VENDOR_UNIQUE,	"Vendor Unique"		},
	{ FC_REASON_INVALID,			NULL			}
};

fc_pkt_reason_t f_busy_reasons [] = {
	{ FC_REASON_FABRIC_BSY,		"Fabric Busy"			},
	{ FC_REASON_N_PORT_BSY,		"N_Port Busy"			},
	{ FC_REASON_INVALID,		NULL				}
};

fc_pkt_reason_t ls_ba_rjt_reasons [] = {
	{ FC_REASON_INVALID_LA_CODE,	"Invalid Link Application Code"	},
	{ FC_REASON_LOGICAL_ERROR,	"Logical Error"			},
	{ FC_REASON_LOGICAL_BSY,	"Logical Busy"			},
	{ FC_REASON_PROTOCOL_ERROR_RJT,	"Protocol Error Reject"		},
	{ FC_REASON_CMD_UNABLE,		"Unable to Perform Command"	},
	{ FC_REASON_CMD_UNSUPPORTED,	"Unsupported Command"		},
	{ FC_REASON_VU_RJT,		"Vendor Unique"			},
	{ FC_REASON_INVALID,		NULL				}
};

fc_pkt_reason_t fs_rjt_reasons [] = {
	{ FC_REASON_FS_INVALID_CMD,	"Invalid Command"		},
	{ FC_REASON_FS_INVALID_VER,	"Invalid Version"		},
	{ FC_REASON_FS_LOGICAL_ERR,	"Logical Error"			},
	{ FC_REASON_FS_INVALID_IUSIZE,	"Invalid IU Size"		},
	{ FC_REASON_FS_LOGICAL_BUSY,	"Logical Busy"			},
	{ FC_REASON_FS_PROTOCOL_ERR,	"Protocol Error"		},
	{ FC_REASON_FS_CMD_UNABLE,	"Unable to Perform Command"	},
	{ FC_REASON_FS_CMD_UNSUPPORTED,	"Unsupported Command"		},
	{ FC_REASON_FS_VENDOR_UNIQUE,	"Vendor Unique"			},
	{ FC_REASON_INVALID,		NULL				}
};

fc_pkt_action_t	n_port_busy_actions [] = {
	{ FC_ACTION_SEQ_TERM_RETRY,	"Retry terminated Sequence"	},
	{ FC_ACTION_SEQ_ACTIVE_RETRY,	"Retry Active Sequence"		},
	{ FC_REASON_INVALID,		NULL				}
};

fc_pkt_action_t rjt_timeout_actions [] = {
	{ FC_ACTION_RETRYABLE,		"Retryable"			},
	{ FC_ACTION_NON_RETRYABLE,	"Non Retryable"			},
	{ FC_REASON_INVALID,		NULL				}
};

fc_pkt_expln_t ba_rjt_explns [] = {
	{ FC_EXPLN_NONE,		"No Explanation"		},
	{ FC_EXPLN_INVALID_OX_RX_ID,	"Invalid X_ID"			},
	{ FC_EXPLN_SEQ_ABORTED,		"Sequence Aborted"		},
	{ FC_EXPLN_INVALID,		NULL				}
};

fc_pkt_error_t fc_pkt_errlist[] = {
	{
		FC_PKT_SUCCESS,
		"Operation Success",
		NULL,
		NULL,
		NULL
	},
	{	FC_PKT_REMOTE_STOP,
	    "Remote Stop",
	    remote_stop_reasons,
	    NULL,
	    NULL
	},
	{
		FC_PKT_LOCAL_RJT,
		"Local Reject",
		general_reasons,
		rjt_timeout_actions,
		NULL
	},
	{
		FC_PKT_NPORT_RJT,
		"N_Port Reject",
		rjt_reasons,
		rjt_timeout_actions,
		NULL
	},
	{
		FC_PKT_FABRIC_RJT,
		"Fabric Reject",
		rjt_reasons,
		rjt_timeout_actions,
		NULL
	},
	{
		FC_PKT_LOCAL_BSY,
		"Local Busy",
		general_reasons,
		NULL,
		NULL,
	},
	{
		FC_PKT_TRAN_BSY,
		"Transport Busy",
		general_reasons,
		NULL,
		NULL,
	},
	{
		FC_PKT_NPORT_BSY,
		"N_Port Busy",
		n_port_busy_reasons,
		n_port_busy_actions,
		NULL
	},
	{
		FC_PKT_FABRIC_BSY,
		"Fabric Busy",
		f_busy_reasons,
		NULL,
		NULL,
	},
	{
		FC_PKT_LS_RJT,
		"Link Service Reject",
		ls_ba_rjt_reasons,
		NULL,
		NULL,
	},
	{
		FC_PKT_BA_RJT,
		"Basic Reject",
		ls_ba_rjt_reasons,
		NULL,
		ba_rjt_explns,
	},
	{
		FC_PKT_TIMEOUT,
		"Timeout",
		general_reasons,
		rjt_timeout_actions,
		NULL
	},
	{
		FC_PKT_FS_RJT,
		"Fabric Switch Reject",
		fs_rjt_reasons,
		NULL,
		NULL
	},
	{
		FC_PKT_TRAN_ERROR,
		"Packet Transport error",
		general_reasons,
		NULL,
		NULL
	},
	{
		FC_PKT_FAILURE,
		"Packet Failure",
		general_reasons,
		NULL,
		NULL
	},
	{
		FC_PKT_PORT_OFFLINE,
		"Port Offline",
		NULL,
		NULL,
		NULL
	},
	{
		FC_PKT_ELS_IN_PROGRESS,
		"ELS is in Progress",
		NULL,
		NULL,
		NULL
	}
};

int
_init()
{
	int rval;

	rw_init(&fctl_ulp_lock, NULL, RW_DRIVER, NULL);
	rw_init(&fctl_mod_ports_lock, NULL, RW_DRIVER, NULL);
	mutex_init(&fctl_port_lock, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&fctl_nwwn_hash_mutex, NULL, MUTEX_DRIVER, NULL);

	fctl_nwwn_hash_table = kmem_zalloc(sizeof (*fctl_nwwn_hash_table) *
	    fctl_nwwn_table_size, KM_SLEEP);

	fctl_ulp_modules = NULL;
	fctl_fca_portlist = NULL;

	fctl_job_cache = kmem_cache_create("fctl_cache",
	    sizeof (job_request_t), 8, fctl_cache_constructor,
	    fctl_cache_destructor, NULL, NULL, NULL, 0);

	if (fctl_job_cache == NULL) {
		kmem_free(fctl_nwwn_hash_table,
		    sizeof (*fctl_nwwn_hash_table) * fctl_nwwn_table_size);
		mutex_destroy(&fctl_nwwn_hash_mutex);
		mutex_destroy(&fctl_port_lock);
		rw_destroy(&fctl_ulp_lock);
		rw_destroy(&fctl_mod_ports_lock);
		return (ENOMEM);
	}

	if ((rval = mod_install(&modlinkage)) != 0) {
		kmem_cache_destroy(fctl_job_cache);
		kmem_free(fctl_nwwn_hash_table,
		    sizeof (*fctl_nwwn_hash_table) * fctl_nwwn_table_size);
		mutex_destroy(&fctl_nwwn_hash_mutex);
		mutex_destroy(&fctl_port_lock);
		rw_destroy(&fctl_ulp_lock);
		rw_destroy(&fctl_mod_ports_lock);
	}

	return (rval);
}


/*
 * The mod_uninstall code doesn't call _fini when
 * there is living dependent module on fctl. So
 * there is no need to be extra careful here ?
 */
int
_fini()
{
	int rval;

	if ((rval = mod_remove(&modlinkage)) != 0) {
		return (rval);
	}

	kmem_cache_destroy(fctl_job_cache);
	kmem_free(fctl_nwwn_hash_table,
	    sizeof (*fctl_nwwn_hash_table) * fctl_nwwn_table_size);
	mutex_destroy(&fctl_nwwn_hash_mutex);
	mutex_destroy(&fctl_port_lock);
	rw_destroy(&fctl_ulp_lock);
	rw_destroy(&fctl_mod_ports_lock);

	return (rval);
}


int
_info(struct modinfo *modinfo_p)
{
	return (mod_info(&modlinkage, modinfo_p));
}


/* ARGSUSED */
static int
fctl_cache_constructor(void *buf, void *cdarg, int kmflag)
{
	job_request_t *job = (job_request_t *)buf;

	mutex_init(&job->job_mutex, NULL, MUTEX_DRIVER, NULL);
	sema_init(&job->job_fctl_sema, 0, NULL, SEMA_DEFAULT, NULL);
	sema_init(&job->job_port_sema, 0, NULL, SEMA_DEFAULT, NULL);

	return (0);
}


/* ARGSUSED */
static void
fctl_cache_destructor(void *buf, void *cdarg)
{
	job_request_t *job = (job_request_t *)buf;

	sema_destroy(&job->job_fctl_sema);
	sema_destroy(&job->job_port_sema);
	mutex_destroy(&job->job_mutex);
}


/*
 * fc_ulp_add:
 *		Add a ULP module
 *
 * Return Codes:
 *		FC_ULP_SAMEMODULE
 *		FC_SUCCESS
 *		FC_FAILURE
 *
 *   fc_ulp_add	 prints	 a warning message if there is	already a
 *   similar ULP type  attached and this is unlikely to change as
 *   we trudge along.  Further, this  function	returns a failure
 *   code if the same  module  attempts to add more than once for
 *   the same FC-4 type.
 */
int
fc_ulp_add(fc_ulp_modinfo_t *ulp_info)
{
	fc_ulp_module_t *mod;
	fc_ulp_module_t *prev;
	job_request_t	*job;
	fc_ulp_list_t	*new;
	fc_fca_port_t	*fca_port;
	int		ntry = 0;

	ASSERT(ulp_info != NULL);

	/*
	 * Make sure ulp_rev matches fctl version.
	 * Whenever non-private data structure or non-static interface changes,
	 * we should use an increased FCTL_ULP_MODREV_# number here and in all
	 * ulps to prevent version mismatch.
	 */
	if (ulp_info->ulp_rev != FCTL_ULP_MODREV_4) {
		cmn_err(CE_WARN, "fctl: ULP %s version mismatch;"
		    " ULP %s would not be loaded", ulp_info->ulp_name,
		    ulp_info->ulp_name);
		return (FC_BADULP);
	}

	new = kmem_zalloc(sizeof (*new), KM_SLEEP);
	ASSERT(new != NULL);

	mutex_enter(&fctl_ulp_list_mutex);
	new->ulp_info = ulp_info;
	if (fctl_ulp_list != NULL) {
		new->ulp_next = fctl_ulp_list;
	}
	fctl_ulp_list = new;
	mutex_exit(&fctl_ulp_list_mutex);

	while (rw_tryenter(&fctl_ulp_lock, RW_WRITER) == 0) {
		delay(drv_usectohz(1000000));
		if (ntry++ > FC_ULP_ADD_RETRY_COUNT) {
			fc_ulp_list_t	*list;
			fc_ulp_list_t	*last;
			mutex_enter(&fctl_ulp_list_mutex);
			for (last = NULL, list = fctl_ulp_list; list != NULL;
			    list = list->ulp_next) {
				if (list->ulp_info == ulp_info) {
					break;
				}
				last = list;
			}

			if (list) {
				if (last) {
					last->ulp_next = list->ulp_next;
				} else {
					fctl_ulp_list = list->ulp_next;
				}
				kmem_free(list, sizeof (*list));
			}
			mutex_exit(&fctl_ulp_list_mutex);
			cmn_err(CE_WARN, "fctl: ULP %s unable to load",
			    ulp_info->ulp_name);
			return (FC_FAILURE);
		}
	}

	for (mod = fctl_ulp_modules, prev = NULL; mod; mod = mod->mod_next) {
		ASSERT(mod->mod_info != NULL);

		if (ulp_info == mod->mod_info &&
		    ulp_info->ulp_type == mod->mod_info->ulp_type) {
			rw_exit(&fctl_ulp_lock);
			return (FC_ULP_SAMEMODULE);
		}

		if (ulp_info->ulp_type == mod->mod_info->ulp_type) {
			cmn_err(CE_NOTE, fctl_greeting, ulp_info->ulp_name,
			    ulp_info->ulp_type);
		}
		prev = mod;
	}

	mod = kmem_zalloc(sizeof (*mod), KM_SLEEP);
	mod->mod_info = ulp_info;
	mod->mod_next = NULL;

	if (prev) {
		prev->mod_next = mod;
	} else {
		fctl_ulp_modules = mod;
	}

	/*
	 * Schedule a job to each port's job_handler
	 * thread to attach their ports with this ULP.
	 */
	mutex_enter(&fctl_port_lock);
	for (fca_port = fctl_fca_portlist; fca_port != NULL;
	    fca_port = fca_port->port_next) {
		job = fctl_alloc_job(JOB_ATTACH_ULP, JOB_TYPE_FCTL_ASYNC,
		    NULL, NULL, KM_SLEEP);

		fctl_enque_job(fca_port->port_handle, job);
	}
	mutex_exit(&fctl_port_lock);

	rw_exit(&fctl_ulp_lock);

	return (FC_SUCCESS);
}


/*
 * fc_ulp_remove
 *	Remove a ULP module
 *
 * A misbehaving ULP may call this routine while I/Os are in progress.
 * Currently there is no mechanism to detect it to fail such a request.
 *
 * Return Codes:
 *		FC_SUCCESS
 *		FC_FAILURE
 */
int
fc_ulp_remove(fc_ulp_modinfo_t *ulp_info)
{
	fc_ulp_module_t *mod;
	fc_ulp_list_t	*list;
	fc_ulp_list_t	*last;
	fc_ulp_module_t *prev;

	mutex_enter(&fctl_ulp_list_mutex);

	for (last = NULL, list = fctl_ulp_list; list != NULL;
	    list = list->ulp_next) {
		if (list->ulp_info == ulp_info) {
			break;
		}
		last = list;
	}

	if (list) {
		if (last) {
			last->ulp_next = list->ulp_next;
		} else {
			fctl_ulp_list = list->ulp_next;
		}
		kmem_free(list, sizeof (*list));
	}

	mutex_exit(&fctl_ulp_list_mutex);

	rw_enter(&fctl_ulp_lock, RW_WRITER);

	for (mod = fctl_ulp_modules, prev = NULL; mod != NULL;
	    mod = mod->mod_next) {
		if (mod->mod_info == ulp_info) {
			break;
		}
		prev = mod;
	}

	if (mod) {
		fc_ulp_ports_t *next;

		if (prev) {
			prev->mod_next = mod->mod_next;
		} else {
			fctl_ulp_modules = mod->mod_next;
		}

		rw_enter(&fctl_mod_ports_lock, RW_WRITER);

		while ((next = mod->mod_ports) != NULL) {
			mod->mod_ports = next->port_next;
			fctl_dealloc_ulp_port(next);
		}

		rw_exit(&fctl_mod_ports_lock);
		rw_exit(&fctl_ulp_lock);

		kmem_free(mod, sizeof (*mod));

		return (FC_SUCCESS);
	}
	rw_exit(&fctl_ulp_lock);

	return (FC_FAILURE);
}


/*
 * The callers typically cache allocate the packet, complete the
 * DMA setup for pkt_cmd and pkt_resp fields of the packet and
 * call this function to see if the FCA is interested in doing
 * its own intialization. For example, socal may like to initialize
 * the soc_hdr which is pointed to by the pkt_fca_private field
 * and sitting right below fc_packet_t in memory.
 *
 * The caller is required to ensure that pkt_pd is populated with the
 * handle that it was given when the transport notified it about the
 * device this packet is associated with.  If there is no associated
 * device, pkt_pd must be set to NULL.	A non-NULL pkt_pd will cause an
 * increment of the reference count for said pd.  When the packet is freed,
 * the reference count will be decremented.  This reference count, in
 * combination with the PD_GIVEN_TO_ULPS flag guarantees that the pd
 * will not wink out of existence while there is a packet outstanding.
 *
 * This function and fca_init_pkt must not perform any operations that
 * would result in a call back to the ULP, as the ULP may be required
 * to hold a mutex across this call to ensure that the pd in question
 * won't go away prior the call to fc_ulp_transport.
 *
 * ULPs are responsible for using the handles they are given during state
 * change callback processing in a manner that ensures consistency.  That
 * is, they must be aware that they could be processing a state change
 * notification that tells them the device associated with a particular
 * handle has gone away at the same time they are being asked to
 * initialize a packet using that handle. ULPs must therefore ensure
 * that their state change processing and packet initialization code
 * paths are sufficiently synchronized to avoid the use of an
 * invalidated handle in any fc_packet_t struct that is passed to the
 * fc_ulp_init_packet() function.
 */
int
fc_ulp_init_packet(opaque_t port_handle, fc_packet_t *pkt, int sleep)
{
	int rval;
	fc_local_port_t *port = port_handle;
	fc_remote_port_t *pd;

	ASSERT(pkt != NULL);

	pd = pkt->pkt_pd;

	/* Call the FCA driver's fca_init_pkt entry point function. */
	rval = port->fp_fca_tran->fca_init_pkt(port->fp_fca_handle, pkt, sleep);

	if ((rval == FC_SUCCESS) && (pd != NULL)) {
		/*
		 * A !NULL pd here must still be a valid
		 * reference to the fc_remote_port_t.
		 */
		mutex_enter(&pd->pd_mutex);
		ASSERT(pd->pd_ref_count >= 0);
		pd->pd_ref_count++;
		mutex_exit(&pd->pd_mutex);
	}

	return (rval);
}


/*
 * This function is called before destroying the cache allocated
 * fc_packet to free up (and uninitialize) any resource specially
 * allocated by the FCA driver during tran_init_pkt().
 *
 * If the pkt_pd field in the given fc_packet_t struct is not NULL, then
 * the pd_ref_count reference count is decremented for the indicated
 * fc_remote_port_t struct.
 */
int
fc_ulp_uninit_packet(opaque_t port_handle, fc_packet_t *pkt)
{
	int rval;
	fc_local_port_t *port = port_handle;
	fc_remote_port_t *pd;

	ASSERT(pkt != NULL);

	pd = pkt->pkt_pd;

	/* Call the FCA driver's fca_un_init_pkt entry point function */
	rval = port->fp_fca_tran->fca_un_init_pkt(port->fp_fca_handle, pkt);

	if ((rval == FC_SUCCESS) && (pd != NULL)) {
		mutex_enter(&pd->pd_mutex);

		ASSERT(pd->pd_ref_count > 0);
		pd->pd_ref_count--;

		/*
		 * If at this point the state of this fc_remote_port_t
		 * struct is PORT_DEVICE_INVALID, it probably means somebody
		 * is cleaning up old (e.g. retried) packets. If the
		 * pd_ref_count has also dropped to zero, it's time to
		 * deallocate this fc_remote_port_t struct.
		 */
		if (pd->pd_state == PORT_DEVICE_INVALID &&
		    pd->pd_ref_count == 0) {
			fc_remote_node_t *node = pd->pd_remote_nodep;

			mutex_exit(&pd->pd_mutex);

			/*
			 * Also deallocate the associated fc_remote_node_t
			 * struct if it has no other associated
			 * fc_remote_port_t structs.
			 */
			if ((fctl_destroy_remote_port(port, pd) == 0) &&
			    (node != NULL)) {
				fctl_destroy_remote_node(node);
			}
			return (rval);
		}

		mutex_exit(&pd->pd_mutex);
	}

	return (rval);
}


int
fc_ulp_getportmap(opaque_t port_handle, fc_portmap_t **map, uint32_t *len,
    int flag)
{
	int		job_code;
	fc_local_port_t *port;
	job_request_t	*job;
	fc_portmap_t	*tmp_map;
	uint32_t	tmp_len;
	fc_portmap_t	*change_list = NULL;
	uint32_t	listlen = 0;

	port = port_handle;

	mutex_enter(&port->fp_mutex);
	if (port->fp_statec_busy) {
		mutex_exit(&port->fp_mutex);
		return (FC_STATEC_BUSY);
	}

	if (FC_PORT_STATE_MASK(port->fp_state) == FC_STATE_OFFLINE) {
		mutex_exit(&port->fp_mutex);
		return (FC_OFFLINE);
	}

	if (port->fp_dev_count && (port->fp_dev_count ==
	    port->fp_total_devices)) {
		mutex_exit(&port->fp_mutex);
		fctl_fillout_map(port, &change_list, &listlen, 1, 1, 0);
		if (listlen > *len) {
			tmp_map = (fc_portmap_t *)kmem_zalloc(
			    listlen * sizeof (fc_portmap_t), KM_NOSLEEP);
			if (tmp_map == NULL) {
				return (FC_NOMEM);
			}
			if (*map) {
				kmem_free(*map, (*len) * sizeof (fc_portmap_t));
			}
			*map = tmp_map;
		}
		if (change_list) {
			bcopy(change_list, *map,
			    listlen * sizeof (fc_portmap_t));
			kmem_free(change_list, listlen * sizeof (fc_portmap_t));
		}
		*len = listlen;
	} else {
		mutex_exit(&port->fp_mutex);

		switch (flag) {
		case FC_ULP_PLOGI_DONTCARE:
			job_code = JOB_PORT_GETMAP;
			break;

		case FC_ULP_PLOGI_PRESERVE:
			job_code = JOB_PORT_GETMAP_PLOGI_ALL;
			break;

		default:
			return (FC_INVALID_REQUEST);
		}
		/*
		 * Submit a job request to the job handler
		 * thread to get the map and wait
		 */
		job = fctl_alloc_job(job_code, 0, NULL, NULL, KM_SLEEP);
		job->job_private = (opaque_t)map;
		job->job_arg = (opaque_t)len;
		fctl_enque_job(port, job);

		fctl_jobwait(job);
		/*
		 * The result of the last I/O operation is
		 * in job_code. We don't care to look at it
		 * Rather we look at the number of devices
		 * that are found to fill out the map for
		 * ULPs.
		 */
		fctl_dealloc_job(job);
	}

	/*
	 * If we're here, we're returning a map to the caller, which means
	 * we'd better make sure every pd in that map has the
	 * PD_GIVEN_TO_ULPS flag set.
	 */

	tmp_len = *len;
	tmp_map = *map;

	while (tmp_len-- != 0) {
		if (tmp_map->map_state != PORT_DEVICE_INVALID) {
			fc_remote_port_t *pd =
			    (fc_remote_port_t *)tmp_map->map_pd;
			mutex_enter(&pd->pd_mutex);
			pd->pd_aux_flags |= PD_GIVEN_TO_ULPS;
			mutex_exit(&pd->pd_mutex);
		}
		tmp_map++;
	}

	return (FC_SUCCESS);
}


int
fc_ulp_login(opaque_t port_handle, fc_packet_t **ulp_pkt, uint32_t listlen)
{
	int			rval = FC_SUCCESS;
	int			job_flags;
	uint32_t		count;
	fc_packet_t		**tmp_array;
	job_request_t		*job;
	fc_local_port_t		*port = port_handle;
	fc_ulp_rscn_info_t	*rscnp =
	    (fc_ulp_rscn_info_t *)(ulp_pkt[0])->pkt_ulp_rscn_infop;

	/*
	 * If the port is OFFLINE, or if the port driver is
	 * being SUSPENDED/PM_SUSPENDED/DETACHED, block all
	 * PLOGI operations
	 */
	mutex_enter(&port->fp_mutex);
	if (port->fp_statec_busy) {
		mutex_exit(&port->fp_mutex);
		return (FC_STATEC_BUSY);
	}

	if ((FC_PORT_STATE_MASK(port->fp_state) == FC_STATE_OFFLINE) ||
	    (port->fp_soft_state &
	    (FP_SOFT_IN_DETACH | FP_SOFT_SUSPEND | FP_SOFT_POWER_DOWN))) {
		mutex_exit(&port->fp_mutex);
		return (FC_OFFLINE);
	}

	/*
	 * If the rscn count in the packet is not the same as the rscn count
	 * in the fc_local_port_t, then one or more new RSCNs has occurred.
	 */
	if ((rscnp != NULL) &&
	    (rscnp->ulp_rscn_count != FC_INVALID_RSCN_COUNT) &&
	    (rscnp->ulp_rscn_count != port->fp_rscn_count)) {
		mutex_exit(&port->fp_mutex);
		return (FC_DEVICE_BUSY_NEW_RSCN);
	}

	mutex_exit(&port->fp_mutex);

	tmp_array = kmem_zalloc(sizeof (*tmp_array) * listlen, KM_SLEEP);
	for (count = 0; count < listlen; count++) {
		tmp_array[count] = ulp_pkt[count];
	}

	job_flags = ((ulp_pkt[0]->pkt_tran_flags) & FC_TRAN_NO_INTR)
	    ? 0 : JOB_TYPE_FCTL_ASYNC;

#ifdef	DEBUG
	{
		int next;
		int count;
		int polled;

		polled = ((ulp_pkt[0]->pkt_tran_flags) &
		    FC_TRAN_NO_INTR) ? 0 : JOB_TYPE_FCTL_ASYNC;

		for (count = 0; count < listlen; count++) {
			next = ((ulp_pkt[count]->pkt_tran_flags)
			    & FC_TRAN_NO_INTR) ? 0 : JOB_TYPE_FCTL_ASYNC;
			ASSERT(next == polled);
		}
	}
#endif

	job = fctl_alloc_job(JOB_PLOGI_GROUP, job_flags, NULL, NULL, KM_SLEEP);
	job->job_ulp_pkts = tmp_array;
	job->job_ulp_listlen = listlen;

	while (listlen--) {
		fc_packet_t *pkt;

		pkt = tmp_array[listlen];
		if (pkt->pkt_pd == NULL) {
			pkt->pkt_state = FC_PKT_SUCCESS;
			continue;
		}

		mutex_enter(&pkt->pkt_pd->pd_mutex);
		if (pkt->pkt_pd->pd_flags == PD_ELS_IN_PROGRESS ||
		    pkt->pkt_pd->pd_flags == PD_ELS_MARK) {
			/*
			 * Set the packet state and let the port
			 * driver call the completion routine
			 * from its thread
			 */
			mutex_exit(&pkt->pkt_pd->pd_mutex);
			pkt->pkt_state = FC_PKT_ELS_IN_PROGRESS;
			continue;
		}

		if (pkt->pkt_pd->pd_state == PORT_DEVICE_INVALID ||
		    pkt->pkt_pd->pd_type == PORT_DEVICE_OLD) {
			mutex_exit(&pkt->pkt_pd->pd_mutex);
			pkt->pkt_state = FC_PKT_LOCAL_RJT;
			continue;
		}
		mutex_exit(&pkt->pkt_pd->pd_mutex);
		pkt->pkt_state = FC_PKT_SUCCESS;
	}

	fctl_enque_job(port, job);

	if (!(job_flags & JOB_TYPE_FCTL_ASYNC)) {
		fctl_jobwait(job);
		rval = job->job_result;
		fctl_dealloc_job(job);
	}

	return (rval);
}


opaque_t
fc_ulp_get_remote_port(opaque_t port_handle, la_wwn_t *pwwn, int *error,
    int create)
{
	fc_local_port_t		*port;
	job_request_t		*job;
	fc_remote_port_t	*pd;

	port = port_handle;
	pd = fctl_get_remote_port_by_pwwn(port, pwwn);

	if (pd != NULL) {
		*error = FC_SUCCESS;
		/*
		 * A ULP now knows about this pd, so mark it
		 */
		mutex_enter(&pd->pd_mutex);
		pd->pd_aux_flags |= PD_GIVEN_TO_ULPS;
		mutex_exit(&pd->pd_mutex);
		return (pd);
	}

	mutex_enter(&port->fp_mutex);
	if (FC_IS_TOP_SWITCH(port->fp_topology) && create) {
		uint32_t	d_id;
		fctl_ns_req_t	*ns_cmd;

		mutex_exit(&port->fp_mutex);

		job = fctl_alloc_job(JOB_NS_CMD, 0, NULL, NULL, KM_SLEEP);

		if (job == NULL) {
			*error = FC_NOMEM;
			return (pd);
		}

		ns_cmd = fctl_alloc_ns_cmd(sizeof (ns_req_gid_pn_t),
		    sizeof (ns_resp_gid_pn_t), sizeof (ns_resp_gid_pn_t),
		    0, KM_SLEEP);

		if (ns_cmd == NULL) {
			fctl_dealloc_job(job);
			*error = FC_NOMEM;
			return (pd);
		}
		ns_cmd->ns_cmd_code = NS_GID_PN;
		((ns_req_gid_pn_t *)(ns_cmd->ns_cmd_buf))->pwwn = *pwwn;

		job->job_result = FC_SUCCESS;
		job->job_private = (void *)ns_cmd;
		job->job_counter = 1;
		fctl_enque_job(port, job);
		fctl_jobwait(job);

		if (job->job_result != FC_SUCCESS) {
			*error = job->job_result;
			fctl_free_ns_cmd(ns_cmd);
			fctl_dealloc_job(job);
			return (pd);
		}
		d_id = ((ns_resp_gid_pn_t *)ns_cmd->ns_data_buf)->pid.port_id;
		fctl_free_ns_cmd(ns_cmd);

		ns_cmd = fctl_alloc_ns_cmd(sizeof (ns_req_gan_t),
		    sizeof (ns_resp_gan_t), 0, FCTL_NS_CREATE_DEVICE,
		    KM_SLEEP);
		ASSERT(ns_cmd != NULL);

		ns_cmd->ns_gan_max = 1;
		ns_cmd->ns_cmd_code = NS_GA_NXT;
		ns_cmd->ns_gan_sid = FCTL_GAN_START_ID;
		((ns_req_gan_t *)(ns_cmd->ns_cmd_buf))->pid.port_id = d_id - 1;
		((ns_req_gan_t *)(ns_cmd->ns_cmd_buf))->pid.priv_lilp_posit = 0;

		job->job_result = FC_SUCCESS;
		job->job_private = (void *)ns_cmd;
		job->job_counter = 1;
		fctl_enque_job(port, job);
		fctl_jobwait(job);

		fctl_free_ns_cmd(ns_cmd);
		if (job->job_result != FC_SUCCESS) {
			*error = job->job_result;
			fctl_dealloc_job(job);
			return (pd);
		}
		fctl_dealloc_job(job);

		/*
		 * Check if the port device is created now.
		 */
		pd = fctl_get_remote_port_by_pwwn(port, pwwn);

		if (pd == NULL) {
			*error = FC_FAILURE;
		} else {
			*error = FC_SUCCESS;

			/*
			 * A ULP now knows about this pd, so mark it
			 */
			mutex_enter(&pd->pd_mutex);
			pd->pd_aux_flags |= PD_GIVEN_TO_ULPS;
			mutex_exit(&pd->pd_mutex);
		}
	} else {
		mutex_exit(&port->fp_mutex);
		*error = FC_FAILURE;
	}

	return (pd);
}


/*
 * If a NS object exists in the host and query is performed
 * on that object, we should retrieve it from our basket
 * and return it right here, there by saving a request going
 * all the up to the Name Server.
 */
int
fc_ulp_port_ns(opaque_t port_handle, opaque_t pd, fc_ns_cmd_t *ns_req)
{
	int		rval;
	int		fabric;
	job_request_t	*job;
	fctl_ns_req_t	*ns_cmd;
	fc_local_port_t	*port = port_handle;

	mutex_enter(&port->fp_mutex);
	fabric = FC_IS_TOP_SWITCH(port->fp_topology) ? 1 : 0;
	mutex_exit(&port->fp_mutex);

	/*
	 * Name server query can't be performed for devices not in Fabric
	 */
	if (!fabric && pd) {
		return (FC_BADOBJECT);
	}

	if (FC_IS_CMD_A_REG(ns_req->ns_cmd)) {
		if (pd == NULL) {
			rval = fctl_update_host_ns_values(port, ns_req);
			if (rval != FC_SUCCESS) {
				return (rval);
			}
		} else {
			/*
			 * Guess what, FC-GS-2 currently prohibits (not
			 * in the strongest language though) setting of
			 * NS object values by other ports. But we might
			 * get that changed to at least accommodate setting
			 * symbolic node/port names - But if disks/tapes
			 * were going to provide a method to set these
			 * values directly (which in turn might register
			 * with the NS when they come up; yep, for that
			 * to happen the disks will have to be very well
			 * behaved Fabric citizen) we won't need to
			 * register the symbolic port/node names for
			 * other ports too (rather send down SCSI commands
			 * to the devices to set the names)
			 *
			 * Be that as it may, let's continue to fail
			 * registration requests for other ports. period.
			 */
			return (FC_BADOBJECT);
		}

		if (!fabric) {
			return (FC_SUCCESS);
		}
	} else if (!fabric) {
		return (fctl_retrieve_host_ns_values(port, ns_req));
	}

	job = fctl_alloc_job(JOB_NS_CMD, 0, NULL, NULL, KM_SLEEP);
	ASSERT(job != NULL);

	ns_cmd = fctl_alloc_ns_cmd(ns_req->ns_req_len,
	    ns_req->ns_resp_len, ns_req->ns_resp_len, 0, KM_SLEEP);
	ASSERT(ns_cmd != NULL);
	ns_cmd->ns_cmd_code = ns_req->ns_cmd;
	bcopy(ns_req->ns_req_payload, ns_cmd->ns_cmd_buf,
	    ns_req->ns_req_len);

	job->job_private = (void *)ns_cmd;
	fctl_enque_job(port, job);
	fctl_jobwait(job);
	rval = job->job_result;

	if (ns_req->ns_resp_len >= ns_cmd->ns_data_len) {
		bcopy(ns_cmd->ns_data_buf, ns_req->ns_resp_payload,
		    ns_cmd->ns_data_len);
	}
	bcopy(&ns_cmd->ns_resp_hdr, &ns_req->ns_resp_hdr,
	    sizeof (fc_ct_header_t));

	fctl_free_ns_cmd(ns_cmd);
	fctl_dealloc_job(job);

	return (rval);
}


int
fc_ulp_transport(opaque_t port_handle, fc_packet_t *pkt)
{
	int			rval;
	fc_local_port_t		*port;
	fc_remote_port_t	*pd, *newpd;
	fc_ulp_rscn_info_t	*rscnp =
	    (fc_ulp_rscn_info_t *)pkt->pkt_ulp_rscn_infop;

	port = port_handle;

	if (pkt->pkt_tran_flags & FC_TRAN_DUMPING) {
		return (port->fp_fca_tran->fca_transport(
		    port->fp_fca_handle, pkt));
	}

	mutex_enter(&port->fp_mutex);
	if (port->fp_statec_busy) {
		mutex_exit(&port->fp_mutex);
		return (FC_STATEC_BUSY);
	}

	/* A locus of race conditions */
	if (((FC_PORT_STATE_MASK(port->fp_state)) == FC_STATE_OFFLINE) ||
	    (port->fp_soft_state &
	    (FP_SOFT_IN_DETACH | FP_SOFT_SUSPEND | FP_SOFT_POWER_DOWN))) {
		mutex_exit(&port->fp_mutex);
		return (FC_OFFLINE);
	}

	/*
	 * If the rscn count in the packet is not the same as the rscn count
	 * in the fc_local_port_t, then one or more new RSCNs has occurred.
	 */
	if ((rscnp != NULL) &&
	    (rscnp->ulp_rscn_count != FC_INVALID_RSCN_COUNT) &&
	    (rscnp->ulp_rscn_count != port->fp_rscn_count)) {
		mutex_exit(&port->fp_mutex);
		return (FC_DEVICE_BUSY_NEW_RSCN);
	}

	pd = pkt->pkt_pd;
	if (pd) {
		if (pd->pd_type == PORT_DEVICE_OLD ||
		    pd->pd_state == PORT_DEVICE_INVALID) {

			newpd = fctl_get_remote_port_by_pwwn_mutex_held(port,
			    &pd->pd_port_name);

			/*
			 * The remote port (pd) in the packet is no longer
			 * usable, as the old pd still exists we can use the
			 * WWN to check if we have a current pd for the device
			 * we want. Either way we continue with the old logic
			 * whether we have a new pd or not, as the new pd
			 * could be bad, or have become unusable.
			 */
			if ((newpd) && (newpd != pd)) {

				/*
				 * There is a better remote port (pd) to try,
				 * so we need to fix the reference counts, etc.
				 */
				mutex_enter(&newpd->pd_mutex);
				newpd->pd_ref_count++;
				pkt->pkt_pd = newpd;
				mutex_exit(&newpd->pd_mutex);

				mutex_enter(&pd->pd_mutex);
				pd->pd_ref_count--;
				if ((pd->pd_state == PORT_DEVICE_INVALID) &&
				    (pd->pd_ref_count == 0)) {
					fc_remote_node_t *node =
					    pd->pd_remote_nodep;

					mutex_exit(&pd->pd_mutex);
					mutex_exit(&port->fp_mutex);

					/*
					 * This will create another PD hole
					 * where we have a reference to a pd,
					 * but someone else could remove it.
					 */
					if ((fctl_destroy_remote_port(port, pd)
					    == 0) && (node != NULL)) {
						fctl_destroy_remote_node(node);
					}
					mutex_enter(&port->fp_mutex);
				} else {
					mutex_exit(&pd->pd_mutex);
				}
				pd = newpd;
			}
		}

		if (pd->pd_state != PORT_DEVICE_LOGGED_IN) {
			rval = (pd->pd_state == PORT_DEVICE_VALID) ?
			    FC_LOGINREQ : FC_BADDEV;
			mutex_exit(&port->fp_mutex);
			return (rval);
		}

		if (pd->pd_flags != PD_IDLE) {
			mutex_exit(&port->fp_mutex);
			return (FC_DEVICE_BUSY);
		}

		if (pd->pd_type == PORT_DEVICE_OLD ||
		    pd->pd_state == PORT_DEVICE_INVALID) {
			mutex_exit(&port->fp_mutex);
			return (FC_BADDEV);
		}

	} else if (FC_IS_REAL_DEVICE(pkt->pkt_cmd_fhdr.d_id)) {
		mutex_exit(&port->fp_mutex);
		return (FC_BADPACKET);
	}
	mutex_exit(&port->fp_mutex);

	return (port->fp_fca_tran->fca_transport(port->fp_fca_handle, pkt));
}


int
fc_ulp_issue_els(opaque_t port_handle, fc_packet_t *pkt)
{
	int			rval;
	fc_local_port_t		*port = port_handle;
	fc_remote_port_t	*pd;
	fc_ulp_rscn_info_t	*rscnp =
	    (fc_ulp_rscn_info_t *)pkt->pkt_ulp_rscn_infop;

	/*
	 * If the port is OFFLINE, or if the port driver is
	 * being SUSPENDED/PM_SUSPENDED/DETACHED, block all
	 * ELS operations
	 */
	mutex_enter(&port->fp_mutex);
	if ((FC_PORT_STATE_MASK(port->fp_state) == FC_STATE_OFFLINE) ||
	    (port->fp_soft_state &
	    (FP_SOFT_IN_DETACH | FP_SOFT_SUSPEND | FP_SOFT_POWER_DOWN))) {
		mutex_exit(&port->fp_mutex);
		return (FC_OFFLINE);
	}

	if (port->fp_statec_busy) {
		mutex_exit(&port->fp_mutex);
		return (FC_STATEC_BUSY);
	}

	/*
	 * If the rscn count in the packet is not the same as the rscn count
	 * in the fc_local_port_t, then one or more new RSCNs has occurred.
	 */
	if ((rscnp != NULL) &&
	    (rscnp->ulp_rscn_count != FC_INVALID_RSCN_COUNT) &&
	    (rscnp->ulp_rscn_count != port->fp_rscn_count)) {
		mutex_exit(&port->fp_mutex);
		return (FC_DEVICE_BUSY_NEW_RSCN);
	}

	mutex_exit(&port->fp_mutex);

	if ((pd = pkt->pkt_pd) != NULL) {
		mutex_enter(&pd->pd_mutex);
		if (pd->pd_state != PORT_DEVICE_LOGGED_IN) {
			rval = (pd->pd_state == PORT_DEVICE_VALID) ?
			    FC_LOGINREQ : FC_BADDEV;
			mutex_exit(&pd->pd_mutex);
			return (rval);
		}

		if (pd->pd_flags != PD_IDLE) {
			mutex_exit(&pd->pd_mutex);
			return (FC_DEVICE_BUSY);
		}
		if (pd->pd_type == PORT_DEVICE_OLD ||
		    pd->pd_state == PORT_DEVICE_INVALID) {
			mutex_exit(&pd->pd_mutex);
			return (FC_BADDEV);
		}
		mutex_exit(&pd->pd_mutex);
	}

	return (port->fp_fca_tran->fca_els_send(port->fp_fca_handle, pkt));
}


int
fc_ulp_uballoc(opaque_t port_handle, uint32_t *count, uint32_t size,
    uint32_t type, uint64_t *tokens)
{
	fc_local_port_t *port = port_handle;

	return (port->fp_fca_tran->fca_ub_alloc(port->fp_fca_handle,
	    tokens, size, count, type));
}


int
fc_ulp_ubfree(opaque_t port_handle, uint32_t count, uint64_t *tokens)
{
	fc_local_port_t *port = port_handle;

	return (port->fp_fca_tran->fca_ub_free(port->fp_fca_handle,
	    count, tokens));
}


int
fc_ulp_ubrelease(opaque_t port_handle, uint32_t count, uint64_t *tokens)
{
	fc_local_port_t *port = port_handle;

	return (port->fp_fca_tran->fca_ub_release(port->fp_fca_handle,
	    count, tokens));
}


int
fc_ulp_abort(opaque_t port_handle, fc_packet_t *pkt, int flags)
{
	fc_local_port_t *port = port_handle;

	return (port->fp_fca_tran->fca_abort(port->fp_fca_handle, pkt, flags));
}


/*
 * Submit an asynchronous request to the job handler if the sleep
 * flag is set to KM_NOSLEEP, as such calls could have been made
 * in interrupt contexts, and the goal is to avoid busy waiting,
 * blocking on a conditional variable, a semaphore or any of the
 * synchronization primitives. A noticeable draw back with this
 * asynchronous request is that an FC_SUCCESS is returned long
 * before the reset is complete (successful or not).
 */
int
fc_ulp_linkreset(opaque_t port_handle, la_wwn_t *pwwn, int sleep)
{
	int		rval;
	fc_local_port_t *port;
	job_request_t	*job;

	port = port_handle;
	/*
	 * Many a times, this function is called from interrupt
	 * contexts and there have been several dead locks and
	 * hangs - One of the simplest work arounds is to fib
	 * if a RESET is in progress.
	 */
	mutex_enter(&port->fp_mutex);
	if (port->fp_soft_state & FP_SOFT_IN_LINK_RESET) {
		mutex_exit(&port->fp_mutex);
		return (FC_SUCCESS);
	}

	/*
	 * Ward off this reset if a state change is in progress.
	 */
	if (port->fp_statec_busy) {
		mutex_exit(&port->fp_mutex);
		return (FC_STATEC_BUSY);
	}
	port->fp_soft_state |= FP_SOFT_IN_LINK_RESET;
	mutex_exit(&port->fp_mutex);

	if (fctl_busy_port(port) != 0) {
		mutex_enter(&port->fp_mutex);
		port->fp_soft_state &= ~FP_SOFT_IN_LINK_RESET;
		mutex_exit(&port->fp_mutex);
		return (FC_FAILURE);
	}

	if (sleep == KM_SLEEP) {
		job = fctl_alloc_job(JOB_LINK_RESET, 0, NULL, NULL, sleep);
		ASSERT(job != NULL);

		job->job_private = (void *)pwwn;
		job->job_counter = 1;
		fctl_enque_job(port, job);
		fctl_jobwait(job);

		mutex_enter(&port->fp_mutex);
		port->fp_soft_state &= ~FP_SOFT_IN_LINK_RESET;
		mutex_exit(&port->fp_mutex);

		fctl_idle_port(port);

		rval = job->job_result;
		fctl_dealloc_job(job);
	} else {
		job = fctl_alloc_job(JOB_LINK_RESET, JOB_TYPE_FCTL_ASYNC,
		    fctl_link_reset_done, port, sleep);
		if (job == NULL) {
			mutex_enter(&port->fp_mutex);
			port->fp_soft_state &= ~FP_SOFT_IN_LINK_RESET;
			mutex_exit(&port->fp_mutex);
			fctl_idle_port(port);
			return (FC_NOMEM);
		}
		job->job_private = (void *)pwwn;
		job->job_counter = 1;
		fctl_priority_enque_job(port, job);
		rval = FC_SUCCESS;
	}

	return (rval);
}


int
fc_ulp_port_reset(opaque_t port_handle, uint32_t cmd)
{
	int		rval = FC_SUCCESS;
	fc_local_port_t *port = port_handle;

	switch (cmd) {
	case FC_RESET_PORT:
		rval = port->fp_fca_tran->fca_reset(
		    port->fp_fca_handle, FC_FCA_LINK_RESET);
		break;

	case FC_RESET_ADAPTER:
		rval = port->fp_fca_tran->fca_reset(
		    port->fp_fca_handle, FC_FCA_RESET);
		break;

	case FC_RESET_DUMP:
		rval = port->fp_fca_tran->fca_reset(
		    port->fp_fca_handle, FC_FCA_CORE);
		break;

	case FC_RESET_CRASH:
		rval = port->fp_fca_tran->fca_reset(
		    port->fp_fca_handle, FC_FCA_RESET_CORE);
		break;

	default:
		rval = FC_FAILURE;
	}

	return (rval);
}


int
fc_ulp_get_port_login_params(opaque_t port_handle, la_els_logi_t *login_params)
{
	fc_local_port_t *port = port_handle;

	/* Copy the login parameters */
	*login_params = port->fp_service_params;
	return (FC_SUCCESS);
}


int
fc_ulp_get_port_instance(opaque_t port_handle)
{
	fc_local_port_t *port = port_handle;

	return (port->fp_instance);
}


opaque_t
fc_ulp_get_port_handle(int port_instance)
{
	opaque_t	port_handle = NULL;
	fc_fca_port_t	*cur;

	mutex_enter(&fctl_port_lock);
	for (cur = fctl_fca_portlist; cur; cur = cur->port_next) {
		if (cur->port_handle->fp_instance == port_instance) {
			port_handle = (opaque_t)cur->port_handle;
			break;
		}
	}
	mutex_exit(&fctl_port_lock);

	return (port_handle);
}


int
fc_ulp_error(int fc_errno, char **errmsg)
{
	return (fctl_error(fc_errno, errmsg));
}


int
fc_ulp_pkt_error(fc_packet_t *pkt, char **state, char **reason,
    char **action, char **expln)
{
	return (fctl_pkt_error(pkt, state, reason, action, expln));
}


/*
 * If an ULP by the specified name exists, return FC_SUCCESS, else FC_FAILURE
 */
int
fc_ulp_is_name_present(caddr_t ulp_name)
{
	int		rval = FC_FAILURE;
	fc_ulp_list_t	*list;

	mutex_enter(&fctl_ulp_list_mutex);
	for (list = fctl_ulp_list; list != NULL; list = list->ulp_next) {
		if (strcmp(list->ulp_info->ulp_name, ulp_name) == 0) {
			rval = FC_SUCCESS;
			break;
		}
	}
	mutex_exit(&fctl_ulp_list_mutex);

	return (rval);
}


/*
 * Return port WWN for a port Identifier
 */
int
fc_ulp_get_pwwn_by_did(opaque_t port_handle, fc_portid_t d_id, la_wwn_t *pwwn)
{
	int			rval = FC_FAILURE;
	fc_remote_port_t	*pd;
	fc_local_port_t		*port = port_handle;

	pd = fctl_get_remote_port_by_did(port, d_id.port_id);
	if (pd != NULL) {
		mutex_enter(&pd->pd_mutex);
		*pwwn = pd->pd_port_name;
		mutex_exit(&pd->pd_mutex);
		rval = FC_SUCCESS;
	}

	return (rval);
}


/*
 * Return a port map for a port WWN
 */
int
fc_ulp_pwwn_to_portmap(opaque_t port_handle, la_wwn_t *bytes, fc_portmap_t *map)
{
	fc_local_port_t		*port = port_handle;
	fc_remote_node_t	*node;
	fc_remote_port_t	*pd;

	pd = fctl_get_remote_port_by_pwwn(port, bytes);
	if (pd == NULL) {
		return (FC_FAILURE);
	}

	mutex_enter(&pd->pd_mutex);
	map->map_pwwn = pd->pd_port_name;
	map->map_did = pd->pd_port_id;
	map->map_hard_addr = pd->pd_hard_addr;
	map->map_state = pd->pd_state;
	map->map_type = pd->pd_type;
	map->map_flags = 0;

	ASSERT(map->map_type <= PORT_DEVICE_DELETE);

	bcopy(pd->pd_fc4types, map->map_fc4_types, sizeof (pd->pd_fc4types));

	node = pd->pd_remote_nodep;
	mutex_exit(&pd->pd_mutex);

	if (node) {
		mutex_enter(&node->fd_mutex);
		map->map_nwwn = node->fd_node_name;
		mutex_exit(&node->fd_mutex);
	}
	map->map_pd = pd;

	return (FC_SUCCESS);
}


opaque_t
fc_ulp_get_fca_device(opaque_t port_handle, fc_portid_t d_id)
{
	fc_local_port_t	*port = port_handle;

	if (port->fp_fca_tran->fca_get_device == NULL) {
		return (NULL);
	}

	return (port->fp_fca_tran->fca_get_device(port->fp_fca_handle, d_id));
}


int
fc_ulp_port_notify(opaque_t port_handle, uint32_t cmd)
{
	int		rval = FC_SUCCESS;
	fc_local_port_t	*port = port_handle;

	if (port->fp_fca_tran->fca_notify) {
		mutex_enter(&port->fp_mutex);
		switch (cmd) {
		case FC_NOTIFY_TARGET_MODE:
			port->fp_options |= FP_TARGET_MODE;
			break;
		case FC_NOTIFY_NO_TARGET_MODE:
			port->fp_options &= ~FP_TARGET_MODE;
			break;
		}
		mutex_exit(&port->fp_mutex);
		rval = port->fp_fca_tran->fca_notify(port->fp_fca_handle, cmd);
	}

	return (rval);
}


void
fc_ulp_disable_relogin(opaque_t *fc_port, la_wwn_t *pwwn)
{
	fc_remote_port_t *pd =
	    fctl_get_remote_port_by_pwwn((fc_local_port_t *)fc_port, pwwn);

	if (pd) {
		mutex_enter(&pd->pd_mutex);
		pd->pd_aux_flags |= PD_DISABLE_RELOGIN;
		mutex_exit(&pd->pd_mutex);
	}
}


void
fc_ulp_enable_relogin(opaque_t *fc_port, la_wwn_t *pwwn)
{
	fc_remote_port_t *pd =
	    fctl_get_remote_port_by_pwwn((fc_local_port_t *)fc_port, pwwn);

	if (pd) {
		mutex_enter(&pd->pd_mutex);
		pd->pd_aux_flags &= ~PD_DISABLE_RELOGIN;
		mutex_exit(&pd->pd_mutex);
	}
}


/*
 * fc_fca_init
 *		Overload the FCA bus_ops vector in its dev_ops with
 *		fctl_fca_busops to handle all the INITchilds for "sf"
 *		in one common place.
 *
 *		Should be called from FCA _init routine.
 */
void
fc_fca_init(struct dev_ops *fca_devops_p)
{
#ifndef	__lock_lint
	fca_devops_p->devo_bus_ops = &fctl_fca_busops;
#endif	/* __lock_lint */
}


/*
 * fc_fca_attach
 */
int
fc_fca_attach(dev_info_t *fca_dip, fc_fca_tran_t *tran)
{
	/*
	 * When we are in a position to offer downward compatibility
	 * we should change the following check to allow lower revision
	 * of FCAs; But we aren't there right now.
	 */
	if (tran->fca_version != FCTL_FCA_MODREV_5) {
		const char *name = ddi_driver_name(fca_dip);

		ASSERT(name != NULL);

		cmn_err(CE_WARN, "fctl: FCA %s version mismatch"
		    " please upgrade %s", name, name);
		return (DDI_FAILURE);
	}

	ddi_set_driver_private(fca_dip, (caddr_t)tran);
	return (DDI_SUCCESS);
}


/*
 * fc_fca_detach
 */
int
fc_fca_detach(dev_info_t *fca_dip)
{
	ddi_set_driver_private(fca_dip, NULL);
	return (DDI_SUCCESS);
}


/*
 * Check if the frame is a Link response Frame; Handle all cases (P_RJT,
 * F_RJT, P_BSY, F_BSY fall into this category). Check also for some Basic
 * Link Service responses such as BA_RJT and Extended Link Service response
 * such as LS_RJT. If the response is a Link_Data Frame or something that
 * this function doesn't understand return FC_FAILURE; Otherwise, fill out
 * various fields (state, action, reason, expln) from the response gotten
 * in the packet and return FC_SUCCESS.
 */
int
fc_fca_update_errors(fc_packet_t *pkt)
{
	int ret = FC_SUCCESS;

	switch (pkt->pkt_resp_fhdr.r_ctl) {
	case R_CTL_P_RJT: {
		uint32_t prjt;

		prjt = pkt->pkt_resp_fhdr.ro;
		pkt->pkt_state = FC_PKT_NPORT_RJT;
		pkt->pkt_action = (prjt & 0xFF000000) >> 24;
		pkt->pkt_reason = (prjt & 0xFF0000) >> 16;
		break;
	}

	case R_CTL_F_RJT: {
		uint32_t frjt;

		frjt = pkt->pkt_resp_fhdr.ro;
		pkt->pkt_state = FC_PKT_FABRIC_RJT;
		pkt->pkt_action = (frjt & 0xFF000000) >> 24;
		pkt->pkt_reason = (frjt & 0xFF0000) >> 16;
		break;
	}

	case R_CTL_P_BSY: {
		uint32_t pbsy;

		pbsy = pkt->pkt_resp_fhdr.ro;
		pkt->pkt_state = FC_PKT_NPORT_BSY;
		pkt->pkt_action = (pbsy & 0xFF000000) >> 24;
		pkt->pkt_reason = (pbsy & 0xFF0000) >> 16;
		break;
	}

	case R_CTL_F_BSY_LC:
	case R_CTL_F_BSY_DF: {
		uchar_t fbsy;

		fbsy = pkt->pkt_resp_fhdr.type;
		pkt->pkt_state = FC_PKT_FABRIC_BSY;
		pkt->pkt_reason = (fbsy & 0xF0) >> 4;
		break;
	}

	case R_CTL_LS_BA_RJT: {
		uint32_t brjt;

		brjt = *(uint32_t *)pkt->pkt_resp;
		pkt->pkt_state = FC_PKT_BA_RJT;
		pkt->pkt_reason = (brjt & 0xFF0000) >> 16;
		pkt->pkt_expln = (brjt & 0xFF00) >> 8;
		break;
	}

	case R_CTL_ELS_RSP: {
		la_els_rjt_t *lsrjt;

		lsrjt = (la_els_rjt_t *)pkt->pkt_resp;
		if (lsrjt->ls_code.ls_code == LA_ELS_RJT) {
			pkt->pkt_state = FC_PKT_LS_RJT;
			pkt->pkt_reason = lsrjt->reason;
			pkt->pkt_action = lsrjt->action;
			break;
		}
	}
	/* FALLTHROUGH */

	default:
		ret = FC_FAILURE;
		break;
	}

	return (ret);
}


int
fc_fca_error(int fc_errno, char **errmsg)
{
	return (fctl_error(fc_errno, errmsg));
}


int
fc_fca_pkt_error(fc_packet_t *pkt, char **state, char **reason,
    char **action, char **expln)
{
	return (fctl_pkt_error(pkt, state, reason, action, expln));
}


/*
 * WWN to string goodie. Unpredictable results will happen
 * if enough memory isn't supplied in str argument. If you
 * are wondering how much does this routine need, it is just
 * (2 * WWN size + 1). So for a WWN size of 8 bytes the str
 * argument should have atleast 17 bytes allocated.
 */
void
fc_wwn_to_str(la_wwn_t *wwn, caddr_t str)
{
	int count;

	for (count = 0; count < FCTL_WWN_SIZE(wwn); count++, str += 2) {
		(void) sprintf(str, "%02x", wwn->raw_wwn[count]);
	}
	*str = '\0';
}

#define	FC_ATOB(x)	(((x) >= '0' && (x) <= '9') ? ((x) - '0') :	\
			((x) >= 'a' && (x) <= 'f') ?			\
			((x) - 'a' + 10) : ((x) - 'A' + 10))

void
fc_str_to_wwn(caddr_t str, la_wwn_t *wwn)
{
	int count = 0;
	uchar_t byte;

	while (*str) {
		byte = FC_ATOB(*str);
		str++;
		byte = byte << 4 | FC_ATOB(*str);
		str++;
		wwn->raw_wwn[count++] = byte;
	}
}

/*
 * FCA driver's intercepted bus control operations.
 */
static int
fctl_fca_bus_ctl(dev_info_t *fca_dip, dev_info_t *rip,
    ddi_ctl_enum_t op, void *arg, void *result)
{
	switch (op) {
	case DDI_CTLOPS_REPORTDEV:
		break;

	case DDI_CTLOPS_IOMIN:
		break;

	case DDI_CTLOPS_INITCHILD:
		return (fctl_initchild(fca_dip, (dev_info_t *)arg));

	case DDI_CTLOPS_UNINITCHILD:
		return (fctl_uninitchild(fca_dip, (dev_info_t *)arg));

	default:
		return (ddi_ctlops(fca_dip, rip, op, arg, result));
	}

	return (DDI_SUCCESS);
}


/*
 * FCAs indicate the maximum number of ports supported in their
 * tran structure. Fail the INITCHILD if the child port number
 * is any greater than the maximum number of ports supported
 * by the FCA.
 */
static int
fctl_initchild(dev_info_t *fca_dip, dev_info_t *port_dip)
{
	int		rval;
	int		port_no;
	int		port_len;
	char		name[20];
	fc_fca_tran_t	*tran;
	dev_info_t	*dip;
	int		portprop;

	port_len = sizeof (port_no);

	/* physical port do not has this property */
	portprop = ddi_prop_get_int(DDI_DEV_T_ANY, port_dip,
	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
	    "phyport-instance", -1);

	if ((portprop == -1) && ndi_dev_is_persistent_node(port_dip)) {
		/*
		 * Clear any addr bindings created by fcode interpreter
		 * in devi_last_addr so that a ndi_devi_find should never
		 * return this fcode node.
		 */
		ddi_set_name_addr(port_dip, NULL);
		return (DDI_FAILURE);
	}

	rval = ddi_prop_op(DDI_DEV_T_ANY, port_dip, PROP_LEN_AND_VAL_BUF,
	    DDI_PROP_DONTPASS | DDI_PROP_CANSLEEP, "port",
	    (caddr_t)&port_no, &port_len);

	if (rval != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}

	tran = (fc_fca_tran_t *)ddi_get_driver_private(fca_dip);
	ASSERT(tran != NULL);

	(void) sprintf((char *)name, "%x,0", port_no);
	ddi_set_name_addr(port_dip, name);

	dip = ndi_devi_find(fca_dip, ddi_binding_name(port_dip), name);

	/*
	 * Even though we never initialize FCode nodes of fp, such a node
	 * could still be there after a DR operation. There will only be
	 * one FCode node, so if this is the one, clear it and issue a
	 * ndi_devi_find again.
	 */
	if ((portprop == -1) && dip && ndi_dev_is_persistent_node(dip)) {
		ddi_set_name_addr(dip, NULL);
		dip = ndi_devi_find(fca_dip, ddi_binding_name(port_dip), name);
	}

	if ((portprop == -1) && dip && (dip != port_dip)) {
		/*
		 * Here we have a duplicate .conf entry. Clear the addr
		 * set previously and return failure.
		 */
		ddi_set_name_addr(port_dip, NULL);
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}


/* ARGSUSED */
static int
fctl_uninitchild(dev_info_t *fca_dip, dev_info_t *port_dip)
{
	ddi_set_name_addr(port_dip, NULL);
	return (DDI_SUCCESS);
}


static dev_info_t *
fctl_findchild(dev_info_t *pdip, char *cname, char *caddr)
{
	dev_info_t *dip;
	char *addr;

	ASSERT(cname != NULL && caddr != NULL);
	/* ASSERT(DEVI_BUSY_OWNED(pdip)); */

	for (dip = ddi_get_child(pdip); dip != NULL;
	    dip = ddi_get_next_sibling(dip)) {
		if (strcmp(cname, ddi_node_name(dip)) != 0) {
			continue;
		}

		if ((addr = ddi_get_name_addr(dip)) == NULL) {
			if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
			    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
			    "bus-addr", &addr) == DDI_PROP_SUCCESS) {
				if (strcmp(caddr, addr) == 0) {
					ddi_prop_free(addr);
					return (dip);
				}
				ddi_prop_free(addr);
			}
		} else {
			if (strcmp(caddr, addr) == 0) {
				return (dip);
			}
		}
	}

	return (NULL);
}

int
fctl_check_npiv_portindex(dev_info_t *dip, int vindex)
{
	int i, instance;
	fc_local_port_t *port;

	instance = ddi_get_instance(dip);
	port = (fc_local_port_t *)fc_ulp_get_port_handle(instance);
	if ((!port) || (vindex <= 0) || (vindex >= FC_NPIV_MAX_PORT)) {
		return (0);
	}

	i = vindex-1;
	mutex_enter(&port->fp_mutex);
	if (port->fp_npiv_portindex[i] == 0) {
		mutex_exit(&port->fp_mutex);
		return (vindex);
	}
	mutex_exit(&port->fp_mutex);
	return (0);
}

int
fctl_get_npiv_portindex(dev_info_t *dip)
{
	int i, instance;
	fc_local_port_t *port;

	instance = ddi_get_instance(dip);
	port = (fc_local_port_t *)fc_ulp_get_port_handle(instance);
	if (!port) {
		return (0);
	}

	mutex_enter(&port->fp_mutex);
	for (i = 0; i < FC_NPIV_MAX_PORT; i++) {
		if (port->fp_npiv_portindex[i] == 0) {
			mutex_exit(&port->fp_mutex);
			return (i+1);
		}
	}
	mutex_exit(&port->fp_mutex);
	return (0);
}


void
fctl_set_npiv_portindex(dev_info_t *dip, int index)
{
	int instance;
	fc_local_port_t *port;

	instance = ddi_get_instance(dip);
	port = (fc_local_port_t *)fc_ulp_get_port_handle(instance);
	if (!port) {
		return;
	}
	mutex_enter(&port->fp_mutex);
	port->fp_npiv_portindex[index - 1] = 1;
	mutex_exit(&port->fp_mutex);
}


int
fctl_fca_create_npivport(dev_info_t *parent,
    dev_info_t *phydip, char *nname, char *pname, uint32_t *vindex)
{
	int rval = 0, devstrlen;
	char	*devname, *cname, *caddr, *devstr;
	dev_info_t	*child = NULL;
	int		portnum;

	if (*vindex == 0) {
		portnum = fctl_get_npiv_portindex(phydip);
		*vindex = portnum;
	} else {
		portnum = fctl_check_npiv_portindex(phydip, *vindex);
	}

	if (portnum == 0) {
		cmn_err(CE_WARN,
		    "Cann't find valid port index, fail to create devnode");
		return (NDI_FAILURE);
	}

	devname = kmem_zalloc(MAXNAMELEN, KM_SLEEP);
	(void) sprintf(devname, "fp@%x,0", portnum);
	devstrlen = strlen(devname) + 1;
	devstr = i_ddi_strdup(devname, KM_SLEEP);
	i_ddi_parse_name(devstr, &cname, &caddr, NULL);

	if (fctl_findchild(parent, cname, caddr) != NULL) {
		rval = NDI_FAILURE;
		goto freememory;
	}

	ndi_devi_alloc_sleep(parent, cname, DEVI_PSEUDO_NODEID, &child);
	if (child == NULL) {
		cmn_err(CE_WARN,
		    "fctl_create_npiv_port fail to create new devinfo");
		rval = NDI_FAILURE;
		goto freememory;
	}

	if (ndi_prop_update_string(DDI_DEV_T_NONE, child,
	    "bus-addr", caddr) != DDI_PROP_SUCCESS) {
		cmn_err(CE_WARN, "fctl%d: prop update bus-addr %s@%s failed",
		    ddi_get_instance(parent), cname, caddr);
		(void) ndi_devi_free(child);
		rval = NDI_FAILURE;
		goto freememory;
	}

	if (strlen(nname) != 0) {
		if (ndi_prop_update_string(DDI_DEV_T_NONE, child,
		    "node-name", nname) != DDI_PROP_SUCCESS) {
			(void) ndi_devi_free(child);
			rval = NDI_FAILURE;
			goto freememory;
		}
	}

	if (strlen(pname) != 0) {
		if (ndi_prop_update_string(DDI_DEV_T_NONE, child,
		    "port-name", pname) != DDI_PROP_SUCCESS) {
			(void) ndi_devi_free(child);
			rval = NDI_FAILURE;
			goto freememory;
		}
	}

	if (ddi_prop_update_int(DDI_DEV_T_NONE, child,
	    "port", portnum) != DDI_PROP_SUCCESS) {
		cmn_err(CE_WARN, "fp%d: prop_update port %s@%s failed",
		    ddi_get_instance(parent), cname, caddr);
		(void) ndi_devi_free(child);
		rval = NDI_FAILURE;
		goto freememory;
	}

	if (ddi_prop_update_int(DDI_DEV_T_NONE, child,
	    "phyport-instance", ddi_get_instance(phydip)) != DDI_PROP_SUCCESS) {
		cmn_err(CE_WARN,
		    "fp%d: prop_update phyport-instance %s@%s failed",
		    ddi_get_instance(parent), cname, caddr);
		(void) ndi_devi_free(child);
		rval = NDI_FAILURE;
		goto freememory;
	}

	rval = ndi_devi_online(child, NDI_ONLINE_ATTACH);
	if (rval != NDI_SUCCESS) {
		cmn_err(CE_WARN, "fp%d: online_driver %s failed",
		    ddi_get_instance(parent), cname);
		rval = NDI_FAILURE;
		goto freememory;
	}

	fctl_set_npiv_portindex(phydip, portnum);
freememory:
	kmem_free(devstr, devstrlen);
	kmem_free(devname, MAXNAMELEN);

	return (rval);
}


void
fctl_add_port(fc_local_port_t *port)
{
	fc_fca_port_t *new;

	new = kmem_zalloc(sizeof (*new), KM_SLEEP);

	mutex_enter(&fctl_port_lock);
	new->port_handle = port;
	new->port_next = fctl_fca_portlist;
	fctl_fca_portlist = new;
	mutex_exit(&fctl_port_lock);
}


void
fctl_remove_port(fc_local_port_t *port)
{
	fc_ulp_module_t		*mod;
	fc_fca_port_t		*prev;
	fc_fca_port_t		*list;
	fc_ulp_ports_t		*ulp_port;

	rw_enter(&fctl_ulp_lock, RW_WRITER);
	rw_enter(&fctl_mod_ports_lock, RW_WRITER);

	for (mod = fctl_ulp_modules; mod; mod = mod->mod_next) {
		ulp_port = fctl_get_ulp_port(mod, port);
		if (ulp_port == NULL) {
			continue;
		}

#ifndef	__lock_lint
		ASSERT((ulp_port->port_dstate & ULP_PORT_ATTACH) == 0);
#endif /* __lock_lint */

		(void) fctl_remove_ulp_port(mod, port);
	}

	rw_exit(&fctl_mod_ports_lock);
	rw_exit(&fctl_ulp_lock);

	mutex_enter(&fctl_port_lock);

	list = fctl_fca_portlist;
	prev = NULL;
	while (list != NULL) {
		if (list->port_handle == port) {
			if (prev == NULL) {
				fctl_fca_portlist = list->port_next;
			} else {
				prev->port_next = list->port_next;
			}
			kmem_free(list, sizeof (*list));
			break;
		}
		prev = list;
		list = list->port_next;
	}
	mutex_exit(&fctl_port_lock);
}


void
fctl_attach_ulps(fc_local_port_t *port, fc_attach_cmd_t cmd,
    struct modlinkage *linkage)
{
	int			rval;
	uint32_t		s_id;
	uint32_t		state;
	fc_ulp_module_t		*mod;
	fc_ulp_port_info_t	info;
	fc_ulp_ports_t		*ulp_port;

	ASSERT(!MUTEX_HELD(&port->fp_mutex));

	info.port_linkage = linkage;
	info.port_dip = port->fp_port_dip;
	info.port_handle = (opaque_t)port;
	info.port_dma_behavior = port->fp_dma_behavior;
	info.port_fcp_dma = port->fp_fcp_dma;
	info.port_acc_attr = port->fp_fca_tran->fca_acc_attr;
	info.port_fca_pkt_size = port->fp_fca_tran->fca_pkt_size;
	info.port_reset_action = port->fp_reset_action;

	mutex_enter(&port->fp_mutex);

	/*
	 * It is still possible that another thread could have gotten
	 * into the detach process before we got here.
	 */
	if (port->fp_soft_state & FP_SOFT_IN_DETACH) {
		mutex_exit(&port->fp_mutex);
		return;
	}

	s_id = port->fp_port_id.port_id;
	if (port->fp_statec_busy) {
		info.port_state = port->fp_bind_state;
	} else {
		info.port_state = port->fp_state;
	}

	switch (state = FC_PORT_STATE_MASK(info.port_state)) {
	case FC_STATE_LOOP:
	case FC_STATE_NAMESERVICE:
		info.port_state &= ~state;
		info.port_state |= FC_STATE_ONLINE;
		break;

	default:
		break;
	}
	ASSERT((info.port_state & FC_STATE_LOOP) == 0);

	info.port_flags = port->fp_topology;
	info.port_pwwn = port->fp_service_params.nport_ww_name;
	info.port_nwwn = port->fp_service_params.node_ww_name;
	mutex_exit(&port->fp_mutex);

	rw_enter(&fctl_ulp_lock, RW_READER);
	rw_enter(&fctl_mod_ports_lock, RW_WRITER);

	for (mod = fctl_ulp_modules; mod; mod = mod->mod_next) {
		if ((port->fp_soft_state & FP_SOFT_FCA_IS_NODMA) &&
		    (mod->mod_info->ulp_type == FC_TYPE_IS8802_SNAP)) {
			/*
			 * We don't support IP over FC on FCOE HBA
			 */
			continue;
		}

		if ((ulp_port = fctl_get_ulp_port(mod, port)) == NULL) {
			ulp_port = fctl_add_ulp_port(mod, port, KM_SLEEP);
			ASSERT(ulp_port != NULL);

			mutex_enter(&ulp_port->port_mutex);
			ulp_port->port_statec = ((info.port_state &
			    FC_STATE_ONLINE) ? FC_ULP_STATEC_ONLINE :
			    FC_ULP_STATEC_OFFLINE);
			mutex_exit(&ulp_port->port_mutex);
		}
	}

	rw_downgrade(&fctl_mod_ports_lock);

	for (mod = fctl_ulp_modules; mod; mod = mod->mod_next) {
		if ((port->fp_soft_state & FP_SOFT_FCA_IS_NODMA) &&
		    (mod->mod_info->ulp_type == FC_TYPE_IS8802_SNAP)) {
			/*
			 * We don't support IP over FC on FCOE HBA
			 */
			continue;
		}

		ulp_port = fctl_get_ulp_port(mod, port);
		ASSERT(ulp_port != NULL);

		if (fctl_pre_attach(ulp_port, cmd) == FC_FAILURE) {
			continue;
		}

		fctl_init_dma_attr(port, mod, &info);

		rval = mod->mod_info->ulp_port_attach(
		    mod->mod_info->ulp_handle, &info, cmd, s_id);

		fctl_post_attach(mod, ulp_port, cmd, rval);

		if (rval == FC_SUCCESS && cmd == FC_CMD_ATTACH &&
		    strcmp(mod->mod_info->ulp_name, "fcp") == 0) {
			ASSERT(ddi_get_driver_private(info.port_dip) != NULL);
		}
	}

	rw_exit(&fctl_mod_ports_lock);
	rw_exit(&fctl_ulp_lock);
}


static int
fctl_pre_attach(fc_ulp_ports_t *ulp_port, fc_attach_cmd_t cmd)
{
	int rval = FC_SUCCESS;

	mutex_enter(&ulp_port->port_mutex);

	switch (cmd) {
	case FC_CMD_ATTACH:
		if (ulp_port->port_dstate & ULP_PORT_ATTACH) {
			rval = FC_FAILURE;
		}
		break;

	case FC_CMD_RESUME:
		ASSERT((ulp_port->port_dstate & ULP_PORT_POWER_DOWN) == 0);
		if (!(ulp_port->port_dstate & ULP_PORT_ATTACH) ||
		    !(ulp_port->port_dstate & ULP_PORT_SUSPEND)) {
			rval = FC_FAILURE;
		}
		break;

	case FC_CMD_POWER_UP:
		if (!(ulp_port->port_dstate & ULP_PORT_ATTACH) ||
		    !(ulp_port->port_dstate & ULP_PORT_POWER_DOWN)) {
			rval = FC_FAILURE;
		}
		break;
	}

	if (rval == FC_SUCCESS) {
		ulp_port->port_dstate |= ULP_PORT_BUSY;
	}
	mutex_exit(&ulp_port->port_mutex);

	return (rval);
}


static void
fctl_post_attach(fc_ulp_module_t *mod, fc_ulp_ports_t *ulp_port,
    fc_attach_cmd_t cmd, int rval)
{
	int	be_chatty;

	ASSERT(cmd == FC_CMD_ATTACH || cmd == FC_CMD_RESUME ||
	    cmd == FC_CMD_POWER_UP);

	mutex_enter(&ulp_port->port_mutex);
	ulp_port->port_dstate &= ~ULP_PORT_BUSY;

	be_chatty = (rval == FC_FAILURE_SILENT) ? 0 : 1;

	if (rval != FC_SUCCESS) {
		caddr_t		op;
		fc_local_port_t *port = ulp_port->port_handle;

		mutex_exit(&ulp_port->port_mutex);

		switch (cmd) {
		case FC_CMD_ATTACH:
			op = "attach";
			break;

		case FC_CMD_RESUME:
			op = "resume";
			break;

		case FC_CMD_POWER_UP:
			op = "power up";
			break;
		}

		if (be_chatty) {
			cmn_err(CE_WARN, "!fctl(%d): %s failed for %s",
			    port->fp_instance, op, mod->mod_info->ulp_name);
		}

		return;
	}

	switch (cmd) {
	case FC_CMD_ATTACH:
		ulp_port->port_dstate |= ULP_PORT_ATTACH;
		break;

	case FC_CMD_RESUME:
		ulp_port->port_dstate &= ~ULP_PORT_SUSPEND;
		break;

	case FC_CMD_POWER_UP:
		ulp_port->port_dstate &= ~ULP_PORT_POWER_DOWN;
		break;
	}
	mutex_exit(&ulp_port->port_mutex);
}


int
fctl_detach_ulps(fc_local_port_t *port, fc_detach_cmd_t cmd,
    struct modlinkage *linkage)
{
	int			rval = FC_SUCCESS;
	fc_ulp_module_t		*mod;
	fc_ulp_port_info_t	info;
	fc_ulp_ports_t		*ulp_port;

	ASSERT(!MUTEX_HELD(&port->fp_mutex));

	info.port_linkage = linkage;
	info.port_dip = port->fp_port_dip;
	info.port_handle = (opaque_t)port;
	info.port_acc_attr = port->fp_fca_tran->fca_acc_attr;
	info.port_fca_pkt_size = port->fp_fca_tran->fca_pkt_size;

	rw_enter(&fctl_ulp_lock, RW_READER);
	rw_enter(&fctl_mod_ports_lock, RW_READER);

	for (mod = fctl_ulp_modules; mod; mod = mod->mod_next) {
		if ((ulp_port = fctl_get_ulp_port(mod, port)) == NULL) {
			continue;
		}

		if (fctl_pre_detach(ulp_port, cmd) != FC_SUCCESS) {
			continue;
		}

		fctl_init_dma_attr(port, mod, &info);

		rval = mod->mod_info->ulp_port_detach(
		    mod->mod_info->ulp_handle, &info, cmd);

		fctl_post_detach(mod, ulp_port, cmd, rval);

		if (rval != FC_SUCCESS) {
			break;
		}

		if (cmd == FC_CMD_DETACH && strcmp(mod->mod_info->ulp_name,
		    "fcp") == 0) {
			ASSERT(ddi_get_driver_private(info.port_dip) == NULL);
		}

		mutex_enter(&ulp_port->port_mutex);
		ulp_port->port_statec = FC_ULP_STATEC_DONT_CARE;
		mutex_exit(&ulp_port->port_mutex);
	}

	rw_exit(&fctl_mod_ports_lock);
	rw_exit(&fctl_ulp_lock);

	return (rval);
}

static	void
fctl_init_dma_attr(fc_local_port_t *port, fc_ulp_module_t *mod,
    fc_ulp_port_info_t	*info)
{

	if ((strcmp(mod->mod_info->ulp_name, "fcp") == 0) ||
	    (strcmp(mod->mod_info->ulp_name, "ltct") == 0)) {
		info->port_cmd_dma_attr =
		    port->fp_fca_tran->fca_dma_fcp_cmd_attr;
		info->port_data_dma_attr =
		    port->fp_fca_tran->fca_dma_fcp_data_attr;
		info->port_resp_dma_attr =
		    port->fp_fca_tran->fca_dma_fcp_rsp_attr;
	} else if (strcmp(mod->mod_info->ulp_name, "fcsm") == 0) {
		info->port_cmd_dma_attr =
		    port->fp_fca_tran->fca_dma_fcsm_cmd_attr;
		info->port_data_dma_attr =
		    port->fp_fca_tran->fca_dma_attr;
		info->port_resp_dma_attr =
		    port->fp_fca_tran->fca_dma_fcsm_rsp_attr;
	} else if (strcmp(mod->mod_info->ulp_name, "fcip") == 0) {
		info->port_cmd_dma_attr =
		    port->fp_fca_tran->fca_dma_fcip_cmd_attr;
		info->port_data_dma_attr =
		    port->fp_fca_tran->fca_dma_attr;
		info->port_resp_dma_attr =
		    port->fp_fca_tran->fca_dma_fcip_rsp_attr;
	} else {
		info->port_cmd_dma_attr = info->port_data_dma_attr =
		    info->port_resp_dma_attr =
		    port->fp_fca_tran->fca_dma_attr; /* default */
	}
}

static int
fctl_pre_detach(fc_ulp_ports_t *ulp_port, fc_detach_cmd_t cmd)
{
	int rval = FC_SUCCESS;

	mutex_enter(&ulp_port->port_mutex);

	switch (cmd) {
	case FC_CMD_DETACH:
		if ((ulp_port->port_dstate & ULP_PORT_ATTACH) == 0) {
			rval = FC_FAILURE;
		}
		break;

	case FC_CMD_SUSPEND:
		if (!(ulp_port->port_dstate & ULP_PORT_ATTACH) ||
		    ulp_port->port_dstate & ULP_PORT_SUSPEND) {
			rval = FC_FAILURE;
		}
		break;

	case FC_CMD_POWER_DOWN:
		if (!(ulp_port->port_dstate & ULP_PORT_ATTACH) ||
		    ulp_port->port_dstate & ULP_PORT_POWER_DOWN) {
			rval = FC_FAILURE;
		}
		break;
	}

	if (rval == FC_SUCCESS) {
		ulp_port->port_dstate |= ULP_PORT_BUSY;
	}
	mutex_exit(&ulp_port->port_mutex);

	return (rval);
}


static void
fctl_post_detach(fc_ulp_module_t *mod, fc_ulp_ports_t *ulp_port,
    fc_detach_cmd_t cmd, int rval)
{
	ASSERT(cmd == FC_CMD_DETACH || cmd == FC_CMD_SUSPEND ||
	    cmd == FC_CMD_POWER_DOWN);

	mutex_enter(&ulp_port->port_mutex);
	ulp_port->port_dstate &= ~ULP_PORT_BUSY;

	if (rval != FC_SUCCESS) {
		caddr_t		op;
		fc_local_port_t *port = ulp_port->port_handle;

		mutex_exit(&ulp_port->port_mutex);

		switch (cmd) {
		case FC_CMD_DETACH:
			op = "detach";
			break;

		case FC_CMD_SUSPEND:
			op = "suspend";
			break;

		case FC_CMD_POWER_DOWN:
			op = "power down";
			break;
		}

		cmn_err(CE_WARN, "!fctl(%d): %s failed for %s",
		    port->fp_instance, op, mod->mod_info->ulp_name);

		return;
	}

	switch (cmd) {
	case FC_CMD_DETACH:
		ulp_port->port_dstate &= ~ULP_PORT_ATTACH;
		break;

	case FC_CMD_SUSPEND:
		ulp_port->port_dstate |= ULP_PORT_SUSPEND;
		break;

	case FC_CMD_POWER_DOWN:
		ulp_port->port_dstate |= ULP_PORT_POWER_DOWN;
		break;
	}
	mutex_exit(&ulp_port->port_mutex);
}


static fc_ulp_ports_t *
fctl_add_ulp_port(fc_ulp_module_t *ulp_module, fc_local_port_t *port_handle,
    int sleep)
{
	fc_ulp_ports_t *last;
	fc_ulp_ports_t *next;
	fc_ulp_ports_t *new;

	ASSERT(RW_READ_HELD(&fctl_ulp_lock));
	ASSERT(RW_WRITE_HELD(&fctl_mod_ports_lock));

	last = NULL;
	next = ulp_module->mod_ports;

	while (next != NULL) {
		last = next;
		next = next->port_next;
	}

	new = fctl_alloc_ulp_port(sleep);
	if (new == NULL) {
		return (new);
	}

	new->port_handle = port_handle;
	if (last == NULL) {
		ulp_module->mod_ports = new;
	} else {
		last->port_next = new;
	}

	return (new);
}


static fc_ulp_ports_t *
fctl_alloc_ulp_port(int sleep)
{
	fc_ulp_ports_t *new;

	new = kmem_zalloc(sizeof (*new), sleep);
	if (new == NULL) {
		return (new);
	}
	mutex_init(&new->port_mutex, NULL, MUTEX_DRIVER, NULL);

	return (new);
}


static int
fctl_remove_ulp_port(struct ulp_module *ulp_module,
    fc_local_port_t *port_handle)
{
	fc_ulp_ports_t *last;
	fc_ulp_ports_t *next;

	ASSERT(RW_WRITE_HELD(&fctl_ulp_lock));
	ASSERT(RW_WRITE_HELD(&fctl_mod_ports_lock));

	last = NULL;
	next = ulp_module->mod_ports;

	while (next != NULL) {
		if (next->port_handle == port_handle) {
			if (next->port_dstate & ULP_PORT_ATTACH) {
				return (FC_FAILURE);
			}
			break;
		}
		last = next;
		next = next->port_next;
	}

	if (next != NULL) {
		ASSERT((next->port_dstate & ULP_PORT_ATTACH) == 0);

		if (last == NULL) {
			ulp_module->mod_ports = next->port_next;
		} else {
			last->port_next = next->port_next;
		}
		fctl_dealloc_ulp_port(next);

		return (FC_SUCCESS);
	} else {
		return (FC_FAILURE);
	}
}


static void
fctl_dealloc_ulp_port(fc_ulp_ports_t *next)
{
	mutex_destroy(&next->port_mutex);
	kmem_free(next, sizeof (*next));
}


static fc_ulp_ports_t *
fctl_get_ulp_port(struct ulp_module *ulp_module, fc_local_port_t *port_handle)
{
	fc_ulp_ports_t *next;

	ASSERT(RW_LOCK_HELD(&fctl_ulp_lock));
	ASSERT(RW_LOCK_HELD(&fctl_mod_ports_lock));

	for (next = ulp_module->mod_ports; next != NULL;
	    next = next->port_next) {
		if (next->port_handle == port_handle) {
			return (next);
		}
	}

	return (NULL);
}


/*
 * Pass state change notfications on to registered ULPs.
 *
 * Can issue wakeups to client callers who might be waiting for completions
 * on other threads.
 *
 * Caution: will silently deallocate any fc_remote_port_t and/or
 * fc_remote_node_t structs it finds that are not in use.
 */
void
fctl_ulp_statec_cb(void *arg)
{
	uint32_t		s_id;
	uint32_t		new_state;
	fc_local_port_t		*port;
	fc_ulp_ports_t		*ulp_port;
	fc_ulp_module_t		*mod;
	fc_port_clist_t		*clist = (fc_port_clist_t *)arg;

	ASSERT(clist != NULL);

	port = clist->clist_port;

	mutex_enter(&port->fp_mutex);
	s_id = port->fp_port_id.port_id;
	mutex_exit(&port->fp_mutex);

	switch (clist->clist_state) {
	case FC_STATE_ONLINE:
		new_state = FC_ULP_STATEC_ONLINE;
		break;

	case FC_STATE_OFFLINE:
		if (clist->clist_len) {
			new_state = FC_ULP_STATEC_OFFLINE_TIMEOUT;
		} else {
			new_state = FC_ULP_STATEC_OFFLINE;
		}
		break;

	default:
		new_state = FC_ULP_STATEC_DONT_CARE;
		break;
	}

#ifdef	DEBUG
	/*
	 * sanity check for presence of OLD devices in the hash lists
	 */
	if (clist->clist_size) {
		int			count;
		fc_remote_port_t	*pd;

		ASSERT(clist->clist_map != NULL);
		for (count = 0; count < clist->clist_len; count++) {
			if (clist->clist_map[count].map_state ==
			    PORT_DEVICE_INVALID) {
				la_wwn_t	pwwn;
				fc_portid_t	d_id;

				pd = clist->clist_map[count].map_pd;
				if (pd != NULL) {
					mutex_enter(&pd->pd_mutex);
					pwwn = pd->pd_port_name;
					d_id = pd->pd_port_id;
					mutex_exit(&pd->pd_mutex);

					pd = fctl_get_remote_port_by_pwwn(port,
					    &pwwn);

					ASSERT(pd != clist->clist_map[count].
					    map_pd);

					pd = fctl_get_remote_port_by_did(port,
					    d_id.port_id);
					ASSERT(pd != clist->clist_map[count].
					    map_pd);
				}
			}
		}
	}
#endif

	/*
	 * Check for duplicate map entries
	 */
	if (clist->clist_size) {
		int			count;
		fc_remote_port_t	*pd1, *pd2;

		ASSERT(clist->clist_map != NULL);
		for (count = 0; count < clist->clist_len-1; count++) {
			int count2;

			pd1 = clist->clist_map[count].map_pd;
			if (pd1 == NULL) {
				continue;
			}

			for (count2 = count+1;
			    count2 < clist->clist_len;
			    count2++) {

				pd2 = clist->clist_map[count2].map_pd;
				if (pd2 == NULL) {
					continue;
				}

				if (pd1 == pd2) {
					clist->clist_map[count].map_flags |=
					    PORT_DEVICE_DUPLICATE_MAP_ENTRY;
					break;
				}
			}
		}
	}


	rw_enter(&fctl_ulp_lock, RW_READER);
	for (mod = fctl_ulp_modules; mod; mod = mod->mod_next) {
		rw_enter(&fctl_mod_ports_lock, RW_READER);
		ulp_port = fctl_get_ulp_port(mod, port);
		rw_exit(&fctl_mod_ports_lock);

		if (ulp_port == NULL) {
			continue;
		}

		mutex_enter(&ulp_port->port_mutex);
		if (FCTL_DISALLOW_CALLBACKS(ulp_port->port_dstate)) {
			mutex_exit(&ulp_port->port_mutex);
			continue;
		}

		switch (ulp_port->port_statec) {
		case FC_ULP_STATEC_DONT_CARE:
			if (ulp_port->port_statec != new_state) {
				ulp_port->port_statec = new_state;
			}
			break;

		case FC_ULP_STATEC_ONLINE:
		case FC_ULP_STATEC_OFFLINE:
			if (ulp_port->port_statec == new_state) {
				mutex_exit(&ulp_port->port_mutex);
				continue;
			}
			ulp_port->port_statec = new_state;
			break;

		case FC_ULP_STATEC_OFFLINE_TIMEOUT:
			if (ulp_port->port_statec == new_state ||
			    new_state == FC_ULP_STATEC_OFFLINE) {
				mutex_exit(&ulp_port->port_mutex);
				continue;
			}
			ulp_port->port_statec = new_state;
			break;

		default:
			ASSERT(0);
			break;
		}

		mod->mod_info->ulp_statec_callback(
		    mod->mod_info->ulp_handle, (opaque_t)port,
		    clist->clist_state, clist->clist_flags,
		    clist->clist_map, clist->clist_len, s_id);

		mutex_exit(&ulp_port->port_mutex);
	}
	rw_exit(&fctl_ulp_lock);

	if (clist->clist_size) {
		int			count;
		fc_remote_node_t	*node;
		fc_remote_port_t	*pd;

		ASSERT(clist->clist_map != NULL);
		for (count = 0; count < clist->clist_len; count++) {

			if ((pd = clist->clist_map[count].map_pd) == NULL) {
				continue;
			}

			mutex_enter(&pd->pd_mutex);

			pd->pd_ref_count--;
			ASSERT(pd->pd_ref_count >= 0);

			if (clist->clist_map[count].map_state !=
			    PORT_DEVICE_INVALID) {
				mutex_exit(&pd->pd_mutex);
				continue;
			}

			node = pd->pd_remote_nodep;
			pd->pd_aux_flags &= ~PD_GIVEN_TO_ULPS;

			mutex_exit(&pd->pd_mutex);

			/*
			 * This fc_remote_port_t is no longer referenced
			 * by any ULPs. Deallocate it if its pd_ref_count
			 * has reached zero.
			 */
			if ((fctl_destroy_remote_port(port, pd) == 0) &&
			    (node != NULL)) {
				fctl_destroy_remote_node(node);
			}
		}

		kmem_free(clist->clist_map,
		    sizeof (*(clist->clist_map)) * clist->clist_size);
	}

	if (clist->clist_wait) {
		mutex_enter(&clist->clist_mutex);
		clist->clist_wait = 0;
		cv_signal(&clist->clist_cv);
		mutex_exit(&clist->clist_mutex);
	} else {
		kmem_free(clist, sizeof (*clist));
	}
}


/*
 * Allocate an fc_remote_node_t struct to represent a remote node for the
 * given nwwn.	This will also add the nwwn to the global nwwn table.
 *
 * Returns a pointer to the newly-allocated struct.  Returns NULL if
 * the kmem_zalloc fails or if the enlist_wwn attempt fails.
 */
fc_remote_node_t *
fctl_create_remote_node(la_wwn_t *nwwn, int sleep)
{
	fc_remote_node_t *rnodep;

	if ((rnodep = kmem_zalloc(sizeof (*rnodep), sleep)) == NULL) {
		return (NULL);
	}

	mutex_init(&rnodep->fd_mutex, NULL, MUTEX_DRIVER, NULL);

	rnodep->fd_node_name = *nwwn;
	rnodep->fd_flags = FC_REMOTE_NODE_VALID;
	rnodep->fd_numports = 1;

	if (fctl_enlist_nwwn_table(rnodep, sleep) != FC_SUCCESS) {
		mutex_destroy(&rnodep->fd_mutex);
		kmem_free(rnodep, sizeof (*rnodep));
		return (NULL);
	}

	return (rnodep);
}

/*
 * Deconstruct and free the given fc_remote_node_t struct (remote node struct).
 * Silently skips the deconstruct/free if there are any fc_remote_port_t
 * (remote port device) structs still referenced by the given
 * fc_remote_node_t struct.
 */
void
fctl_destroy_remote_node(fc_remote_node_t *rnodep)
{
	mutex_enter(&rnodep->fd_mutex);

	/*
	 * Look at the count and linked list of of remote ports
	 * (fc_remote_port_t structs); bail if these indicate that
	 * given fc_remote_node_t may be in use.
	 */
	if (rnodep->fd_numports != 0 || rnodep->fd_portlistp) {
		mutex_exit(&rnodep->fd_mutex);
		return;
	}

	mutex_exit(&rnodep->fd_mutex);

	mutex_destroy(&rnodep->fd_mutex);
	kmem_free(rnodep, sizeof (*rnodep));
}


/*
 * Add the given fc_remote_node_t to the global fctl_nwwn_hash_table[]. This
 * uses the nwwn in the fd_node_name.raw_wwn of the given struct.
 * This only fails if the kmem_zalloc fails.  This does not check for a
 * unique or pre-existing nwwn in the fctl_nwwn_hash_table[].
 * This is only called from fctl_create_remote_node().
 */
int
fctl_enlist_nwwn_table(fc_remote_node_t *rnodep, int sleep)
{
	int			index;
	fctl_nwwn_elem_t	*new;
	fctl_nwwn_list_t	*head;

	ASSERT(!MUTEX_HELD(&rnodep->fd_mutex));

	if ((new = kmem_zalloc(sizeof (*new), sleep)) == NULL) {
		return (FC_FAILURE);
	}

	mutex_enter(&fctl_nwwn_hash_mutex);
	new->fne_nodep = rnodep;

	mutex_enter(&rnodep->fd_mutex);
	ASSERT(fctl_is_wwn_zero(&rnodep->fd_node_name) == FC_FAILURE);
	index = HASH_FUNC(WWN_HASH_KEY(rnodep->fd_node_name.raw_wwn),
	    fctl_nwwn_table_size);
	mutex_exit(&rnodep->fd_mutex);

	head = &fctl_nwwn_hash_table[index];

	/* Link it in at the head of the hash list */
	new->fne_nextp = head->fnl_headp;
	head->fnl_headp = new;

	mutex_exit(&fctl_nwwn_hash_mutex);

	return (FC_SUCCESS);
}


/*
 * Remove the given fc_remote_node_t from the global fctl_nwwn_hash_table[].
 * This uses the nwwn in the fd_node_name.raw_wwn of the given struct.
 */
void
fctl_delist_nwwn_table(fc_remote_node_t *rnodep)
{
	int			index;
	fctl_nwwn_list_t	*head;
	fctl_nwwn_elem_t	*elem;
	fctl_nwwn_elem_t	*prev;

	ASSERT(MUTEX_HELD(&fctl_nwwn_hash_mutex));
	ASSERT(MUTEX_HELD(&rnodep->fd_mutex));

	index = HASH_FUNC(WWN_HASH_KEY(rnodep->fd_node_name.raw_wwn),
	    fctl_nwwn_table_size);

	head = &fctl_nwwn_hash_table[index];
	elem = head->fnl_headp;
	prev = NULL;

	while (elem != NULL) {
		if (elem->fne_nodep == rnodep) {
			/*
			 * Found it -- unlink it from the list & decrement
			 * the count for the hash chain.
			 */
			if (prev == NULL) {
				head->fnl_headp = elem->fne_nextp;
			} else {
				prev->fne_nextp = elem->fne_nextp;
			}
			break;
		}
		prev = elem;
		elem = elem->fne_nextp;
	}

	if (elem != NULL) {
		kmem_free(elem, sizeof (*elem));
	}
}


/*
 * Returns a reference to an fc_remote_node_t struct for the given node_wwn.
 * Looks in the global fctl_nwwn_hash_table[]. Identical to the
 * fctl_lock_remote_node_by_nwwn() function, except that this does NOT increment
 * the fc_count reference count in the f_device_t before returning.
 *
 * This function is called by: fctl_create_remote_port_t().
 *
 * OLD COMMENT:
 * Note: The calling thread needs to make sure it isn't holding any device
 * mutex (more so the fc_remote_node_t that could potentially have this wwn).
 */
fc_remote_node_t *
fctl_get_remote_node_by_nwwn(la_wwn_t *node_wwn)
{
	int			index;
	fctl_nwwn_elem_t	*elem;
	fc_remote_node_t	*next;
	fc_remote_node_t	*rnodep = NULL;

	index = HASH_FUNC(WWN_HASH_KEY(node_wwn->raw_wwn),
	    fctl_nwwn_table_size);
	ASSERT(index >= 0 && index < fctl_nwwn_table_size);

	mutex_enter(&fctl_nwwn_hash_mutex);
	elem = fctl_nwwn_hash_table[index].fnl_headp;
	while (elem != NULL) {
		next = elem->fne_nodep;
		if (next != NULL) {
			mutex_enter(&next->fd_mutex);
			if (fctl_wwn_cmp(node_wwn, &next->fd_node_name) == 0) {
				rnodep = next;
				mutex_exit(&next->fd_mutex);
				break;
			}
			mutex_exit(&next->fd_mutex);
		}
		elem = elem->fne_nextp;
	}
	mutex_exit(&fctl_nwwn_hash_mutex);

	return (rnodep);
}


/*
 * Returns a reference to an fc_remote_node_t struct for the given node_wwn.
 * Looks in the global fctl_nwwn_hash_table[]. Increments the fd_numports
 * reference count in the f_device_t before returning.
 *
 * This function is only called by fctl_create_remote_port_t().
 */
fc_remote_node_t *
fctl_lock_remote_node_by_nwwn(la_wwn_t *node_wwn)
{
	int			index;
	fctl_nwwn_elem_t	*elem;
	fc_remote_node_t	*next;
	fc_remote_node_t	*rnodep = NULL;

	index = HASH_FUNC(WWN_HASH_KEY(node_wwn->raw_wwn),
	    fctl_nwwn_table_size);
	ASSERT(index >= 0 && index < fctl_nwwn_table_size);

	mutex_enter(&fctl_nwwn_hash_mutex);
	elem = fctl_nwwn_hash_table[index].fnl_headp;
	while (elem != NULL) {
		next = elem->fne_nodep;
		if (next != NULL) {
			mutex_enter(&next->fd_mutex);
			if (fctl_wwn_cmp(node_wwn, &next->fd_node_name) == 0) {
				rnodep = next;
				rnodep->fd_numports++;
				mutex_exit(&next->fd_mutex);
				break;
			}
			mutex_exit(&next->fd_mutex);
		}
		elem = elem->fne_nextp;
	}
	mutex_exit(&fctl_nwwn_hash_mutex);

	return (rnodep);
}


/*
 * Allocate and initialize an fc_remote_port_t struct & returns a pointer to
 * the newly allocated struct.	Only fails if the kmem_zalloc() fails.
 */
fc_remote_port_t *
fctl_alloc_remote_port(fc_local_port_t *port, la_wwn_t *port_wwn,
    uint32_t d_id, uchar_t recepient, int sleep)
{
	fc_remote_port_t *pd;

	ASSERT(MUTEX_HELD(&port->fp_mutex));
	ASSERT(FC_IS_REAL_DEVICE(d_id));

	if ((pd = kmem_zalloc(sizeof (*pd), sleep)) == NULL) {
		return (NULL);
	}
	fctl_tc_constructor(&pd->pd_logo_tc, FC_LOGO_TOLERANCE_LIMIT,
	    FC_LOGO_TOLERANCE_TIME_LIMIT);

	mutex_init(&pd->pd_mutex, NULL, MUTEX_DRIVER, NULL);

	pd->pd_port_id.port_id = d_id;
	pd->pd_port_name = *port_wwn;
	pd->pd_port = port;
	pd->pd_state = PORT_DEVICE_VALID;
	pd->pd_type = PORT_DEVICE_NEW;
	pd->pd_recepient = recepient;

	return (pd);
}


/*
 * Deconstruct and free the given fc_remote_port_t struct (unconditionally).
 */
void
fctl_dealloc_remote_port(fc_remote_port_t *pd)
{
	ASSERT(!MUTEX_HELD(&pd->pd_mutex));

	fctl_tc_destructor(&pd->pd_logo_tc);
	mutex_destroy(&pd->pd_mutex);
	kmem_free(pd, sizeof (*pd));
}

/*
 * Add the given fc_remote_port_t onto the linked list of remote port
 * devices associated with the given fc_remote_node_t. Does NOT add the
 * fc_remote_port_t to the list if already exists on the list.
 */
void
fctl_link_remote_port_to_remote_node(fc_remote_node_t *rnodep,
    fc_remote_port_t *pd)
{
	fc_remote_port_t *last;
	fc_remote_port_t *ports;

	mutex_enter(&rnodep->fd_mutex);

	last = NULL;
	for (ports = rnodep->fd_portlistp; ports != NULL;
	    ports = ports->pd_port_next) {
		if (ports == pd) {
			/*
			 * The given fc_remote_port_t is already on the linked
			 * list chain for the given remote node, so bail now.
			 */
			mutex_exit(&rnodep->fd_mutex);
			return;
		}
		last = ports;
	}

	/* Add the fc_remote_port_t to the tail of the linked list */
	if (last != NULL) {
		last->pd_port_next = pd;
	} else {
		rnodep->fd_portlistp = pd;
	}
	pd->pd_port_next = NULL;

	/*
	 * Link the fc_remote_port_t back to the associated fc_remote_node_t.
	 */
	mutex_enter(&pd->pd_mutex);
	pd->pd_remote_nodep = rnodep;