xref: /illumos-gate/usr/src/uts/common/io/ib/clients/ibd/ibd.c (revision 6a634c9d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * An implementation of the IPoIB standard based on PSARC 2001/289.
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/stropts.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/strsubr.h>
#include <sys/dlpi.h>
#include <sys/mac_provider.h>

#include <sys/pattr.h>		/* for HCK_FULLCKSUM */
#include <sys/sysmacros.h>	/* for offsetof */
#include <sys/disp.h>		/* for async thread pri */
#include <sys/atomic.h>		/* for atomic_add*() */
#include <sys/ethernet.h>	/* for ETHERTYPE_IPV6 */
#include <netinet/in.h>		/* for netinet/ip.h below */
#include <netinet/ip.h>		/* for struct ip */
#include <netinet/udp.h>	/* for struct udphdr */
#include <inet/common.h>	/* for inet/ip.h below */
#include <inet/ip.h>		/* for ipha_t */
#include <inet/ip6.h>		/* for ip6_t */
#include <inet/tcp.h>		/* for tcph_t */
#include <netinet/icmp6.h>	/* for icmp6_t */
#include <sys/callb.h>
#include <sys/modhash.h>

#include <sys/ib/clients/ibd/ibd.h>
#include <sys/ib/mgt/sm_attr.h>	/* for SM_INIT_TYPE_* */
#include <sys/note.h>
#include <sys/multidata.h>

#include <sys/ib/mgt/ibmf/ibmf.h>	/* for ibd_get_portspeed */

#include <sys/priv_names.h>
#include <sys/dls.h>
#include <sys/dld_ioc.h>
#include <sys/policy.h>
#include <sys/ibpart.h>
#include <sys/file.h>

/*
 * The write-up below includes details on the following:
 * 1. The dladm administrative model.
 * 2. Late HCA initialization feature.
 * 3. Brussels support and its implications to the current architecture.
 *
 * 1. The dladm administrative model.
 * ------------------------------------------
 * With the dladm model, ibnex will create one ibd instance per port. These
 * instances will be created independent of the port state.
 *
 * The ibd driver is two faceted: One side of it working as the port driver and
 * the other as the partition object driver.
 *
 * The port instance is a child of the HCA, and will have an entry in the devfs.
 * A DDI attach only happens for the port driver, and its attach is
 * handled in ibd_port_attach(). Similary, a DDI detach for the port driver is
 * handled in ibd_port_unattach().
 *
 * The partition object is only a registrant to the mac layer via mac_register()
 * and does not have an entry in the device tree. There is no DDI softstate
 * managed by the DDI framework for the partition objects. However, the state is
 * managed inside the ibd driver, and every partition object hangs off the
 * "ibd_objlist_head".
 *
 * The partition object first comes into existence when a user runs the
 * 'create-part' subcommand of dladm. This is like invoking the attach entry
 * point of the partition object. The partition object goes away with the
 * 'delete-part' subcommand of dladm. This is like invoking the detach entry
 * point of the partition object.
 *
 * The create-part and delete-part subcommands result in dld ioctls that end up
 * calling ibd_create_parition() and ibd_delete_partition respectively.
 * There ioctls are registered with the dld layer in _init() via a call to
 * dld_ioc_register().
 *
 * The port instance by itself cannot be plumbed. It is only the partition
 * objects that can be plumbed and they alone participate in I/O and not the
 * port driver.
 *
 * There are some info ioctls supported in ibd which are used by dladm(1M) to
 * display useful information. The info entry point for ibd is
 * ibd_get_partition_info().
 *
 * 2. Late HCA initialization feature.
 * ------------------------------------
 * As mentioned in section 1, the user creates the partition objects via
 * dladm(1M). It is possible that:
 * a) The physical port itself is down and the SM cannot be reached.
 * b) The PKEY specified by the used has not been created in the SM yet.
 * c) An IPoIB broadcast group for the specified PKEY is not present.
 *
 * In all of the above cases, complete initialization of the partition object is
 * not possible. However, the new model allows the creation of partition
 * objects even in such cases but will defer the initialization for later.
 * When such a partition object is plumbed, the link state will be displayed as
 * "down".
 * The driver, at this point, is listening to events that herald the
 * availability of resources -
 * i)   LINK_UP when the link becomes available
 * ii)  PORT_CHANGE when the PKEY has been created
 * iii) MCG_CREATED when the IPoIB broadcast group for the given pkey has been
 * created
 * via ibd_async_handler() for events i) and ii), and via
 * ibd_snet_notices_handler() for iii.
 * The driver handles these events (as and when they arrive) and completes the
 * initialization of the partition object and transitions it to a usable state.
 *
 * 3. Brussels support and its implications to the current architecture.
 * ---------------------------------------------------------------------
 * The brussels support introduces two new interfaces to the ibd driver -
 * ibd_m_getprop() and ibd_m_setprop().
 * These interfaces allow setting and retrieval of certain properties.
 * Some of them are public properties while most other are private properties
 * meant to be used by developers. Tuning the latter kind can cause
 * performance issues and should not be used without understanding the
 * implications. All properties are specific to an instance of either the
 * partition object or the port driver.
 *
 * The public properties are : mtu and linkmode.
 * mtu is a read-only property.
 * linkmode can take two values - UD and CM.
 *
 * Changing the linkmode requires some bookkeeping in the driver. The
 * capabilities need to be re-reported to the mac layer. This is done by
 * calling mac_capab_update().  The maxsdu is updated by calling
 * mac_maxsdu_update2().
 * The private properties retain their values across the change of linkmode.
 * NOTE:
 * - The port driver does not support any property apart from mtu.
 * - All other properties are only meant for the partition object.
 * - The properties cannot be set when an instance is plumbed. The
 * instance has to be unplumbed to effect any setting.
 */

/*
 * Driver wide tunables
 *
 * ibd_tx_softintr
 * ibd_rx_softintr
 *     The softintr mechanism allows ibd to avoid event queue overflows if
 *     the receive/completion handlers are to be expensive. These are enabled
 *     by default.
 *
 * ibd_log_sz
 *     This specifies the size of the ibd log buffer in bytes. The buffer is
 *     allocated and logging is enabled only when IBD_LOGGING is defined.
 *
 */
uint_t ibd_rx_softintr = 1;
uint_t ibd_tx_softintr = 1;

#ifdef IBD_LOGGING
uint_t ibd_log_sz = 0x20000;
#endif

#ifdef IBD_LOGGING
#define	IBD_LOG_SZ			ibd_log_sz
#endif

/* Post IBD_RX_POST_CNT receive work requests at a time. */
#define	IBD_RX_POST_CNT			8

/* Hash into 1 << IBD_LOG_RX_POST number of rx post queues */
#define	IBD_LOG_RX_POST			4

/* Minimum number of receive work requests driver needs to always have */
#define	IBD_RWQE_MIN	((IBD_RX_POST_CNT << IBD_LOG_RX_POST) * 4)

/*
 * LSO parameters
 */
#define	IBD_LSO_MAXLEN			65536
#define	IBD_LSO_BUFSZ			8192

/*
 * Async operation states
 */
#define	IBD_OP_NOTSTARTED		0
#define	IBD_OP_ONGOING			1
#define	IBD_OP_COMPLETED		2
#define	IBD_OP_ERRORED			3
#define	IBD_OP_ROUTERED			4

/*
 * Start/stop in-progress flags; note that restart must always remain
 * the OR of start and stop flag values.
 */
#define	IBD_DRV_START_IN_PROGRESS	0x10000000
#define	IBD_DRV_STOP_IN_PROGRESS	0x20000000
#define	IBD_DRV_RESTART_IN_PROGRESS	0x30000000
#define	IBD_DRV_DELETE_IN_PROGRESS	IBD_DRV_RESTART_IN_PROGRESS

/*
 * Miscellaneous constants
 */
#define	IB_MGID_IPV4_LOWGRP_MASK	0xFFFFFFFF
#define	IBD_DEF_MAX_SDU			2044
#define	IBD_DEF_MAX_MTU			(IBD_DEF_MAX_SDU + IPOIB_HDRSIZE)
#define	IBD_DEF_RC_MAX_SDU		65520
#define	IBD_DEF_RC_MAX_MTU		(IBD_DEF_RC_MAX_SDU + IPOIB_HDRSIZE)
#define	IBD_DEFAULT_QKEY		0xB1B
#ifdef IBD_LOGGING
#define	IBD_DMAX_LINE			100
#endif

/*
 * Enumerations for link states
 */
typedef enum {
	IBD_LINK_DOWN,
	IBD_LINK_UP,
	IBD_LINK_UP_ABSENT
} ibd_link_op_t;

/*
 * Driver State Pointer
 */
void *ibd_list;

/*
 * Driver Global Data
 */
ibd_global_state_t ibd_gstate;

/*
 * Partition object list
 */
ibd_state_t	*ibd_objlist_head = NULL;
kmutex_t	ibd_objlist_lock;

int ibd_rc_conn_timeout = 60 * 10;	/* 10 minutes */

/*
 * Logging
 */
#ifdef IBD_LOGGING
kmutex_t ibd_lbuf_lock;
uint8_t *ibd_lbuf;
uint32_t ibd_lbuf_ndx;
#endif

/*
 * Required system entry points
 */
static int ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
static int ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);

/*
 * Required driver entry points for GLDv3
 */
static int ibd_m_stat(void *, uint_t, uint64_t *);
static int ibd_m_start(void *);
static void ibd_m_stop(void *);
static int ibd_m_promisc(void *, boolean_t);
static int ibd_m_multicst(void *, boolean_t, const uint8_t *);
static int ibd_m_unicst(void *, const uint8_t *);
static mblk_t *ibd_m_tx(void *, mblk_t *);
static boolean_t ibd_m_getcapab(void *, mac_capab_t, void *);

static int ibd_m_setprop(void *, const char *, mac_prop_id_t, uint_t,
    const void *);
static int ibd_m_getprop(void *, const char *, mac_prop_id_t, uint_t, void *);
static void ibd_m_propinfo(void *, const char *, mac_prop_id_t,
    mac_prop_info_handle_t);
static int ibd_set_priv_prop(ibd_state_t *, const char *, uint_t,
    const void *);
static int ibd_get_priv_prop(ibd_state_t *, const char *, uint_t, void *);

/*
 * Private driver entry points for GLDv3
 */

/*
 * Initialization
 */
static int ibd_state_init(ibd_state_t *, dev_info_t *);
static int ibd_init_txlist(ibd_state_t *);
static int ibd_init_rxlist(ibd_state_t *);
static int ibd_acache_init(ibd_state_t *);
#ifdef IBD_LOGGING
static void ibd_log_init(void);
#endif

/*
 * Termination/cleanup
 */
static void ibd_state_fini(ibd_state_t *);
static void ibd_fini_txlist(ibd_state_t *);
static void ibd_fini_rxlist(ibd_state_t *);
static void ibd_tx_cleanup(ibd_state_t *, ibd_swqe_t *);
static void ibd_tx_cleanup_list(ibd_state_t *, ibd_swqe_t *, ibd_swqe_t *);
static void ibd_acache_fini(ibd_state_t *);
#ifdef IBD_LOGGING
static void ibd_log_fini(void);
#endif

/*
 * Allocation/acquire/map routines
 */
static int ibd_alloc_tx_copybufs(ibd_state_t *);
static int ibd_alloc_rx_copybufs(ibd_state_t *);
static int ibd_alloc_tx_lsobufs(ibd_state_t *);
static ibd_swqe_t *ibd_acquire_swqe(ibd_state_t *);
static int ibd_acquire_lsobufs(ibd_state_t *, uint_t, ibt_wr_ds_t *,
    uint32_t *);

/*
 * Free/release/unmap routines
 */
static void ibd_free_rwqe(ibd_state_t *, ibd_rwqe_t *);
static void ibd_free_tx_copybufs(ibd_state_t *);
static void ibd_free_rx_copybufs(ibd_state_t *);
static void ibd_free_rx_rsrcs(ibd_state_t *);
static void ibd_free_tx_lsobufs(ibd_state_t *);
static void ibd_release_swqe(ibd_state_t *, ibd_swqe_t *, ibd_swqe_t *, int);
static void ibd_release_lsobufs(ibd_state_t *, ibt_wr_ds_t *, uint32_t);
static void ibd_free_lsohdr(ibd_swqe_t *, mblk_t *);

/*
 * Handlers/callback routines
 */
static uint_t ibd_intr(caddr_t);
static uint_t ibd_tx_recycle(caddr_t);
static void ibd_rcq_handler(ibt_cq_hdl_t, void *);
static void ibd_scq_handler(ibt_cq_hdl_t, void *);
static void ibd_poll_rcq(ibd_state_t *, ibt_cq_hdl_t);
static void ibd_poll_scq(ibd_state_t *, ibt_cq_hdl_t);
static void ibd_drain_rcq(ibd_state_t *, ibt_cq_hdl_t);
static void ibd_drain_scq(ibd_state_t *, ibt_cq_hdl_t);
static void ibd_freemsg_cb(char *);
static void ibd_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
    ibt_async_event_t *);
static void ibdpd_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
    ibt_async_event_t *);
static void ibd_snet_notices_handler(void *, ib_gid_t,
    ibt_subnet_event_code_t, ibt_subnet_event_t *);

/*
 * Send/receive routines
 */
static boolean_t ibd_send(ibd_state_t *, mblk_t *);
static void ibd_post_send(ibd_state_t *, ibd_swqe_t *);
static void ibd_post_recv(ibd_state_t *, ibd_rwqe_t *);
static mblk_t *ibd_process_rx(ibd_state_t *, ibd_rwqe_t *, ibt_wc_t *);

/*
 * Threads
 */
static void ibd_async_work(ibd_state_t *);

/*
 * Async tasks
 */
static void ibd_async_acache(ibd_state_t *, ipoib_mac_t *);
static void ibd_async_multicast(ibd_state_t *, ib_gid_t, int);
static void ibd_async_setprom(ibd_state_t *);
static void ibd_async_unsetprom(ibd_state_t *);
static void ibd_async_reap_group(ibd_state_t *, ibd_mce_t *, ib_gid_t, uint8_t);
static void ibd_async_trap(ibd_state_t *, ibd_req_t *);
static void ibd_async_txsched(ibd_state_t *);
static void ibd_async_link(ibd_state_t *, ibd_req_t *);

/*
 * Async task helpers
 */
static ibd_mce_t *ibd_async_mcache(ibd_state_t *, ipoib_mac_t *, boolean_t *);
static ibd_mce_t *ibd_join_group(ibd_state_t *, ib_gid_t, uint8_t);
static ibd_mce_t *ibd_mcache_find(ib_gid_t, struct list *);
static boolean_t ibd_get_allroutergroup(ibd_state_t *,
    ipoib_mac_t *, ipoib_mac_t *);
static void ibd_leave_group(ibd_state_t *, ib_gid_t, uint8_t);
static void ibd_reacquire_group(ibd_state_t *, ibd_mce_t *);
static ibt_status_t ibd_iba_join(ibd_state_t *, ib_gid_t, ibd_mce_t *);
static ibt_status_t ibd_find_bgroup(ibd_state_t *);
static void ibd_n2h_gid(ipoib_mac_t *, ib_gid_t *);
static void ibd_h2n_mac(ipoib_mac_t *, ib_qpn_t, ib_sn_prefix_t, ib_guid_t);
static uint64_t ibd_get_portspeed(ibd_state_t *);
static boolean_t ibd_async_safe(ibd_state_t *);
static void ibd_async_done(ibd_state_t *);
static ibd_ace_t *ibd_acache_lookup(ibd_state_t *, ipoib_mac_t *, int *, int);
static ibd_ace_t *ibd_acache_get_unref(ibd_state_t *);
static void ibd_link_mod(ibd_state_t *, ibt_async_code_t);
static int ibd_locate_pkey(ib_pkey_t *, uint16_t, ib_pkey_t, uint16_t *);

/*
 * Helpers for attach/start routines
 */
static int ibd_register_mac(ibd_state_t *, dev_info_t *);
static int ibd_record_capab(ibd_state_t *);
static int ibd_get_port_details(ibd_state_t *);
static int ibd_alloc_cqs(ibd_state_t *);
static int ibd_setup_ud_channel(ibd_state_t *);
static int ibd_start(ibd_state_t *);
static int ibd_undo_start(ibd_state_t *, link_state_t);
static void ibd_set_mac_progress(ibd_state_t *, uint_t);
static void ibd_clr_mac_progress(ibd_state_t *, uint_t);
static int ibd_part_attach(ibd_state_t *state, dev_info_t *dip);
static void ibd_part_unattach(ibd_state_t *state);
static int ibd_port_attach(dev_info_t *);
static int ibd_port_unattach(ibd_state_t *state, dev_info_t *dip);
static int ibd_get_port_state(ibd_state_t *, link_state_t *);
static int ibd_part_busy(ibd_state_t *);

/*
 * Miscellaneous helpers
 */
static int ibd_sched_poll(ibd_state_t *, int, int);
static void ibd_resume_transmission(ibd_state_t *);
static int ibd_setup_lso(ibd_swqe_t *, mblk_t *, uint32_t, ibt_ud_dest_hdl_t);
static int ibd_prepare_sgl(ibd_state_t *, mblk_t *, ibd_swqe_t *, uint_t);
static void *list_get_head(list_t *);
static int ibd_hash_key_cmp(mod_hash_key_t, mod_hash_key_t);
static uint_t ibd_hash_by_id(void *, mod_hash_key_t);

ibt_status_t ibd_get_part_attr(datalink_id_t, ibt_part_attr_t *);
ibt_status_t ibd_get_all_part_attr(ibt_part_attr_t **, int *);

#ifdef IBD_LOGGING
static void ibd_log(const char *, ...);
#endif

DDI_DEFINE_STREAM_OPS(ibd_dev_ops, nulldev, nulldev, ibd_attach, ibd_detach,
    nodev, NULL, D_MP, NULL, ddi_quiesce_not_needed);

/* Module Driver Info */
static struct modldrv ibd_modldrv = {
	&mod_driverops,			/* This one is a driver */
	"InfiniBand GLDv3 Driver",	/* short description */
	&ibd_dev_ops			/* driver specific ops */
};

/* Module Linkage */
static struct modlinkage ibd_modlinkage = {
	MODREV_1, (void *)&ibd_modldrv, NULL
};

/*
 * Module (static) info passed to IBTL during ibt_attach
 */
static struct ibt_clnt_modinfo_s ibd_clnt_modinfo = {
	IBTI_V_CURR,
	IBT_NETWORK,
	ibd_async_handler,
	NULL,
	"IBPART"
};

static struct ibt_clnt_modinfo_s ibdpd_clnt_modinfo = {
	IBTI_V_CURR,
	IBT_NETWORK,
	ibdpd_async_handler,
	NULL,
	"IPIB"
};

/*
 * GLDv3 entry points
 */
#define	IBD_M_CALLBACK_FLAGS	\
	(MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)

static mac_callbacks_t ibd_m_callbacks = {
	IBD_M_CALLBACK_FLAGS,
	ibd_m_stat,
	ibd_m_start,
	ibd_m_stop,
	ibd_m_promisc,
	ibd_m_multicst,
	ibd_m_unicst,
	ibd_m_tx,
	NULL,
	NULL,
	ibd_m_getcapab,
	NULL,
	NULL,
	ibd_m_setprop,
	ibd_m_getprop,
	ibd_m_propinfo
};

/* Private properties */
char *ibd_priv_props[] = {
	"_ibd_broadcast_group",
	"_ibd_coalesce_completions",
	"_ibd_create_broadcast_group",
	"_ibd_hash_size",
	"_ibd_lso_enable",
	"_ibd_num_ah",
	"_ibd_num_lso_bufs",
	"_ibd_rc_enable_srq",
	"_ibd_rc_num_rwqe",
	"_ibd_rc_num_srq",
	"_ibd_rc_num_swqe",
	"_ibd_rc_rx_comp_count",
	"_ibd_rc_rx_comp_usec",
	"_ibd_rc_rx_copy_thresh",
	"_ibd_rc_rx_rwqe_thresh",
	"_ibd_rc_tx_comp_count",
	"_ibd_rc_tx_comp_usec",
	"_ibd_rc_tx_copy_thresh",
	"_ibd_ud_num_rwqe",
	"_ibd_ud_num_swqe",
	"_ibd_ud_rx_comp_count",
	"_ibd_ud_rx_comp_usec",
	"_ibd_ud_tx_comp_count",
	"_ibd_ud_tx_comp_usec",
	"_ibd_ud_tx_copy_thresh",
	NULL
};

static int ibd_create_partition(void *, intptr_t, int, cred_t *, int *);
static int ibd_delete_partition(void *, intptr_t, int, cred_t *, int *);
static int ibd_get_partition_info(void *, intptr_t, int, cred_t *, int *);

static dld_ioc_info_t ibd_dld_ioctl_list[] = {
	{IBD_CREATE_IBPART, DLDCOPYINOUT, sizeof (ibpart_ioctl_t),
	    ibd_create_partition, secpolicy_dl_config},
	{IBD_DELETE_IBPART, DLDCOPYIN, sizeof (ibpart_ioctl_t),
	    ibd_delete_partition, secpolicy_dl_config},
	{IBD_INFO_IBPART, DLDCOPYIN, sizeof (ibd_ioctl_t),
	    ibd_get_partition_info, NULL}
};

/*
 * Fill/clear <scope> and <p_key> in multicast/broadcast address
 */
#define	IBD_FILL_SCOPE_PKEY(maddr, scope, pkey)		\
{							\
	*(uint32_t *)((char *)(maddr) + 4) |=		\
	    htonl((uint32_t)(scope) << 16);		\
	*(uint32_t *)((char *)(maddr) + 8) |=		\
	    htonl((uint32_t)(pkey) << 16);		\
}

#define	IBD_CLEAR_SCOPE_PKEY(maddr)			\
{							\
	*(uint32_t *)((char *)(maddr) + 4) &=		\
	    htonl(~((uint32_t)0xF << 16));		\
	*(uint32_t *)((char *)(maddr) + 8) &=		\
	    htonl(~((uint32_t)0xFFFF << 16));		\
}

/*
 * Rudimentary debugging support
 */
#ifdef DEBUG
int ibd_debuglevel = 100;
void
debug_print(int l, char *fmt, ...)
{
	va_list ap;

	if (l < ibd_debuglevel)
		return;
	va_start(ap, fmt);
	vcmn_err(CE_CONT, fmt, ap);
	va_end(ap);
}
#endif

/*
 * Common routine to print warning messages; adds in hca guid, port number
 * and pkey to be able to identify the IBA interface.
 */
void
ibd_print_warn(ibd_state_t *state, char *fmt, ...)
{
	ib_guid_t hca_guid;
	char ibd_print_buf[MAXNAMELEN + 256];
	int len;
	va_list ap;
	char part_name[MAXNAMELEN];
	datalink_id_t linkid = state->id_plinkid;

	hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, state->id_dip,
	    0, "hca-guid", 0);
	(void) dls_mgmt_get_linkinfo(linkid, part_name, NULL, NULL, NULL);
	len = snprintf(ibd_print_buf, sizeof (ibd_print_buf),
	    "%s%d: HCA GUID %016llx port %d PKEY %02x link %s ",
	    ddi_driver_name(state->id_dip), ddi_get_instance(state->id_dip),
	    (u_longlong_t)hca_guid, state->id_port, state->id_pkey,
	    part_name);
	va_start(ap, fmt);
	(void) vsnprintf(ibd_print_buf + len, sizeof (ibd_print_buf) - len,
	    fmt, ap);
	cmn_err(CE_NOTE, "!%s", ibd_print_buf);
	va_end(ap);
}

/*
 * Warlock directives
 */

/*
 * id_lso_lock
 *
 * state->id_lso->bkt_nfree may be accessed without a lock to
 * determine the threshold at which we have to ask the nw layer
 * to resume transmission (see ibd_resume_transmission()).
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_lso_lock,
    ibd_state_t::id_lso))
_NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_lso))
_NOTE(SCHEME_PROTECTS_DATA("init", ibd_state_t::id_lso_policy))
_NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_lsobkt_t::bkt_nfree))

/*
 * id_scq_poll_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_scq_poll_lock,
    ibd_state_t::id_scq_poll_busy))

/*
 * id_txpost_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock,
    ibd_state_t::id_tx_head))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_txpost_lock,
    ibd_state_t::id_tx_busy))

/*
 * id_acache_req_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock,
    ibd_state_t::id_acache_req_cv))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_acache_req_lock,
    ibd_state_t::id_req_list))
_NOTE(SCHEME_PROTECTS_DATA("atomic",
    ibd_acache_s::ac_ref))

/*
 * id_ac_mutex
 *
 * This mutex is actually supposed to protect id_ah_op as well,
 * but this path of the code isn't clean (see update of id_ah_op
 * in ibd_async_acache(), immediately after the call to
 * ibd_async_mcache()). For now, we'll skip this check by
 * declaring that id_ah_op is protected by some internal scheme
 * that warlock isn't aware of.
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
    ibd_state_t::id_ah_active))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
    ibd_state_t::id_ah_free))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
    ibd_state_t::id_ah_addr))
_NOTE(SCHEME_PROTECTS_DATA("ac mutex should protect this",
    ibd_state_t::id_ah_op))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
    ibd_state_t::id_ah_error))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_ac_mutex,
    ibd_state_t::id_ac_hot_ace))
_NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_ah_error))

/*
 * id_mc_mutex
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex,
    ibd_state_t::id_mc_full))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_mc_mutex,
    ibd_state_t::id_mc_non))

/*
 * id_trap_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
    ibd_state_t::id_trap_cv))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
    ibd_state_t::id_trap_stop))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_trap_lock,
    ibd_state_t::id_trap_inprog))

/*
 * id_prom_op
 */
_NOTE(SCHEME_PROTECTS_DATA("only by async thread",
    ibd_state_t::id_prom_op))

/*
 * id_sched_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_sched_lock,
    ibd_state_t::id_sched_needed))

/*
 * id_link_mutex
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_link_mutex,
    ibd_state_t::id_link_state))
_NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_link_state))
_NOTE(SCHEME_PROTECTS_DATA("only async thr and ibd_m_start",
    ibd_state_t::id_link_speed))
_NOTE(DATA_READABLE_WITHOUT_LOCK(ibd_state_t::id_sgid))

/*
 * id_tx_list.dl_mutex
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
    ibd_state_t::id_tx_list.dl_head))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
    ibd_state_t::id_tx_list.dl_pending_sends))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::id_tx_list.dl_mutex,
    ibd_state_t::id_tx_list.dl_cnt))

/*
 * id_rx_list.dl_mutex
 */
_NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
    ibd_state_t::id_rx_list.dl_bufs_outstanding))
_NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
    ibd_state_t::id_rx_list.dl_cnt))

/*
 * rc_timeout_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::rc_timeout_lock,
    ibd_state_t::rc_timeout_start))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_t::rc_timeout_lock,
    ibd_state_t::rc_timeout))


/*
 * Items protected by atomic updates
 */
_NOTE(SCHEME_PROTECTS_DATA("atomic update only",
    ibd_state_s::id_brd_rcv
    ibd_state_s::id_brd_xmt
    ibd_state_s::id_multi_rcv
    ibd_state_s::id_multi_xmt
    ibd_state_s::id_num_intrs
    ibd_state_s::id_rcv_bytes
    ibd_state_s::id_rcv_pkt
    ibd_state_s::id_rx_post_queue_index
    ibd_state_s::id_tx_short
    ibd_state_s::id_xmt_bytes
    ibd_state_s::id_xmt_pkt
    ibd_state_s::rc_rcv_trans_byte
    ibd_state_s::rc_rcv_trans_pkt
    ibd_state_s::rc_rcv_copy_byte
    ibd_state_s::rc_rcv_copy_pkt
    ibd_state_s::rc_xmt_bytes
    ibd_state_s::rc_xmt_small_pkt
    ibd_state_s::rc_xmt_fragmented_pkt
    ibd_state_s::rc_xmt_map_fail_pkt
    ibd_state_s::rc_xmt_map_succ_pkt
    ibd_rc_chan_s::rcq_invoking))

/*
 * Non-mutex protection schemes for data elements. Almost all of
 * these are non-shared items.
 */
_NOTE(SCHEME_PROTECTS_DATA("unshared or single-threaded",
    callb_cpr
    ib_gid_s
    ib_header_info
    ibd_acache_rq
    ibd_acache_s::ac_mce
    ibd_acache_s::ac_chan
    ibd_mcache::mc_fullreap
    ibd_mcache::mc_jstate
    ibd_mcache::mc_req
    ibd_rwqe_s
    ibd_swqe_s
    ibd_wqe_s
    ibt_wr_ds_s::ds_va
    ibt_wr_lso_s
    ipoib_mac::ipoib_qpn
    mac_capab_lso_s
    msgb::b_next
    msgb::b_cont
    msgb::b_rptr
    msgb::b_wptr
    ibd_state_s::id_bgroup_created
    ibd_state_s::id_mac_state
    ibd_state_s::id_mtu
    ibd_state_s::id_ud_num_rwqe
    ibd_state_s::id_ud_num_swqe
    ibd_state_s::id_qpnum
    ibd_state_s::id_rcq_hdl
    ibd_state_s::id_rx_buf_sz
    ibd_state_s::id_rx_bufs
    ibd_state_s::id_rx_mr_hdl
    ibd_state_s::id_rx_wqes
    ibd_state_s::id_rxwcs
    ibd_state_s::id_rxwcs_size
    ibd_state_s::id_rx_nqueues
    ibd_state_s::id_rx_queues
    ibd_state_s::id_scope
    ibd_state_s::id_scq_hdl
    ibd_state_s::id_tx_buf_sz
    ibd_state_s::id_tx_bufs
    ibd_state_s::id_tx_mr_hdl
    ibd_state_s::id_tx_rel_list.dl_cnt
    ibd_state_s::id_tx_wqes
    ibd_state_s::id_txwcs
    ibd_state_s::id_txwcs_size
    ibd_state_s::rc_listen_hdl
    ibd_state_s::rc_listen_hdl_OFED_interop
    ibd_state_s::rc_srq_size
    ibd_state_s::rc_srq_rwqes
    ibd_state_s::rc_srq_rx_bufs
    ibd_state_s::rc_srq_rx_mr_hdl
    ibd_state_s::rc_tx_largebuf_desc_base
    ibd_state_s::rc_tx_mr_bufs
    ibd_state_s::rc_tx_mr_hdl
    ipha_s
    icmph_s
    ibt_path_info_s::pi_sid
    ibd_rc_chan_s::ace
    ibd_rc_chan_s::chan_hdl
    ibd_rc_chan_s::state
    ibd_rc_chan_s::chan_state
    ibd_rc_chan_s::is_tx_chan
    ibd_rc_chan_s::rcq_hdl
    ibd_rc_chan_s::rcq_size
    ibd_rc_chan_s::scq_hdl
    ibd_rc_chan_s::scq_size
    ibd_rc_chan_s::rx_bufs
    ibd_rc_chan_s::rx_mr_hdl
    ibd_rc_chan_s::rx_rwqes
    ibd_rc_chan_s::tx_wqes
    ibd_rc_chan_s::tx_mr_bufs
    ibd_rc_chan_s::tx_mr_hdl
    ibd_rc_chan_s::tx_rel_list.dl_cnt
    ibd_rc_chan_s::is_used
    ibd_rc_tx_largebuf_s::lb_buf
    ibd_rc_msg_hello_s
    ibt_cm_return_args_s))

/*
 * ibd_rc_chan_s::next is protected by two mutexes:
 * 1) ibd_state_s::rc_pass_chan_list.chan_list_mutex
 * 2) ibd_state_s::rc_obs_act_chan_list.chan_list_mutex.
 */
_NOTE(SCHEME_PROTECTS_DATA("protected by two mutexes",
    ibd_rc_chan_s::next))

/*
 * ibd_state_s.rc_tx_large_bufs_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock,
    ibd_state_s::rc_tx_largebuf_free_head))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock,
    ibd_state_s::rc_tx_largebuf_nfree))
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_tx_large_bufs_lock,
    ibd_rc_tx_largebuf_s::lb_next))

/*
 * ibd_acache_s.tx_too_big_mutex
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_acache_s::tx_too_big_mutex,
    ibd_acache_s::tx_too_big_ongoing))

/*
 * tx_wqe_list.dl_mutex
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex,
    ibd_rc_chan_s::tx_wqe_list.dl_head))
_NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex,
    ibd_rc_chan_s::tx_wqe_list.dl_pending_sends))
_NOTE(MUTEX_PROTECTS_DATA(ibd_rc_chan_s::tx_wqe_list.dl_mutex,
    ibd_rc_chan_s::tx_wqe_list.dl_cnt))

/*
 * ibd_state_s.rc_ace_recycle_lock
 */
_NOTE(MUTEX_PROTECTS_DATA(ibd_state_s::rc_ace_recycle_lock,
    ibd_state_s::rc_ace_recycle))

/*
 * rc_srq_rwqe_list.dl_mutex
 */
_NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
    ibd_state_t::rc_srq_rwqe_list.dl_bufs_outstanding))
_NOTE(SCHEME_PROTECTS_DATA("atomic or dl mutex or single thr",
    ibd_state_t::rc_srq_rwqe_list.dl_cnt))

/*
 * Non-mutex protection schemes for data elements. They are counters
 * for problem diagnosis. Don't need be protected.
 */
_NOTE(SCHEME_PROTECTS_DATA("counters for problem diagnosis",
    ibd_state_s::rc_rcv_alloc_fail
    ibd_state_s::rc_rcq_err
    ibd_state_s::rc_ace_not_found
    ibd_state_s::rc_xmt_drop_too_long_pkt
    ibd_state_s::rc_xmt_icmp_too_long_pkt
    ibd_state_s::rc_xmt_reenter_too_long_pkt
    ibd_state_s::rc_swqe_short
    ibd_state_s::rc_swqe_mac_update
    ibd_state_s::rc_xmt_buf_short
    ibd_state_s::rc_xmt_buf_mac_update
    ibd_state_s::rc_scq_no_swqe
    ibd_state_s::rc_scq_no_largebuf
    ibd_state_s::rc_conn_succ
    ibd_state_s::rc_conn_fail
    ibd_state_s::rc_null_conn
    ibd_state_s::rc_no_estab_conn
    ibd_state_s::rc_act_close
    ibd_state_s::rc_pas_close
    ibd_state_s::rc_delay_ace_recycle
    ibd_state_s::rc_act_close_simultaneous
    ibd_state_s::rc_act_close_not_clean
    ibd_state_s::rc_pas_close_rcq_invoking
    ibd_state_s::rc_reset_cnt
    ibd_state_s::rc_timeout_act
    ibd_state_s::rc_timeout_pas
    ibd_state_s::rc_stop_connect))

#ifdef DEBUG
/*
 * Non-mutex protection schemes for data elements. They are counters
 * for problem diagnosis. Don't need be protected.
 */
_NOTE(SCHEME_PROTECTS_DATA("counters for problem diagnosis",
    ibd_state_s::rc_rwqe_short
    ibd_rc_stat_s::rc_rcv_trans_byte
    ibd_rc_stat_s::rc_rcv_trans_pkt
    ibd_rc_stat_s::rc_rcv_copy_byte
    ibd_rc_stat_s::rc_rcv_copy_pkt
    ibd_rc_stat_s::rc_rcv_alloc_fail
    ibd_rc_stat_s::rc_rcq_err
    ibd_rc_stat_s::rc_rwqe_short
    ibd_rc_stat_s::rc_xmt_bytes
    ibd_rc_stat_s::rc_xmt_small_pkt
    ibd_rc_stat_s::rc_xmt_fragmented_pkt
    ibd_rc_stat_s::rc_xmt_map_fail_pkt
    ibd_rc_stat_s::rc_xmt_map_succ_pkt
    ibd_rc_stat_s::rc_ace_not_found
    ibd_rc_stat_s::rc_scq_no_swqe
    ibd_rc_stat_s::rc_scq_no_largebuf
    ibd_rc_stat_s::rc_swqe_short
    ibd_rc_stat_s::rc_swqe_mac_update
    ibd_rc_stat_s::rc_xmt_buf_short
    ibd_rc_stat_s::rc_xmt_buf_mac_update
    ibd_rc_stat_s::rc_conn_succ
    ibd_rc_stat_s::rc_conn_fail
    ibd_rc_stat_s::rc_null_conn
    ibd_rc_stat_s::rc_no_estab_conn
    ibd_rc_stat_s::rc_act_close
    ibd_rc_stat_s::rc_pas_close
    ibd_rc_stat_s::rc_delay_ace_recycle
    ibd_rc_stat_s::rc_act_close_simultaneous
    ibd_rc_stat_s::rc_reset_cnt
    ibd_rc_stat_s::rc_timeout_act
    ibd_rc_stat_s::rc_timeout_pas))
#endif

int
_init()
{
	int status;

	status = ddi_soft_state_init(&ibd_list, max(sizeof (ibd_state_t),
	    PAGESIZE), 0);
	if (status != 0) {
		DPRINT(10, "_init:failed in ddi_soft_state_init()");
		return (status);
	}

	mutex_init(&ibd_objlist_lock, NULL, MUTEX_DRIVER, NULL);

	mac_init_ops(&ibd_dev_ops, "ibp");
	status = mod_install(&ibd_modlinkage);
	if (status != 0) {
		DPRINT(10, "_init:failed in mod_install()");
		ddi_soft_state_fini(&ibd_list);
		mac_fini_ops(&ibd_dev_ops);
		return (status);
	}

	mutex_init(&ibd_gstate.ig_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_enter(&ibd_gstate.ig_mutex);
	ibd_gstate.ig_ibt_hdl = NULL;
	ibd_gstate.ig_ibt_hdl_ref_cnt = 0;
	ibd_gstate.ig_service_list = NULL;
	mutex_exit(&ibd_gstate.ig_mutex);

	if (dld_ioc_register(IBPART_IOC, ibd_dld_ioctl_list,
	    DLDIOCCNT(ibd_dld_ioctl_list)) != 0) {
		return (EIO);
	}

	ibt_register_part_attr_cb(ibd_get_part_attr, ibd_get_all_part_attr);

#ifdef IBD_LOGGING
	ibd_log_init();
#endif
	return (0);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&ibd_modlinkage, modinfop));
}

int
_fini()
{
	int status;

	status = mod_remove(&ibd_modlinkage);
	if (status != 0)
		return (status);

	ibt_unregister_part_attr_cb();

	mac_fini_ops(&ibd_dev_ops);
	mutex_destroy(&ibd_objlist_lock);
	ddi_soft_state_fini(&ibd_list);
	mutex_destroy(&ibd_gstate.ig_mutex);
#ifdef IBD_LOGGING
	ibd_log_fini();
#endif
	return (0);
}

/*
 * Convert the GID part of the mac address from network byte order
 * to host order.
 */
static void
ibd_n2h_gid(ipoib_mac_t *mac, ib_gid_t *dgid)
{
	ib_sn_prefix_t nbopref;
	ib_guid_t nboguid;

	bcopy(mac->ipoib_gidpref, &nbopref, sizeof (ib_sn_prefix_t));
	bcopy(mac->ipoib_gidsuff, &nboguid, sizeof (ib_guid_t));
	dgid->gid_prefix = b2h64(nbopref);
	dgid->gid_guid = b2h64(nboguid);
}

/*
 * Create the IPoIB address in network byte order from host order inputs.
 */
static void
ibd_h2n_mac(ipoib_mac_t *mac, ib_qpn_t qpn, ib_sn_prefix_t prefix,
    ib_guid_t guid)
{
	ib_sn_prefix_t nbopref;
	ib_guid_t nboguid;

	mac->ipoib_qpn = htonl(qpn);
	nbopref = h2b64(prefix);
	nboguid = h2b64(guid);
	bcopy(&nbopref, mac->ipoib_gidpref, sizeof (ib_sn_prefix_t));
	bcopy(&nboguid, mac->ipoib_gidsuff, sizeof (ib_guid_t));
}

/*
 * Send to the appropriate all-routers group when the IBA multicast group
 * does not exist, based on whether the target group is v4 or v6.
 */
static boolean_t
ibd_get_allroutergroup(ibd_state_t *state, ipoib_mac_t *mcmac,
    ipoib_mac_t *rmac)
{
	boolean_t retval = B_TRUE;
	uint32_t adjscope = state->id_scope << 16;
	uint32_t topword;

	/*
	 * Copy the first 4 bytes in without assuming any alignment of
	 * input mac address; this will have IPoIB signature, flags and
	 * scope bits.
	 */
	bcopy(mcmac->ipoib_gidpref, &topword, sizeof (uint32_t));
	topword = ntohl(topword);

	/*
	 * Generate proper address for IPv4/v6, adding in the Pkey properly.
	 */
	if ((topword == (IB_MCGID_IPV4_PREFIX | adjscope)) ||
	    (topword == (IB_MCGID_IPV6_PREFIX | adjscope)))
		ibd_h2n_mac(rmac, IB_MC_QPN, (((uint64_t)topword << 32) |
		    ((uint32_t)(state->id_pkey << 16))),
		    (INADDR_ALLRTRS_GROUP - INADDR_UNSPEC_GROUP));
	else
		/*
		 * Does not have proper bits in the mgid address.
		 */
		retval = B_FALSE;

	return (retval);
}

/*
 * Membership states for different mcg's are tracked by two lists:
 * the "non" list is used for promiscuous mode, when all mcg traffic
 * needs to be inspected. This type of membership is never used for
 * transmission, so there can not be an AH in the active list
 * corresponding to a member in this list. This list does not need
 * any protection, since all operations are performed by the async
 * thread.
 *
 * "Full" and "SendOnly" membership is tracked using a single list,
 * the "full" list. This is because this single list can then be
 * searched during transmit to a multicast group (if an AH for the
 * mcg is not found in the active list), since at least one type
 * of membership must be present before initiating the transmit.
 * This list is also emptied during driver detach, since sendonly
 * membership acquired during transmit is dropped at detach time
 * along with ipv4 broadcast full membership. Insert/deletes to
 * this list are done only by the async thread, but it is also
 * searched in program context (see multicast disable case), thus
 * the id_mc_mutex protects the list. The driver detach path also
 * deconstructs the "full" list, but it ensures that the async
 * thread will not be accessing the list (by blocking out mcg
 * trap handling and making sure no more Tx reaping will happen).
 *
 * Currently, an IBA attach is done in the SendOnly case too,
 * although this is not required.
 */
#define	IBD_MCACHE_INSERT_FULL(state, mce) \
	list_insert_head(&state->id_mc_full, mce)
#define	IBD_MCACHE_INSERT_NON(state, mce) \
	list_insert_head(&state->id_mc_non, mce)
#define	IBD_MCACHE_FIND_FULL(state, mgid) \
	ibd_mcache_find(mgid, &state->id_mc_full)
#define	IBD_MCACHE_FIND_NON(state, mgid) \
	ibd_mcache_find(mgid, &state->id_mc_non)
#define	IBD_MCACHE_PULLOUT_FULL(state, mce) \
	list_remove(&state->id_mc_full, mce)
#define	IBD_MCACHE_PULLOUT_NON(state, mce) \
	list_remove(&state->id_mc_non, mce)

static void *
list_get_head(list_t *list)
{
	list_node_t *lhead = list_head(list);

	if (lhead != NULL)
		list_remove(list, lhead);
	return (lhead);
}

/*
 * This is always guaranteed to be able to queue the work.
 */
void
ibd_queue_work_slot(ibd_state_t *state, ibd_req_t *ptr, int op)
{
	/* Initialize request */
	DPRINT(1, "ibd_queue_work_slot : op: %d \n", op);
	ptr->rq_op = op;

	/*
	 * Queue provided slot onto request pool.
	 */
	mutex_enter(&state->id_acache_req_lock);
	list_insert_tail(&state->id_req_list, ptr);

	/* Go, fetch, async thread */
	cv_signal(&state->id_acache_req_cv);
	mutex_exit(&state->id_acache_req_lock);
}

/*
 * Main body of the per interface async thread.
 */
static void
ibd_async_work(ibd_state_t *state)
{
	ibd_req_t *ptr;
	callb_cpr_t cprinfo;

	mutex_enter(&state->id_acache_req_lock);
	CALLB_CPR_INIT(&cprinfo, &state->id_acache_req_lock,
	    callb_generic_cpr, "ibd_async_work");

	for (;;) {
		ptr = list_get_head(&state->id_req_list);
		if (ptr != NULL) {
			mutex_exit(&state->id_acache_req_lock);

			/*
			 * If we are in late hca initialization mode, do not
			 * process any other async request other than TRAP. TRAP
			 * is used for indicating creation of a broadcast group;
			 * in which case, we need to join/create the group.
			 */
			if ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) &&
			    (ptr->rq_op != IBD_ASYNC_TRAP)) {
				goto free_req_and_continue;
			}

			/*
			 * Once we have done the operation, there is no
			 * guarantee the request slot is going to be valid,
			 * it might be freed up (as in IBD_ASYNC_LEAVE, REAP,
			 * TRAP).
			 *
			 * Perform the request.
			 */
			switch (ptr->rq_op) {
				case IBD_ASYNC_GETAH:
					ibd_async_acache(state, &ptr->rq_mac);
					break;
				case IBD_ASYNC_JOIN:
				case IBD_ASYNC_LEAVE:
					ibd_async_multicast(state,
					    ptr->rq_gid, ptr->rq_op);
					break;
				case IBD_ASYNC_PROMON:
					ibd_async_setprom(state);
					break;
				case IBD_ASYNC_PROMOFF:
					ibd_async_unsetprom(state);
					break;
				case IBD_ASYNC_REAP:
					ibd_async_reap_group(state,
					    ptr->rq_ptr, ptr->rq_gid,
					    IB_MC_JSTATE_FULL);
					/*
					 * the req buf contains in mce
					 * structure, so we do not need
					 * to free it here.
					 */
					ptr = NULL;
					break;
				case IBD_ASYNC_TRAP:
					ibd_async_trap(state, ptr);
					break;
				case IBD_ASYNC_SCHED:
					ibd_async_txsched(state);
					break;
				case IBD_ASYNC_LINK:
					ibd_async_link(state, ptr);
					break;
				case IBD_ASYNC_EXIT:
					mutex_enter(&state->id_acache_req_lock);
#ifndef __lock_lint
					CALLB_CPR_EXIT(&cprinfo);
#else
					mutex_exit(&state->id_acache_req_lock);
#endif
					return;
				case IBD_ASYNC_RC_TOO_BIG:
					ibd_async_rc_process_too_big(state,
					    ptr);
					break;
				case IBD_ASYNC_RC_CLOSE_ACT_CHAN:
					ibd_async_rc_close_act_chan(state, ptr);
					break;
				case IBD_ASYNC_RC_RECYCLE_ACE:
					ibd_async_rc_recycle_ace(state, ptr);
					break;
				case IBD_ASYNC_RC_CLOSE_PAS_CHAN:
					(void) ibd_rc_pas_close(ptr->rq_ptr,
					    B_TRUE, B_TRUE);
					break;
			}
free_req_and_continue:
			if (ptr != NULL)
				kmem_cache_free(state->id_req_kmc, ptr);

			mutex_enter(&state->id_acache_req_lock);
		} else {
#ifndef __lock_lint
			/*
			 * Nothing to do: wait till new request arrives.
			 */
			CALLB_CPR_SAFE_BEGIN(&cprinfo);
			cv_wait(&state->id_acache_req_cv,
			    &state->id_acache_req_lock);
			CALLB_CPR_SAFE_END(&cprinfo,
			    &state->id_acache_req_lock);
#endif
		}
	}

	/*NOTREACHED*/
	_NOTE(NOT_REACHED)
}

/*
 * Return when it is safe to queue requests to the async daemon; primarily
 * for subnet trap and async event handling. Disallow requests before the
 * daemon is created, and when interface deinitilization starts.
 */
static boolean_t
ibd_async_safe(ibd_state_t *state)
{
	mutex_enter(&state->id_trap_lock);
	if (state->id_trap_stop) {
		mutex_exit(&state->id_trap_lock);
		return (B_FALSE);
	}
	state->id_trap_inprog++;
	mutex_exit(&state->id_trap_lock);
	return (B_TRUE);
}

/*
 * Wake up ibd_m_stop() if the unplumb code is waiting for pending subnet
 * trap or event handling to complete to kill the async thread and deconstruct
 * the mcg/ace list.
 */
static void
ibd_async_done(ibd_state_t *state)
{
	mutex_enter(&state->id_trap_lock);
	if (--state->id_trap_inprog == 0)
		cv_signal(&state->id_trap_cv);
	mutex_exit(&state->id_trap_lock);
}

/*
 * Hash functions:
 * ibd_hash_by_id: Returns the qpn as the hash entry into bucket.
 * ibd_hash_key_cmp: Compares two keys, return 0 on success or else 1.
 * These operate on mac addresses input into ibd_send, but there is no
 * guarantee on the alignment of the ipoib_mac_t structure.
 */
/*ARGSUSED*/
static uint_t
ibd_hash_by_id(void *hash_data, mod_hash_key_t key)
{
	ulong_t ptraddr = (ulong_t)key;
	uint_t hval;

	/*
	 * If the input address is 4 byte aligned, we can just dereference
	 * it. This is most common, since IP will send in a 4 byte aligned
	 * IP header, which implies the 24 byte IPoIB psuedo header will be
	 * 4 byte aligned too.
	 */
	if ((ptraddr & 3) == 0)
		return ((uint_t)((ipoib_mac_t *)key)->ipoib_qpn);

	bcopy(&(((ipoib_mac_t *)key)->ipoib_qpn), &hval, sizeof (uint_t));
	return (hval);
}

static int
ibd_hash_key_cmp(mod_hash_key_t key1, mod_hash_key_t key2)
{
	if (bcmp((char *)key1, (char *)key2, IPOIB_ADDRL) == 0)
		return (0);
	else
		return (1);
}

/*
 * Initialize all the per interface caches and lists; AH cache,
 * MCG list etc.
 */
static int
ibd_acache_init(ibd_state_t *state)
{
	ibd_ace_t *ce;
	int i;

	mutex_init(&state->id_ac_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->id_mc_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_enter(&state->id_ac_mutex);
	list_create(&state->id_ah_free, sizeof (ibd_ace_t),
	    offsetof(ibd_ace_t, ac_list));
	list_create(&state->id_ah_active, sizeof (ibd_ace_t),
	    offsetof(ibd_ace_t, ac_list));
	state->id_ah_active_hash = mod_hash_create_extended("IBD AH hash",
	    state->id_hash_size, mod_hash_null_keydtor, mod_hash_null_valdtor,
	    ibd_hash_by_id, NULL, ibd_hash_key_cmp, KM_SLEEP);
	list_create(&state->id_mc_full, sizeof (ibd_mce_t),
	    offsetof(ibd_mce_t, mc_list));
	list_create(&state->id_mc_non, sizeof (ibd_mce_t),
	    offsetof(ibd_mce_t, mc_list));
	state->id_ac_hot_ace = NULL;

	state->id_ac_list = ce = (ibd_ace_t *)kmem_zalloc(sizeof (ibd_ace_t) *
	    state->id_num_ah, KM_SLEEP);
	for (i = 0; i < state->id_num_ah; i++, ce++) {
		if (ibt_alloc_ud_dest(state->id_hca_hdl, IBT_UD_DEST_NO_FLAGS,
		    state->id_pd_hdl, &ce->ac_dest) != IBT_SUCCESS) {
			mutex_exit(&state->id_ac_mutex);
			ibd_acache_fini(state);
			return (DDI_FAILURE);
		} else {
			CLEAR_REFCYCLE(ce);
			ce->ac_mce = NULL;
			mutex_init(&ce->tx_too_big_mutex, NULL,
			    MUTEX_DRIVER, NULL);
			IBD_ACACHE_INSERT_FREE(state, ce);
		}
	}
	mutex_exit(&state->id_ac_mutex);
	return (DDI_SUCCESS);
}

static void
ibd_acache_fini(ibd_state_t *state)
{
	ibd_ace_t *ptr;

	mutex_enter(&state->id_ac_mutex);

	while ((ptr = IBD_ACACHE_GET_ACTIVE(state)) != NULL) {
		ASSERT(GET_REF(ptr) == 0);
		mutex_destroy(&ptr->tx_too_big_mutex);
		(void) ibt_free_ud_dest(ptr->ac_dest);
	}

	while ((ptr = IBD_ACACHE_GET_FREE(state)) != NULL) {
		ASSERT(GET_REF(ptr) == 0);
		mutex_destroy(&ptr->tx_too_big_mutex);
		(void) ibt_free_ud_dest(ptr->ac_dest);
	}

	list_destroy(&state->id_ah_free);
	list_destroy(&state->id_ah_active);
	list_destroy(&state->id_mc_full);
	list_destroy(&state->id_mc_non);
	kmem_free(state->id_ac_list, sizeof (ibd_ace_t) * state->id_num_ah);
	mutex_exit(&state->id_ac_mutex);
	mutex_destroy(&state->id_ac_mutex);
	mutex_destroy(&state->id_mc_mutex);
}

/*
 * Search AH active hash list for a cached path to input destination.
 * If we are "just looking", hold == F. When we are in the Tx path,
 * we set hold == T to grab a reference on the AH so that it can not
 * be recycled to a new destination while the Tx request is posted.
 */
ibd_ace_t *
ibd_acache_find(ibd_state_t *state, ipoib_mac_t *mac, boolean_t hold, int num)
{
	ibd_ace_t *ptr;

	ASSERT(mutex_owned(&state->id_ac_mutex));

	/*
	 * Do hash search.
	 */
	if (mod_hash_find(state->id_ah_active_hash,
	    (mod_hash_key_t)mac, (mod_hash_val_t)&ptr) == 0) {
		if (hold)
			INC_REF(ptr, num);
		return (ptr);
	}
	return (NULL);
}

/*
 * This is called by the tx side; if an initialized AH is found in
 * the active list, it is locked down and can be used; if no entry
 * is found, an async request is queued to do path resolution.
 */
static ibd_ace_t *
ibd_acache_lookup(ibd_state_t *state, ipoib_mac_t *mac, int *err, int numwqe)
{
	ibd_ace_t *ptr;
	ibd_req_t *req;

	/*
	 * Only attempt to print when we can; in the mdt pattr case, the
	 * address is not aligned properly.
	 */
	if (((ulong_t)mac & 3) == 0) {
		DPRINT(4,
		    "ibd_acache_lookup : lookup for %08X:%08X:%08X:%08X:%08X",
		    htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
		    htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]),
		    htonl(mac->ipoib_gidsuff[1]));
	}

	mutex_enter(&state->id_ac_mutex);

	if (((ptr = state->id_ac_hot_ace) != NULL) &&
	    (memcmp(&ptr->ac_mac, mac, sizeof (*mac)) == 0)) {
		INC_REF(ptr, numwqe);
		mutex_exit(&state->id_ac_mutex);
		return (ptr);
	}
	if (((ptr = ibd_acache_find(state, mac, B_TRUE, numwqe)) != NULL)) {
		state->id_ac_hot_ace = ptr;
		mutex_exit(&state->id_ac_mutex);
		return (ptr);
	}

	/*
	 * Implementation of a single outstanding async request; if
	 * the operation is not started yet, queue a request and move
	 * to ongoing state. Remember in id_ah_addr for which address
	 * we are queueing the request, in case we need to flag an error;
	 * Any further requests, for the same or different address, until
	 * the operation completes, is sent back to GLDv3 to be retried.
	 * The async thread will update id_ah_op with an error indication
	 * or will set it to indicate the next look up can start; either
	 * way, it will mac_tx_update() so that all blocked requests come
	 * back here.
	 */
	*err = EAGAIN;
	if (state->id_ah_op == IBD_OP_NOTSTARTED) {
		req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
		if (req != NULL) {
			/*
			 * We did not even find the entry; queue a request
			 * for it.
			 */
			bcopy(mac, &(req->rq_mac), IPOIB_ADDRL);
			state->id_ah_op = IBD_OP_ONGOING;
			ibd_queue_work_slot(state, req, IBD_ASYNC_GETAH);
			bcopy(mac, &state->id_ah_addr, IPOIB_ADDRL);
		}
	} else if ((state->id_ah_op != IBD_OP_ONGOING) &&
	    (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) == 0)) {
		/*
		 * Check the status of the pathrecord lookup request
		 * we had queued before.
		 */
		if (state->id_ah_op == IBD_OP_ERRORED) {
			*err = EFAULT;
			state->id_ah_error++;
		} else {
			/*
			 * IBD_OP_ROUTERED case: We need to send to the
			 * all-router MCG. If we can find the AH for
			 * the mcg, the Tx will be attempted. If we
			 * do not find the AH, we return NORESOURCES
			 * to retry.
			 */
			ipoib_mac_t routermac;

			(void) ibd_get_allroutergroup(state, mac, &routermac);
			ptr = ibd_acache_find(state, &routermac, B_TRUE,
			    numwqe);
		}
		state->id_ah_op = IBD_OP_NOTSTARTED;
	} else if ((state->id_ah_op != IBD_OP_ONGOING) &&
	    (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) != 0)) {
		/*
		 * This case can happen when we get a higher band
		 * packet. The easiest way is to reset the state machine
		 * to accommodate the higher priority packet.
		 */
		state->id_ah_op = IBD_OP_NOTSTARTED;
	}
	mutex_exit(&state->id_ac_mutex);

	return (ptr);
}

/*
 * Grab a not-currently-in-use AH/PathRecord from the active
 * list to recycle to a new destination. Only the async thread
 * executes this code.
 */
static ibd_ace_t *
ibd_acache_get_unref(ibd_state_t *state)
{
	ibd_ace_t *ptr = list_tail(&state->id_ah_active);
	boolean_t try_rc_chan_recycle = B_FALSE;

	ASSERT(mutex_owned(&state->id_ac_mutex));

	/*
	 * Do plain linear search.
	 */
	while (ptr != NULL) {
		/*
		 * Note that it is possible that the "cycle" bit
		 * is set on the AH w/o any reference count. The
		 * mcg must have been deleted, and the tx cleanup
		 * just decremented the reference count to 0, but
		 * hasn't gotten around to grabbing the id_ac_mutex
		 * to move the AH into the free list.
		 */
		if (GET_REF(ptr) == 0) {
			if (ptr->ac_chan != NULL) {
				ASSERT(state->id_enable_rc == B_TRUE);
				if (!try_rc_chan_recycle) {
					try_rc_chan_recycle = B_TRUE;
					ibd_rc_signal_ace_recycle(state, ptr);
				}
			} else {
				IBD_ACACHE_PULLOUT_ACTIVE(state, ptr);
				break;
			}
		}
		ptr = list_prev(&state->id_ah_active, ptr);
	}
	return (ptr);
}

/*
 * Invoked to clean up AH from active list in case of multicast
 * disable and to handle sendonly memberships during mcg traps.
 * And for port up processing for multicast and unicast AHs.
 * Normally, the AH is taken off the active list, and put into
 * the free list to be recycled for a new destination. In case
 * Tx requests on the AH have not completed yet, the AH is marked
 * for reaping (which will put the AH on the free list) once the Tx's
 * complete; in this case, depending on the "force" input, we take
 * out the AH from the active list right now, or leave it also for
 * the reap operation. Returns TRUE if the AH is taken off the active
 * list (and either put into the free list right now, or arranged for
 * later), FALSE otherwise.
 */
boolean_t
ibd_acache_recycle(ibd_state_t *state, ipoib_mac_t *acmac, boolean_t force)
{
	ibd_ace_t *acactive;
	boolean_t ret = B_TRUE;

	ASSERT(mutex_owned(&state->id_ac_mutex));

	if ((acactive = ibd_acache_find(state, acmac, B_FALSE, 0)) != NULL) {

		/*
		 * Note that the AH might already have the cycle bit set
		 * on it; this might happen if sequences of multicast
		 * enables and disables are coming so fast, that posted
		 * Tx's to the mcg have not completed yet, and the cycle
		 * bit is set successively by each multicast disable.
		 */
		if (SET_CYCLE_IF_REF(acactive)) {
			if (!force) {
				/*
				 * The ace is kept on the active list, further
				 * Tx's can still grab a reference on it; the
				 * ace is reaped when all pending Tx's
				 * referencing the AH complete.
				 */
				ret = B_FALSE;
			} else {
				/*
				 * In the mcg trap case, we always pull the
				 * AH from the active list. And also the port
				 * up multi/unicast case.
				 */
				ASSERT(acactive->ac_chan == NULL);
				IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
				acactive->ac_mce = NULL;
			}
		} else {
			/*
			 * Determined the ref count is 0, thus reclaim
			 * immediately after pulling out the ace from
			 * the active list.
			 */
			ASSERT(acactive->ac_chan == NULL);
			IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
			acactive->ac_mce = NULL;
			IBD_ACACHE_INSERT_FREE(state, acactive);
		}

	}
	return (ret);
}

/*
 * Helper function for async path record lookup. If we are trying to
 * Tx to a MCG, check our membership, possibly trying to join the
 * group if required. If that fails, try to send the packet to the
 * all router group (indicated by the redirect output), pointing
 * the input mac address to the router mcg address.
 */
static ibd_mce_t *
ibd_async_mcache(ibd_state_t *state, ipoib_mac_t *mac, boolean_t *redirect)
{
	ib_gid_t mgid;
	ibd_mce_t *mce;
	ipoib_mac_t routermac;

	*redirect = B_FALSE;
	ibd_n2h_gid(mac, &mgid);

	/*
	 * Check the FullMember+SendOnlyNonMember list.
	 * Since we are the only one who manipulates the
	 * id_mc_full list, no locks are needed.
	 */
	mce = IBD_MCACHE_FIND_FULL(state, mgid);
	if (mce != NULL) {
		DPRINT(4, "ibd_async_mcache : already joined to group");
		return (mce);
	}

	/*
	 * Not found; try to join(SendOnlyNonMember) and attach.
	 */
	DPRINT(4, "ibd_async_mcache : not joined to group");
	if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
	    NULL) {
		DPRINT(4, "ibd_async_mcache : nonmem joined to group");
		return (mce);
	}

	/*
	 * MCGroup not present; try to join the all-router group. If
	 * any of the following steps succeed, we will be redirecting
	 * to the all router group.
	 */
	DPRINT(4, "ibd_async_mcache : nonmem join failed");
	if (!ibd_get_allroutergroup(state, mac, &routermac))
		return (NULL);
	*redirect = B_TRUE;
	ibd_n2h_gid(&routermac, &mgid);
	bcopy(&routermac, mac, IPOIB_ADDRL);
	DPRINT(4, "ibd_async_mcache : router mgid : %016llx:%016llx\n",
	    mgid.gid_prefix, mgid.gid_guid);

	/*
	 * Are we already joined to the router group?
	 */
	if ((mce = IBD_MCACHE_FIND_FULL(state, mgid)) != NULL) {
		DPRINT(4, "ibd_async_mcache : using already joined router"
		    "group\n");
		return (mce);
	}

	/*
	 * Can we join(SendOnlyNonMember) the router group?
	 */
	DPRINT(4, "ibd_async_mcache : attempting join to router grp");
	if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
	    NULL) {
		DPRINT(4, "ibd_async_mcache : joined to router grp");
		return (mce);
	}

	return (NULL);
}

/*
 * Async path record lookup code.
 */
static void
ibd_async_acache(ibd_state_t *state, ipoib_mac_t *mac)
{
	ibd_ace_t *ce;
	ibd_mce_t *mce = NULL;
	ibt_path_attr_t path_attr;
	ibt_path_info_t path_info;
	ib_gid_t destgid;
	char ret = IBD_OP_NOTSTARTED;

	DPRINT(4, "ibd_async_acache :  %08X:%08X:%08X:%08X:%08X",
	    htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
	    htonl(mac->ipoib_gidpref[1]), htonl(mac->ipoib_gidsuff[0]),
	    htonl(mac->ipoib_gidsuff[1]));

	/*
	 * Check whether we are trying to transmit to a MCG.
	 * In that case, we need to make sure we are a member of
	 * the MCG.
	 */
	if (mac->ipoib_qpn == htonl(IB_MC_QPN)) {
		boolean_t redirected;

		/*
		 * If we can not find or join the group or even
		 * redirect, error out.
		 */
		if ((mce = ibd_async_mcache(state, mac, &redirected)) ==
		    NULL) {
			state->id_ah_op = IBD_OP_ERRORED;
			return;
		}

		/*
		 * If we got redirected, we need to determine whether
		 * the AH for the new mcg is in the cache already, and
		 * not pull it in then; otherwise proceed to get the
		 * path for the new mcg. There is no guarantee that
		 * if the AH is currently in the cache, it will still be
		 * there when we look in ibd_acache_lookup(), but that's
		 * okay, we will come back here.
		 */
		if (redirected) {
			ret = IBD_OP_ROUTERED;
			DPRINT(4, "ibd_async_acache :  redirected to "
			    "%08X:%08X:%08X:%08X:%08X",
			    htonl(mac->ipoib_qpn), htonl(mac->ipoib_gidpref[0]),
			    htonl(mac->ipoib_gidpref[1]),
			    htonl(mac->ipoib_gidsuff[0]),
			    htonl(mac->ipoib_gidsuff[1]));

			mutex_enter(&state->id_ac_mutex);
			if (ibd_acache_find(state, mac, B_FALSE, 0) != NULL) {
				state->id_ah_op = IBD_OP_ROUTERED;
				mutex_exit(&state->id_ac_mutex);
				DPRINT(4, "ibd_async_acache : router AH found");
				return;
			}
			mutex_exit(&state->id_ac_mutex);
		}
	}

	/*
	 * Get an AH from the free list.
	 */
	mutex_enter(&state->id_ac_mutex);
	if ((ce = IBD_ACACHE_GET_FREE(state)) == NULL) {
		/*
		 * No free ones; try to grab an unreferenced active
		 * one. Maybe we need to make the active list LRU,
		 * but that will create more work for Tx callbacks.
		 * Is there a way of not having to pull out the
		 * entry from the active list, but just indicate it
		 * is being recycled? Yes, but that creates one more
		 * check in the fast lookup path.
		 */
		if ((ce = ibd_acache_get_unref(state)) == NULL) {
			/*
			 * Pretty serious shortage now.
			 */
			state->id_ah_op = IBD_OP_NOTSTARTED;
			mutex_exit(&state->id_ac_mutex);
			DPRINT(10, "ibd_async_acache : failed to find AH "
			    "slot\n");
			return;
		}
		/*
		 * We could check whether ac_mce points to a SendOnly
		 * member and drop that membership now. Or do it lazily
		 * at detach time.
		 */
		ce->ac_mce = NULL;
	}
	mutex_exit(&state->id_ac_mutex);
	ASSERT(ce->ac_mce == NULL);

	/*
	 * Update the entry.
	 */
	bcopy((char *)mac, &ce->ac_mac, IPOIB_ADDRL);

	bzero(&path_info, sizeof (path_info));
	bzero(&path_attr, sizeof (ibt_path_attr_t));
	path_attr.pa_sgid = state->id_sgid;
	path_attr.pa_num_dgids = 1;
	ibd_n2h_gid(&ce->ac_mac, &destgid);
	path_attr.pa_dgids = &destgid;
	path_attr.pa_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
	path_attr.pa_pkey = state->id_pkey;
	if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_PKEY, &path_attr, 1,
	    &path_info, NULL) != IBT_SUCCESS) {
		DPRINT(10, "ibd_async_acache : failed in ibt_get_paths");
		goto error;
	}
	if (ibt_modify_ud_dest(ce->ac_dest, state->id_mcinfo->mc_qkey,
	    ntohl(ce->ac_mac.ipoib_qpn),
	    &path_info.pi_prim_cep_path.cep_adds_vect) != IBT_SUCCESS) {
		DPRINT(10, "ibd_async_acache : failed in ibt_modify_ud_dest");
		goto error;
	}

	/*
	 * mce is set whenever an AH is being associated with a
	 * MCG; this will come in handy when we leave the MCG. The
	 * lock protects Tx fastpath from scanning the active list.
	 */
	if (mce != NULL)
		ce->ac_mce = mce;

	/*
	 * initiate a RC mode connection for unicast address
	 */
	if (state->id_enable_rc && (mac->ipoib_qpn != htonl(IB_MC_QPN)) &&
	    (htonl(mac->ipoib_qpn) & IBD_MAC_ADDR_RC)) {
		ASSERT(ce->ac_chan == NULL);
		DPRINT(10, "ibd_async_acache: call "
		    "ibd_rc_try_connect(ace=%p)", ce);
		ibd_rc_try_connect(state, ce, &path_info);
		if (ce->ac_chan == NULL) {
			DPRINT(10, "ibd_async_acache: fail to setup RC"
			    " channel");
			state->rc_conn_fail++;
			goto error;
		}
	}

	mutex_enter(&state->id_ac_mutex);
	IBD_ACACHE_INSERT_ACTIVE(state, ce);
	state->id_ah_op = ret;
	mutex_exit(&state->id_ac_mutex);
	return;
error:
	/*
	 * We might want to drop SendOnly membership here if we
	 * joined above. The lock protects Tx callbacks inserting
	 * into the free list.
	 */
	mutex_enter(&state->id_ac_mutex);
	state->id_ah_op = IBD_OP_ERRORED;
	IBD_ACACHE_INSERT_FREE(state, ce);
	mutex_exit(&state->id_ac_mutex);
}

/*
 * While restoring port's presence on the subnet on a port up, it is possible
 * that the port goes down again.
 */
static void
ibd_async_link(ibd_state_t *state, ibd_req_t *req)
{
	ibd_link_op_t opcode = (ibd_link_op_t)req->rq_ptr;
	link_state_t lstate = (opcode == IBD_LINK_DOWN) ? LINK_STATE_DOWN :
	    LINK_STATE_UP;
	ibd_mce_t *mce, *pmce;
	ibd_ace_t *ace, *pace;

	DPRINT(10, "ibd_async_link(): %d", opcode);

	/*
	 * On a link up, revalidate the link speed/width. No point doing
	 * this on a link down, since we will be unable to do SA operations,
	 * defaulting to the lowest speed. Also notice that we update our
	 * notion of speed before calling mac_link_update(), which will do
	 * necessary higher level notifications for speed changes.
	 */
	if ((opcode == IBD_LINK_UP_ABSENT) || (opcode == IBD_LINK_UP)) {
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
		state->id_link_speed = ibd_get_portspeed(state);
		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state))
	}

	/*
	 * Do all the work required to establish our presence on
	 * the subnet.
	 */
	if (opcode == IBD_LINK_UP_ABSENT) {
		/*
		 * If in promiscuous mode ...
		 */
		if (state->id_prom_op == IBD_OP_COMPLETED) {
			/*
			 * Drop all nonmembership.
			 */
			ibd_async_unsetprom(state);

			/*
			 * Then, try to regain nonmembership to all mcg's.
			 */
			ibd_async_setprom(state);

		}

		/*
		 * Drop all sendonly membership (which also gets rid of the
		 * AHs); try to reacquire all full membership.
		 */
		mce = list_head(&state->id_mc_full);
		while ((pmce = mce) != NULL) {
			mce = list_next(&state->id_mc_full, mce);
			if (pmce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON)
				ibd_leave_group(state,
				    pmce->mc_info.mc_adds_vect.av_dgid,
				    IB_MC_JSTATE_SEND_ONLY_NON);
			else
				ibd_reacquire_group(state, pmce);
		}

		/*
		 * Recycle all active AHs to free list (and if there are
		 * pending posts, make sure they will go into the free list
		 * once the Tx's complete). Grab the lock to prevent
		 * concurrent Tx's as well as Tx cleanups.
		 */
		mutex_enter(&state->id_ac_mutex);
		ace = list_head(&state->id_ah_active);
		while ((pace = ace) != NULL) {
			boolean_t cycled;

			ace = list_next(&state->id_ah_active, ace);
			mce = pace->ac_mce;
			if (pace->ac_chan != NULL) {
				ASSERT(mce == NULL);
				ASSERT(state->id_enable_rc == B_TRUE);
				if (pace->ac_chan->chan_state ==
				    IBD_RC_STATE_ACT_ESTAB) {
					INC_REF(pace, 1);
					IBD_ACACHE_PULLOUT_ACTIVE(state, pace);
					pace->ac_chan->chan_state =
					    IBD_RC_STATE_ACT_CLOSING;
					ibd_rc_signal_act_close(state, pace);
				} else {
					state->rc_act_close_simultaneous++;
					DPRINT(40, "ibd_async_link: other "
					    "thread is closing it, ace=%p, "
					    "ac_chan=%p, chan_state=%d",
					    pace, pace->ac_chan,
					    pace->ac_chan->chan_state);
				}
			} else {
				cycled = ibd_acache_recycle(state,
				    &pace->ac_mac, B_TRUE);
			}
			/*
			 * If this is for an mcg, it must be for a fullmember,
			 * since we got rid of send-only members above when
			 * processing the mce list.
			 */
			ASSERT(cycled && ((mce == NULL) || (mce->mc_jstate ==
			    IB_MC_JSTATE_FULL)));

			/*
			 * Check if the fullmember mce needs to be torn down,
			 * ie whether the DLPI disable has already been done.
			 * If so, do some of the work of tx_cleanup, namely
			 * causing leave (which will fail), detach and
			 * mce-freeing. tx_cleanup will put the AH into free
			 * list. The reason to duplicate some of this
			 * tx_cleanup work is because we want to delete the
			 * AH right now instead of waiting for tx_cleanup, to
			 * force subsequent Tx's to reacquire an AH.
			 */
			if ((mce != NULL) && (mce->mc_fullreap))
				ibd_async_reap_group(state, mce,
				    mce->mc_info.mc_adds_vect.av_dgid,
				    mce->mc_jstate);
		}
		mutex_exit(&state->id_ac_mutex);
	}

	/*
	 * mac handle is guaranteed to exist since driver does ibt_close_hca()
	 * (which stops further events from being delivered) before
	 * mac_unregister(). At this point, it is guaranteed that mac_register
	 * has already been done.
	 */
	mutex_enter(&state->id_link_mutex);
	state->id_link_state = lstate;
	mac_link_update(state->id_mh, lstate);
	mutex_exit(&state->id_link_mutex);

	ibd_async_done(state);
}

/*
 * Check the pkey table to see if we can find the pkey we're looking for.
 * Set the pkey index in 'pkix' if found. Return 0 on success and -1 on
 * failure.
 */
static int
ibd_locate_pkey(ib_pkey_t *pkey_tbl, uint16_t pkey_tbl_sz, ib_pkey_t pkey,
    uint16_t *pkix)
{
	uint16_t ndx;

	ASSERT(pkix != NULL);

	for (ndx = 0; ndx < pkey_tbl_sz; ndx++) {
		if (pkey_tbl[ndx] == pkey) {
			*pkix = ndx;
			return (0);
		}
	}
	return (-1);
}

/*
 * Late HCA Initialization:
 * If plumb had succeeded without the availability of an active port or the
 * pkey, and either of their availability is now being indicated via PORT_UP
 * or PORT_CHANGE respectively, try a start of the interface.
 *
 * Normal Operation:
 * When the link is notified up, we need to do a few things, based
 * on the port's current p_init_type_reply claiming a reinit has been
 * done or not. The reinit steps are:
 * 1. If in InitTypeReply, NoLoadReply == PreserveContentReply == 0, verify
 *    the old Pkey and GID0 are correct.
 * 2. Register for mcg traps (already done by ibmf).
 * 3. If PreservePresenceReply indicates the SM has restored port's presence
 *    in subnet, nothing more to do. Else go to next steps (on async daemon).
 * 4. Give up all sendonly memberships.
 * 5. Acquire all full memberships.
 * 6. In promiscuous mode, acquire all non memberships.
 * 7. Recycle all AHs to free list.
 */
static void
ibd_link_mod(ibd_state_t *state, ibt_async_code_t code)
{
	ibt_hca_portinfo_t *port_infop = NULL;
	ibt_status_t ibt_status;
	uint_t psize, port_infosz;
	ibd_link_op_t opcode;
	ibd_req_t *req;
	link_state_t new_link_state = LINK_STATE_UP;
	uint8_t itreply;
	uint16_t pkix;
	int ret;

	/*
	 * Let's not race with a plumb or an unplumb; if we detect a
	 * pkey relocation event later on here, we may have to restart.
	 */
	ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);

	mutex_enter(&state->id_link_mutex);

	/*
	 * If the link state is unknown, a plumb has not yet been attempted
	 * on the interface. Nothing to do.
	 */
	if (state->id_link_state == LINK_STATE_UNKNOWN) {
		mutex_exit(&state->id_link_mutex);
		goto link_mod_return;
	}

	/*
	 * If link state is down because of plumb failure, and we are not in
	 * late HCA init, and we were not successfully plumbed, nothing to do.
	 */
	if ((state->id_link_state == LINK_STATE_DOWN) &&
	    ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 0) &&
	    ((state->id_mac_state & IBD_DRV_STARTED) == 0)) {
		mutex_exit(&state->id_link_mutex);
		goto link_mod_return;
	}

	/*
	 * If this routine was called in response to a port down event,
	 * we just need to see if this should be informed.
	 */
	if (code == IBT_ERROR_PORT_DOWN) {
		new_link_state = LINK_STATE_DOWN;
		goto update_link_state;
	}

	/*
	 * If it's not a port down event we've received, try to get the port
	 * attributes first. If we fail here, the port is as good as down.
	 * Otherwise, if the link went down by the time the handler gets
	 * here, give up - we cannot even validate the pkey/gid since those
	 * are not valid and this is as bad as a port down anyway.
	 */
	ibt_status = ibt_query_hca_ports(state->id_hca_hdl, state->id_port,
	    &port_infop, &psize, &port_infosz);
	if ((ibt_status != IBT_SUCCESS) || (psize != 1) ||
	    (port_infop->p_linkstate != IBT_PORT_ACTIVE)) {
		new_link_state = LINK_STATE_DOWN;
		goto update_link_state;
	}

	/*
	 * If in the previous attempt, the pkey was not found either due to the
	 * port state being down, or due to it's absence in the pkey table,
	 * look for it now and try to start the interface.
	 */
	if (state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) {
		mutex_exit(&state->id_link_mutex);
		if ((ret = ibd_start(state)) != 0) {
			DPRINT(10, "ibd_linkmod: cannot start from late HCA "
			    "init, ret=%d", ret);
		}
		ibt_free_portinfo(port_infop, port_infosz);
		goto link_mod_return;
	}

	/*
	 * Check the SM InitTypeReply flags. If both NoLoadReply and
	 * PreserveContentReply are 0, we don't know anything about the
	 * data loaded into the port attributes, so we need to verify
	 * if gid0 and pkey are still valid.
	 */
	itreply = port_infop->p_init_type_reply;
	if (((itreply & SM_INIT_TYPE_REPLY_NO_LOAD_REPLY) == 0) &&
	    ((itreply & SM_INIT_TYPE_PRESERVE_CONTENT_REPLY) == 0)) {
		/*
		 * Check to see if the subnet part of GID0 has changed. If
		 * not, check the simple case first to see if the pkey
		 * index is the same as before; finally check to see if the
		 * pkey has been relocated to a different index in the table.
		 */
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid))
		if (bcmp(port_infop->p_sgid_tbl,
		    &state->id_sgid, sizeof (ib_gid_t)) != 0) {

			new_link_state = LINK_STATE_DOWN;

		} else if (port_infop->p_pkey_tbl[state->id_pkix] ==
		    state->id_pkey) {

			new_link_state = LINK_STATE_UP;

		} else if (ibd_locate_pkey(port_infop->p_pkey_tbl,
		    port_infop->p_pkey_tbl_sz, state->id_pkey, &pkix) == 0) {

			ibt_free_portinfo(port_infop, port_infosz);
			mutex_exit(&state->id_link_mutex);

			/*
			 * Currently a restart is required if our pkey has moved
			 * in the pkey table. If we get the ibt_recycle_ud() to
			 * work as documented (expected), we may be able to
			 * avoid a complete restart.  Note that we've already
			 * marked both the start and stop 'in-progress' flags,
			 * so it is ok to go ahead and do this restart.
			 */
			(void) ibd_undo_start(state, LINK_STATE_DOWN);
			if ((ret = ibd_start(state)) != 0) {
				DPRINT(10, "ibd_restart: cannot restart, "
				    "ret=%d", ret);
			}

			goto link_mod_return;
		} else {
			new_link_state = LINK_STATE_DOWN;
		}
		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid))
	}

update_link_state:
	if (port_infop) {
		ibt_free_portinfo(port_infop, port_infosz);
	}

	/*
	 * If we're reporting a link up, check InitTypeReply to see if
	 * the SM has ensured that the port's presence in mcg, traps,
	 * etc. is intact.
	 */
	if (new_link_state == LINK_STATE_DOWN) {
		opcode = IBD_LINK_DOWN;
	} else {
		if ((itreply & SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY) ==
		    SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY) {
			opcode = IBD_LINK_UP;
		} else {
			opcode = IBD_LINK_UP_ABSENT;
		}
	}

	/*
	 * If the old state is the same as the new state, and the SM indicated
	 * no change in the port parameters, nothing to do.
	 */
	if ((state->id_link_state == new_link_state) && (opcode !=
	    IBD_LINK_UP_ABSENT)) {
		mutex_exit(&state->id_link_mutex);
		goto link_mod_return;
	}

	/*
	 * Ok, so there was a link state change; see if it's safe to ask
	 * the async thread to do the work
	 */
	if (!ibd_async_safe(state)) {
		state->id_link_state = new_link_state;
		mutex_exit(&state->id_link_mutex);
		goto link_mod_return;
	}

	mutex_exit(&state->id_link_mutex);

	/*
	 * Queue up a request for ibd_async_link() to handle this link
	 * state change event
	 */
	req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP);
	req->rq_ptr = (void *)opcode;
	ibd_queue_work_slot(state, req, IBD_ASYNC_LINK);

link_mod_return:
	ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
}

/*
 * For the port up/down events, IBTL guarantees there will not be concurrent
 * invocations of the handler. IBTL might coalesce link transition events,
 * and not invoke the handler for _each_ up/down transition, but it will
 * invoke the handler with last known state
 */
static void
ibd_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
    ibt_async_code_t code, ibt_async_event_t *event)
{
	ibd_state_t *state = (ibd_state_t *)clnt_private;

	switch (code) {
	case IBT_ERROR_CATASTROPHIC_CHAN:
		ibd_print_warn(state, "catastrophic channel error");
		break;
	case IBT_ERROR_CQ:
		ibd_print_warn(state, "completion queue error");
		break;
	case IBT_PORT_CHANGE_EVENT:
		/*
		 * Events will be delivered to all instances that have
		 * done ibt_open_hca() but not yet done ibt_close_hca().
		 * Only need to do work for our port; IBTF will deliver
		 * events for other ports on the hca we have ibt_open_hca'ed
		 * too. Note that id_port is initialized in ibd_attach()
		 * before we do an ibt_open_hca() in ibd_attach().
		 */
		ASSERT(state->id_hca_hdl == hca_hdl);
		if (state->id_port != event->ev_port)
			break;

		if ((event->ev_port_flags & IBT_PORT_CHANGE_PKEY) ==
		    IBT_PORT_CHANGE_PKEY) {
			ibd_link_mod(state, code);
		}
		break;
	case IBT_ERROR_PORT_DOWN:
	case IBT_CLNT_REREG_EVENT:
	case IBT_EVENT_PORT_UP:
		/*
		 * Events will be delivered to all instances that have
		 * done ibt_open_hca() but not yet done ibt_close_hca().
		 * Only need to do work for our port; IBTF will deliver
		 * events for other ports on the hca we have ibt_open_hca'ed
		 * too. Note that id_port is initialized in ibd_attach()
		 * before we do an ibt_open_hca() in ibd_attach().
		 */
		ASSERT(state->id_hca_hdl == hca_hdl);
		if (state->id_port != event->ev_port)
			break;

		ibd_link_mod(state, code);
		break;

	case IBT_HCA_ATTACH_EVENT:
	case IBT_HCA_DETACH_EVENT:
		/*
		 * When a new card is plugged to the system, attach_event is
		 * invoked. Additionally, a cfgadm needs to be run to make the
		 * card known to the system, and an ifconfig needs to be run to
		 * plumb up any ibd interfaces on the card. In the case of card
		 * unplug, a cfgadm is run that will trigger any RCM scripts to
		 * unplumb the ibd interfaces on the card; when the card is
		 * actually unplugged, the detach_event is invoked;
		 * additionally, if any ibd instances are still active on the
		 * card (eg there were no associated RCM scripts), driver's
		 * detach routine is invoked.
		 */
		break;
	default:
		break;
	}
}

static int
ibd_register_mac(ibd_state_t *state, dev_info_t *dip)
{
	mac_register_t *macp;
	int ret;

	if ((macp = mac_alloc(MAC_VERSION)) == NULL) {
		DPRINT(10, "ibd_register_mac: mac_alloc() failed");
		return (DDI_FAILURE);
	}

	/*
	 * Note that when we register with mac during attach, we don't
	 * have the id_macaddr yet, so we'll simply be registering a
	 * zero macaddr that we'll overwrite later during plumb (in
	 * ibd_m_start()). Similar is the case with id_mtu - we'll
	 * update the mac layer with the correct mtu during plumb.
	 */
	macp->m_type_ident = MAC_PLUGIN_IDENT_IB;
	macp->m_driver = state;
	macp->m_dip = dip;
	macp->m_src_addr = (uint8_t *)&state->id_macaddr;
	macp->m_callbacks = &ibd_m_callbacks;
	macp->m_min_sdu = 0;
	macp->m_multicast_sdu = IBD_DEF_MAX_SDU;
	if (state->id_type == IBD_PORT_DRIVER) {
		macp->m_max_sdu = IBD_DEF_RC_MAX_SDU;
	} else if (state->id_enable_rc) {
		macp->m_max_sdu = state->rc_mtu - IPOIB_HDRSIZE;
	} else {
		macp->m_max_sdu = IBD_DEF_MAX_SDU;
	}
	macp->m_priv_props = ibd_priv_props;

	/*
	 *  Register ourselves with the GLDv3 interface
	 */
	if ((ret = mac_register(macp, &state->id_mh)) != 0) {
		mac_free(macp);
		DPRINT(10,
		    "ibd_register_mac: mac_register() failed, ret=%d", ret);
		return (DDI_FAILURE);
	}

	mac_free(macp);
	return (DDI_SUCCESS);
}

static int
ibd_record_capab(ibd_state_t *state)
{
	ibt_hca_attr_t hca_attrs;
	ibt_status_t ibt_status;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))

	/*
	 * Query the HCA and fetch its attributes
	 */
	ibt_status = ibt_query_hca(state->id_hca_hdl, &hca_attrs);
	ASSERT(ibt_status == IBT_SUCCESS);

	/*
	 * 1. Set the Hardware Checksum capability. Currently we only consider
	 *    full checksum offload.
	 */
	if (state->id_enable_rc) {
			state->id_hwcksum_capab = 0;
	} else {
		if ((hca_attrs.hca_flags & IBT_HCA_CKSUM_FULL)
		    == IBT_HCA_CKSUM_FULL) {
			state->id_hwcksum_capab = IBT_HCA_CKSUM_FULL;
		}
	}

	/*
	 * 2. Set LSO policy, capability and maximum length
	 */
	if (state->id_enable_rc) {
		state->id_lso_capable = B_FALSE;
		state->id_lso_maxlen = 0;
	} else {
		if (hca_attrs.hca_max_lso_size > 0) {
			state->id_lso_capable = B_TRUE;
			if (hca_attrs.hca_max_lso_size > IBD_LSO_MAXLEN)
				state->id_lso_maxlen = IBD_LSO_MAXLEN;
			else
				state->id_lso_maxlen =
				    hca_attrs.hca_max_lso_size;
		} else {
			state->id_lso_capable = B_FALSE;
			state->id_lso_maxlen = 0;
		}
	}

	/*
	 * 3. Set Reserved L_Key capability
	 */
	if ((hca_attrs.hca_flags2 & IBT_HCA2_RES_LKEY) == IBT_HCA2_RES_LKEY) {
		state->id_hca_res_lkey_capab = 1;
		state->id_res_lkey = hca_attrs.hca_reserved_lkey;
		state->rc_enable_iov_map = B_TRUE;
	} else {
		/* If no reserved lkey, we will not use ibt_map_mem_iov */
		state->rc_enable_iov_map = B_FALSE;
	}

	/*
	 * 4. Set maximum sqseg value after checking to see if extended sgl
	 *    size information is provided by the hca
	 */
	if (hca_attrs.hca_flags & IBT_HCA_WQE_SIZE_INFO) {
		state->id_max_sqseg = hca_attrs.hca_ud_send_sgl_sz;
		state->rc_tx_max_sqseg = hca_attrs.hca_conn_send_sgl_sz;
	} else {
		state->id_max_sqseg = hca_attrs.hca_max_sgl;
		state->rc_tx_max_sqseg = hca_attrs.hca_max_sgl;
	}
	if (state->id_max_sqseg > IBD_MAX_SQSEG) {
		state->id_max_sqseg = IBD_MAX_SQSEG;
	} else if (state->id_max_sqseg < IBD_MAX_SQSEG) {
		ibd_print_warn(state, "Set #sgl = %d instead of default %d",
		    state->id_max_sqseg, IBD_MAX_SQSEG);
	}
	if (state->rc_tx_max_sqseg > IBD_MAX_SQSEG) {
		state->rc_tx_max_sqseg = IBD_MAX_SQSEG;
	} else if (state->rc_tx_max_sqseg < IBD_MAX_SQSEG) {
		ibd_print_warn(state, "RC mode: Set #sgl = %d instead of "
		    "default %d", state->rc_tx_max_sqseg, IBD_MAX_SQSEG);
	}

	/*
	 * Translating the virtual address regions into physical regions
	 * for using the Reserved LKey feature results in a wr sgl that
	 * is a little longer. Since failing ibt_map_mem_iov() is costly,
	 * we'll fix a high-water mark (65%) for when we should stop.
	 */
	state->id_max_sqseg_hiwm = (state->id_max_sqseg * 65) / 100;
	state->rc_max_sqseg_hiwm = (state->rc_tx_max_sqseg * 65) / 100;

	/*
	 * 5. Set number of recv and send wqes after checking hca maximum
	 *    channel size. Store the max channel size in the state so that it
	 *    can be referred to when the swqe/rwqe change is requested via
	 *    dladm.
	 */

	state->id_hca_max_chan_sz = hca_attrs.hca_max_chan_sz;

	if (hca_attrs.hca_max_chan_sz < state->id_ud_num_rwqe)
		state->id_ud_num_rwqe = hca_attrs.hca_max_chan_sz;

	state->id_rx_bufs_outstanding_limit = state->id_ud_num_rwqe -
	    IBD_RWQE_MIN;

	if (hca_attrs.hca_max_chan_sz < state->id_ud_num_swqe)
		state->id_ud_num_swqe = hca_attrs.hca_max_chan_sz;

	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state))

	return (DDI_SUCCESS);
}

static int
ibd_part_busy(ibd_state_t *state)
{
	if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) != 0) {
		DPRINT(10, "ibd_part_busy: failed: rx bufs outstanding\n");
		return (DDI_FAILURE);
	}

	if (state->rc_srq_rwqe_list.dl_bufs_outstanding != 0) {
		DPRINT(10, "ibd_part_busy: failed: srq bufs outstanding\n");
		return (DDI_FAILURE);
	}

	/*
	 * "state->id_ah_op == IBD_OP_ONGOING" means this IPoIB port is
	 * connecting to a remote IPoIB port. We can't remove this port.
	 */
	if (state->id_ah_op == IBD_OP_ONGOING) {
		DPRINT(10, "ibd_part_busy: failed: connecting\n");
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}


static void
ibd_part_unattach(ibd_state_t *state)
{
	uint32_t progress = state->id_mac_state;
	ibt_status_t ret;

	/* make sure rx resources are freed */
	ibd_free_rx_rsrcs(state);

	if (progress & IBD_DRV_RC_SRQ_ALLOCD) {
		ASSERT(state->id_enable_rc);
		ibd_rc_fini_srq_list(state);
		state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD);
	}

	if (progress & IBD_DRV_MAC_REGISTERED) {
		(void) mac_unregister(state->id_mh);
		state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED);
	}

	if (progress & IBD_DRV_ASYNC_THR_CREATED) {
		/*
		 * No new async requests will be posted since the device
		 * link state has been marked as unknown; completion handlers
		 * have been turned off, so Tx handler will not cause any
		 * more IBD_ASYNC_REAP requests.
		 *
		 * Queue a request for the async thread to exit, which will
		 * be serviced after any pending ones. This can take a while,
		 * specially if the SM is unreachable, since IBMF will slowly
		 * timeout each SM request issued by the async thread.  Reap
		 * the thread before continuing on, we do not want it to be
		 * lingering in modunloaded code.
		 */
		ibd_queue_work_slot(state, &state->id_ah_req, IBD_ASYNC_EXIT);
		thread_join(state->id_async_thrid);

		state->id_mac_state &= (~IBD_DRV_ASYNC_THR_CREATED);
	}

	if (progress & IBD_DRV_REQ_LIST_INITED) {
		list_destroy(&state->id_req_list);
		mutex_destroy(&state->id_acache_req_lock);
		cv_destroy(&state->id_acache_req_cv);
		state->id_mac_state &= ~IBD_DRV_REQ_LIST_INITED;
	}

	if (progress & IBD_DRV_PD_ALLOCD) {
		if ((ret = ibt_free_pd(state->id_hca_hdl,
		    state->id_pd_hdl)) != IBT_SUCCESS) {
			ibd_print_warn(state, "failed to free "
			    "protection domain, ret=%d", ret);
		}
		state->id_pd_hdl = NULL;
		state->id_mac_state &= (~IBD_DRV_PD_ALLOCD);
	}

	if (progress & IBD_DRV_HCA_OPENED) {
		if ((ret = ibt_close_hca(state->id_hca_hdl)) !=
		    IBT_SUCCESS) {
			ibd_print_warn(state, "failed to close "
			    "HCA device, ret=%d", ret);
		}
		state->id_hca_hdl = NULL;
		state->id_mac_state &= (~IBD_DRV_HCA_OPENED);
	}

	mutex_enter(&ibd_gstate.ig_mutex);
	if (progress & IBD_DRV_IBTL_ATTACH_DONE) {
		if ((ret = ibt_detach(state->id_ibt_hdl)) !=
		    IBT_SUCCESS) {
			ibd_print_warn(state,
			    "ibt_detach() failed, ret=%d", ret);
		}
		state->id_ibt_hdl = NULL;
		state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE);
		ibd_gstate.ig_ibt_hdl_ref_cnt--;
	}
	if ((ibd_gstate.ig_ibt_hdl_ref_cnt == 0) &&
	    (ibd_gstate.ig_ibt_hdl != NULL)) {
		if ((ret = ibt_detach(ibd_gstate.ig_ibt_hdl)) !=
		    IBT_SUCCESS) {
			ibd_print_warn(state, "ibt_detach(): global "
			    "failed, ret=%d", ret);
		}
		ibd_gstate.ig_ibt_hdl = NULL;
	}
	mutex_exit(&ibd_gstate.ig_mutex);

	if (progress & IBD_DRV_TXINTR_ADDED) {
		ddi_remove_softintr(state->id_tx);
		state->id_tx = NULL;
		state->id_mac_state &= (~IBD_DRV_TXINTR_ADDED);
	}

	if (progress & IBD_DRV_RXINTR_ADDED) {
		ddi_remove_softintr(state->id_rx);
		state->id_rx = NULL;
		state->id_mac_state &= (~IBD_DRV_RXINTR_ADDED);
	}

#ifdef DEBUG
	if (progress & IBD_DRV_RC_PRIVATE_STATE) {
		kstat_delete(state->rc_ksp);
		state->id_mac_state &= (~IBD_DRV_RC_PRIVATE_STATE);
	}
#endif

	if (progress & IBD_DRV_STATE_INITIALIZED) {
		ibd_state_fini(state);
		state->id_mac_state &= (~IBD_DRV_STATE_INITIALIZED);
	}
}

int
ibd_part_attach(ibd_state_t *state, dev_info_t *dip)
{
	ibt_status_t ret;
	int rv;
	kthread_t *kht;

	/*
	 * Initialize mutexes and condition variables
	 */
	if (ibd_state_init(state, dip) != DDI_SUCCESS) {
		DPRINT(10, "ibd_part_attach: failed in ibd_state_init()");
		return (DDI_FAILURE);
	}
	state->id_mac_state |= IBD_DRV_STATE_INITIALIZED;

	/*
	 * Allocate rx,tx softintr
	 */
	if (ibd_rx_softintr == 1) {
		if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_rx,
		    NULL, NULL, ibd_intr, (caddr_t)state)) != DDI_SUCCESS) {
			DPRINT(10, "ibd_part_attach: failed in "
			    "ddi_add_softintr(id_rx),  ret=%d", rv);
			return (DDI_FAILURE);
		}
		state->id_mac_state |= IBD_DRV_RXINTR_ADDED;
	}
	if (ibd_tx_softintr == 1) {
		if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_tx,
		    NULL, NULL, ibd_tx_recycle,
		    (caddr_t)state)) != DDI_SUCCESS) {
			DPRINT(10, "ibd_part_attach: failed in "
			    "ddi_add_softintr(id_tx), ret=%d", rv);
			return (DDI_FAILURE);
		}
		state->id_mac_state |= IBD_DRV_TXINTR_ADDED;
	}

	/*
	 * Attach to IBTL
	 */
	mutex_enter(&ibd_gstate.ig_mutex);
	if (ibd_gstate.ig_ibt_hdl == NULL) {
		if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state,
		    &ibd_gstate.ig_ibt_hdl)) != IBT_SUCCESS) {
			DPRINT(10, "ibd_part_attach: global: failed in "
			    "ibt_attach(), ret=%d", ret);
			mutex_exit(&ibd_gstate.ig_mutex);
			return (DDI_FAILURE);
		}
	}
	if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state,
	    &state->id_ibt_hdl)) != IBT_SUCCESS) {
		DPRINT(10, "ibd_part_attach: failed in ibt_attach(), ret=%d",
		    ret);
		mutex_exit(&ibd_gstate.ig_mutex);
		return (DDI_FAILURE);
	}
	ibd_gstate.ig_ibt_hdl_ref_cnt++;
	mutex_exit(&ibd_gstate.ig_mutex);
	state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE;

	/*
	 * Open the HCA
	 */
	if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid,
	    &state->id_hca_hdl)) != IBT_SUCCESS) {
		DPRINT(10, "ibd_part_attach: ibt_open_hca() failed, ret=%d",
		    ret);
		return (DDI_FAILURE);
	}
	state->id_mac_state |= IBD_DRV_HCA_OPENED;

#ifdef DEBUG
	/* Initialize Driver Counters for Reliable Connected Mode */
	if (state->id_enable_rc) {
		if (ibd_rc_init_stats(state) != DDI_SUCCESS) {
			DPRINT(10, "ibd_part_attach: failed in "
			    "ibd_rc_init_stats");
			return (DDI_FAILURE);
		}
		state->id_mac_state |= IBD_DRV_RC_PRIVATE_STATE;
	}
#endif

	/*
	 * Record capabilities
	 */
	(void) ibd_record_capab(state);

	/*
	 * Allocate a protection domain on the HCA
	 */
	if ((ret = ibt_alloc_pd(state->id_hca_hdl, IBT_PD_NO_FLAGS,
	    &state->id_pd_hdl)) != IBT_SUCCESS) {
		DPRINT(10, "ibd_part_attach: ibt_alloc_pd() failed, ret=%d",
		    ret);
		return (DDI_FAILURE);
	}
	state->id_mac_state |= IBD_DRV_PD_ALLOCD;


	/*
	 * We need to initialise the req_list that is required for the
	 * operation of the async_thread.
	 */
	mutex_init(&state->id_acache_req_lock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&state->id_acache_req_cv, NULL, CV_DEFAULT, NULL);
	list_create(&state->id_req_list, sizeof (ibd_req_t),
	    offsetof(ibd_req_t, rq_list));
	state->id_mac_state |= IBD_DRV_REQ_LIST_INITED;

	/*
	 * Create the async thread; thread_create never fails.
	 */
	kht = thread_create(NULL, 0, ibd_async_work, state, 0, &p0,
	    TS_RUN, minclsyspri);
	state->id_async_thrid = kht->t_did;
	state->id_mac_state |= IBD_DRV_ASYNC_THR_CREATED;

	return (DDI_SUCCESS);
}

/*
 * Attach device to the IO framework.
 */
static int
ibd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	int ret;

	switch (cmd) {
		case DDI_ATTACH:
			ret = ibd_port_attach(dip);
			break;
		default:
			ret = DDI_FAILURE;
			break;
	}
	return (ret);
}

/*
 * Detach device from the IO framework.
 */
static int
ibd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	ibd_state_t *state;
	int instance;

	/*
	 * IBD doesn't support suspend/resume
	 */
	if (cmd != DDI_DETACH)
		return (DDI_FAILURE);

	/*
	 * Get the instance softstate
	 */
	instance = ddi_get_instance(dip);
	state = ddi_get_soft_state(ibd_list, instance);

	/*
	 * Release all resources we're holding still.  Note that if we'd
	 * done ibd_attach(), ibd_m_start() and ibd_m_stop() correctly
	 * so far, we should find all the flags we need in id_mac_state.
	 */
	return (ibd_port_unattach(state, dip));
}

/*
 * Pre ibt_attach() driver initialization
 */
static int
ibd_state_init(ibd_state_t *state, dev_info_t *dip)
{
	char buf[64];

	mutex_init(&state->id_link_mutex, NULL, MUTEX_DRIVER, NULL);
	state->id_link_state = LINK_STATE_UNKNOWN;

	mutex_init(&state->id_trap_lock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&state->id_trap_cv, NULL, CV_DEFAULT, NULL);
	state->id_trap_stop = B_TRUE;
	state->id_trap_inprog = 0;

	mutex_init(&state->id_scq_poll_lock, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->id_rcq_poll_lock, NULL, MUTEX_DRIVER, NULL);
	state->id_dip = dip;

	mutex_init(&state->id_sched_lock, NULL, MUTEX_DRIVER, NULL);

	mutex_init(&state->id_tx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->id_tx_rel_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->id_txpost_lock, NULL, MUTEX_DRIVER, NULL);
	state->id_tx_busy = 0;
	mutex_init(&state->id_lso_lock, NULL, MUTEX_DRIVER, NULL);

	state->id_rx_list.dl_bufs_outstanding = 0;
	state->id_rx_list.dl_cnt = 0;
	mutex_init(&state->id_rx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->id_rx_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
	(void) sprintf(buf, "ibd_req%d_%x_%u", ddi_get_instance(dip),
	    state->id_pkey, state->id_plinkid);
	state->id_req_kmc = kmem_cache_create(buf, sizeof (ibd_req_t),
	    0, NULL, NULL, NULL, NULL, NULL, 0);

	/* For Reliable Connected Mode */
	mutex_init(&state->rc_rx_lock, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->rc_tx_large_bufs_lock, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->rc_srq_rwqe_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->rc_srq_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&state->rc_pass_chan_list.chan_list_mutex, NULL,
	    MUTEX_DRIVER, NULL);
	mutex_init(&state->rc_timeout_lock, NULL, MUTEX_DRIVER, NULL);

	/*
	 * Make the default link mode as RC. If this fails during connection
	 * setup, the link mode is automatically transitioned to UD.
	 * Also set the RC MTU.
	 */
	state->id_enable_rc = IBD_DEF_LINK_MODE;
	state->rc_mtu = IBD_DEF_RC_MAX_MTU;
	state->id_mtu = IBD_DEF_MAX_MTU;

	/* Iniatialize all tunables to default */
	state->id_lso_policy = IBD_DEF_LSO_POLICY;
	state->id_num_lso_bufs = IBD_DEF_NUM_LSO_BUFS;
	state->id_num_ah = IBD_DEF_NUM_AH;
	state->id_hash_size = IBD_DEF_HASH_SIZE;
	state->id_create_broadcast_group = IBD_DEF_CREATE_BCAST_GROUP;
	state->id_allow_coalesce_comp_tuning = IBD_DEF_COALESCE_COMPLETIONS;
	state->id_ud_rx_comp_count = IBD_DEF_UD_RX_COMP_COUNT;
	state->id_ud_rx_comp_usec = IBD_DEF_UD_RX_COMP_USEC;
	state->id_ud_tx_comp_count = IBD_DEF_UD_TX_COMP_COUNT;
	state->id_ud_tx_comp_usec = IBD_DEF_UD_TX_COMP_USEC;
	state->id_rc_rx_comp_count = IBD_DEF_RC_RX_COMP_COUNT;
	state->id_rc_rx_comp_usec = IBD_DEF_RC_RX_COMP_USEC;
	state->id_rc_tx_comp_count = IBD_DEF_RC_TX_COMP_COUNT;
	state->id_rc_tx_comp_usec = IBD_DEF_RC_TX_COMP_USEC;
	state->id_ud_tx_copy_thresh = IBD_DEF_UD_TX_COPY_THRESH;
	state->id_rc_rx_copy_thresh = IBD_DEF_RC_RX_COPY_THRESH;
	state->id_rc_tx_copy_thresh = IBD_DEF_RC_TX_COPY_THRESH;
	state->id_ud_num_rwqe = IBD_DEF_UD_NUM_RWQE;
	state->id_ud_num_swqe = IBD_DEF_UD_NUM_SWQE;
	state->id_rc_num_rwqe = IBD_DEF_RC_NUM_RWQE;
	state->id_rc_num_swqe = IBD_DEF_RC_NUM_SWQE;
	state->rc_enable_srq = IBD_DEF_RC_ENABLE_SRQ;
	state->id_rc_num_srq = IBD_DEF_RC_NUM_SRQ;
	state->id_rc_rx_rwqe_thresh = IBD_DEF_RC_RX_RWQE_THRESH;

	return (DDI_SUCCESS);
}

/*
 * Post ibt_detach() driver deconstruction
 */
static void
ibd_state_fini(ibd_state_t *state)
{
	kmem_cache_destroy(state->id_req_kmc);

	mutex_destroy(&state->id_rx_list.dl_mutex);
	mutex_destroy(&state->id_rx_free_list.dl_mutex);

	mutex_destroy(&state->id_txpost_lock);
	mutex_destroy(&state->id_tx_list.dl_mutex);
	mutex_destroy(&state->id_tx_rel_list.dl_mutex);
	mutex_destroy(&state->id_lso_lock);

	mutex_destroy(&state->id_sched_lock);
	mutex_destroy(&state->id_scq_poll_lock);
	mutex_destroy(&state->id_rcq_poll_lock);

	cv_destroy(&state->id_trap_cv);
	mutex_destroy(&state->id_trap_lock);
	mutex_destroy(&state->id_link_mutex);

	/* For Reliable Connected Mode */
	mutex_destroy(&state->rc_timeout_lock);
	mutex_destroy(&state->rc_srq_free_list.dl_mutex);
	mutex_destroy(&state->rc_srq_rwqe_list.dl_mutex);
	mutex_destroy(&state->rc_pass_chan_list.chan_list_mutex);
	mutex_destroy(&state->rc_tx_large_bufs_lock);
	mutex_destroy(&state->rc_rx_lock);
}

/*
 * Fetch link speed from SA for snmp ifspeed reporting.
 */
static uint64_t
ibd_get_portspeed(ibd_state_t *state)
{
	int			ret;
	ibt_path_info_t		path;
	ibt_path_attr_t		path_attr;
	uint8_t			num_paths;
	uint64_t		ifspeed;

	/*
	 * Due to serdes 8b10b encoding on the wire, 2.5 Gbps on wire
	 * translates to 2 Gbps data rate. Thus, 1X single data rate is
	 * 2000000000. Start with that as default.
	 */
	ifspeed = 2000000000;

	bzero(&path_attr, sizeof (path_attr));

	/*
	 * Get the port speed from Loopback path information.
	 */
	path_attr.pa_dgids = &state->id_sgid;
	path_attr.pa_num_dgids = 1;
	path_attr.pa_sgid = state->id_sgid;

	if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_NO_FLAGS,
	    &path_attr, 1, &path, &num_paths) != IBT_SUCCESS)
		goto earlydone;

	if (num_paths < 1)
		goto earlydone;

	/*
	 * In case SA does not return an expected value, report the default
	 * speed as 1X.
	 */
	ret = 1;
	switch (path.pi_prim_cep_path.cep_adds_vect.av_srate) {
		case IBT_SRATE_2:	/*  1X SDR i.e 2.5 Gbps */
			ret = 1;
			break;
		case IBT_SRATE_10:	/*  4X SDR or 1X QDR i.e 10 Gbps */
			ret = 4;
			break;
		case IBT_SRATE_30:	/* 12X SDR i.e 30 Gbps */
			ret = 12;
			break;
		case IBT_SRATE_5:	/*  1X DDR i.e  5 Gbps */
			ret = 2;
			break;
		case IBT_SRATE_20:	/*  4X DDR or 8X SDR i.e 20 Gbps */
			ret = 8;
			break;
		case IBT_SRATE_40:	/*  8X DDR or 4X QDR i.e 40 Gbps */
			ret = 16;
			break;
		case IBT_SRATE_60:	/* 12X DDR i.e 60 Gbps */
			ret = 24;
			break;
		case IBT_SRATE_80:	/*  8X QDR i.e 80 Gbps */
			ret = 32;
			break;
		case IBT_SRATE_120:	/* 12X QDR i.e 120 Gbps */
			ret = 48;
			break;
	}

	ifspeed *= ret;

earlydone:
	return (ifspeed);
}

/*
 * Search input mcg list (id_mc_full or id_mc_non) for an entry
 * representing the input mcg mgid.
 */
static ibd_mce_t *
ibd_mcache_find(ib_gid_t mgid, struct list *mlist)
{
	ibd_mce_t *ptr = list_head(mlist);

	/*
	 * Do plain linear search.
	 */
	while (ptr != NULL) {
		if (bcmp(&mgid, &ptr->mc_info.mc_adds_vect.av_dgid,
		    sizeof (ib_gid_t)) == 0)
			return (ptr);
		ptr = list_next(mlist, ptr);
	}
	return (NULL);
}

/*
 * Execute IBA JOIN.
 */
static ibt_status_t
ibd_iba_join(ibd_state_t *state, ib_gid_t mgid, ibd_mce_t *mce)
{
	ibt_mcg_attr_t mcg_attr;

	bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
	mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey;
	mcg_attr.mc_mgid = mgid;
	mcg_attr.mc_join_state = mce->mc_jstate;
	mcg_attr.mc_scope = state->id_scope;
	mcg_attr.mc_pkey = state->id_pkey;
	mcg_attr.mc_flow = state->id_mcinfo->mc_adds_vect.av_flow;
	mcg_attr.mc_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
	mcg_attr.mc_tclass = state->id_mcinfo->mc_adds_vect.av_tclass;
	return (ibt_join_mcg(state->id_sgid, &mcg_attr, &mce->mc_info,
	    NULL, NULL));
}

/*
 * This code JOINs the port in the proper way (depending on the join
 * state) so that IBA fabric will forward mcg packets to/from the port.
 * It also attaches the QPN to the mcg so it can receive those mcg
 * packets. This code makes sure not to attach the mcg to the QP if
 * that has been previously done due to the mcg being joined with a
 * different join state, even though this is not required by SWG_0216,
 * refid 3610.
 */
static ibd_mce_t *
ibd_join_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
{
	ibt_status_t ibt_status;
	ibd_mce_t *mce, *tmce, *omce = NULL;
	boolean_t do_attach = B_TRUE;

	DPRINT(2, "ibd_join_group : join_group state %d : %016llx:%016llx\n",
	    jstate, mgid.gid_prefix, mgid.gid_guid);

	/*
	 * For enable_multicast Full member joins, we need to do some
	 * extra work. If there is already an mce on the list that
	 * indicates full membership, that means the membership has
	 * not yet been dropped (since the disable_multicast was issued)
	 * because there are pending Tx's to the mcg; in that case, just
	 * mark the mce not to be reaped when the Tx completion queues
	 * an async reap operation.
	 *
	 * If there is already an mce on the list indicating sendonly
	 * membership, try to promote to full membership. Be careful
	 * not to deallocate the old mce, since there might be an AH
	 * pointing to it; instead, update the old mce with new data
	 * that tracks the full membership.
	 */
	if ((jstate == IB_MC_JSTATE_FULL) && ((omce =
	    IBD_MCACHE_FIND_FULL(state, mgid)) != NULL)) {
		if (omce->mc_jstate == IB_MC_JSTATE_FULL) {
			ASSERT(omce->mc_fullreap);
			omce->mc_fullreap = B_FALSE;
			return (omce);
		} else {
			ASSERT(omce->mc_jstate == IB_MC_JSTATE_SEND_ONLY_NON);
		}
	}

	/*
	 * Allocate the ibd_mce_t to track this JOIN.
	 */
	mce = kmem_zalloc(sizeof (ibd_mce_t), KM_SLEEP);
	mce->mc_fullreap = B_FALSE;
	mce->mc_jstate = jstate;

	if ((ibt_status = ibd_iba_join(state, mgid, mce)) != IBT_SUCCESS) {
		DPRINT(10, "ibd_join_group : failed ibt_join_mcg() %d",
		    ibt_status);
		kmem_free(mce, sizeof (ibd_mce_t));
		return (NULL);
	}

	/*
	 * Is an IBA attach required? Not if the interface is already joined
	 * to the mcg in a different appropriate join state.
	 */
	if (jstate == IB_MC_JSTATE_NON) {
		tmce = IBD_MCACHE_FIND_FULL(state, mgid);
		if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL))
			do_attach = B_FALSE;
	} else if (jstate == IB_MC_JSTATE_FULL) {
		if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
			do_attach = B_FALSE;
	} else {	/* jstate == IB_MC_JSTATE_SEND_ONLY_NON */
		do_attach = B_FALSE;
	}

	if (do_attach) {
		/*
		 * Do the IBA attach.
		 */
		DPRINT(10, "ibd_join_group: ibt_attach_mcg \n");
		if ((ibt_status = ibt_attach_mcg(state->id_chnl_hdl,
		    &mce->mc_info)) != IBT_SUCCESS) {
			DPRINT(10, "ibd_join_group : failed qp attachment "
			    "%d\n", ibt_status);
			/*
			 * NOTE that we should probably preserve the join info
			 * in the list and later try to leave again at detach
			 * time.
			 */
			(void) ibt_leave_mcg(state->id_sgid, mgid,
			    state->id_sgid, jstate);
			kmem_free(mce, sizeof (ibd_mce_t));
			return (NULL);
		}
	}

	/*
	 * Insert the ibd_mce_t in the proper list.
	 */
	if (jstate == IB_MC_JSTATE_NON) {
		IBD_MCACHE_INSERT_NON(state, mce);
	} else {
		/*
		 * Set up the mc_req fields used for reaping the
		 * mcg in case of delayed tx completion (see
		 * ibd_tx_cleanup()). Also done for sendonly join in
		 * case we are promoted to fullmembership later and
		 * keep using the same mce.
		 */
		mce->mc_req.rq_gid = mgid;
		mce->mc_req.rq_ptr = mce;
		/*
		 * Check whether this is the case of trying to join
		 * full member, and we were already joined send only.
		 * We try to drop our SendOnly membership, but it is
		 * possible that the mcg does not exist anymore (and
		 * the subnet trap never reached us), so the leave
		 * operation might fail.
		 */
		if (omce != NULL) {
			(void) ibt_leave_mcg(state->id_sgid, mgid,
			    state->id_sgid, IB_MC_JSTATE_SEND_ONLY_NON);
			omce->mc_jstate = IB_MC_JSTATE_FULL;
			bcopy(&mce->mc_info, &omce->mc_info,
			    sizeof (ibt_mcg_info_t));
			kmem_free(mce, sizeof (ibd_mce_t));
			return (omce);
		}
		mutex_enter(&state->id_mc_mutex);
		IBD_MCACHE_INSERT_FULL(state, mce);
		mutex_exit(&state->id_mc_mutex);
	}

	return (mce);
}

/*
 * Called during port up event handling to attempt to reacquire full
 * membership to an mcg. Stripped down version of ibd_join_group().
 * Note that it is possible that the mcg might have gone away, and
 * gets recreated at this point.
 */
static void
ibd_reacquire_group(ibd_state_t *state, ibd_mce_t *mce)
{
	ib_gid_t mgid;

	/*
	 * If the mc_fullreap flag is set, or this join fails, a subsequent
	 * reap/leave is going to try to leave the group. We could prevent
	 * that by adding a boolean flag into ibd_mce_t, if required.
	 */
	if (mce->mc_fullreap)
		return;

	mgid = mce->mc_info.mc_adds_vect.av_dgid;

	DPRINT(2, "ibd_reacquire_group : %016llx:%016llx\n", mgid.gid_prefix,
	    mgid.gid_guid);

	/* While reacquiring, leave and then join the MCG */
	(void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid,
	    mce->mc_jstate);
	if (ibd_iba_join(state, mgid, mce) != IBT_SUCCESS)
		ibd_print_warn(state, "Failure on port up to rejoin "
		    "multicast gid %016llx:%016llx",
		    (u_longlong_t)mgid.gid_prefix,
		    (u_longlong_t)mgid.gid_guid);
}

/*
 * This code handles delayed Tx completion cleanups for mcg's to which
 * disable_multicast has been issued, regular mcg related cleanups during
 * disable_multicast, disable_promiscuous and mcg traps, as well as
 * cleanups during driver detach time. Depending on the join state,
 * it deletes the mce from the appropriate list and issues the IBA
 * leave/detach; except in the disable_multicast case when the mce
 * is left on the active list for a subsequent Tx completion cleanup.
 */
static void
ibd_async_reap_group(ibd_state_t *state, ibd_mce_t *mce, ib_gid_t mgid,
    uint8_t jstate)
{
	ibd_mce_t *tmce;
	boolean_t do_detach = B_TRUE;

	/*
	 * Before detaching, we must check whether the other list
	 * contains the mcg; if we detach blindly, the consumer
	 * who set up the other list will also stop receiving
	 * traffic.
	 */
	if (jstate == IB_MC_JSTATE_FULL) {
		/*
		 * The following check is only relevant while coming
		 * from the Tx completion path in the reap case.
		 */
		if (!mce->mc_fullreap)
			return;
		mutex_enter(&state->id_mc_mutex);
		IBD_MCACHE_PULLOUT_FULL(state, mce);
		mutex_exit(&state->id_mc_mutex);
		if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
			do_detach = B_FALSE;
	} else if (jstate == IB_MC_JSTATE_NON) {
		IBD_MCACHE_PULLOUT_NON(state, mce);
		tmce = IBD_MCACHE_FIND_FULL(state, mgid);
		if ((tmce != NULL) && (tmce->mc_jstate == IB_MC_JSTATE_FULL))
			do_detach = B_FALSE;
	} else {	/* jstate == IB_MC_JSTATE_SEND_ONLY_NON */
		mutex_enter(&state->id_mc_mutex);
		IBD_MCACHE_PULLOUT_FULL(state, mce);
		mutex_exit(&state->id_mc_mutex);
		do_detach = B_FALSE;
	}

	/*
	 * If we are reacting to a mcg trap and leaving our sendonly or
	 * non membership, the mcg is possibly already gone, so attempting
	 * to leave might fail. On the other hand, we must try to leave
	 * anyway, since this might be a trap from long ago, and we could
	 * have potentially sendonly joined to a recent incarnation of
	 * the mcg and are about to loose track of this information.
	 */
	if (do_detach) {
		DPRINT(2, "ibd_async_reap_group : ibt_detach_mcg : "
		    "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid);
		(void) ibt_detach_mcg(state->id_chnl_hdl, &mce->mc_info);
	}

	(void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, jstate);
	kmem_free(mce, sizeof (ibd_mce_t));
}

/*
 * Async code executed due to multicast and promiscuous disable requests
 * and mcg trap handling; also executed during driver detach. Mostly, a
 * leave and detach is done; except for the fullmember case when Tx
 * requests are pending, whence arrangements are made for subsequent
 * cleanup on Tx completion.
 */
static void
ibd_leave_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
{
	ipoib_mac_t mcmac;
	boolean_t recycled;
	ibd_mce_t *mce;

	DPRINT(2, "ibd_leave_group : leave_group state %d : %016llx:%016llx\n",
	    jstate, mgid.gid_prefix, mgid.gid_guid);

	if (jstate == IB_MC_JSTATE_NON) {
		recycled = B_TRUE;
		mce = IBD_MCACHE_FIND_NON(state, mgid);
		/*
		 * In case we are handling a mcg trap, we might not find
		 * the mcg in the non list.
		 */
		if (mce == NULL) {
			return;
		}
	} else {
		mce = IBD_MCACHE_FIND_FULL(state, mgid);

		/*
		 * In case we are handling a mcg trap, make sure the trap
		 * is not arriving late; if we have an mce that indicates
		 * that we are already a fullmember, that would be a clear
		 * indication that the trap arrived late (ie, is for a
		 * previous incarnation of the mcg).
		 */
		if (jstate == IB_MC_JSTATE_SEND_ONLY_NON) {
			if ((mce == NULL) || (mce->mc_jstate ==
			    IB_MC_JSTATE_FULL)) {
				return;
			}
		} else {
			ASSERT(jstate == IB_MC_JSTATE_FULL);

			/*
			 * If join group failed, mce will be NULL here.
			 * This is because in GLDv3 driver, set multicast
			 *  will always return success.
			 */
			if (mce == NULL) {
				return;
			}

			mce->mc_fullreap = B_TRUE;
		}

		/*
		 * If no pending Tx's remain that reference the AH
		 * for the mcg, recycle it from active to free list.
		 * Else in the IB_MC_JSTATE_FULL case, just mark the AH,
		 * so the last completing Tx will cause an async reap
		 * operation to be invoked, at which time we will drop our
		 * membership to the mcg so that the pending Tx's complete
		 * successfully. Refer to comments on "AH and MCE active
		 * list manipulation" at top of this file. The lock protects
		 * against Tx fast path and Tx cleanup code.
		 */
		mutex_enter(&state->id_ac_mutex);
		ibd_h2n_mac(&mcmac, IB_MC_QPN, mgid.gid_prefix, mgid.gid_guid);
		recycled = ibd_acache_recycle(state, &mcmac, (jstate ==
		    IB_MC_JSTATE_SEND_ONLY_NON));
		mutex_exit(&state->id_ac_mutex);
	}

	if (recycled) {
		DPRINT(2, "ibd_leave_group : leave_group reaping : "
		    "%016llx:%016llx\n", mgid.gid_prefix, mgid.gid_guid);
		ibd_async_reap_group(state, mce, mgid, jstate);
	}
}

/*
 * Find the broadcast address as defined by IPoIB; implicitly
 * determines the IBA scope, mtu, tclass etc of the link the
 * interface is going to be a member of.
 */
static ibt_status_t
ibd_find_bgroup(ibd_state_t *state)
{
	ibt_mcg_attr_t mcg_attr;
	uint_t numg;
	uchar_t scopes[] = { IB_MC_SCOPE_SUBNET_LOCAL,
	    IB_MC_SCOPE_SITE_LOCAL, IB_MC_SCOPE_ORG_LOCAL,
	    IB_MC_SCOPE_GLOBAL };
	int i, mcgmtu;
	boolean_t found = B_FALSE;
	int ret;
	ibt_mcg_info_t mcg_info;

	state->id_bgroup_created = B_FALSE;
	state->id_bgroup_present = B_FALSE;

query_bcast_grp:
	bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
	mcg_attr.mc_pkey = state->id_pkey;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
	state->id_mgid.gid_guid = IB_MGID_IPV4_LOWGRP_MASK;
	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))

	for (i = 0; i < sizeof (scopes)/sizeof (scopes[0]); i++) {
		state->id_scope = mcg_attr.mc_scope = scopes[i];

		/*
		 * Look for the IPoIB broadcast group.
		 */
		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
		state->id_mgid.gid_prefix =
		    (((uint64_t)IB_MCGID_IPV4_PREFIX << 32) |
		    ((uint64_t)state->id_scope << 48) |
		    ((uint32_t)(state->id_pkey << 16)));
		mcg_attr.mc_mgid = state->id_mgid;
		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))
		if (ibt_query_mcg(state->id_sgid, &mcg_attr, 1,
		    &state->id_mcinfo, &numg) == IBT_SUCCESS) {
			found = B_TRUE;
			break;
		}
	}

	if (!found) {
		if (state->id_create_broadcast_group) {
			/*
			 * If we created the broadcast group, but failed to
			 * find it, we can't do anything except leave the
			 * one we created and return failure.
			 */
			if (state->id_bgroup_created) {
				ibd_print_warn(state, "IPoIB broadcast group "
				    "absent. Unable to query after create.");
				goto find_bgroup_fail;
			}

			/*
			 * Create the ipoib broadcast group if it didn't exist
			 */
			bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
			mcg_attr.mc_qkey = IBD_DEFAULT_QKEY;
			mcg_attr.mc_join_state = IB_MC_JSTATE_FULL;
			mcg_attr.mc_scope = IB_MC_SCOPE_SUBNET_LOCAL;
			mcg_attr.mc_pkey = state->id_pkey;
			mcg_attr.mc_flow = 0;
			mcg_attr.mc_sl = 0;
			mcg_attr.mc_tclass = 0;
			_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
			state->id_mgid.gid_prefix =
			    (((uint64_t)IB_MCGID_IPV4_PREFIX << 32) |
			    ((uint64_t)IB_MC_SCOPE_SUBNET_LOCAL << 48) |
			    ((uint32_t)(state->id_pkey << 16)));
			mcg_attr.mc_mgid = state->id_mgid;
			_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))

			if ((ret = ibt_join_mcg(state->id_sgid, &mcg_attr,
			    &mcg_info, NULL, NULL)) != IBT_SUCCESS) {
				ibd_print_warn(state, "IPoIB broadcast group "
				    "absent, create failed: ret = %d\n", ret);
				state->id_bgroup_created = B_FALSE;
				return (IBT_FAILURE);
			}
			state->id_bgroup_created = B_TRUE;
			goto query_bcast_grp;
		} else {
			ibd_print_warn(state, "IPoIB broadcast group absent");
			return (IBT_FAILURE);
		}
	}

	/*
	 * Assert that the mcg mtu <= id_mtu. Fill in updated id_mtu.
	 */
	mcgmtu = (128 << state->id_mcinfo->mc_mtu);
	if (state->id_mtu < mcgmtu) {
		ibd_print_warn(state, "IPoIB broadcast group MTU %d "
		    "greater than port's maximum MTU %d", mcgmtu,
		    state->id_mtu);
		ibt_free_mcg_info(state->id_mcinfo, 1);
		goto find_bgroup_fail;
	}
	state->id_mtu = mcgmtu;
	state->id_bgroup_present = B_TRUE;

	return (IBT_SUCCESS);

find_bgroup_fail:
	if (state->id_bgroup_created) {
		(void) ibt_leave_mcg(state->id_sgid,
		    mcg_info.mc_adds_vect.av_dgid, state->id_sgid,
		    IB_MC_JSTATE_FULL);
	}

	return (IBT_FAILURE);
}

static int
ibd_alloc_tx_copybufs(ibd_state_t *state)
{
	ibt_mr_attr_t mem_attr;

	/*
	 * Allocate one big chunk for all regular tx copy bufs
	 */
	state->id_tx_buf_sz = state->id_mtu;
	if (state->id_lso_policy && state->id_lso_capable &&
	    (state->id_ud_tx_copy_thresh > state->id_mtu)) {
		state->id_tx_buf_sz = state->id_ud_tx_copy_thresh;
	}

	state->id_tx_bufs = kmem_zalloc(state->id_ud_num_swqe *
	    state->id_tx_buf_sz, KM_SLEEP);

	state->id_tx_wqes = kmem_zalloc(state->id_ud_num_swqe *
	    sizeof (ibd_swqe_t), KM_SLEEP);

	/*
	 * Do one memory registration on the entire txbuf area
	 */
	mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_tx_bufs;
	mem_attr.mr_len = state->id_ud_num_swqe * state->