of/sol_ofs/sol_cma.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * sol_cma is a part of sol_ofs misc module. This file
 * provides interfaces for supporting the communication
 * management API defined in "rdma_cm.h". In-Kernel
 * consumers of the "rdma_cm.h" API should link sol_ofs
 * misc module using :
 *	-N misc/sol_ofs
 * Solaris uCMA (sol_ucma) driver is the current consumer for
 * sol_cma.
 */

/* Standard driver includes */
#include <sys/types.h>
#include <sys/modctl.h>
#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>

#include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
#include <sys/ib/clients/of/ofed_kernel.h>
#include <sys/ib/clients/of/rdma/ib_addr.h>
#include <sys/ib/clients/of/rdma/rdma_cm.h>

#include <sys/ib/clients/of/sol_ofs/sol_cma.h>
#include <sys/ib/clients/of/sol_ofs/sol_kverb_impl.h>

/* Modload support */
static struct modlmisc sol_ofs_modmisc	= {
	&mod_miscops,
	"Solaris OFS Misc module"
};

struct modlinkage sol_ofs_modlinkage = {
	MODREV_1,
	(void *)&sol_ofs_modmisc,
	NULL
};

static ib_client_t	*sol_cma_ib_client;
sol_cma_glbl_listen_t	sol_cma_glbl_listen;
avl_tree_t		sol_cma_glbl_listen_tree;

static void		sol_cma_add_dev(struct ib_device *);
static void		sol_cma_rem_dev(struct ib_device *);

static llist_head_t	sol_cma_dev_list = LLIST_HEAD_INIT(sol_cma_dev_list);
kmutex_t		sol_cma_dev_mutex;
kmutex_t		sol_cma_glob_mutex;

char	*sol_rdmacm_dbg_str = "sol_rdmacm";
char	*sol_ofs_dbg_str = "sol_ofs_mod";

/*
 * Local functions defines.
 */
int sol_cma_req_cmid_cmp(const void *p1, const void *p2);
int sol_cma_cmid_cmp(const void *p1, const void *p2);
int sol_cma_svc_cmp(const void *, const void *);

static struct rdma_cm_id *cma_alloc_chan(rdma_cm_event_handler,
    void *, enum rdma_port_space);
static void cma_set_chan_state(sol_cma_chan_t *, cma_chan_state_t);
static int cma_cas_chan_state(sol_cma_chan_t *, cma_chan_state_t,
    cma_chan_state_t);
static void cma_free_listen_list(struct rdma_cm_id *);
static void cma_destroy_id(struct rdma_cm_id *);
static void cma_handle_nomore_events(sol_cma_chan_t *);

extern void sol_ofs_dprintf_init();
extern void sol_ofs_dprintf_fini();

cma_chan_state_t cma_get_chan_state(sol_cma_chan_t *);
extern int ibcma_init_root_chan(sol_cma_chan_t *, sol_cma_glbl_listen_t *);
extern int ibcma_fini_root_chan(sol_cma_chan_t *);
extern void ibcma_copy_srv_hdl(sol_cma_chan_t *, sol_cma_glbl_listen_t *);
extern int ibcma_fini_ep_chan(sol_cma_chan_t *);
extern uint64_t ibcma_init_root_sid(sol_cma_chan_t *);
extern void rdma_ib_destroy_id(struct rdma_cm_id *);
extern int rdma_ib_bind_addr(struct rdma_cm_id *, struct sockaddr *);
extern int rdma_ib_resolve_addr(struct rdma_cm_id *, struct sockaddr *,
    struct sockaddr *, int);
extern int rdma_ib_resolve_route(struct rdma_cm_id *, int);
extern int rdma_ib_init_qp_attr(struct rdma_cm_id *, struct ib_qp_attr *,
    int *);
extern int rdma_ib_connect(struct rdma_cm_id *, struct rdma_conn_param *);
extern int rdma_ib_listen(struct rdma_cm_id *, int);
extern int rdma_ib_accept(struct rdma_cm_id *, struct rdma_conn_param *);
extern int rdma_ib_reject(struct rdma_cm_id *, const void *, uint8_t);
extern int rdma_ib_disconnect(struct rdma_cm_id *);
extern int rdma_ib_join_multicast(struct rdma_cm_id *, struct sockaddr *,
    void *);
extern void rdma_ib_leave_multicast(struct rdma_cm_id *, struct sockaddr *);

int
_init(void)
{
	int		err;

	sol_ofs_dprintf_init();
	SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_init()");

	mutex_init(&sol_cma_glob_mutex, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&sol_cma_dev_mutex, NULL, MUTEX_DRIVER, NULL);
	avl_create(&sol_cma_glbl_listen_tree,
	    sol_cma_svc_cmp, sizeof (sol_cma_glbl_listen_t),
	    offsetof(sol_cma_glbl_listen_t, cma_listen_node));

	sol_cma_ib_client = kmem_zalloc(sizeof (ib_client_t), KM_NOSLEEP);
	if (!sol_cma_ib_client) {
		SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str,
		    "_init() - mem alloc failed");
		avl_destroy(&sol_cma_glbl_listen_tree);
		mutex_destroy(&sol_cma_dev_mutex);
		mutex_destroy(&sol_cma_glob_mutex);
		sol_ofs_dprintf_fini();
		return (ENOMEM);
	}

	sol_cma_ib_client->name = "sol_ofs";
	sol_cma_ib_client->add = sol_cma_add_dev;
	sol_cma_ib_client->remove = sol_cma_rem_dev;
	sol_cma_ib_client->dip = NULL;

	if ((err = ib_register_client(sol_cma_ib_client)) != 0) {
		SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str,
		    "_init() ib_register_client() failed with err %d",
		    err);
		kmem_free(sol_cma_ib_client, sizeof (ib_client_t));
		avl_destroy(&sol_cma_glbl_listen_tree);
		mutex_destroy(&sol_cma_dev_mutex);
		mutex_destroy(&sol_cma_glob_mutex);
		sol_ofs_dprintf_fini();
		return (err);
	}

	if ((err = mod_install(&sol_ofs_modlinkage)) != 0) {
		SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str,
		    "_init() - mod_install() failed");
		ib_unregister_client(sol_cma_ib_client);
		kmem_free(sol_cma_ib_client, sizeof (ib_client_t));
		avl_destroy(&sol_cma_glbl_listen_tree);
		mutex_destroy(&sol_cma_dev_mutex);
		mutex_destroy(&sol_cma_glob_mutex);
		sol_ofs_dprintf_fini();
		return (err);
	}

	SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_init() - ret");
	return (err);
}

int
_fini(void)
{
	int		err;

	SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_fini()");

	if (avl_numnodes(&sol_cma_glbl_listen_tree)) {
		SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str, "_fini - "
		    "listen CMIDs still active");
		return (EBUSY);
	}
	if ((err = mod_remove(&sol_ofs_modlinkage)) != 0) {
		SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str,
		    "_fini: mod_remove failed");
		return (err);
	}

	ib_unregister_client(sol_cma_ib_client);
	kmem_free(sol_cma_ib_client, sizeof (ib_client_t));
	avl_destroy(&sol_cma_glbl_listen_tree);
	mutex_destroy(&sol_cma_dev_mutex);
	mutex_destroy(&sol_cma_glob_mutex);
	SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_fini() - ret");
	sol_ofs_dprintf_fini();
	return (err);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&sol_ofs_modlinkage, modinfop));
}

typedef struct cma_device {
	kmutex_t		cma_mutex;
	llist_head_t		cma_list;
	genlist_t		cma_epchan_list;
	struct ib_device	*cma_device;
	uint_t			cma_ref_count;
	enum {
		SOL_CMA_DEV_ADDED,
		SOL_CMA_DEV_REM_IN_PROGRESS
	} cma_dev_state;
} cma_device_t;

static void
sol_cma_add_dev(struct ib_device *dev)
{
	cma_device_t	*new_device;

	new_device = kmem_zalloc(sizeof (cma_device_t), KM_NOSLEEP);
	if (!new_device) {
		SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str, "sol_cma_add_dev() "
		    "alloc failed!!");
		return;
	}
	mutex_init(&new_device->cma_mutex, NULL, MUTEX_DRIVER, NULL);
	llist_head_init(&new_device->cma_list, new_device);
	init_genlist(&new_device->cma_epchan_list);
	new_device->cma_device = dev;

	ib_set_client_data(dev, sol_cma_ib_client, new_device);

	mutex_enter(&sol_cma_dev_mutex);
	llist_add_tail(&new_device->cma_list, &sol_cma_dev_list);
	mutex_exit(&sol_cma_dev_mutex);
}

static void
sol_cma_rem_dev(struct ib_device *dev)
{
	cma_device_t	*rem_device;
	genlist_entry_t	*entry;

	SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "sol_rem_dev(%p)", dev);

	rem_device = (cma_device_t *)ib_get_client_data(dev, sol_cma_ib_client);
	if (!rem_device) {
		SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str, "sol_cma_rem_dev() "
		    "NULL cma_dev!!");
		return;
	}

	mutex_enter(&rem_device->cma_mutex);
	rem_device->cma_dev_state = SOL_CMA_DEV_REM_IN_PROGRESS;
	if (rem_device->cma_ref_count) {
		mutex_exit(&rem_device->cma_mutex);
		SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str, "sol_cma_rem_dev() "
		    "BUSY cma_dev!!");
		return;
	}
	entry = remove_genlist_head(&rem_device->cma_epchan_list);
	while (entry) {
		sol_cma_chan_t	*ep_chanp;

		ep_chanp = (sol_cma_chan_t *)entry->data;
		if (ibcma_fini_ep_chan(ep_chanp) == 0) {
			genlist_entry_t	*entry1;
			sol_cma_chan_t	*root_chanp;

			ASSERT(ep_chanp->chan_listenp);
			entry1 = ep_chanp->chan_listenp->listen_ep_root_entry;
			root_chanp = (sol_cma_chan_t *)ep_chanp->listen_root;
			root_chanp->chan_listenp->listen_eps--;
			delete_genlist(&root_chanp->chan_listenp->listen_list,
			    entry1);

			kmem_free(ep_chanp, sizeof (sol_cma_chan_t));
			kmem_free(entry, sizeof (genlist_entry_t));
		}

		entry = remove_genlist_head(&rem_device->cma_epchan_list);
	}
	mutex_exit(&rem_device->cma_mutex);

	mutex_enter(&sol_cma_dev_mutex);
	llist_del(&rem_device->cma_list);
	mutex_exit(&sol_cma_dev_mutex);

	kmem_free(rem_device, sizeof (cma_device_t));
}

struct ib_device *
sol_cma_acquire_device(ib_guid_t hca_guid)
{
	llist_head_t	*entry;
	cma_device_t	*cma_devp;

	mutex_enter(&sol_cma_dev_mutex);
	list_for_each(entry, &sol_cma_dev_list) {
		cma_devp = (cma_device_t *)entry->ptr;

		if (cma_devp->cma_device->node_guid != hca_guid)
			continue;

		mutex_enter(&cma_devp->cma_mutex);
		if (cma_devp->cma_dev_state == SOL_CMA_DEV_REM_IN_PROGRESS) {
			SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str,
			    "sol_cma_acquire_dev() - Device getting removed!!");
			mutex_exit(&cma_devp->cma_mutex);
			mutex_exit(&sol_cma_dev_mutex);
			return (NULL);
		}
		cma_devp->cma_ref_count++;
		mutex_exit(&cma_devp->cma_mutex);
		mutex_exit(&sol_cma_dev_mutex);
		return (cma_devp->cma_device);

	}
	mutex_exit(&sol_cma_dev_mutex);
	return (NULL);
}

static void
sol_cma_release_device(struct rdma_cm_id *id)
{
	ib_device_t	*device = id->device;
	llist_head_t	*entry;
	cma_device_t	*cma_devp;

	mutex_enter(&sol_cma_dev_mutex);
	list_for_each(entry, &sol_cma_dev_list) {
		cma_devp = (cma_device_t *)entry->ptr;

		if (cma_devp->cma_device != device)
			continue;

		mutex_enter(&cma_devp->cma_mutex);
		cma_devp->cma_ref_count--;
		if (cma_devp->cma_dev_state == SOL_CMA_DEV_REM_IN_PROGRESS &&
		    cma_devp->cma_ref_count == 0) {
			SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str,
			    "sol_cma_release_dev() - Device free removed!!");
			mutex_exit(&cma_devp->cma_mutex);
			llist_del(&cma_devp->cma_list);
			kmem_free(cma_devp, sizeof (cma_device_t));
			mutex_exit(&sol_cma_dev_mutex);
			return;
		}
		mutex_exit(&cma_devp->cma_mutex);
	}
	mutex_exit(&sol_cma_dev_mutex);
}

void
sol_cma_add_hca_list(sol_cma_chan_t *ep_chanp, ib_guid_t hca_guid)
{
	llist_head_t	*entry;
	cma_device_t	*cma_devp;

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "add_hca_list(%p, %llx)",
	    ep_chanp, hca_guid);
	mutex_enter(&sol_cma_dev_mutex);
	list_for_each(entry, &sol_cma_dev_list) {
		cma_devp = (cma_device_t *)entry->ptr;

		if ((cma_devp->cma_device)->node_guid != hca_guid)
			continue;

		mutex_enter(&cma_devp->cma_mutex);
		ep_chanp->chan_listenp->listen_ep_dev_entry =
		    add_genlist(&cma_devp->cma_epchan_list,
		    (uintptr_t)ep_chanp, NULL);
		ep_chanp->chan_listenp->listen_ep_device = cma_devp->cma_device;
		mutex_exit(&cma_devp->cma_mutex);
		mutex_exit(&sol_cma_dev_mutex);
		return;
	}
	mutex_exit(&sol_cma_dev_mutex);
	SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "add_hca_list(%p, %llx): "
	    "No matching HCA in list!!", ep_chanp, hca_guid);
}

/*
 * rdma_cm.h API functions.
 */
struct rdma_cm_id *
rdma_create_id(rdma_cm_event_handler evt_hdlr, void *context,
    enum rdma_port_space ps)
{
	struct rdma_cm_id 	*rdma_idp;

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_create_id(%p, %p, %x)",
	    evt_hdlr, context, ps);

	if (ps != RDMA_PS_TCP && ps != RDMA_PS_UDP && ps != RDMA_PS_IPOIB) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_create_id: unsupported protocol %x", ps);
		return (NULL);
	}

	rdma_idp = cma_alloc_chan(evt_hdlr, context, ps);
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
	    "rdma_create_id : ret %p", rdma_idp);

	return (rdma_idp);
}

void
rdma_map_id2clnthdl(struct rdma_cm_id *rdma_idp, void *ib_client_hdl,
    void *iw_client_hdl)
{
	sol_cma_chan_t	*chanp = (sol_cma_chan_t *)rdma_idp;

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
	    "rdma_map_id2clnthdl(%p, %p, %p)",
	    rdma_idp, ib_client_hdl, iw_client_hdl);
	ASSERT(ib_client_hdl != NULL || iw_client_hdl != NULL);
	chanp->chan_ib_client_hdl = ib_client_hdl;
	chanp->chan_iw_client_hdl = iw_client_hdl;
}

void
rdma_map_id2qphdl(struct rdma_cm_id *rdma_idp, void *qp_hdl)
{
	sol_cma_chan_t	*chanp = (sol_cma_chan_t *)rdma_idp;

	ASSERT(rdma_idp);
	ASSERT(qp_hdl);
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_mapid2qphdl(%p, %p)",
	    rdma_idp, qp_hdl);
	chanp->chan_qp_hdl = qp_hdl;
}


void
rdma_destroy_id(struct rdma_cm_id *rdma_idp)
{
	sol_cma_chan_t		*chanp, *root_chanp;
	cma_chan_state_t	state;
	int			rc, is_root_cmid, do_wait, is_passive;

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id(%p)", rdma_idp);

	if (!rdma_idp)
		return;

	is_root_cmid = do_wait = is_passive = 0;

	chanp = (sol_cma_chan_t *)rdma_idp;
	root_chanp = (sol_cma_chan_t *)chanp->listen_root;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id(%p), %p",
	    rdma_idp, root_chanp);

	mutex_enter(&chanp->chan_mutex);
	chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_CMID_DESTROYED;

	/*
	 * Wait in destroy of CMID when rdma_resolve_addr() / rdma_listen()
	 * rdma_resolve_route() API is in progress.
	 */
	while (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_API_PROGRESS)
		cv_wait(&chanp->chan_destroy_cv, &chanp->chan_mutex);

	/* Wait if Event is been notified to consumer */
	while (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_EVENT_PROGRESS)
		cv_wait(&chanp->chan_destroy_cv, &chanp->chan_mutex);

	if (rdma_idp->device)
		sol_cma_release_device(rdma_idp);

	if (chanp->chan_listenp && chanp->chan_listenp->listen_is_root)
		is_root_cmid = 1;
	if (root_chanp == NULL && is_root_cmid == 0)
		is_passive = 1;

	/*
	 * Skip Active side handling for passive CMIDs and listen CMID
	 * for which REQ CMIDs have not been created.
	 */
	if (is_passive || (is_root_cmid && chanp->chan_req_state !=
	    REQ_CMID_CREATED)) {
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
		    "Skipping passive %p, %x, %x", chanp->chan_listenp,
		    is_root_cmid, chanp->chan_req_state);
		goto skip_passive_handling;
	}

	/*
	 * destroy_id() called for listening CMID and there are REQ
	 * CMIDs not yet notified. Reject such CMIDs and decrement
	 * the count.
	 */
	if (is_root_cmid && chanp->chan_req_cnt) {
		sol_cma_chan_t	*req_cmid_chan, *next_chan;

		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
		    "not notified handling");
		for (req_cmid_chan = (sol_cma_chan_t *)avl_first(
		    &chanp->chan_req_avl_tree); req_cmid_chan &&
		    chanp->chan_req_cnt; req_cmid_chan = next_chan) {
			next_chan = AVL_NEXT(
			    &chanp->chan_req_avl_tree, req_cmid_chan);
			if (req_cmid_chan->chan_req_state ==
			    REQ_CMID_CREATED) {
				avl_remove(&chanp->chan_req_avl_tree,
				    req_cmid_chan);
				chanp->chan_req_cnt--;
				chanp->chan_req_total_cnt--;
				mutex_exit(&chanp->chan_mutex);
				mutex_enter(&req_cmid_chan->chan_mutex);
				req_cmid_chan->chan_req_state =
				    REQ_CMID_NONE;
				mutex_exit(&req_cmid_chan->chan_mutex);
				(void) rdma_disconnect(
				    (struct rdma_cm_id *)req_cmid_chan);
				mutex_enter(&chanp->chan_mutex);
				if (rdma_idp->ps == RDMA_PS_TCP) {
					mutex_enter(
					    &req_cmid_chan->chan_mutex);
					req_cmid_chan->listen_root =
					    rdma_idp;
					cma_set_chan_state(req_cmid_chan,
					    SOL_CMA_CHAN_DESTROY_PENDING);
					mutex_exit(
					    &req_cmid_chan->chan_mutex);
				} else {
					mutex_destroy(
					    &req_cmid_chan->chan_mutex);
					cv_destroy(
					    &req_cmid_chan->chan_destroy_cv);
					kmem_free(req_cmid_chan,
					    sizeof (sol_cma_chan_t));
				}
			}
		}
	}

	/*
	 * destroy_id() called for :
	 * 	listening CMID and all REQ CMIDs destroy_id() called
	 *	REQ CMID and 1 more REQ CMID not yet destroyed.
	 * wait till the CMID is completly destroyed.
	 */
	if (is_root_cmid && chanp->chan_req_total_cnt == 0) {
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
		    "root idp waiting");
		cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROY_WAIT);
		cv_wait(&chanp->chan_destroy_cv, &chanp->chan_mutex);
	}

	if (root_chanp)
		mutex_enter(&root_chanp->chan_mutex);
#ifdef	DEBUG
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
	    "root_idp %p, cnt %x, state %x", root_chanp,
	    root_chanp ? root_chanp->chan_req_total_cnt : 0,
	    root_chanp ? cma_get_chan_state(root_chanp) : 0);
#endif

	if (root_chanp && root_chanp->chan_req_total_cnt == 1 &&
	    cma_get_chan_state(root_chanp) == SOL_CMA_CHAN_DESTROY_PENDING)
		do_wait = 1;
	if (root_chanp)
		mutex_exit(&root_chanp->chan_mutex);

skip_passive_handling :
	state = cma_get_chan_state(chanp);
	if (is_root_cmid == 0 && state != SOL_CMA_CHAN_DISCONNECT &&
	    SOL_CMA_DISCONNECT_OK(chanp)) {
		/*
		 * A connected CM ID has not been disconnected.
		 * Call rdma_disconnect() to disconnect it.
		 */
		mutex_exit(&chanp->chan_mutex);
		rc = rdma_disconnect(rdma_idp);
		if (rc) {
			SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
			    "rdma_destroy_id(%p)- disconnect failed!!",
			    rdma_idp);
			return;
		}
		mutex_enter(&chanp->chan_mutex);
		if (root_chanp && chanp->listen_root == NULL)
			chanp->listen_root = (struct rdma_cm_id *)root_chanp;
		mutex_exit(&chanp->chan_mutex);
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
		    "rdma_destroy_id(chanp %p, connect %x, ps %x)",
		    chanp, chanp->chan_connect_flag, rdma_idp->ps);
		if ((SOL_CMAID_IS_CONNECTED(chanp) &&
		    rdma_idp->ps == RDMA_PS_TCP) ||
		    (IS_UDP_CMID(rdma_idp) &&
		    chanp->chan_connect_flag == SOL_CMA_CONNECT_INITIATED)) {
			if (do_wait) {
				mutex_enter(&chanp->chan_mutex);
				cma_set_chan_state(chanp,
				    SOL_CMA_CHAN_DESTROY_WAIT);
				cv_wait(&chanp->chan_destroy_cv,
				    &chanp->chan_mutex);
				mutex_exit(&chanp->chan_mutex);
				cma_destroy_id(rdma_idp);
			} else {
				mutex_enter(&chanp->chan_mutex);
				cma_set_chan_state(chanp,
				    SOL_CMA_CHAN_DESTROY_PENDING);
				mutex_exit(&chanp->chan_mutex);
			}
		} else {
			/*
			 * Disconnected a CMID for which CONNECT has been
			 * Initiated but not complete.
			 * No more callbacks are expected for this CMID.
			 * Free this CMID.
			 */
			cma_destroy_id(rdma_idp);
		}
	} else if (is_root_cmid == 0 && state ==
	    SOL_CMA_CHAN_DISCONNECT && SOL_CMAID_IS_CONNECTED(chanp)) {
		/*
		 * CM ID was connected and disconnect is process.
		 * Free of this CM ID is done for the DISCONNECT
		 * notification for this CMID.
		 */
		cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROY_PENDING);
		mutex_exit(&chanp->chan_mutex);
	} else if (state != SOL_CMA_CHAN_DESTROY_PENDING) {
		/* CM ID, not connected, just free it. */
		mutex_exit(&chanp->chan_mutex);
		cma_destroy_id(rdma_idp);
	} else
		mutex_exit(&chanp->chan_mutex);

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: ret");
}

/*
 * State transitions for Address resolution :
 *	Active Side (Client) :
 *	1. CREATE_ID-->BIND_ADDR-->RESOLVE_ADDR-->RESOLVE_ROUTE
 *
 *	Passive Side (Server) :
 *	2. CREATE_ID-->RESOLVE_ADDR-->RESOLVE_ROUTE
 *	IF_ADDR_ANY can be passed as local address in RESOLVE_ADDR
 */
int
rdma_bind_addr(struct rdma_cm_id *idp, struct sockaddr *addr)
{
	sol_cma_chan_t		*chanp;
	struct rdma_addr	*addrp;
	int			ret;

	ASSERT(idp);
	ASSERT(addr);
	chanp = (sol_cma_chan_t *)idp;
	addrp = &(idp->route.addr);
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_bind_addr(%p, %p)",
	    idp, addr);

	mutex_enter(&chanp->chan_mutex);
	ret = cma_cas_chan_state(chanp, SOL_CMA_CHAN_IDLE, SOL_CMA_CHAN_BOUND);
	if (ret) {
		mutex_exit(&chanp->chan_mutex);
		return (ret);
	}
	/* Copy the local address to rdma_id structure */
	bcopy((void *)addr, (void *)&(addrp->src_addr),
	    sizeof (struct sockaddr));
	mutex_exit(&chanp->chan_mutex);

	/*
	 * First call rdma_ib_bind_addr() to bind this address.
	 * Next call rdma_iw_bind_addr() to bind this address.
	 * For IF_ADDR_ANY, IB address is given priority over
	 * iWARP.
	 */
	if (chanp->chan_ib_client_hdl == NULL) {
		ofs_client_t	*ofs_clnt;

		ofs_clnt = (ofs_client_t *)sol_cma_ib_client->clnt_hdl;
		chanp->chan_ib_client_hdl = ofs_clnt->ibt_hdl;
	}
	if (chanp->chan_ib_client_hdl && rdma_ib_bind_addr(idp, addr) == 0) {
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
		    "rdma_bind_addr: ret IB @");
		return (0);
#ifdef	IWARP_SUPPORT
	} else if (chanp->chan_iw_client_hdl && rdma_iw_bind_addr(idp, addr)
	    == 0) {
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
		    "rdma_bind_addr: ret iWARP @");
		return (0);
#endif	/* IWARP_SUPPORT */
	}

	mutex_enter(&chanp->chan_mutex);
	cma_set_chan_state(chanp, SOL_CMA_CHAN_IDLE);
	mutex_exit(&chanp->chan_mutex);
	SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str, "rdma_bind_addr: ret failure!");
	return (EINVAL);
}

int
rdma_resolve_addr(struct rdma_cm_id *idp, struct sockaddr *src_addr,
    struct sockaddr *dst_addr, int timeout_ms)
{
	sol_cma_chan_t		*chanp;
	struct rdma_addr	*addrp;
	cma_chan_state_t	state;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	addrp = &(idp->route.addr);
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_resolve_addr(%p, %p, "
	    "%p, %x)", idp, src_addr, dst_addr, timeout_ms);

	mutex_enter(&chanp->chan_mutex);
	state = cma_get_chan_state(chanp);
	if (state != SOL_CMA_CHAN_IDLE && state != SOL_CMA_CHAN_BOUND) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_resolve_addr : invalid chan state %x", state);
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	if (chanp->chan_cmid_destroy_state &
	    SOL_CMA_CALLER_CMID_DESTROYED) {
		SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
		    "rdma_resolve_addr : CMID %p, destroy called", chanp);
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_API_PROGRESS;

	if (chanp->chan_xport_type == SOL_CMA_XPORT_NONE) {
		bcopy((void *)src_addr, (void *)&(addrp->src_addr),
		    sizeof (struct sockaddr));
	}
	bcopy((void *)dst_addr, (void *)&(addrp->dst_addr),
	    sizeof (struct sockaddr));
	mutex_exit(&chanp->chan_mutex);

	/*
	 * First resolve this as an @ corresponding to IB fabric
	 * if this fails, resolve this as an @ corresponding to iWARP
	 */
	if (chanp->chan_ib_client_hdl == NULL) {
		ofs_client_t	*ofs_clnt;

		ofs_clnt = (ofs_client_t *)sol_cma_ib_client->clnt_hdl;
		chanp->chan_ib_client_hdl = ofs_clnt->ibt_hdl;
	}
	if (chanp->chan_ib_client_hdl && rdma_ib_resolve_addr(idp, src_addr,
	    dst_addr, timeout_ms) == 0) {
		SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str,
		    "rdma_resolve_addr: ret IB @");
#ifdef IWARP_SUPPORT
	} else if (chanp->chan_iw_client_hdl && rdma_iw_resolve_addr(idp,
	    src_addr, dst_addr, timeout_ms) == 0) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_resolve_addr: ret iWARP @");
#endif	/* IWARP_SUPPORT */
	} else {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_resolve_addr: Invalid @");
		return (EINVAL);
	}
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_resolve_addr: ret 0");
	return (0);
}

static void cma_generate_event_sync(struct rdma_cm_id *,
    enum rdma_cm_event_type, int, struct rdma_conn_param *,
    struct rdma_ud_param *);

void
cma_resolve_addr_callback(sol_cma_chan_t *chanp, int rc)
{
	enum rdma_cm_event_type	event;

	mutex_enter(&chanp->chan_mutex);
	if (chanp->chan_cmid_destroy_state &
	    SOL_CMA_CALLER_CMID_DESTROYED) {
		SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
		    "cma_resolve_addr : CMID %p, destroy called", chanp);
		chanp->chan_cmid_destroy_state &=
		    ~SOL_CMA_CALLER_API_PROGRESS;
		cv_broadcast(&chanp->chan_destroy_cv);
		mutex_exit(&chanp->chan_mutex);
		return;
	}
	if (rc == 0) {
		cma_set_chan_state(chanp, SOL_CMA_CHAN_ADDR_RESLVD);
		event = RDMA_CM_EVENT_ADDR_RESOLVED;
	} else
		event = RDMA_CM_EVENT_ADDR_ERROR;

	/*
	 * Generate RDMA_CM_EVENT_ADDR_RESOLVED event
	 * This will result in RDMA_USER_CM_CMD_RESOLVE_ROUTE in
	 * userland.
	 */
	chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_EVENT_PROGRESS;
	mutex_exit(&chanp->chan_mutex);
	cma_generate_event_sync((struct rdma_cm_id *)chanp, event, 0,
	    NULL, NULL);

	mutex_enter(&chanp->chan_mutex);
	chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_API_PROGRESS;
	if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
		cv_broadcast(&chanp->chan_destroy_cv);
	mutex_exit(&chanp->chan_mutex);
}

int
rdma_resolve_route(struct rdma_cm_id *idp, int timeout_ms)
{
	sol_cma_chan_t		*chanp;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "resolve_route(%p, %x)", idp,
	    timeout_ms);

	mutex_enter(&chanp->chan_mutex);
	if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_ADDR_RESLVD,
	    SOL_CMA_CHAN_ROUTE_RESLVD) != 0) {
		mutex_exit(&chanp->chan_mutex);
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "resolve_route: Invalid state");
		return (EINVAL);
	}
	if (chanp->chan_cmid_destroy_state &
	    SOL_CMA_CALLER_CMID_DESTROYED) {
		SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
		    "rdma_resolve_route : CMID %p, destroy called", chanp);
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_API_PROGRESS;
	mutex_exit(&chanp->chan_mutex);

	/*
	 * Generate RDMA_CM_EVENT_ROUTE_RESOLVED event
	 * This will result in RDMA_USER_CM_CMD_RESOLVE_ROUTE in
	 * userland
	 */
	cma_generate_event(idp, RDMA_CM_EVENT_ROUTE_RESOLVED, 0,
	    NULL, NULL);

	mutex_enter(&chanp->chan_mutex);
	chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_API_PROGRESS;
	if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
		cv_broadcast(&chanp->chan_destroy_cv);
	mutex_exit(&chanp->chan_mutex);

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "resolve_route: ret 0");
	return (0);
}

/*
 * Connect or Listen request should be send after Route is resolved
 *
 *	Active Side (Client) :
 *	1. (State ROUTE_RESOLVED)-->CONNECT-->ACCEPT/REJECT-->DISCONNECT
 *	       -->DESTROY_ID-->close(9E)
 *	2. Same as (1), DESTROY_ID without DISCONNECT
 *	3. Same as (1), close(9e) without DESTROY_ID.
 *
 *	Passive Side (Server) :
 *	4. (State ROUTE_RESOLVED)-->LISTEN->DISCONNECT
 *		-->DESTROY_ID-->close(9E)
 *	5. Same as (4), DESTROY_ID without DISCONNECT
 *	6. Same as (4), close(9e) without DESTROY_ID.
 */
int
rdma_connect(struct rdma_cm_id *idp, struct rdma_conn_param *conn_param)
{
	sol_cma_chan_t		*chanp;
	int			ret = EINVAL;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_connect(%p, %p)", idp,
	    conn_param);

	mutex_enter(&chanp->chan_mutex);
	if (chanp->chan_xport_type == SOL_CMA_XPORT_NONE) {
		mutex_exit(&chanp->chan_mutex);
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_connect, Invalid Xport");
		return (EINVAL);
	}
	if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_ROUTE_RESLVD,
	    SOL_CMA_CHAN_CONNECT)) {
		mutex_exit(&chanp->chan_mutex);
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_connect, Invalid state");
		return (EINVAL);
	}

	if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
		ret = rdma_ib_connect(idp, conn_param);
#ifdef	IWARP_SUPPORT
	} else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP) {
		ret = rdma_iw_connect(idp, conn_param);
#endif	/* IWARP_SUPPORT */
	}
	mutex_exit(&chanp->chan_mutex);

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_connect: ret %x", ret);
	return (ret);
}

static int cma_init_listen_root(sol_cma_chan_t *);
static void cma_fini_listen_root(sol_cma_chan_t *);

int
rdma_listen(struct rdma_cm_id *idp, int bklog)
{
	sol_cma_chan_t		*chanp;
	int			ret;
	genlist_entry_t		*entry;
	cma_chan_state_t	state;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_listen(%p, %x)",
	    idp, bklog);

	mutex_enter(&chanp->chan_mutex);
	state = cma_get_chan_state(chanp);
	if (state == SOL_CMA_CHAN_IDLE) {
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	cma_set_chan_state(chanp, SOL_CMA_CHAN_LISTEN);

	if (chanp->chan_cmid_destroy_state &
	    SOL_CMA_CALLER_CMID_DESTROYED) {
		SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
		    "rdma_listen : CMID %p, destroy called", chanp);
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_API_PROGRESS;

	ASSERT(chanp->chan_listenp == NULL);

	chanp->chan_listenp = kmem_zalloc(sizeof (sol_cma_listen_info_t),
	    KM_SLEEP);
	init_genlist(&(CHAN_LISTEN_LIST(chanp)));
	(chanp->chan_listenp)->listen_is_root = 1;
	ret = cma_init_listen_root(chanp);
	if (ret) {
		chanp->chan_listenp = NULL;
		mutex_exit(&chanp->chan_mutex);
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "rdma_listen: "
		    "cma_init_listen_root: failed");
		kmem_free(chanp->chan_listenp,
		    sizeof (sol_cma_listen_info_t));
		return (EINVAL);
	}

	if (chanp->chan_xport_type == SOL_CMA_XPORT_NONE) {
		ibcma_append_listen_list(idp);
#ifdef IWARP_SUPPORT
		iwcma_append_listen_list(idp);
#endif
	} else if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
		ibcma_append_listen_list(idp);
#ifdef	IWARP_SUPPORT
	} else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP) {
		iwcma_append_listen_list(idp);
#endif	/* IWARP_SUPPORT */
	}

	if (genlist_empty(&(CHAN_LISTEN_LIST(chanp)))) {
		cma_fini_listen_root(chanp);
		kmem_free((void *)chanp->chan_listenp,
		    sizeof (sol_cma_listen_info_t));
		chanp->chan_listenp = NULL;
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "rdma_listen: "
		    "No listeners");
		mutex_exit(&chanp->chan_mutex);
		return (0);
	}

	if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED) {
		chanp->chan_cmid_destroy_state &=
		    ~SOL_CMA_CALLER_API_PROGRESS;
		cv_broadcast(&chanp->chan_destroy_cv);
	}

	genlist_for_each(entry, &(CHAN_LISTEN_LIST(chanp))) {
		struct rdma_cm_id	*ep_idp;
		sol_cma_chan_t		*ep_chanp;

		ep_idp = (struct rdma_cm_id *)entry->data;
		ep_chanp = (sol_cma_chan_t *)ep_idp;
		if (ep_chanp->chan_xport_type == SOL_CMA_XPORT_IB)
			ret = rdma_ib_listen(ep_idp, bklog);
#ifdef IWARP_SUPPORT
		if (ep_chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
			ret = rdma_iw_listen(ep_idp, bklog);
#endif
		if (ret)
			break;
	}

	chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_API_PROGRESS;
	if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
		cv_broadcast(&chanp->chan_destroy_cv);
	mutex_exit(&chanp->chan_mutex);

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_listen: ret %x", ret);
	return (ret);
}

int
rdma_accept(struct rdma_cm_id *idp, struct rdma_conn_param *conn_param)
{
	struct rdma_cm_id	*root_idp;
	sol_cma_chan_t		*root_chanp, *chanp;
	int			ret = EINVAL;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_accept(%p, %p)",
	    idp, conn_param);

	mutex_enter(&chanp->chan_mutex);
	if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_LISTEN,
	    SOL_CMA_CHAN_ACCEPT) && cma_cas_chan_state(chanp,
	    SOL_CMA_CHAN_CONNECT, SOL_CMA_CHAN_ACCEPT)) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_accept, Invalid state");
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	mutex_exit(&chanp->chan_mutex);

	root_idp = CHAN_LISTEN_ROOT(chanp);
	root_chanp = (sol_cma_chan_t *)root_idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "accept: root_idp %p",
	    root_idp);

	/* Delete from REQ_AVL_TREE on passive side */
	if (root_idp) {
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "accept: root_idp %p"
		    "REQ AVL remove %p", root_chanp, idp);
		mutex_enter(&root_chanp->chan_mutex);
		avl_remove(&root_chanp->chan_req_avl_tree, idp);

		/* For TCP, insert into ACPT_AVL_TREE */
		if (idp->ps == RDMA_PS_TCP) {
			void		*find_ret;
			avl_index_t	where;

			SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
			    "Add to ACPT AVL of %p IDP, idp %p, qp_hdl %p",
			    root_idp, idp, chanp->chan_qp_hdl);
			find_ret = avl_find(&root_chanp->chan_acpt_avl_tree,
			    (void *)chanp->chan_qp_hdl, &where);
			if (find_ret) {
				mutex_exit(&root_chanp->chan_mutex);
				SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
				    "DUPLICATE ENTRY in ACPT AVL : root %p, "
				    "idp %p, qp_hdl %p",
				    root_idp, idp, chanp->chan_qp_hdl);
				return (EINVAL);
			}
			avl_insert(&root_chanp->chan_acpt_avl_tree,
			    (void *)idp, where);
		}
		mutex_exit(&root_chanp->chan_mutex);

		mutex_enter(&chanp->chan_mutex);
		/* Update chan_req_state to ACCEPTED */
		chanp->chan_req_state = REQ_CMID_ACCEPTED;
		mutex_exit(&chanp->chan_mutex);
	}

	if (root_idp && IS_UDP_CMID(root_idp)) {
		cma_chan_state_t	chan_state;

		/*
		 * Accepting the connect request, no more events for this
		 * connection.
		 */
		mutex_enter(&chanp->chan_mutex);
		cma_handle_nomore_events(chanp);
		chan_state = cma_get_chan_state(chanp);
		mutex_exit(&chanp->chan_mutex);
		/* If rdma_destroy_id() was called, destroy CMID */
		if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING)
			cma_destroy_id((struct rdma_cm_id *)chanp);
	}

	if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
		ret = rdma_ib_accept(idp, conn_param);
#ifdef	IWARP_SUPPORT
	if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
		ret = rdma_iw_accept(idp, conn_param);
#endif	/* IWARP_SUPPORT */

	if (ret && root_idp && idp->ps == RDMA_PS_TCP) {
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
		    "Delete from REQ AVL of %p IDP, idp %p",
		    root_idp, idp);
		mutex_enter(&root_chanp->chan_mutex);
		avl_remove(&root_chanp->chan_acpt_avl_tree, idp);
		mutex_exit(&root_chanp->chan_mutex);
		mutex_enter(&chanp->chan_mutex);
		chanp->chan_req_state = REQ_CMID_NONE;
		mutex_exit(&chanp->chan_mutex);
	}

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_accept: ret %x", ret);
	return (ret);
}

int
rdma_notify(struct rdma_cm_id *idp, enum ib_event_type evt)
{
	sol_cma_chan_t		*chanp;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_notify(%p, %x)", idp, evt);

	mutex_enter(&chanp->chan_mutex);
	if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_ROUTE_RESLVD,
	    SOL_CMA_CHAN_EVENT_NOTIFIED)) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_notify, Invalid state");
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	mutex_exit(&chanp->chan_mutex);

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_notify: ret 0");
	return (0);
}

int
rdma_reject(struct rdma_cm_id *idp, const void *priv_data,
    uint8_t priv_data_len)
{
	struct rdma_cm_id	*root_idp;
	sol_cma_chan_t		*root_chanp, *chanp;
	int			ret = EINVAL;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	root_idp = CHAN_LISTEN_ROOT(chanp);
	root_chanp = (sol_cma_chan_t *)root_idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_reject(%p, %p)", idp,
	    priv_data, priv_data_len);

	mutex_enter(&chanp->chan_mutex);
	if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_LISTEN,
	    SOL_CMA_CHAN_REJECT)) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_accept, Invalid state");
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	mutex_exit(&chanp->chan_mutex);

	if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
		ret = rdma_ib_reject(idp, priv_data, priv_data_len);
#ifdef	IWARP_SUPPORT
	if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
		ret = rdma_iw_reject(idp, priv_data, priv_data_len);
#endif	/* IWARP_SUPPORT */


	if (!ret && root_idp) {
		cma_chan_state_t	chan_state;

		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "reject: root_idp %p"
		    "REQ AVL remove %p", root_chanp, idp);
		/* Remove from Req AVL tree */
		mutex_enter(&root_chanp->chan_mutex);
		avl_remove(&root_chanp->chan_req_avl_tree, idp);
		mutex_exit(&root_chanp->chan_mutex);

		/* Update chan_req_state to REJECTED */
		mutex_enter(&chanp->chan_mutex);
		chanp->chan_req_state = REQ_CMID_REJECTED;

		/*
		 * Rejecting connect request, no more events for this
		 * connection.
		 */
		cma_handle_nomore_events(chanp);
		chan_state = cma_get_chan_state(chanp);
		mutex_exit(&chanp->chan_mutex);
		/* If rdma_destroy_id() was called, destroy CMID */
		if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING)
			cma_destroy_id((struct rdma_cm_id *)chanp);
	}

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_reject: ret %x", ret);
	return (ret);
}

int
rdma_disconnect(struct rdma_cm_id *idp)
{
	sol_cma_chan_t		*chanp;
	int			ret = EINVAL;
	cma_chan_state_t	state;

	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_disconnect(%p)", idp);

	if (!idp)
		return (0);

	mutex_enter(&chanp->chan_mutex);
	if (!(SOL_CMA_DISCONNECT_OK(chanp))) {
		SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
		    "rdma_disconnect(%p) - Not connected!!", idp);
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}
	state = cma_get_chan_state(chanp);
	cma_set_chan_state(chanp, SOL_CMA_CHAN_DISCONNECT);
	mutex_exit(&chanp->chan_mutex);

	if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
		ret = rdma_ib_disconnect(idp);
#ifdef	IWARP_SUPPORT
	} else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP) {
		ret = rdma_iw_disconnect(idp);
#endif	/* IWARP_SUPPORT */
	}

	if (ret) {
		mutex_enter(&chanp->chan_mutex);
		cma_set_chan_state(chanp, state);
		mutex_exit(&chanp->chan_mutex);
		return (ret);
	}

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_disconnect: ret %x", ret);
	return (ret);
}

int
rdma_init_qp_attr(struct rdma_cm_id *idp, struct ib_qp_attr *qpattr,
    int *qp_attr_mask)
{
	sol_cma_chan_t		*chanp;
	int			ret = EINVAL;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_init_qp_attr(%p, %p, %p)",
	    idp, qpattr, qp_attr_mask);

	if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
		ret = rdma_ib_init_qp_attr(idp, qpattr, qp_attr_mask);
#ifdef	IWARP_SUPPORT
	} else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
		ret = rdma_iw_init_qp_attr(idp, qpattr, qp_attr_mask);
#endif	/* IWARP_SUPPORT */
	} else {
		ret = EINVAL;
	}

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
	    "rdma_init_qp_attr: ret %x", ret);

	return (ret);
}

int
rdma_join_multicast(struct rdma_cm_id *idp, struct sockaddr *addr,
    void *context)
{
	sol_cma_chan_t		*chanp;
	int			ret = ENODEV;
	cma_chan_state_t	state;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
	    "rdma_join_multicast(%p, %p, %p)",
	    idp, addr, context);

	mutex_enter(&chanp->chan_mutex);
	state = cma_get_chan_state(chanp);
	if (state != SOL_CMA_CHAN_BOUND &&
	    state != SOL_CMA_CHAN_ROUTE_RESLVD &&
	    state != SOL_CMA_CHAN_ADDR_RESLVD) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_join_multicast, Invalid state");
		mutex_exit(&chanp->chan_mutex);
		return (EINVAL);
	}

	if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
		ret = rdma_ib_join_multicast(idp, addr, context);
#ifdef	IWARP_SUPPORT
	/* No support for Multicast on iWARP */
	else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
		ret = ENOTSUP;
#endif	/* IWARP_SUPPORT */
	mutex_exit(&chanp->chan_mutex);

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
	    "rdma_join_multicast: ret %x", ret);
	return (ret);
}

void
rdma_leave_multicast(struct rdma_cm_id *idp, struct sockaddr *addr)
{
	sol_cma_chan_t		*chanp;
	cma_chan_state_t	state;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_leave_multicast(%p, %p)",
	    idp, addr);

	mutex_enter(&chanp->chan_mutex);
	state = cma_get_chan_state(chanp);
	if (state != SOL_CMA_CHAN_BOUND &&
	    state != SOL_CMA_CHAN_ROUTE_RESLVD &&
	    state != SOL_CMA_CHAN_ADDR_RESLVD) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_leave_multicast, Invalid state");
		mutex_exit(&chanp->chan_mutex);
		return;
	}

	if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
		rdma_ib_leave_multicast(idp, addr);
#ifdef	IWARP_SUPPORT
	/* No support for Multicast on iWARP */
	else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "rdma_leave_multicast, iWARP");
#endif	/* IWARP_SUPPORT */
	mutex_exit(&chanp->chan_mutex);

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_join_multicast: ret");
}

/*
 * Functions to compare to rdma_cm_id *, used by AVL tree
 * routines.
 */
int
sol_cma_req_cmid_cmp(const void *p1, const void *p2)
{
	sol_cma_chan_t		*chanp;

	chanp = (sol_cma_chan_t *)p2;
	if (chanp->chan_session_id > p1)
		return (+1);
	else if (chanp->chan_session_id < p1)
		return (-1);
	else
		return (0);
}

int
sol_cma_cmid_cmp(const void *p1, const void *p2)
{
	sol_cma_chan_t		*chanp;

	chanp = (sol_cma_chan_t *)p2;
	if (chanp->chan_qp_hdl > p1)
		return (+1);
	else if (chanp->chan_qp_hdl < p1)
		return (-1);
	else
		return (0);
}

/*
 * Function to compare two sol_cma_glbl_listen_t *, used by
 * AVL tree routines.
 */
int
sol_cma_svc_cmp(const void *p1, const void *p2)
{
	sol_cma_glbl_listen_t	*listenp;
	uint64_t		sid;

	sid = *(uint64_t *)p1;
	listenp = (sol_cma_glbl_listen_t *)p2;
	if (listenp->cma_listen_chan_sid > sid)
		return (+1);
	else if (listenp->cma_listen_chan_sid < sid)
		return (-1);
	else
		return (0);
}

static int
cma_init_listen_root(sol_cma_chan_t *chanp)
{
	sol_cma_glbl_listen_t	*cma_listenp;
	sol_cma_listen_info_t	*chan_listenp;
	int			rc = 0;
	avl_index_t		where = 0;
	uint64_t		listen_sid;

	ASSERT(chanp);
	ASSERT(chanp->chan_listenp);
	chan_listenp = chanp->chan_listenp;

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
	    "cma_init_listen_root(%p)", chanp);

	/*
	 * First search for matching global listen_info for this SID.
	 * If found with the same client handle, reuse the service
	 * handle, if matching SID is found with different client
	 * handle, return EINVAL.
	 */
	listen_sid = ibcma_init_root_sid(chanp);
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
	    "cma_init_listen_root: search SID 0x%llx",
	    listen_sid);

	mutex_enter(&sol_cma_glob_mutex);
	cma_listenp = avl_find(&sol_cma_glbl_listen_tree,
	    (void *) &listen_sid, &where);
	if (cma_listenp && cma_listenp->cma_listen_clnt_hdl ==
	    chanp->chan_ib_client_hdl) {
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
		    "cma_init_listen_root: matching listenp %p SID 0x%llx",
		    cma_listenp, listen_sid);
		chan_listenp->listen_entry = add_genlist(
		    &cma_listenp->cma_listen_chan_list,
		    (uintptr_t)chanp, NULL);
		chan_listenp->chan_glbl_listen_info = cma_listenp;
		ibcma_copy_srv_hdl(chanp, cma_listenp);
		mutex_exit(&sol_cma_glob_mutex);
		return (0);
	} else if (cma_listenp) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "cma_init_listen_root: listenp %p, SID 0x%llx match, "
		    "client hdl prev %p, new %p mismatch",
		    cma_listenp, listen_sid,
		    cma_listenp->cma_listen_clnt_hdl,
		    chanp->chan_ib_client_hdl);
		mutex_exit(&sol_cma_glob_mutex);
		return (EINVAL);
	}

	cma_listenp = kmem_zalloc(sizeof (sol_cma_glbl_listen_t), KM_SLEEP);
	init_genlist(&cma_listenp->cma_listen_chan_list);
	chan_listenp->listen_entry = add_genlist(
	    &cma_listenp->cma_listen_chan_list, (uintptr_t)chanp, NULL);
	chan_listenp->chan_glbl_listen_info = cma_listenp;
	cma_listenp->cma_listen_clnt_hdl = chanp->chan_ib_client_hdl;
	cma_listenp->cma_listen_chan_sid = listen_sid;

	rc = ibcma_init_root_chan(chanp, cma_listenp);
	if (rc) {
		mutex_exit(&sol_cma_glob_mutex);
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "cma_init_listen_root: ibcma_init_root_chan failed!!");
		delete_genlist(&cma_listenp->cma_listen_chan_list,
		    chan_listenp->listen_entry);
		kmem_free(cma_listenp, sizeof (sol_cma_glbl_listen_t));
		return (rc);
	}
	avl_insert(&sol_cma_glbl_listen_tree, cma_listenp, where);
	mutex_exit(&sol_cma_glob_mutex);
	return (0);
}

static void
cma_fini_listen_root(sol_cma_chan_t *chanp)
{
	sol_cma_glbl_listen_t	*cma_listenp;
	sol_cma_listen_info_t	*chan_listenp;

	ASSERT(chanp);
	ASSERT(chanp->chan_listenp);
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_fini_listen_root(%p)",
	    chanp);
	chan_listenp = chanp->chan_listenp;
	cma_listenp = chan_listenp->chan_glbl_listen_info;
	ASSERT(cma_listenp);
	mutex_enter(&sol_cma_glob_mutex);
	delete_genlist(&cma_listenp->cma_listen_chan_list,
	    chan_listenp->listen_entry);
	if (genlist_empty(&cma_listenp->cma_listen_chan_list)) {
		if (ibcma_fini_root_chan(chanp) == 0) {
			avl_remove(&sol_cma_glbl_listen_tree,
			    cma_listenp);
			kmem_free(cma_listenp,
			    sizeof (sol_cma_glbl_listen_t));
		} else
			SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
			    "cma_fini_listen_root: "
			    "ibcma_fini_root_chan failed");
	}

	mutex_exit(&sol_cma_glob_mutex);
}

typedef struct cma_event_async_arg {
	struct rdma_cm_id	*idp;
	enum rdma_cm_event_type	event;
	int			status;
	union {
		struct rdma_conn_param	conn;
		struct rdma_ud_param	param;
	} un;
	struct rdma_conn_param	*conn_param;
	struct rdma_ud_param	*ud_paramp;
} cma_event_async_arg_t;

static void cma_generate_event_sync(struct rdma_cm_id *,
    enum rdma_cm_event_type, int, struct rdma_conn_param *,
    struct rdma_ud_param *);

void
cma_generate_event_thr(void *arg)
{
	cma_event_async_arg_t	*event_arg = (cma_event_async_arg_t *)arg;

	cma_generate_event_sync(event_arg->idp, event_arg->event,
	    event_arg->status, event_arg->conn_param,
	    event_arg->ud_paramp);

	if (event_arg->conn_param && event_arg->conn_param->private_data_len)
		kmem_free((void *)event_arg->conn_param->private_data,
		    event_arg->conn_param->private_data_len);
	if (event_arg->ud_paramp && event_arg->ud_paramp->private_data_len)
		kmem_free((void *)event_arg->ud_paramp->private_data,
		    event_arg->ud_paramp->private_data_len);
	kmem_free(arg, sizeof (cma_event_async_arg_t));
}

void
cma_generate_event(struct rdma_cm_id *idp, enum rdma_cm_event_type event,
    int status, struct rdma_conn_param *conn_param,
    struct rdma_ud_param *ud_paramp)
{
	cma_event_async_arg_t	*event_arg;
	sol_cma_chan_t		*chanp = (sol_cma_chan_t *)idp;

	/*
	 * Set SOL_CMA_CALLER_EVENT_PROGRESS to indicate event
	 * notification is in progress, so that races between
	 * rdma_destroy_id() and event notification is taken care.
	 *
	 * If rdma_destroy_id() has been called for this CMID, call
	 * cma_generate_event_sync() which skips notification to the
	 * consumer and handles the event.
	 */
	mutex_enter(&chanp->chan_mutex);
	chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_EVENT_PROGRESS;
	if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED) {
		mutex_exit(&chanp->chan_mutex);
		cma_generate_event_sync(idp, event, status, conn_param,
		    ud_paramp);
		return;
	}
	mutex_exit(&chanp->chan_mutex);

	event_arg = kmem_zalloc(sizeof (cma_event_async_arg_t), KM_SLEEP);
	event_arg->idp = idp;
	event_arg->event = event;
	event_arg->status = status;
	event_arg->conn_param = NULL;
	event_arg->ud_paramp = NULL;
	if (conn_param && conn_param->private_data_len) {
		bcopy(conn_param, &(event_arg->un.conn),
		    sizeof (struct rdma_conn_param));
		event_arg->conn_param = &(event_arg->un.conn);
		event_arg->conn_param->private_data = kmem_zalloc(
		    conn_param->private_data_len, KM_SLEEP);
		bcopy(conn_param->private_data,
		    (void *)event_arg->conn_param->private_data,
		    conn_param->private_data_len);
	} else if (conn_param && conn_param->private_data_len == 0) {
		bcopy(conn_param, &(event_arg->un.conn),
		    sizeof (struct rdma_conn_param));
	} else if (ud_paramp) {
		bcopy(ud_paramp, &(event_arg->un.param),
		    sizeof (struct rdma_ud_param));
		event_arg->ud_paramp = &(event_arg->un.param);
		if (ud_paramp->private_data_len) {
			event_arg->ud_paramp->private_data = kmem_zalloc(
			    ud_paramp->private_data_len, KM_SLEEP);
			bcopy(ud_paramp->private_data,
			    (void *)event_arg->ud_paramp->private_data,
			    ud_paramp->private_data_len);
		} else if (ud_paramp->private_data) {
			event_arg->ud_paramp->private_data =
			    ud_paramp->private_data;
		}
	}

	if (taskq_dispatch(system_taskq, cma_generate_event_thr,
	    (void *)event_arg, TQ_SLEEP) == 0) {
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "generate_event_async: taskq_dispatch() failed!!");
		mutex_enter(&chanp->chan_mutex);
		chanp->chan_cmid_destroy_state &=
		    ~SOL_CMA_CALLER_EVENT_PROGRESS;
		if (chanp->chan_cmid_destroy_state &
		    SOL_CMA_CALLER_CMID_DESTROYED)
			cv_broadcast(&chanp->chan_destroy_cv);
		mutex_exit(&chanp->chan_mutex);
	}
}

static void
cma_generate_event_sync(struct rdma_cm_id *idp, enum rdma_cm_event_type event,
    int status, struct rdma_conn_param *conn_param,
    struct rdma_ud_param *ud_paramp)
{
	struct rdma_cm_event	cm_event;
	sol_cma_chan_t		*chanp = (sol_cma_chan_t *)idp;
	struct rdma_cm_id	*root_idp = NULL;
	sol_cma_chan_t		*root_chanp;
	int			ret;
	cma_chan_state_t	chan_state;

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "generate_event_sync(%p, %x, "
	    "%x, %p, %p", idp, event, status, conn_param, ud_paramp);

	bzero(&cm_event, sizeof (cm_event));
	cm_event.event = event;
	cm_event.status = status;
	if (conn_param)
		bcopy((void *)conn_param, (void *)(&(cm_event.param.conn)),
		    sizeof (struct rdma_conn_param));
	else if (ud_paramp)
		bcopy((void *)ud_paramp, (void *)(&(cm_event.param.ud)),
		    sizeof (struct rdma_ud_param));

	/*
	 * If the consumer has destroyed the context for this CMID -
	 * do not notify, skip to handling the sol_ofs specific
	 * handling of the event.
	 */
	mutex_enter(&chanp->chan_mutex);
	if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED) {
		mutex_exit(&chanp->chan_mutex);
		goto ofs_consume_event;
	}
	mutex_exit(&chanp->chan_mutex);

	root_idp = CHAN_LISTEN_ROOT(chanp);
	root_chanp = (sol_cma_chan_t *)root_idp;
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "gen_event: root_idp %p",
	    root_idp);

	if (event == RDMA_CM_EVENT_CONNECT_REQUEST) {
		/*
		 * Update chan_req_state for the REQ CMID. Decrement
		 * count of REQ CMIDs not notifed to consumer.
		 */
		ASSERT(root_idp);
		mutex_enter(&root_chanp->chan_mutex);
		root_chanp->chan_req_cnt--;
#ifdef	DEBUG
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
		    "Dec req_cnt of %p IDP, idp %p, req_cnt %x",
		    root_idp, idp, root_chanp->chan_req_cnt);
#endif
		mutex_exit(&root_chanp->chan_mutex);
	}

	/* Pass the event to the client */
	ret = (idp->event_handler) (idp, &cm_event);

	if (ret) {
		if (event == RDMA_CM_EVENT_CONNECT_REQUEST) {
			SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str,
			    "cma_generate_event_async: consumer failed %d "
			    "event", event);
			/*
			 * Disconnect if the consumer returned non zero.
			 * rdma_disconnect will send a REJ to the active
			 * side / client.
			 */
			if (rdma_disconnect(idp))
				SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
				    "generate_event_async: rdma_disconnect "
				    "failed");
		} else
			SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
			    "generate_event_async: consumer failed %d event",
			    event);

		mutex_enter(&chanp->chan_mutex);
		chanp->chan_req_state = REQ_CMID_NONE;
		chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE;
		chanp->chan_cmid_destroy_state &=
		    ~SOL_CMA_CALLER_EVENT_PROGRESS;
		if (chanp->chan_cmid_destroy_state &
		    SOL_CMA_CALLER_CMID_DESTROYED) {
			cv_broadcast(&chanp->chan_destroy_cv);
			mutex_exit(&chanp->chan_mutex);
		} else {
			mutex_exit(&chanp->chan_mutex);
			rdma_destroy_id(idp);
		}
		return;
	}
ofs_consume_event:
	if (event == RDMA_CM_EVENT_DISCONNECTED || event ==
	    RDMA_CM_EVENT_REJECTED) {
		mutex_enter(&chanp->chan_mutex);
		chanp->chan_connect_flag = SOL_CMA_CONNECT_NONE;
		chanp->chan_qp_hdl = NULL;
		mutex_exit(&chanp->chan_mutex);
	}
	if (event == RDMA_CM_EVENT_DISCONNECTED && root_idp) {
		cma_chan_state_t	chan_state;

		mutex_enter(&chanp->chan_mutex);
		cma_handle_nomore_events(chanp);
		chan_state = cma_get_chan_state(chanp);
		chanp->chan_cmid_destroy_state &=
		    ~SOL_CMA_CALLER_EVENT_PROGRESS;
		if (chanp->chan_cmid_destroy_state &
		    SOL_CMA_CALLER_CMID_DESTROYED) {
			cv_broadcast(&chanp->chan_destroy_cv);
			mutex_exit(&chanp->chan_mutex);
		} else if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING) {
			/* If rdma_destroy_id() was called, destroy CMID */
			mutex_exit(&chanp->chan_mutex);
			cma_destroy_id((struct rdma_cm_id *)chanp);
		} else
			mutex_exit(&chanp->chan_mutex);
		return;
	} else if (event == RDMA_CM_EVENT_DISCONNECTED && !root_idp) {
		/*
		 * Client side TCP CMID :
		 *	If rdma_destroy_id() was called, destroy CMID.
		 *
		 *	If not chan_connect_flag is set to CONNECT_NONE
		 *	so it can be deleted when rdma_destroy_id is
		 *	called.
		 */
		mutex_enter(&chanp->chan_mutex);
		chan_state = cma_get_chan_state(chanp);
		chanp->chan_cmid_destroy_state &=
		    ~SOL_CMA_CALLER_EVENT_PROGRESS;
		if (chanp->chan_cmid_destroy_state &
		    SOL_CMA_CALLER_CMID_DESTROYED) {
			cv_broadcast(&chanp->chan_destroy_cv);
			mutex_exit(&chanp->chan_mutex);
		} else if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING) {
			mutex_exit(&chanp->chan_mutex);
			cma_destroy_id(idp);
		} else
			mutex_exit(&chanp->chan_mutex);
		return;
	} else if (IS_UDP_CMID(idp) && event == RDMA_CM_EVENT_UNREACHABLE) {
		/*
		 * If rdma_destroy_id() was called, destroy CMID
		 * If not chan_connect_flag is set to CONNECT_NONE
		 * so it can be deleted when rdma_destroy_id is
		 * called.
		 */
		mutex_enter(&chanp->chan_mutex);
		chan_state = cma_get_chan_state(chanp);
		chanp->chan_cmid_destroy_state &=
		    ~SOL_CMA_CALLER_EVENT_PROGRESS;
		if (chanp->chan_cmid_destroy_state &
		    SOL_CMA_CALLER_CMID_DESTROYED) {
			cv_broadcast(&chanp->chan_destroy_cv);
			mutex_exit(&chanp->chan_mutex);
		} else if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING) {
			mutex_exit(&chanp->chan_mutex);
			cma_destroy_id(idp);
		} else
			mutex_exit(&chanp->chan_mutex);
		return;
	}

	mutex_enter(&chanp->chan_mutex);
	chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_EVENT_PROGRESS;
	if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
		cv_broadcast(&chanp->chan_destroy_cv);
	mutex_exit(&chanp->chan_mutex);
}

/* Local Static functions */
static struct rdma_cm_id *
cma_alloc_chan(rdma_cm_event_handler evt_hdlr, void *context,
    enum rdma_port_space ps)
{
	struct rdma_cm_id	*rdma_idp;
	sol_cma_chan_t		*chanp;

	chanp = kmem_zalloc(sizeof (sol_cma_chan_t), KM_SLEEP);
	mutex_init(&chanp->chan_mutex, NULL, MUTEX_DRIVER, NULL);
	cv_init(&chanp->chan_destroy_cv, NULL, CV_DRIVER, NULL);
	rdma_idp = &(chanp->chan_rdma_cm);
	rdma_idp->context = context;
	rdma_idp->ps = ps;
	rdma_idp->event_handler = evt_hdlr;
	mutex_enter(&chanp->chan_mutex);
	cma_set_chan_state(chanp, SOL_CMA_CHAN_IDLE);
	avl_create(&chanp->chan_req_avl_tree, sol_cma_req_cmid_cmp,
	    sizeof (sol_cma_chan_t),
	    offsetof(sol_cma_chan_t, chan_req_avl_node));
	avl_create(&chanp->chan_acpt_avl_tree, sol_cma_cmid_cmp,
	    sizeof (sol_cma_chan_t),
	    offsetof(sol_cma_chan_t, chan_acpt_avl_node));
	mutex_exit(&chanp->chan_mutex);

	return (rdma_idp);
}

/* Change the state of sol_cma_chan_t */
static void
cma_set_chan_state(sol_cma_chan_t *chanp, cma_chan_state_t newstate)
{
	ASSERT(MUTEX_HELD(&chanp->chan_mutex));
	chanp->chan_state = newstate;
}

cma_chan_state_t
cma_get_chan_state(sol_cma_chan_t *chanp)
{
	ASSERT(MUTEX_HELD(&chanp->chan_mutex));
	return (chanp->chan_state);
}

/* Check & Swap the state of sol_ucma_chan_t */
static int
cma_cas_chan_state(sol_cma_chan_t *chanp, cma_chan_state_t prevstate,
    cma_chan_state_t newstate)
{
	int	ret = 0;

	ASSERT(MUTEX_HELD(&chanp->chan_mutex));
	if (chanp->chan_state != prevstate)
		ret = -1;
	else
		chanp->chan_state = newstate;

	return (ret);
}

static void
cma_free_listen_list(struct rdma_cm_id *idp)
{
	genlist_entry_t	*entry;
	sol_cma_chan_t	*chanp = (sol_cma_chan_t *)idp;

	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_free_listen_list(%p)", idp);
	mutex_enter(&chanp->chan_mutex);
	entry = remove_genlist_head(&(CHAN_LISTEN_LIST(chanp)));
	mutex_exit(&chanp->chan_mutex);
	while (entry) {
		sol_cma_chan_t	*ep_chanp;

		ep_chanp = (sol_cma_chan_t *)entry->data;
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "fini_ep_chan: %p",
		    ep_chanp);
		if (ibcma_fini_ep_chan(ep_chanp) == 0) {
			genlist_entry_t		*entry1;
			struct ib_device	*device;
			cma_device_t		*cma_device;

			ASSERT(ep_chanp->chan_listenp);
			mutex_enter(&ep_chanp->chan_mutex);
			entry1 = ep_chanp->chan_listenp->listen_ep_dev_entry;
			device = ep_chanp->chan_listenp->listen_ep_device;
			ASSERT(device);
			cma_device = device->data;
			delete_genlist(&cma_device->cma_epchan_list,
			    entry1);
			sol_cma_release_device(
			    (struct rdma_cm_id *)ep_chanp);
			mutex_exit(&ep_chanp->chan_mutex);
			if (ep_chanp->chan_listenp)
				kmem_free(ep_chanp->chan_listenp,
				    sizeof (sol_cma_listen_info_t));

			mutex_destroy(&ep_chanp->chan_mutex);
			cv_destroy(&ep_chanp->chan_destroy_cv);
			kmem_free(ep_chanp, sizeof (sol_cma_chan_t));
			kmem_free(entry, sizeof (genlist_entry_t));
		}

		mutex_enter(&chanp->chan_mutex);
		entry = remove_genlist_head(&(CHAN_LISTEN_LIST(chanp)));
		mutex_exit(&chanp->chan_mutex);
	}
}

/*
 * Destroy a listening CMID when :
 *	a. All CONNECTION REQUEST recieved have been rejected
 *	   or closed.
 *	b. No CONNECTION REQUEST recieved.
 * Do not destroy a listening CMID when :
 *	a. CONNECTION REQUEST has been recieved and not been
 *	   accepted from the passive / server side.
 *	b. CONNECTION REQUEST has been recieved and has been
 *	   accepted from the passive server side.
 *	Mark the listening CMID as destroy pending.
 *
 * For CMIDs created for rdma_connect() or created for a
 * CONNECT request, destroy the CMID only when :
 *       CONNECTION has been closed or rejected.
 *
 *       Mark the CMID as destroy pending.
 *
 * When a connection is rejected or closed :
 *	Check if flag indicates - destroy pending,
 *	cma_destroy_id() is called, this also does
 *
 *	If there is a listening CMID assosiated with it,
 *	   call cma_destroy_if(listen_cmid);
 */
void
cma_destroy_id(struct rdma_cm_id *idp)
{
	sol_cma_chan_t		*chanp = (sol_cma_chan_t *)idp;
	cma_chan_state_t	state;
	ulong_t			acpt_nodes, req_nodes;

	mutex_enter(&chanp->chan_mutex);
	acpt_nodes = avl_numnodes(&chanp->chan_acpt_avl_tree);
	req_nodes = avl_numnodes(&chanp->chan_req_avl_tree);
	state = cma_get_chan_state(chanp);
	SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_destroy_id(%p)- "
	    "est CMIDs %ld, req CMID %ld, listen_root %p, state %x, %x",
	    idp, acpt_nodes, req_nodes, chanp->listen_root,
	    state, chanp->chan_req_state);

	/*
	 * If there are either REQ recieved or Established CMIDs just return.
	 * rdma_destroy() for these CMIDs can be called by client later.
	 */
	if (acpt_nodes || req_nodes) {
		cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROY_PENDING);
		mutex_exit(&chanp->chan_mutex);
		return;
	}
	cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROYING);
	avl_destroy(&chanp->chan_req_avl_tree);
	avl_destroy(&chanp->chan_acpt_avl_tree);

	mutex_exit(&chanp->chan_mutex);
	if (idp->route.path_rec) {
		kmem_free(idp->route.path_rec,
		    sizeof (struct ib_sa_path_rec) * idp->route.num_paths);
		idp->route.path_rec = NULL;
	}

	switch (chanp->chan_xport_type) {
	case SOL_CMA_XPORT_NONE :
		break;
	case SOL_CMA_XPORT_IB :
		rdma_ib_destroy_id(idp);
		break;
#ifdef	IWARP_SUPPORT
	case SOL_CMA_XPORT_IWARP :
		rdma_iw_destroy_id(idp);
		break;
#endif	/* IWARP_SUPPORT */
	default :
		SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
		    "cma_destroy_id: Unsupported xport type %x",
		    chanp->chan_xport_type);
		break;
	}

	/*
	 * Flush out & Free all listeners wrt to this ID
	 * No locking is required as this code is executed
	 * all REQ CMIDs have been destroyed. listen_list
	 * will therefore not be modified during this loop.
	 */
	if (chanp->chan_listenp) {
		cma_free_listen_list(idp);
		cma_fini_listen_root(chanp);
		kmem_free((void *)chanp->chan_listenp,
		    sizeof (sol_cma_listen_info_t));
		chanp->chan_listenp = NULL;
	}

	if (chanp->listen_root) {
		struct rdma_cm_id	*root_idp;
		sol_cma_chan_t		*root_chanp;

		root_idp = chanp->listen_root;
		root_chanp = (sol_cma_chan_t *)root_idp;
		mutex_enter(&root_chanp->chan_mutex);
		state = cma_get_chan_state(root_chanp);
		acpt_nodes = avl_numnodes(&root_chanp->chan_acpt_avl_tree);
		req_nodes = avl_numnodes(&root_chanp->chan_req_avl_tree);
		mutex_exit(&root_chanp->chan_mutex);
		SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_destroy_id(%p)-"
		    " root idp %p, state %x, acpt_nodes %ld, req_nodes %ld",
		    idp, root_idp, state, acpt_nodes, req_nodes);

		if (state == SOL_CMA_CHAN_DESTROY_PENDING &&
		    req_nodes == 0UL && acpt_nodes == 0UL) {
			mutex_enter(&root_chanp->chan_mutex);
			root_chanp->chan_req_state = REQ_CMID_NONE;
			mutex_exit(&root_chanp->chan_mutex);
			cma_destroy_id(root_idp);
		} else if (state == SOL_CMA_CHAN_DESTROY_WAIT &&
		    req_nodes == 0UL && acpt_nodes == 0UL) {
			mutex_enter(&root_chanp->chan_mutex);
			cma_set_chan_state(root_chanp,
			    SOL_CMA_CHAN_DESTROY_PENDING);
			root_chanp->chan_req_state = REQ_CMID_NONE;
			cv_broadcast(&root_chanp->chan_destroy_cv);
			mutex_exit(&root_chanp->chan_mutex);
		}
	}

	mutex_destroy(&chanp->chan_mutex);
	cv_destroy(&chanp->chan_destroy_cv);
	kmem_free(chanp, sizeof (sol_cma_chan_t));
}

/*
 * Server TCP disconnect for an established channel.
 *	Remove from EST AVL tree.
 *
 *	If destroy_id() has been called for the listening
 *	CMID and there are no more CMIDs with pending
 *	events corresponding to the listening CMID, free
 *	the listening CMID.
 *
 *
 *	If not chan_connect_flag is set to CONNECT_NONE
 *	so it can be deleted when rdma_destroy_id is
 *	called.
 */
static void
cma_handle_nomore_events(sol_cma_chan_t *chanp)
{
	struct rdma_cm_id	*idp, *root_idp;
	sol_cma_chan_t		*root_chanp;
	cma_chan_state_t	state;
	ulong_t			req_nodes, acpt_nodes;

	idp = (struct rdma_cm_id *)chanp;
	root_idp = CHAN_LISTEN_ROOT(chanp);
	root_chanp = (sol_cma_chan_t *)root_idp;
	if (!root_chanp)
		return;

	CHAN_LISTEN_ROOT(chanp) = NULL;
	mutex_enter(&root_chanp->chan_mutex);
	root_chanp->chan_req_total_cnt--;
	if (!root_chanp->chan_req_total_cnt)
		root_chanp->chan_req_state = REQ_CMID_NONE;
	if (root_idp->ps == RDMA_PS_TCP && chanp->chan_req_state ==
	    REQ_CMID_ACCEPTED) {
		avl_remove(&root_chanp->chan_acpt_avl_tree, idp);
		chanp->chan_req_state = REQ_CMID_NONE;
	}
	if (chanp->chan_req_state == REQ_CMID_CREATED ||
	    chanp->chan_req_state == REQ_CMID_NOTIFIED) {
		avl_remove(&root_chanp->chan_req_avl_tree, idp);
		chanp->chan_req_state = REQ_CMID_NONE;
	}
	state = cma_get_chan_state(root_chanp);
	req_nodes = avl_numnodes(&root_chanp->chan_req_avl_tree);
	acpt_nodes = avl_numnodes(&root_chanp->chan_acpt_avl_tree);
	mutex_exit(&root_chanp->chan_mutex);
	if (state == SOL_CMA_CHAN_DESTROY_PENDING && req_nodes == 0UL &&
	    acpt_nodes == 0UL)
		cma_destroy_id(root_idp);
}

extern int ib_modify_qp(struct ib_qp *, struct ib_qp_attr *, int);
extern int rdma_init_qp_attr(struct rdma_cm_id *, struct ib_qp_attr *,
    int *);

static int
cma_init_ud_qp(sol_cma_chan_t *chanp, struct ib_qp *qp)
{
	struct ib_qp_attr qp_attr;
	int qp_attr_mask, ret;

	qp_attr.qp_state = IB_QPS_INIT;
	ret = rdma_init_qp_attr(&chanp->chan_rdma_cm, &qp_attr, &qp_attr_mask);
	if (ret)
		return (ret);

	ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
	if (ret)
		return (ret);

	qp_attr.qp_state = IB_QPS_RTR;
	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
	if (ret)
		return (ret);

	qp_attr.qp_state = IB_QPS_RTS;
	qp_attr.sq_psn = 0;
	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);

	return (ret);
}

static int
cma_init_conn_qp(sol_cma_chan_t *chanp, struct ib_qp *qp)
{
	struct ib_qp_attr qp_attr;
	int qp_attr_mask, ret;

	qp_attr.qp_state = IB_QPS_INIT;
	ret = rdma_init_qp_attr(&chanp->chan_rdma_cm, &qp_attr, &qp_attr_mask);
	if (ret)
		return (ret);

	return (ib_modify_qp(qp, &qp_attr, qp_attr_mask));
}

static inline int
cma_is_ud_ps(enum rdma_port_space ps)
{
	return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
}

int
rdma_create_qp(struct rdma_cm_id *idp, struct ib_pd *pd,
    struct ib_qp_init_attr *qp_init_attr)
{
	sol_cma_chan_t	*chanp;
	struct ib_qp	*qp;
	int		ret;
	ofs_client_t	*dev_ofs_client;

	ASSERT(idp);
	chanp = (sol_cma_chan_t *)idp;
	if (idp->device->node_guid != pd->device->node_guid)
		return (-EINVAL);

	dev_ofs_client = (ofs_client_t *)pd->device->clnt_hdl;
	rdma_map_id2clnthdl(idp, dev_ofs_client->ibt_hdl, NULL);

	qp = ib_create_qp(pd, qp_init_attr);
	if ((uintptr_t)qp >= (uintptr_t)-0xFFF) {
		return ((intptr_t)qp);
	}
	rdma_map_id2qphdl(idp, (void *)qp->ibt_qp);

	if (cma_is_ud_ps(idp->ps)) {
		ret = cma_init_ud_qp(chanp, qp);
	} else {
		ret = cma_init_conn_qp(chanp, qp);
	}

	if (ret) {
		goto err;
	}

	idp->qp = qp;
	chanp->chan_qp_num = qp->qp_num;
	chanp->chan_is_srq = (qp->srq != NULL);
	return (0);
err:
	(void) ib_destroy_qp(qp);
	return (ret);
}

void
rdma_destroy_qp(struct rdma_cm_id *idp)
{
	ASSERT(idp);
	(void) ib_destroy_qp(idp->qp);
	idp->qp = NULL;
}