/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "dapl.h"
#include "dapl_adapter_util.h"
#include "dapl_evd_util.h"
#include "dapl_cr_util.h"
#include "dapl_lmr_util.h"
#include "dapl_rmr_util.h"
#include "dapl_cookie.h"
#include "dapl_ring_buffer_util.h"
#include "dapl_vendor.h"
#include "dapl_tavor_ibtf_impl.h"

/* Function prototypes */
static DAT_RETURN dapli_ib_cq_resize_internal(DAPL_EVD *, DAT_COUNT);

/*
 * The following declarations/fn are to used by the base library
 * place holder for now
 */

int	g_loopback_connection = 0;

/*
 * dapl_ib_cq_alloc
 *
 * Alloc a CQ
 *
 * Input:
 *	ia_handle		IA handle
 *	evd_ptr			pointer to EVD struct
 *	cno_ptr			pointer to CNO struct
 *	cqlen			minimum QLen
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cq_alloc(
	IN  DAPL_IA		*ia_ptr,
	IN  DAPL_EVD		*evd_ptr,
	IN  DAPL_CNO		*cno_ptr,
	IN  DAT_COUNT		*cqlen)
{
	dapl_evd_create_t	create_msg;
	dapl_evd_free_t		free_msg;
	ib_cq_handle_t		cq_handle = IB_INVALID_HANDLE;
	int			ia_fd;
	int			hca_fd;
	int			retval;
	mlnx_umap_cq_data_out_t	*mcq;

	/* cq handle is created even for non-cq type events */
	/* since cq handle is where the evd fd gets stored. */
	cq_handle = (ib_cq_handle_t)
	    dapl_os_alloc(sizeof (struct dapls_ib_cq_handle));
	if (cq_handle == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "cq_alloc: evd_ptr 0x%p, cq_handle == NULL\n",
		    evd_ptr);
		return (DAT_INSUFFICIENT_RESOURCES);
	}

	(void) dapl_os_memzero(cq_handle, sizeof (*cq_handle));

	/* get the hca information from ia_ptr */
	(void) dapl_os_memzero(&create_msg, sizeof (create_msg));
	create_msg.evd_flags = evd_ptr->evd_flags;
	create_msg.evd_cookie = (uintptr_t)evd_ptr;
	if (cno_ptr != NULL) {
		create_msg.evd_cno_hkey =
		    (uint64_t)cno_ptr->ib_cno_handle;
	}
	if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) {
		create_msg.evd_cq_size = (uint32_t)*cqlen;
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "cq_alloc: evd 0x%p, flags 0x%x, cookie 0x%llx, hkey 0x%llx,\n"
	    "          cno_hkey 0x%llx, cq_size %d\n", evd_ptr,
	    create_msg.evd_flags, create_msg.evd_cookie, create_msg.evd_hkey,
	    create_msg.evd_cno_hkey, create_msg.evd_cq_size);

	ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
	hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
	mcq = (mlnx_umap_cq_data_out_t *)create_msg.evd_cq_data_out;

	/* The next line is only needed for backward compatibility */
	mcq->mcq_rev = MLNX_UMAP_IF_VERSION;

	/* call into driver to allocate cq */
	retval = ioctl(ia_fd, DAPL_EVD_CREATE, &create_msg);
	if (retval != 0 || mcq->mcq_rev != MLNX_UMAP_IF_VERSION) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "cq_alloc: evd_create failed, %s\n", strerror(errno));
		dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle));
		return (dapls_convert_error(errno, retval));
	}
	(void) dapl_os_memzero(cq_handle, sizeof (struct dapls_ib_cq_handle));
	dapl_os_lock_init(&cq_handle->cq_wrid_wqhdr_lock);

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "cq_alloc: created, evd 0x%p, hkey 0x%016llx\n\n", evd_ptr,
	    create_msg.evd_hkey);

	cq_handle->evd_hkey = create_msg.evd_hkey;

	if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) {

		/*
		 * allocate a hash table for wrid management, the key is
		 * a combination of QPnumber and SEND/RECV type. This is
		 * required only for evd which have a CQ mapped to
		 * it.
		 */
		if (DAT_SUCCESS != dapls_hash_create(DAPL_MED_HASHSIZE,
		    DAT_FALSE, &cq_handle->cq_wrid_wqhdr_list)) {
			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
			    "cq_alloc: hash_create failed\n");
			dapl_os_free(cq_handle,
			    sizeof (struct dapls_ib_cq_handle));
			return (DAT_INSUFFICIENT_RESOURCES |
			    DAT_RESOURCE_MEMORY);
		}

		dapl_os_assert(create_msg.evd_cq_real_size > 0);

		/* In the case of Arbel or Hermon */
		if (mcq->mcq_polldbr_mapoffset != 0 ||
		    mcq->mcq_polldbr_maplen != 0)
			cq_handle->cq_poll_dbp = dapls_ib_get_dbp(
			    mcq->mcq_polldbr_maplen, hca_fd,
			    mcq->mcq_polldbr_mapoffset,
			    mcq->mcq_polldbr_offset);
		if (mcq->mcq_armdbr_mapoffset != 0 ||
		    mcq->mcq_armdbr_maplen != 0)
			cq_handle->cq_arm_dbp = dapls_ib_get_dbp(
			    mcq->mcq_armdbr_maplen, hca_fd,
			    mcq->mcq_armdbr_mapoffset,
			    mcq->mcq_armdbr_offset);

		cq_handle->cq_addr = (tavor_hw_cqe_t *)(void *) mmap64(
		    (void *)0, mcq->mcq_maplen,
		    (PROT_READ | PROT_WRITE), MAP_SHARED, hca_fd,
		    mcq->mcq_mapoffset);

		if (cq_handle->cq_addr == MAP_FAILED ||
		    cq_handle->cq_poll_dbp == MAP_FAILED ||
		    cq_handle->cq_arm_dbp == MAP_FAILED) {
			free_msg.evf_hkey = cq_handle->evd_hkey;
			retval = ioctl(ia_fd, DAPL_EVD_FREE, &free_msg);
			if (retval != 0) {
				dapl_dbg_log(DAPL_DBG_TYPE_ERR,
				    "cq_alloc: EVD_FREE err:%s\n",
				    strerror(errno));
			}

			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
			    "cq_alloc: DAPL_CQ_ALLOC failed\n");
			/* free the hash table we created */
			(void) dapls_hash_free(cq_handle->cq_wrid_wqhdr_list);
			dapl_os_free(cq_handle,
			    sizeof (struct dapls_ib_cq_handle));
			return (DAT_INSUFFICIENT_RESOURCES);
		}

		cq_handle->cq_map_offset = mcq->mcq_mapoffset;
		cq_handle->cq_map_len = mcq->mcq_maplen;
		cq_handle->cq_num = mcq->mcq_cqnum;
		/*
		 * cq_size is the actual depth of the CQ which is 1 more
		 * than what ibt_alloc_cq reports. However the application
		 * can only use (cq_size - 1) entries.
		 */
		cq_handle->cq_size = create_msg.evd_cq_real_size + 1;
		cq_handle->cq_cqesz = mcq->mcq_cqesz;
		cq_handle->cq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
		*cqlen = create_msg.evd_cq_real_size;

		DAPL_INIT_CQ(ia_ptr)(cq_handle);
	}

	evd_ptr->ib_cq_handle = cq_handle;
	return (DAT_SUCCESS);
}


/*
 * dapl_ib_cq_resize
 *
 * Resize a CQ
 *
 * Input:
 *	evd_ptr			pointer to EVD struct
 *	cqlen			new length of the cq
 * Output:
 *	none
 *
 * Returns:
 *	DAT_SUCCESS
 *	DAT_INVALID_HANDLE
 *	DAT_INTERNAL_ERROR
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cq_resize(
	IN  DAPL_EVD		*evd_ptr,
	IN  DAT_COUNT		cqlen)
{
	ib_cq_handle_t	cq_handle;
	DAT_RETURN	dat_status;

	dat_status = dapli_ib_cq_resize_internal(evd_ptr, cqlen);
	if (DAT_INSUFFICIENT_RESOURCES == dat_status) {
		cq_handle = evd_ptr->ib_cq_handle;
		/* attempt to resize back to the current size */
		dat_status = dapli_ib_cq_resize_internal(evd_ptr,
		    cq_handle->cq_size - 1);
		if (DAT_SUCCESS != dat_status) {
			/*
			 * XXX this is catastrophic need to post an event
			 * to the async evd
			 */
			return (DAT_INTERNAL_ERROR);
		}
	}

	return (dat_status);
}

/*
 * dapli_ib_cq_resize_internal
 *
 * An internal routine to resize a CQ.
 *
 * Input:
 *	evd_ptr			pointer to EVD struct
 *	cqlen			new length of the cq
 * Output:
 *	none
 *
 * Returns:
 *	DAT_SUCCESS
 *	DAT_INVALID_HANDLE
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
static DAT_RETURN
dapli_ib_cq_resize_internal(
	IN  DAPL_EVD		*evd_ptr,
	IN  DAT_COUNT		cqlen)
{
	ib_cq_handle_t		cq_handle;
	dapl_cq_resize_t	resize_msg;
	int			ia_fd;
	int			hca_fd;
	int			retval;
	mlnx_umap_cq_data_out_t	*mcq;
	DAPL_HCA		*hca_ptr;
	dapls_hw_cqe_t		cq_addr;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "dapls_ib_cq_resize: evd 0x%p cq 0x%p "
	    "evd_hkey 0x%016llx cqlen %d\n",
	    evd_ptr, (void *)evd_ptr->ib_cq_handle,
	    evd_ptr->ib_cq_handle->evd_hkey, cqlen);

	cq_handle = evd_ptr->ib_cq_handle;
	/*
	 * Since CQs are created in powers of 2 with one non-usable slot,
	 * its possible that the previously allocated CQ has sufficient
	 * entries. If the current cq is big enough and it is mapped in
	 * we are done.
	 */
	if ((cqlen < cq_handle->cq_size) && (cq_handle->cq_addr)) {
		return (DAT_SUCCESS);
	}

	hca_ptr = evd_ptr->header.owner_ia->hca_ptr;

	/* unmap the CQ before resizing it */
	if (hca_ptr->hermon_resize_cq == 0) {
		if ((cq_handle->cq_addr) &&
		    (munmap((char *)cq_handle->cq_addr,
		    cq_handle->cq_map_len) < 0)) {
			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
			    "cq_resize: munmap(%p:0x%llx) failed(%d)\n",
			    cq_handle->cq_addr, cq_handle->cq_map_len, errno);
			return (DAT_INVALID_HANDLE);
		}
		/* cq_addr is unmapped and no longer valid */
		cq_handle->cq_addr = NULL;
	}

	ia_fd = hca_ptr->ib_hca_handle->ia_fd;
	hca_fd = hca_ptr->ib_hca_handle->hca_fd;

	(void) dapl_os_memzero(&resize_msg, sizeof (resize_msg));
	mcq = (mlnx_umap_cq_data_out_t *)resize_msg.cqr_cq_data_out;
	resize_msg.cqr_evd_hkey = cq_handle->evd_hkey;
	resize_msg.cqr_cq_new_size = cqlen;

	/* The next line is only needed for backward compatibility */
	mcq->mcq_rev = MLNX_UMAP_IF_VERSION;
	retval = ioctl(ia_fd, DAPL_CQ_RESIZE, &resize_msg);
	if (retval != 0 || mcq->mcq_rev != MLNX_UMAP_IF_VERSION) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "dapls_ib_cq_resize: evd 0x%p, err: %s\n",
		    evd_ptr, strerror(errno));
		if (errno == EINVAL) { /* Couldn't find evd for this cq */
			return (DAT_INVALID_HANDLE);
		} else { /* Need to retry resize with a smaller qlen */
			return (DAT_INSUFFICIENT_RESOURCES);
		}
	}

	dapl_os_assert(cq_handle->cq_num == mcq->mcq_cqnum);

	/* In the case of Arbel or Hermon */
	if (mcq->mcq_polldbr_mapoffset != 0 ||
	    mcq->mcq_polldbr_maplen != 0)
		cq_handle->cq_poll_dbp = dapls_ib_get_dbp(
		    mcq->mcq_polldbr_maplen, hca_fd,
		    mcq->mcq_polldbr_mapoffset,
		    mcq->mcq_polldbr_offset);
	if (mcq->mcq_armdbr_mapoffset != 0 ||
	    mcq->mcq_armdbr_maplen != 0)
		cq_handle->cq_arm_dbp = dapls_ib_get_dbp(
		    mcq->mcq_armdbr_maplen, hca_fd,
		    mcq->mcq_armdbr_mapoffset,
		    mcq->mcq_armdbr_offset);

	cq_addr = (tavor_hw_cqe_t *)(void *)mmap64((void *)0,
	    mcq->mcq_maplen, (PROT_READ | PROT_WRITE),
	    MAP_SHARED, hca_fd, mcq->mcq_mapoffset);

	if (cq_addr == MAP_FAILED ||
	    cq_handle->cq_poll_dbp == MAP_FAILED ||
	    cq_handle->cq_arm_dbp == MAP_FAILED) {
		if (hca_ptr->hermon_resize_cq == 0)
			cq_handle->cq_addr = NULL;
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "cq_resize: mmap failed(%d)\n", errno);
		/* Need to retry resize with a smaller qlen */
		return (DAT_INSUFFICIENT_RESOURCES);
	}

	if (hca_ptr->hermon_resize_cq == 0) {
		cq_handle->cq_addr = cq_addr;
		cq_handle->cq_map_offset = mcq->mcq_mapoffset;
		cq_handle->cq_map_len = mcq->mcq_maplen;
		cq_handle->cq_size = resize_msg.cqr_cq_real_size + 1;
		cq_handle->cq_cqesz = mcq->mcq_cqesz;
		/*
		 * upon resize the old events are moved to the start of the CQ
		 * hence we need to reset the consumer index too
		 */
		cq_handle->cq_consindx = 0;
	} else {	/* Hermon */
		cq_handle->cq_resize_addr = cq_addr;
		cq_handle->cq_resize_map_offset = mcq->mcq_mapoffset;
		cq_handle->cq_resize_map_len = mcq->mcq_maplen;
		cq_handle->cq_resize_size = resize_msg.cqr_cq_real_size + 1;
		cq_handle->cq_resize_cqesz = mcq->mcq_cqesz;
	}

	return (DAT_SUCCESS);
}

/*
 * dapl_ib_cq_free
 *
 * Free a CQ
 *
 * Input:
 *	ia_handle		IA handle
 *	evd_ptr			pointer to EVD struct
 * Output:
 *	none
 *
 * Returns:
 *	DAT_SUCCESS
 *	DAT_INVALID_HANDLE
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cq_free(
	IN  DAPL_IA		*ia_ptr,
	IN  DAPL_EVD		*evd_ptr)
{
	dapl_evd_free_t		args;
	int			retval;
	ib_cq_handle_t		cq_handle = evd_ptr->ib_cq_handle;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "dapls_ib_cq_free: evd 0x%p cq 0x%p hkey %016llx\n", evd_ptr,
	    (void *)evd_ptr->ib_cq_handle, evd_ptr->ib_cq_handle->evd_hkey);

	/* If the cq was mmap'd unmap it before freeing it */
	if ((cq_handle->cq_addr) &&
	    (munmap((char *)cq_handle->cq_addr, cq_handle->cq_map_len) < 0)) {
			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
			    "cq_free: (%p:0x%llx)\n", cq_handle->cq_addr,
			    cq_handle->cq_map_len);
	}


	args.evf_hkey = cq_handle->evd_hkey;

	retval = ioctl(ia_ptr->hca_ptr->ib_hca_handle->ia_fd,
	    DAPL_EVD_FREE, &args);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "dapls_ib_cq_free: evd 0x%p, err: %s\n",
		    evd_ptr, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle));
	evd_ptr->ib_cq_handle = NULL;

	return (DAT_SUCCESS);
}

/*
 * dapl_set_cq_notify
 *
 * Set up CQ completion notifications
 *
 * Input:
 *	ia_handle		IA handle
 *	evd_ptr			pointer to EVD struct
 *
 * Output:
 *	none
 *
 * Returns:
 *	DAT_SUCCESS
 *	DAT_INVALID_HANDLE
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
/* ARGSUSED */
DAT_RETURN
dapls_set_cq_notify(
	IN  DAPL_IA		*ia_ptr,
	IN  DAPL_EVD		*evd_ptr)
{
	int			retval;
	ib_cq_handle_t		cq_handle = evd_ptr->ib_cq_handle;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "dapls_ib_cq_notify: evd 0x%p cq 0x%p\n", evd_ptr,
	    (void *)cq_handle);

	retval = DAPL_NOTIFY(evd_ptr)(cq_handle, IB_NOTIFY_ON_NEXT_COMP, 0);

	return (retval);

}

/* ARGSUSED */
DAT_RETURN
dapls_set_cqN_notify(
	IN  DAPL_IA		*ia_ptr,
	IN  DAPL_EVD		*evd_ptr,
	IN  uint32_t		num_events)
{
	int			retval;
	ib_cq_handle_t		cq_handle = evd_ptr->ib_cq_handle;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "dapls_set_cqN_notify:evd %p cq %p num_events %d\n", evd_ptr,
	    (void *)cq_handle, num_events);

	retval = DAPL_NOTIFY(evd_ptr)(cq_handle, IB_NOTIFY_ON_NEXT_NCOMP,
	    num_events);

	return (retval);

}

/*
 * dapls_ib_cqd_create
 *
 * Set up CQ notification event thread
 *
 * Input:
 *	ia_handle		IA handle
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INVALID_HANDLE
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
/* ARGSUSED */
DAT_RETURN
dapls_ib_cqd_create(
	IN  DAPL_HCA		*hca_ptr)
{
	return (DAT_SUCCESS);
}


/*
 * dapl_cqd_destroy
 *
 * Destroy CQ notification event thread
 *
 * Input:
 *	ia_handle		IA handle
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INVALID_HANDLE
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cqd_destroy(
	IN  DAPL_HCA		*hca_ptr)
{
	dapl_evd_free_t		args;
	ib_cq_handle_t		cq_handle;
	int			retval;

	if (hca_ptr->null_ib_cq_handle != IB_INVALID_HANDLE) {
		/* free up the dummy cq */
		cq_handle = hca_ptr->null_ib_cq_handle;
		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
		    "dapls_ib_cqd_destroy: cq %p\n", (void *)cq_handle);

		args.evf_hkey = cq_handle->evd_hkey;

		retval = ioctl(hca_ptr->ib_hca_handle->ia_fd,
		    DAPL_EVD_FREE, &args);
		if (retval != 0) {
			dapl_dbg_log(DAPL_DBG_TYPE_ERR,
			    "dapls_ib_cqd_destroy: EVD_FREE err:%d errno:%d\n",
			    retval, errno);
		}

		dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle));
		hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
	}

	return (DAT_SUCCESS);
}


/*
 * dapl_ib_pd_alloc
 *
 * Alloc a PD
 *
 * Input:
 *	ia_handle		IA handle
 *	PZ_ptr			pointer to PZEVD struct
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_pd_alloc(
	IN  DAPL_IA 		*ia,
	IN  DAPL_PZ 		*pz)
{
	struct dapls_ib_pd_handle *pd_p;
	dapl_pd_alloc_t args;
	int retval;

	pd_p = (struct dapls_ib_pd_handle *)dapl_os_alloc(sizeof (*pd_p));
	if (pd_p == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "pd_alloc: ia 0x%p, pz 0x%p, cannot allocate pd\n",
		    ia, pz);
		return (DAT_INSUFFICIENT_RESOURCES);
	}
	retval = ioctl(ia->hca_ptr->ib_hca_handle->ia_fd,
	    DAPL_PD_ALLOC, &args);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "pd_alloc: ia 0x%p, pz 0x%p, cannot create pd, "
		    "err: %s\n", ia, pz, strerror(errno));
		dapl_os_free(pd_p, sizeof (*pd_p));
		return (dapls_convert_error(errno, retval));
	}

	pd_p->pd_hkey = args.pda_hkey;
	pz->pd_handle = pd_p;
	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "pd_alloc: successful, ia 0x%p, pz 0x%p, hkey %016llx\n",
	    ia, pz, args.pda_hkey);

	return (DAT_SUCCESS);
}


/*
 * dapl_ib_pd_free
 *
 * Free a PD
 *
 * Input:
 *	ia_handle		IA handle
 *	PZ_ptr			pointer to PZ struct
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_pd_free(
	IN  DAPL_PZ 		*pz)
{
	struct dapls_ib_pd_handle *pd_p;
	dapl_pd_free_t args;
	int retval;

	pd_p = (struct dapls_ib_pd_handle *)pz->pd_handle;
	args.pdf_hkey = pd_p->pd_hkey;

	retval = ioctl(pz->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd,
	    DAPL_PD_FREE, &args);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "pd_free: pz 0x%p, cannot free pd\n", pz);
		return (dapls_convert_error(errno, retval));
	}
	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "pd_free: pz 0x%p, hkey %016llx, freed\n", pz, pd_p->pd_hkey);
	dapl_os_free((void *)pd_p, sizeof (*pd_p));
	pz->pd_handle = NULL;
	return (DAT_SUCCESS);
}


/*
 * dapl_ib_mr_register
 *
 * Register a virtual memory region
 *
 * Input:
 *	ia_handle		IA handle
 *	lmr			pointer to dapl_lmr struct
 *	virt_addr		virtual address of beginning of mem region
 *	length			length of memory region
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_register(
	IN  DAPL_IA 		*ia,
	IN  DAPL_LMR		*lmr,
	IN  DAT_PVOID		virt_addr,
	IN  DAT_VLEN		length,
	IN  DAT_MEM_PRIV_FLAGS  privileges)
{
	dapl_mr_register_t	reg_msg;
	ib_mr_handle_t		mr_handle;
	DAPL_PZ *		pz_handle;
	int			ia_fd;
	int			retval;

	ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd;
	mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle));
	if (mr_handle == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register: lmr 0x%p, ia 0x%p, "
		    "cannot alloc mr_handle\n", lmr, ia);
		return (DAT_INSUFFICIENT_RESOURCES);
	}
	pz_handle = ((DAPL_PZ *)lmr->param.pz_handle);
	if (pz_handle == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register: lmr 0x%p, ia 0x%p, "
		    "pz_handle == NULL!\n", lmr, ia);
		dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
		return (DAT_INVALID_PARAMETER);
	}
	reg_msg.mr_pd_hkey = pz_handle->pd_handle->pd_hkey;
	reg_msg.mr_vaddr = (ib_vaddr_t)(uintptr_t)virt_addr;
	reg_msg.mr_len = (ib_memlen_t)length;
	reg_msg.mr_flags = (ibt_mr_flags_t)
	    dapl_lmr_convert_privileges(privileges);
	reg_msg.mr_flags |= IBT_MR_ENABLE_WINDOW_BIND;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_register: lmr 0x%p, pd_hkey 0x%016llx, vaddr 0x%016llx, "
	    "len %llu, flags 0x%x\n", lmr, reg_msg.mr_pd_hkey,
	    reg_msg.mr_vaddr, reg_msg.mr_len, reg_msg.mr_flags);

	/* call into driver to allocate MR resource */
	retval = ioctl(ia_fd, DAPL_MR_REGISTER, &reg_msg);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register: lmr 0x%p, failed (%s)\n",
		    lmr, strerror(errno));
		dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
		return (dapls_convert_error(errno, retval));
	}
	mr_handle->mr_hkey = reg_msg.mr_hkey;
	lmr->param.lmr_context = (DAT_LMR_CONTEXT)reg_msg.mr_lkey;
	lmr->param.rmr_context = (DAT_RMR_CONTEXT)reg_msg.mr_rkey;
	lmr->param.registered_address = reg_msg.mr_vaddr;
	lmr->param.registered_size = reg_msg.mr_len;
	lmr->mr_handle = mr_handle;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_register: successful, lmr 0x%p, mr_hkey 0x%016llx, "
	    "lmr_ctx 0x%08x\n\n", lmr, reg_msg.mr_hkey,
	    reg_msg.mr_lkey);
	return (DAT_SUCCESS);
}

/*
 * dapl_ib_mr_register_shared
 *
 * Register a shared virtual memory region
 *
 * Input:
 *	ia_handle		IA handle
 *	lmr			pointer to dapl_lmr struct
 *	virt_addr		virtual address of beginning of mem region
 *	cookie			shared memory identifer
 *	length			length of memory region
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_register_shared(
	IN  DAPL_IA 		*ia,
	IN  DAPL_LMR		*lmr,
	IN  DAT_PVOID		virt_addr,
	IN  DAT_VLEN		length,
	IN  DAT_LMR_COOKIE	cookie,
	IN  DAT_MEM_PRIV_FLAGS  privileges)
{
	dapl_mr_register_shared_t	reg_msg;
	ib_mr_handle_t			mr_handle;
	DAPL_PZ				*pz_handle;
	int				ia_fd, i;
	int				retval;

	ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd;
	mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle));
	if (mr_handle == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register_shared: lmr 0x%p, ia 0x%p, "
		    "cannot alloc mr_handle\n", lmr, ia);
		return (DAT_INSUFFICIENT_RESOURCES);
	}
	pz_handle = ((DAPL_PZ *)lmr->param.pz_handle);
	if (pz_handle == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register_shared: lmr 0x%p, ia 0x%p, "
		    "pz_handle == NULL!\n", lmr, ia);
		dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
		return (DAT_INVALID_PARAMETER);
	}
	reg_msg.mrs_pd_hkey = pz_handle->pd_handle->pd_hkey;
	reg_msg.mrs_vaddr = (ib_vaddr_t)(uintptr_t)virt_addr;
	reg_msg.mrs_len = (ib_memlen_t)length;
	reg_msg.mrs_flags = (ibt_mr_flags_t)
	    dapl_lmr_convert_privileges(privileges);
	reg_msg.mrs_flags |= IBT_MR_ENABLE_WINDOW_BIND;
	/*CONSTCOND*/
	dapl_os_assert(DAT_LMR_COOKIE_SIZE == sizeof (reg_msg.mrs_shm_cookie));
	(void) dapl_os_memcpy((void *)&reg_msg.mrs_shm_cookie, (void *)cookie,
	    DAT_LMR_COOKIE_SIZE);

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_register_shared: lmr 0x%p, pd_hkey 0x%016llx, "
	    "vaddr 0x%016llx, len %llu, flags 0x%x\n",
	    lmr, reg_msg.mrs_pd_hkey, reg_msg.mrs_vaddr, reg_msg.mrs_len,
	    reg_msg.mrs_flags);

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_register_shared: cookie \n0x");
	for (i = 4; i >= 0; i--) {
		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
		    "%016llx", reg_msg.mrs_shm_cookie.mc_uint_arr[i]);
	}
	dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "\n");

	/* call into driver to allocate MR resource */
	retval = ioctl(ia_fd, DAPL_MR_REGISTER_SHARED, &reg_msg);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register_shared: lmr 0x%p, failed (%s)\n",
		    lmr, strerror(errno));
		dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
		return (dapls_convert_error(errno, retval));
	}
	mr_handle->mr_hkey = reg_msg.mrs_hkey;
	lmr->param.lmr_context = (DAT_LMR_CONTEXT)reg_msg.mrs_lkey;
	lmr->param.rmr_context = (DAT_RMR_CONTEXT)reg_msg.mrs_rkey;
	lmr->param.registered_address = reg_msg.mrs_vaddr;
	lmr->param.registered_size = reg_msg.mrs_len;
	lmr->mr_handle = mr_handle;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_register_shared: successful, lmr 0x%p, mr_hkey 0x%016llx, "
	    "lmr_ctx 0x%08x\n\n", lmr, reg_msg.mrs_hkey,
	    reg_msg.mrs_lkey);
	return (DAT_SUCCESS);
}

/*
 * dapl_ib_mr_deregister
 *
 * Free a memory region
 *
 * Input:
 *	lmr			pointer to dapl_lmr struct
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_deregister(
	IN  DAPL_LMR		*lmr)
{
	dapl_mr_deregister_t args;
	int retval;

	args.mrd_hkey = lmr->mr_handle->mr_hkey;
	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_deregister: lmr 0x%p, hkey 0x%016llx, lmr_ctx 0x%08x\n"
	    "               vaddr 0x%016llx, len %llu, flags 0x%x\n",
	    lmr, args.mrd_hkey, lmr->param.lmr_context,
	    lmr->param.registered_address, lmr->param.registered_size,
	    dapl_lmr_convert_privileges(lmr->param.mem_priv) |
	    IBT_MR_ENABLE_WINDOW_BIND);

	/* call into driver to do MR deregister */
	retval = ioctl(lmr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd,
	    DAPL_MR_DEREGISTER, &args);

	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_deregister: lmr 0x%p, failed (%s)\n",
		    lmr, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_deregister: successful\n\n");
	dapl_os_free(lmr->mr_handle, sizeof (struct dapls_ib_mr_handle));
	lmr->mr_handle = NULL;
	return (DAT_SUCCESS);
}


/*
 * dapl_ib_mr_register_lmr
 *
 * Register a memory region based on attributes of an existing one
 *
 * Input:
 *	ia_handle		IA handle
 *	lmr			pointer to dapl_lmr struct
 *	virt_addr		virtual address of beginning of mem region
 *	length			length of memory region
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_register_lmr(
	IN  DAPL_IA 			*ia,
	IN  DAPL_LMR			*lmr,
	IN  DAT_MEM_PRIV_FLAGS		privileges)
{
	dapl_mr_register_lmr_t		regl_msg;
	DAPL_LMR			*orig_lmr;
	struct dapls_ib_mr_handle	*orig_mr_handle;
	ib_mr_handle_t			mr_handle;
	int				ia_fd;
	int				retval;

	ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd;
	mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle));
	if (mr_handle == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register_lmr: lmr 0x%p, ia 0x%p, "
		    "cannot alloc mr_handle\n", lmr, ia);
		return (DAT_INSUFFICIENT_RESOURCES);
	}

	orig_lmr = (DAPL_LMR *)lmr->param.region_desc.for_lmr_handle;
	orig_mr_handle = (struct dapls_ib_mr_handle *)orig_lmr->mr_handle;
	regl_msg.mrl_orig_hkey = orig_mr_handle->mr_hkey;
	regl_msg.mrl_flags = (ibt_mr_flags_t)
	    dapl_lmr_convert_privileges(privileges);
	regl_msg.mrl_flags |= IBT_MR_ENABLE_WINDOW_BIND;
	regl_msg.mrl_lkey = regl_msg.mrl_rkey = 0;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_register_lmr: lmr 0x%p, hkey 0x%016llx, lmr_ctx 0x%08x\n"
	    "                 vaddr 0x%016llx, len %llu, flags 0x%x\n",
	    lmr, mr_handle->mr_hkey, lmr->param.lmr_context,
	    orig_lmr->param.registered_address,
	    orig_lmr->param.registered_size,
	    dapl_lmr_convert_privileges(orig_lmr->param.mem_priv) |
	    IBT_MR_ENABLE_WINDOW_BIND);


	/* call into driver to allocate MR resource */
	retval = ioctl(ia_fd, DAPL_MR_REGISTER_LMR, &regl_msg);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mr_register_lmr: failed (%s), orig_hkey (%016llx)\n",
		    strerror(errno), orig_mr_handle->mr_hkey);
		dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
		return (dapls_convert_error(errno, retval));
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mr_registered_lmr: successful, lmr 0x%p, hkey 0x%016llx\n",
	    lmr, regl_msg.mrl_hkey);

	mr_handle->mr_hkey = regl_msg.mrl_hkey;
	lmr->param.lmr_context = (DAT_LMR_CONTEXT)regl_msg.mrl_lkey;
	lmr->param.rmr_context = (DAT_RMR_CONTEXT)regl_msg.mrl_rkey;
	lmr->param.registered_address = orig_lmr->param.registered_address;
	lmr->param.registered_size = orig_lmr->param.registered_size;
	lmr->mr_handle = mr_handle;

	return (DAT_SUCCESS);
}


/*
 * dapls_ib_mw_alloc
 *
 * Bind a protection domain to a memory window
 *
 * Input:
 *	rmr			Initialized rmr to hold binding handles
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_alloc(
	IN  DAPL_RMR	*rmr)
{
	DAPL_IA		*ia_hdl = (DAPL_IA *)rmr->param.ia_handle;
	DAPL_PZ		*pz_hdl = rmr->param.pz_handle;
	dapl_mw_alloc_t	args;
	ib_mw_handle_t	mw_handle;
	int		ia_fd;
	int		retval;

	ia_fd = ((struct dapls_ib_hca_handle *)(ia_hdl->hca_ptr->
	    ib_hca_handle))->ia_fd;

	mw_handle = dapl_os_alloc(sizeof (struct dapls_ib_mw_handle));
	if (mw_handle == NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mw_alloc: rmr 0x%p, cannot alloc mw_handle\n", rmr);
		return (DAT_INSUFFICIENT_RESOURCES);
	}
	args.mw_pd_hkey = ((struct dapls_ib_pd_handle *)
	    (pz_hdl->pd_handle))->pd_hkey;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mw_alloc: rmr 0x%p, pd_hkey 0x%016llx\n",
	    rmr, args.mw_pd_hkey);

	retval = ioctl(ia_fd, DAPL_MW_ALLOC, &args);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mw_alloc: rmr 0x%p, failed (%s)\n", rmr, strerror(errno));
		dapl_os_free(mw_handle, sizeof (struct dapls_ib_mr_handle));
		return (dapls_convert_error(errno, retval));
	}

	mw_handle->mw_hkey = args.mw_hkey;
	rmr->mw_handle = mw_handle;
	rmr->param.rmr_context = (DAT_RMR_CONTEXT) args.mw_rkey;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mw_alloc: successful, rmr 0x%p, mw_hkey 0x%llx, "
	    "rmr_ctx 0x%x\n\n", rmr, (uint64_t)args.mw_hkey,
	    rmr->param.rmr_context);

	return (DAT_SUCCESS);
}


/*
 * dapls_ib_mw_free
 *
 * Release bindings of a protection domain to a memory window
 *
 * Input:
 *	rmr			Initialized rmr to hold binding handles
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_free(
	IN  DAPL_RMR	*rmr)
{
	DAPL_IA		*ia_hdl = rmr->param.ia_handle;
	dapl_mw_free_t	args;
	int		ia_fd;
	int		retval;

	ia_fd = ((struct dapls_ib_hca_handle *)(ia_hdl->hca_ptr->
	    ib_hca_handle))->ia_fd;

	args.mw_hkey = rmr->mw_handle->mw_hkey;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mw_free: rmr 0x%p, mw_hkey 0x%016llx\n", rmr, args.mw_hkey);

	retval = ioctl(ia_fd, DAPL_MW_FREE, &args);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mw_free: rmr 0x%p, failed (%s)\n", rmr, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "mw_free: successful\n\n");
	dapl_os_free(rmr->mw_handle, sizeof (struct dapls_ib_mr_handle));
	rmr->mw_handle = NULL;

	return (DAT_SUCCESS);
}

/*
 * dapls_ib_mw_bind
 *
 * Bind a protection domain to a memory window
 *
 * Input:
 *	rmr			Initialized rmr to hold binding handles
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_bind(
	IN  DAPL_RMR		*rmr,
	IN  DAT_LMR_CONTEXT	lmr_context,
	IN  DAPL_EP		*ep,
	IN  DAPL_COOKIE		*cookie,
	IN  DAT_VADDR		virtual_address,
	IN  DAT_VLEN		length,
	IN  DAT_MEM_PRIV_FLAGS	mem_priv,
	IN  DAT_COMPLETION_FLAGS completion_flags)
{
	ibt_send_wr_t	wre;
	ibt_wr_bind_t	wrbind;
	boolean_t	suppress_notification;
	int		retval;

	if (length > 0) {
		wrbind.bind_flags = (ibt_bind_flags_t)
		    (dapl_rmr_convert_privileges(mem_priv) |
		    IBT_WR_BIND_ATOMIC);
	} else {
		wrbind.bind_flags = (ibt_bind_flags_t)NULL;
	}
	wrbind.bind_rkey = rmr->param.rmr_context;
	wrbind.bind_va = virtual_address;
	wrbind.bind_len = length;
	wrbind.bind_lkey = lmr_context;

	wre.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
	/*
	 * wre.wr_flags = (is_signaled) ? IBT_WR_SEND_SIGNAL :
	 *   IBT_WR_NO_FLAGS;
	 * Till we fix the chan alloc flags do the following -
	 */
	/* Translate dapl flags */
	wre.wr_flags = (DAT_COMPLETION_BARRIER_FENCE_FLAG &
	    completion_flags) ? IBT_WR_SEND_FENCE : 0;
	/* suppress completions */
	wre.wr_flags |= (DAT_COMPLETION_SUPPRESS_FLAG &
	    completion_flags) ? 0 : IBT_WR_SEND_SIGNAL;

	wre.wr_trans = IBT_RC_SRV;
	wre.wr_opcode = IBT_WRC_BIND;
	wre.wr_nds = 0;
	wre.wr_sgl = NULL;
	wre.wr.rc.rcwr.bind = &wrbind;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mw_bind: rmr 0x%p, wr_flags 0x%x, rkey 0x%x, bind_flags 0x%x\n"
	    "         bind_va 0x%llx, bind_len 0x%llx, mem_priv 0x%x\n",
	    rmr, wre.wr_flags, wrbind.bind_rkey, wrbind.bind_flags,
	    wrbind.bind_va, wrbind.bind_len, mem_priv);

	if (ep->param.ep_attr.recv_completion_flags &
	    DAT_COMPLETION_UNSIGNALLED_FLAG) {
		/* This flag is used to control notification of completions */
		suppress_notification = (completion_flags &
		    DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
	} else {
		/*
		 * The evd waiter will use threshold to control wakeups
		 * Hence the event notification will be done via arming the
		 * CQ so we do not need special notification generation
		 * hence set suppression to true
		 */
		suppress_notification = B_TRUE;
	}

	retval = DAPL_SEND(ep)(ep, &wre, suppress_notification);

	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "mw_bind: rmr 0x%p, failed (%s)\n", rmr, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mw_bind: new_rkey = 0x%x\n", wrbind.bind_rkey_out);
	rmr->param.rmr_context = (DAT_RMR_CONTEXT) wrbind.bind_rkey_out;

	return (DAT_SUCCESS);
}

/*
 * dapls_ib_mw_unbind
 *
 * Unbind a protection domain from a memory window
 *
 * Input:
 *	rmr			Initialized rmr to hold binding handles
 *
 * Output:
 *	none
 *
 * Returns:
 *	DAT_SUCCESS
 *	DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_unbind(
	IN  DAPL_RMR		*rmr,
	IN  DAT_LMR_CONTEXT	lmr_context,
	IN  DAPL_EP		*ep,
	IN  DAPL_COOKIE		*cookie,
	IN  DAT_COMPLETION_FLAGS completion_flags)
{
	DAT_RETURN retval;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mw_unbind: rmr 0x%p, enter\n", rmr);

	retval = dapls_ib_mw_bind(rmr, lmr_context, ep, cookie,
	    (DAT_VADDR)0, (DAT_VLEN)0, (DAT_MEM_PRIV_FLAGS)NULL,
	    completion_flags);

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "mw_unbind: rmr 0x%p, exit\n\n", rmr);

	return (retval);
}

/*
 * Processes async events and calls appropriate callbacks so that events
 * can be posted to the async evd.
 */
void
dapls_ib_async_callback(
	IN    DAPL_EVD		  *async_evd,
	IN    ib_hca_handle_t	  hca_handle,
	IN    ib_error_record_t	  *event_ptr,
	IN    void		  *context)
{
	DAPL_IA			*ia_ptr;
	DAPL_EP			*ep_ptr;
	DAPL_EVD		*evd_ptr;
	dapl_ib_async_event_t	*async_evp;

	ia_ptr = (DAPL_IA *)context;

	dapl_os_assert(event_ptr != NULL);
	async_evp = (dapl_ib_async_event_t *)event_ptr;

	switch (async_evp->ibae_type) {
	case IBT_ERROR_INVALID_REQUEST_CHAN:
	case IBT_ERROR_CATASTROPHIC_CHAN:
		/*
		 * Walk the EPs to match this EP, then invoke the
		 * routine when we have the EP we need
		 */
		dapl_os_assert(!dapl_llist_is_empty(&ia_ptr->ep_list_head));
		dapl_os_lock(&ia_ptr->header.lock);

		ep_ptr = (DAPL_EP *)dapl_llist_next_entry(&ia_ptr->ep_list_head,
		    NULL);
		while (ep_ptr != NULL) {
			if (ep_ptr ==
			    (DAPL_EP *)(uintptr_t)async_evp->ibae_cookie) {
				break;
			}

			ep_ptr = (DAPL_EP *) dapl_llist_next_entry(
			    &ia_ptr->ep_list_head,
			    &ep_ptr->header.ia_list_entry);
		}

		dapl_os_unlock(&ia_ptr->header.lock);
		dapl_os_assert(ep_ptr != NULL);
		dapl_evd_qp_async_error_callback(hca_handle, NULL, event_ptr,
		    (void *)ep_ptr);
		break;
	case IBT_ERROR_CQ:
		/*
		 * Walk the EVDs to match this EVD, then invoke the
		 * routine when we have the EVD we need
		 */
		dapl_os_assert(!dapl_llist_is_empty(&ia_ptr->evd_list_head));
		dapl_os_lock(&ia_ptr->header.lock);

		evd_ptr = (DAPL_EVD *) dapl_llist_next_entry(
		    &ia_ptr->evd_list_head, NULL);
		while (evd_ptr != NULL) {
			if (evd_ptr ==
			    (DAPL_EVD *)(uintptr_t)async_evp->ibae_cookie) {
				break;
			}
			evd_ptr = (DAPL_EVD *)
			    dapl_llist_next_entry(&ia_ptr->evd_list_head,
			    &evd_ptr->header.ia_list_entry);
		}
		dapl_os_unlock(&ia_ptr->header.lock);
		dapl_os_assert(evd_ptr != NULL);
		dapl_evd_cq_async_error_callback(hca_handle, NULL, event_ptr,
		    (void *)evd_ptr);
		break;
	case IBT_ERROR_PORT_DOWN:
	case IBT_ERROR_LOCAL_CATASTROPHIC:
		dapl_evd_un_async_error_callback(hca_handle, event_ptr,
		    (void *)async_evd);
		break;
	default:
		/*
		 * We are not interested in the following events
		 * case IBT_EVENT_PATH_MIGRATED:
		 * case IBT_EVENT_COM_EST:
		 * case IBT_EVENT_SQD:
		 * case IBT_ERROR_PATH_MIGRATE_REQ:
		 * case IBT_EVENT_PORT_UP:
		 */
		dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
		    "dapls_ib_async_callback: unhandled async code:%x\n",
		    async_evp->ibae_type);
		break;
	}
}

/*
 * dapls_ib_setup_async_callback
 * The reference implementation calls this to register callbacks,
 * but since our model of polling for events is based on retrieving
 * events by the waiting thread itself this is a NOOP for us.
 */
/* ARGSUSED */
DAT_RETURN
dapls_ib_setup_async_callback(
	IN  DAPL_IA			*ia_ptr,
	IN  DAPL_ASYNC_HANDLER_TYPE	handler_type,
	IN  unsigned int		*callback_handle,
	IN  ib_async_handler_t		callback,
	IN  void			*context)
{
	return (DAT_SUCCESS);
}

/*
 * dapls_ib_query_hca
 *
 * Set up an asynchronous callbacks of various kinds
 *
 * Input:
 *	hca_handl		hca handle
 *	ep_attr			attribute of the ep
 *
 * Output:
 * 	none
 *
 * Returns:
 * 	DAT_SUCCESS
 *	DAT_INVALID_PARAMETER
 */

/* these are just arbitrary values for now */

static DAT_RETURN
dapls_ib_query_ia(
	IN  dapl_hca_attr_t		*hca_attr,
	IN  DAPL_HCA			*hca_ptr,
	OUT DAT_IA_ATTR			*ia_attr)
{
	(void) dapl_os_memzero(ia_attr, sizeof (*ia_attr));

	(void) dapl_os_strcpy(ia_attr->adapter_name, VN_ADAPTER_NAME);

	(void) sprintf(ia_attr->vendor_name, "0x%08x:0x%08x",
	    hca_attr->dhca_vendor_id, hca_attr->dhca_device_id);

	ia_attr->hardware_version_major = (DAT_UINT32)hca_attr->dhca_version_id;
	ia_attr->ia_address_ptr = (DAT_IA_ADDRESS_PTR)&hca_ptr->hca_address;
	ia_attr->max_eps = (DAT_COUNT)hca_attr->dhca_max_chans;
	ia_attr->max_dto_per_ep = (DAT_COUNT)hca_attr->dhca_max_chan_sz;
	ia_attr->max_rdma_read_per_ep_in = hca_attr->dhca_max_rdma_in_chan;
	ia_attr->max_rdma_read_per_ep_out = hca_attr->dhca_max_rdma_out_chan;
	ia_attr->max_evds = (DAT_COUNT)hca_attr->dhca_max_cq;
	ia_attr->max_evd_qlen = (DAT_COUNT)hca_attr->dhca_max_cq_sz;
	/* max_iov_segments_per_dto is for non-RDMA */
	ia_attr->max_iov_segments_per_dto = (DAT_COUNT)hca_attr->dhca_max_sgl;
	ia_attr->max_lmrs = (DAT_COUNT)hca_attr->dhca_max_memr;
	ia_attr->max_lmr_block_size = (DAT_VLEN)hca_attr->dhca_max_memr_len;
	ia_attr->max_lmr_virtual_address = (DAT_VADDR)DAPL_MAX_ADDRESS;
	ia_attr->max_pzs = (DAT_COUNT)hca_attr->dhca_max_pd;
	ia_attr->max_mtu_size = (DAT_VLEN)DAPL_IB_MAX_MESSAGE_SIZE;
	ia_attr->max_rdma_size = (DAT_VLEN)DAPL_IB_MAX_MESSAGE_SIZE;
	ia_attr->max_rmrs = (DAT_COUNT)hca_attr->dhca_max_mem_win;
	ia_attr->max_rmr_target_address = (DAT_VADDR)DAPL_MAX_ADDRESS;
	ia_attr->max_iov_segments_per_rdma_read =
	    (DAT_COUNT)hca_attr->dhca_max_sgl;
	ia_attr->max_iov_segments_per_rdma_write =
	    (DAT_COUNT)hca_attr->dhca_max_sgl;
	/* all instances of IA */
	ia_attr->max_rdma_read_in = hca_attr->dhca_max_rdma_in_chan *
	    hca_attr->dhca_max_chans;
	ia_attr->max_rdma_read_out = hca_attr->dhca_max_rdma_out_chan *
	    hca_attr->dhca_max_chans;
	ia_attr->max_rdma_read_per_ep_in_guaranteed = DAT_TRUE;
	ia_attr->max_rdma_read_per_ep_out_guaranteed = DAT_TRUE;
	ia_attr->max_srqs = (DAT_COUNT)hca_attr->dhca_max_srqs;
	ia_attr->max_ep_per_srq = ia_attr->max_eps;
	ia_attr->max_recv_per_srq = (DAT_COUNT)hca_attr->dhca_max_srqs_sz;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "IA Attributes:\n"
	    "\tadapter_name %s\n "
	    "\tvendor_name %s\n "
	    "\thardware_version_major 0x%08x\n"
	    "\tmax_eps %d\n"
	    "\tmax_dto_per_ep %d\n"
	    "\tmax_rdma_read_per_ep_in %d\n"
	    "\tmax_rdma_read_per_ep_out %d\n"
	    "\tmax_evds %d\n"
	    "\tmax_evd_qlen %d\n"
	    "\tmax_iov_segments_per_dto %d\n"
	    "\tmax_lmrs %d\n"
	    "\tmax_lmr_block_size 0x%016llx\n"
	    "\tmax_lmr_virtual_address 0x%016llx\n"
	    "\tmax_pzs %d\n"
	    "\tmax_mtu_size 0x%016llx\n"
	    "\tmax_rdma_size 0x%016llx\n"
	    "\tmax_rmrs %d\n"
	    "\tmax_rmr_target_address 0x%016llx\n"
	    "\tmax_iov_segments_per_rdma_read %d\n"
	    "\tmax_iov_segments_per_rdma_write %d\n"
	    "\tmax_rdma_read_in %d\n"
	    "\tmax_rdma_read_out %d\n"
	    "\tmax_srqs %d\n"
	    "\tmax_ep_per_srq %d\n"
	    "\tmax_recv_per_srq %d\n"
	    "\n",
	    ia_attr->adapter_name,
	    ia_attr->vendor_name,
	    ia_attr->hardware_version_major,
	    ia_attr->max_eps,
	    ia_attr->max_dto_per_ep,
	    ia_attr->max_rdma_read_per_ep_in,
	    ia_attr->max_rdma_read_per_ep_out,
	    ia_attr->max_evds,
	    ia_attr->max_evd_qlen,
	    ia_attr->max_iov_segments_per_dto,
	    ia_attr->max_lmrs,
	    ia_attr->max_lmr_block_size,
	    ia_attr->max_lmr_virtual_address,
	    ia_attr->max_pzs,
	    ia_attr->max_mtu_size,
	    ia_attr->max_rdma_size,
	    ia_attr->max_rmrs,
	    ia_attr->max_rmr_target_address,
	    ia_attr->max_iov_segments_per_rdma_read,
	    ia_attr->max_iov_segments_per_rdma_write,
	    ia_attr->max_rdma_read_in,
	    ia_attr->max_rdma_read_out,
	    ia_attr->max_srqs,
	    ia_attr->max_ep_per_srq,
	    ia_attr->max_recv_per_srq);

	return (DAT_SUCCESS);
}

/* ARGSUSED */
static DAT_RETURN
dapls_ib_query_ep(
	IN  dapl_hca_attr_t		*hca_attr,
	IN  DAPL_HCA			*hca_ptr,
	OUT DAT_EP_ATTR			*ep_attr)
{
	(void) dapl_os_memzero(ep_attr, sizeof (*ep_attr));
	ep_attr->service_type = DAT_SERVICE_TYPE_RC;
	ep_attr->max_mtu_size = DAPL_IB_MAX_MESSAGE_SIZE;
	ep_attr->max_rdma_size = DAPL_IB_MAX_MESSAGE_SIZE;
	ep_attr->qos = DAT_QOS_BEST_EFFORT;
	ep_attr->max_recv_dtos = hca_attr->dhca_max_chan_sz;
	ep_attr->max_request_dtos = hca_attr->dhca_max_chan_sz;
	ep_attr->max_recv_iov = hca_attr->dhca_max_sgl;
	ep_attr->max_request_iov = hca_attr->dhca_max_sgl;
	ep_attr->request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
	ep_attr->recv_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
	ep_attr->srq_soft_hw = DAT_HW_DEFAULT;
	return (DAT_SUCCESS);
}

static void
dapls_ib_query_srq(
	IN  dapl_hca_attr_t		*hca_attr,
	OUT DAT_SRQ_ATTR		*srq_attr)
{
	(void) dapl_os_memzero(srq_attr, sizeof (*srq_attr));
	srq_attr->max_recv_dtos = hca_attr->dhca_max_srqs_sz;
	srq_attr->max_recv_iov = hca_attr->dhca_max_srq_sgl;
	srq_attr->low_watermark = DAT_SRQ_LW_DEFAULT;
}

/* ARGSUSED */
DAT_RETURN
dapls_ib_query_hca(
	IN  DAPL_HCA			*hca_ptr,
	OUT DAT_IA_ATTR			*ia_attr,
	OUT DAT_EP_ATTR			*ep_attr,
	OUT DAT_SOCK_ADDR6		*ip_addr,
	OUT DAT_SRQ_ATTR		*srq_attr)
{
	dapl_ia_query_t args;
	int ia_fd, retval;

	if (hca_ptr == NULL) {
		return (DAT_INVALID_PARAMETER);
	}

	ia_fd = hca_ptr->ib_hca_handle->ia_fd;
	retval = ioctl(ia_fd, DAPL_IA_QUERY, &args);
	if (retval != 0) {
		return (dapls_convert_error(errno, retval));
	}

	if (ia_attr != NULL) {
		(void) dapls_ib_query_ia(&args.hca_attr, hca_ptr, ia_attr);
	}
	if (ep_attr != NULL) {
		(void) dapls_ib_query_ep(&args.hca_attr, hca_ptr, ep_attr);
	}
	if (srq_attr != NULL) {
		(void) dapls_ib_query_srq(&args.hca_attr, srq_attr);
	}
	if (ia_attr == NULL && ep_attr == NULL && srq_attr == NULL) {
		return (DAT_INVALID_PARAMETER);
	}
	return (DAT_SUCCESS);
}

void
dapls_ib_store_premature_events(
	IN ib_qp_handle_t	qp_ptr,
	IN ib_work_completion_t	*cqe_ptr)
{
	ib_srq_handle_t	srqp;
	int		head;

	if (qp_ptr->qp_srq_enabled) {
		/*
		 * For QPs with SRQ attached store the premature event in the
		 * SRQ's premature event list
		 */
		srqp = qp_ptr->qp_srq;
		dapl_os_assert(srqp->srq_freepr_num_events > 0);
		head = srqp->srq_freepr_events[srqp->srq_freepr_head];
		/*
		 * mark cqe as valid before storing it in the
		 * premature events list
		 */
		DAPL_SET_CQE_VALID(cqe_ptr);
		(void) dapl_os_memcpy(&(srqp->srq_premature_events[head]),
		    cqe_ptr, sizeof (*cqe_ptr));
		srqp->srq_freepr_head = (srqp->srq_freepr_head + 1) %
		    srqp->srq_wq_numwqe;
		srqp->srq_freepr_num_events--;
	} else {
		(void) dapl_os_memcpy(&(qp_ptr->qp_premature_events[
		    qp_ptr->qp_num_premature_events]),
		    cqe_ptr, sizeof (*cqe_ptr));
	}
	qp_ptr->qp_num_premature_events++;
}

void
dapls_ib_poll_premature_events(
	IN  DAPL_EP			*ep_ptr,
	OUT ib_work_completion_t	**cqe_ptr,
	OUT int				*nevents)
{
	ib_qp_handle_t qp = ep_ptr->qp_handle;

	if (qp->qp_srq_enabled) {
		*cqe_ptr = qp->qp_srq->srq_premature_events;
	} else {
		*cqe_ptr = qp->qp_premature_events;
	}

	*nevents = qp->qp_num_premature_events;
	qp->qp_num_premature_events = 0;
}

/*
 * Return the premature events to the free list after processing it
 * This function is called only for premature events on the SRQ
 */
void
dapls_ib_free_premature_events(
	IN  DAPL_EP	*ep_ptr,
	IN  int		free_index)
{
	ib_qp_handle_t	qp_ptr;
	ib_srq_handle_t	srq_ptr;
	int		tail;

	qp_ptr = ep_ptr->qp_handle;
	srq_ptr = qp_ptr->qp_srq;

	dapl_os_assert(qp_ptr->qp_srq_enabled);

	tail = srq_ptr->srq_freepr_tail;
	srq_ptr->srq_freepr_events[tail] = free_index;
	srq_ptr->srq_freepr_tail = (tail + 1) % srq_ptr->srq_wq_numwqe;
	srq_ptr->srq_freepr_num_events++;
	DAPL_SET_CQE_INVALID(&srq_ptr->srq_premature_events[free_index]);
}

/*
 * dapls_ib_get_async_event
 *
 * Translate an asynchronous event type to the DAT event.
 * Note that different providers have different sets of errors.
 *
 * Input:
 *	cause_ptr		provider event cause
 *
 * Output:
 *	async_event		DAT mapping of error
 *
 * Returns:
 *	DAT_SUCCESS
 *	DAT_NOT_IMPLEMENTED	Caller is not interested this event
 */

DAT_RETURN dapls_ib_get_async_event(
	IN  ib_error_record_t		*cause_ptr,
	OUT DAT_EVENT_NUMBER		*async_event)
{
	ibt_async_code_t	code;
	DAT_RETURN		dat_status;

	dat_status = DAT_SUCCESS;
	code = (ibt_async_code_t)((dapl_ib_async_event_t *)cause_ptr->
	    ibae_type);

	switch (code) {
	case IBT_ERROR_CQ:
	case IBT_ERROR_ACCESS_VIOLATION_CHAN:
	case IBT_ERROR_INVALID_REQUEST_CHAN:
		*async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR;
		break;
	/* CATASTROPHIC errors */
	case IBT_ERROR_CATASTROPHIC_CHAN:
	case IBT_ERROR_LOCAL_CATASTROPHIC:
	case IBT_ERROR_PORT_DOWN:
		*async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC;
		break;
	default:
		/*
		 * Errors we are not interested in reporting:
		 * IBT_EVENT_PATH_MIGRATED
		 * IBT_ERROR_PATH_MIGRATE_REQ
		 * IBT_EVENT_COM_EST
		 * IBT_EVENT_SQD
		 * IBT_EVENT_PORT_UP
		 */
		dat_status = DAT_NOT_IMPLEMENTED;
	}
	return (dat_status);
}

DAT_RETURN
dapls_ib_event_poll(
	IN DAPL_EVD		*evd_ptr,
	IN uint64_t		timeout,
	IN uint_t		threshold,
	OUT dapl_ib_event_t	*evp_ptr,
	OUT int			*num_events)
{
	dapl_event_poll_t	evp_msg;
	int			ia_fd;
	int			retval;

	*num_events = 0;
	ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;

	evp_msg.evp_evd_hkey = evd_ptr->ib_cq_handle->evd_hkey;
	evp_msg.evp_threshold = threshold;
	evp_msg.evp_timeout = timeout;
	evp_msg.evp_ep = evp_ptr;
	if (evp_ptr) {
		evp_msg.evp_num_ev =
		    DAPL_MAX(evd_ptr->threshold, NUM_EVENTS_PER_POLL);
	} else {
		evp_msg.evp_num_ev = 0;
	}
	evp_msg.evp_num_polled = 0;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "event_poll: evd 0x%p, hkey 0x%llx, threshold %d,\n"
	    "            timeout 0x%llx, evp_ptr 0x%p, num_ev %d\n",
	    evd_ptr, evp_msg.evp_evd_hkey, evp_msg.evp_threshold,
	    timeout, evp_ptr, evp_msg.evp_num_ev);

	/*
	 * Poll the EVD and if there are no events then we wait in
	 * the kernel.
	 */
	retval = ioctl(ia_fd, DAPL_EVENT_POLL, &evp_msg);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_EVD,
		    "event_poll: evd 0x%p, retval %d err: %s\n",
		    evd_ptr, retval, strerror(errno));
		*num_events = evp_msg.evp_num_polled;
		return (dapls_convert_error(errno, retval));
	}

	dapl_dbg_log(DAPL_DBG_TYPE_EVD,
	    "dapls_ib_event_poll: evd %p nevents %d\n", evd_ptr,
	    evp_msg.evp_num_polled);

	*num_events = evp_msg.evp_num_polled;

	return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_event_wakeup(
	IN DAPL_EVD		*evd_ptr)
{
	dapl_event_wakeup_t	evw_msg;
	int			ia_fd;
	int			retval;

	ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;

	evw_msg.evw_hkey = evd_ptr->ib_cq_handle->evd_hkey;

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "event_wakeup: evd 0x%p, hkey 0x%llx\n",
	    evd_ptr, evw_msg.evw_hkey);

	/*
	 * Wakeup any thread waiting in the kernel on this EVD
	 */
	retval = ioctl(ia_fd, DAPL_EVENT_WAKEUP, &evw_msg);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_EVD,
		    "event_wakeup: evd 0x%p, retval %d err: %s\n",
		    evd_ptr, retval, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	return (DAT_SUCCESS);
}

/*
 * dapls_ib_cq_peek is used by dapl_cno_wait(). After the CQ has been
 * inspected we arm the CQ if it was empty.
 *
 */
void dapls_ib_cq_peek(
	IN DAPL_EVD	*evd_ptr,
	OUT int		*num_cqe)
{
	DAPL_IA		*ia_ptr;

	*num_cqe = 0;
	if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) {
		DAPL_PEEK(evd_ptr)(evd_ptr->ib_cq_handle, num_cqe);
		/* No events found in CQ arm it now */
		if (*num_cqe == 0) {
			ia_ptr = evd_ptr->header.owner_ia;
			(void) dapls_set_cq_notify(ia_ptr, evd_ptr);
			dapl_dbg_log(DAPL_DBG_TYPE_EVD,
			    "dapls_ib_cq_peek: set_cq_notify\n");
		}
	}
}

/*
 * Modifies the CNO associated to an EVD
 */
DAT_RETURN dapls_ib_modify_cno(
	IN DAPL_EVD	*evd_ptr,
	IN DAPL_CNO	*cno_ptr)
{
	dapl_evd_modify_cno_t	evmc_msg;
	int			ia_fd;
	int			retval;

	ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;

	evmc_msg.evmc_hkey = evd_ptr->ib_cq_handle->evd_hkey;

	if (cno_ptr) {
		evmc_msg.evmc_cno_hkey = (uint64_t)cno_ptr->ib_cno_handle;
	} else {
		evmc_msg.evmc_cno_hkey = 0;
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "modify_cno: evd 0x%p, hkey 0x%llx, cno 0x%p, cno_hkey 0x%llx\n",
	    evd_ptr, evmc_msg.evmc_hkey, cno_ptr, evmc_msg.evmc_cno_hkey);

	/*
	 * modify CNO associated with the EVD
	 */
	retval = ioctl(ia_fd, DAPL_EVD_MODIFY_CNO, &evmc_msg);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_EVD,
		    "modify_cno: evd 0x%p, cno %p retval %d err: %s\n",
		    evd_ptr, cno_ptr, retval, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_cno_wait(
	IN DAPL_CNO	*cno_ptr,
	IN DAT_TIMEOUT	timeout,
	IN DAPL_EVD	**evd_ptr_p)
{
	dapl_cno_wait_t		args;
	int			retval;

	args.cnw_hkey = (uint64_t)cno_ptr->ib_cno_handle;
	if (timeout == DAT_TIMEOUT_INFINITE) {
		args.cnw_timeout = UINT64_MAX;
	} else {
		args.cnw_timeout = (uint64_t)timeout & 0x00000000ffffffff;
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "cno_wait: cno 0x%p, hkey 0x%016llx, timeout 0x%016llx\n",
	    cno_ptr, args.cnw_hkey, args.cnw_timeout);

	retval = ioctl(cno_ptr->header.owner_ia->hca_ptr->
	    ib_hca_handle->ia_fd, DAPL_CNO_WAIT, &args);

	if (retval != 0) {
		*evd_ptr_p = (DAPL_EVD *)NULL;
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "cno_wait: cno 0x%p ioctl err: %s\n",
		    cno_ptr, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	*evd_ptr_p = (DAPL_EVD *)(uintptr_t)args.cnw_evd_cookie;
	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "cno_wait: woken up, cno 0x%p, evd 0x%p\n\n",
	    cno_ptr, *evd_ptr_p);

	return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_cno_alloc(
	IN DAPL_IA	*ia_ptr,
	IN DAPL_CNO	*cno_ptr)
{
	dapl_cno_alloc_t	args;
	int			retval;

	if (cno_ptr->cno_wait_agent.instance_data != NULL ||
	    cno_ptr->cno_wait_agent.proxy_agent_func != NULL) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "cno_alloc: cno 0x%p, wait_agent != NULL\n", cno_ptr);
		return (DAT_NOT_IMPLEMENTED);
	}

	retval = ioctl(ia_ptr->hca_ptr->ib_hca_handle->ia_fd,
	    DAPL_CNO_ALLOC, &args);
	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "cno_alloc: cno 0x%p ioctl err: %s\n",
		    cno_ptr, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	cno_ptr->ib_cno_handle = (ib_cno_handle_t)args.cno_hkey;
	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "cno_alloc: cno 0x%p allocated, ia_ptr 0x%p, hkey 0x%016llx\n",
	    cno_ptr, ia_ptr, args.cno_hkey);

	return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_cno_free(
	IN DAPL_CNO	*cno_ptr)
{
	dapl_cno_free_t		args;
	int			retval;

	args.cnf_hkey = (uint64_t)cno_ptr->ib_cno_handle;
	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "cno_free: cno 0x%p, hkey 0x%016llx\n",
	    cno_ptr, args.cnf_hkey);

	retval = ioctl(cno_ptr->header.owner_ia->hca_ptr->
	    ib_hca_handle->ia_fd, DAPL_CNO_FREE, &args);

	if (retval != 0) {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "cno_free: cno 0x%p ioctl err: %s\n",
		    cno_ptr, strerror(errno));
		return (dapls_convert_error(errno, retval));
	}

	dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
	    "cno_free: cno 0x%p freed\n", cno_ptr);

	return (DAT_SUCCESS);
}

DAT_RETURN
dapls_convert_error(int errnum, int retval)
{
	if (retval < 0) {
		switch (errnum) {
		case EINVAL:
			return (DAT_INVALID_PARAMETER);
		case ENOMEM:
			return (DAT_INSUFFICIENT_RESOURCES);
		case ETIME:
			return (DAT_TIMEOUT_EXPIRED);
		case EINTR:
			return (DAT_INTERRUPTED_CALL);
		case EFAULT:
			return (DAT_INTERNAL_ERROR);
		default:
			return (DAT_INTERNAL_ERROR);
		}
	} else {
		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
		    "ERROR: got IBTF error %d\n", retval);
		switch (retval) {
		case IBT_SERVICE_RECORDS_NOT_FOUND:
			/*
			 * Connecting to a non-existant conn qual gets
			 * us here
			 */
			return (DAT_ERROR(DAT_INVALID_PARAMETER,
			    DAT_INVALID_ADDRESS_UNREACHABLE));
		case IBT_INSUFF_RESOURCE:
		case IBT_INSUFF_KERNEL_RESOURCE:
			return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
		case IBT_AR_NOT_REGISTERED:
			/*
			 * forward ipaddr lookup failed
			 */
			return (DAT_ERROR(DAT_INVALID_ADDRESS, 0));
		default:
			return (DAT_INTERNAL_ERROR);
		}
	}
}

typedef struct dapls_ib_dbp_page_s {
	uint32_t			*dbp_page_addr;
	uint64_t			dbp_mapoffset;
	struct dapls_ib_dbp_page_s	*next;
	int				fd;
} dapls_ib_dbp_page_t;

dapls_ib_dbp_page_t	*dapls_ib_pagelist = NULL;

/* Function that returns a pointer to the specified doorbell entry */
uint32_t *dapls_ib_get_dbp(uint64_t maplen, int fd, uint64_t mapoffset,
    uint32_t offset)
{
	dapls_ib_dbp_page_t	*new_page;
	dapls_ib_dbp_page_t	*cur_page;

	dapl_os_lock(&dapls_ib_dbp_lock);
	/* Check to see if page already mapped for entry */
	for (cur_page = dapls_ib_pagelist; cur_page != NULL;
	    cur_page = cur_page->next)
		if (cur_page->dbp_mapoffset == mapoffset &&
		    cur_page->fd == fd) {
			dapl_os_unlock(&dapls_ib_dbp_lock);
			return ((uint32_t *)
			    (offset + (uintptr_t)cur_page->dbp_page_addr));
		}

	/* If not, map a new page and prepend to pagelist */
	new_page = malloc(sizeof (dapls_ib_dbp_page_t));
	if (new_page == NULL) {
		dapl_os_unlock(&dapls_ib_dbp_lock);
		return (MAP_FAILED);
	}
	new_page->dbp_page_addr = (uint32_t *)(void *)mmap64((void *)0,
	    maplen, (PROT_READ | PROT_WRITE), MAP_SHARED, fd, mapoffset);
	if (new_page->dbp_page_addr == MAP_FAILED) {
		free(new_page);
		dapl_os_unlock(&dapls_ib_dbp_lock);
		return (MAP_FAILED);
	}
	new_page->next = dapls_ib_pagelist;
	new_page->dbp_mapoffset = mapoffset;
	new_page->fd = fd;
	dapls_ib_pagelist = new_page;
	dapl_os_unlock(&dapls_ib_dbp_lock);
	return ((uint32_t *)(offset + (uintptr_t)new_page->dbp_page_addr));
}