xref: /illumos-gate/usr/src/lib/udapl/udapl_tavor/tavor/dapl_tavor_wr.c (revision 9e39c5ba)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "dapl.h"
#include "dapl_tavor_wr.h"
#include "dapl_hash.h"
#include "dapl_tavor_ibtf_impl.h"

static dapls_tavor_wrid_entry_t *dapli_tavor_wrid_find_match(
	dapls_tavor_workq_hdr_t *, tavor_hw_cqe_t *);
static dapls_tavor_wrid_list_hdr_t *dapli_tavor_wrid_get_list(uint32_t, int);
static void dapli_tavor_wrid_reaplist_add(ib_cq_handle_t,
    dapls_tavor_workq_hdr_t *);
static dapls_tavor_workq_hdr_t *dapli_tavor_wrid_wqhdr_find(ib_cq_handle_t,
    uint_t, uint_t);
static uint32_t dapli_tavor_wrid_get_wqeaddrsz(dapls_tavor_workq_hdr_t *);
static dapls_tavor_workq_hdr_t *dapli_tavor_wrid_list_reap(
	dapls_tavor_wrid_list_hdr_t *);
static dapls_tavor_workq_hdr_t *dapli_tavor_wrid_wqhdr_create(ib_cq_handle_t,
    uint_t, uint_t, uint_t);
static void dapli_tavor_wrid_wqhdr_add(dapls_tavor_workq_hdr_t *,
    dapls_tavor_wrid_list_hdr_t *);
static void dapli_tavor_wrid_wqhdr_remove(dapls_tavor_workq_hdr_t *,
    dapls_tavor_wrid_list_hdr_t *);
static void dapli_tavor_wrid_wqhdr_lock_both(ib_qp_handle_t);
static void dapli_tavor_wrid_wqhdr_unlock_both(ib_qp_handle_t);
static DAT_RETURN dapli_tavor_cq_wqhdr_add(ib_cq_handle_t,
    dapls_tavor_workq_hdr_t *);
static void dapli_tavor_cq_wqhdr_remove(ib_cq_handle_t,
    dapls_tavor_workq_hdr_t *);

/*
 * dapls_tavor_wrid_get_entry()
 */
uint64_t
dapls_tavor_wrid_get_entry(ib_cq_handle_t cq, tavor_hw_cqe_t *cqe,
    uint_t send_or_recv, uint_t error, dapls_tavor_wrid_entry_t *wre)
{
	dapls_tavor_workq_hdr_t	*wq;
	dapls_tavor_wrid_entry_t	*wre_tmp;
	uint64_t		wrid;
	uint_t			qpnum;

	/* Lock the list of work queues associated with this CQ */
	dapl_os_lock(&cq->cq_wrid_wqhdr_lock);

	/* Find the work queue for this QP number (send or receive side) */
	qpnum = TAVOR_CQE_QPNUM_GET(cqe);
	wq = dapli_tavor_wrid_wqhdr_find(cq, qpnum, send_or_recv);

	dapl_os_assert(wq != NULL);

	/*
	 * Regardless of whether the completion is the result of a "success"
	 * or a "failure", we lock the list of "containers" and attempt to
	 * search for the the first matching completion (i.e. the first WR
	 * with a matching WQE addr and size).  Once we find it, we pull out
	 * the "wrid" field and return it (see below).  Note: One possible
	 * future enhancement would be to enable this routine to skip over
	 * any "unsignaled" completions to go directly to the next "signaled"
	 * entry on success. XXX
	 */
	dapl_os_lock(&wq->wq_wrid_lock->wrl_lock);
	wre_tmp = dapli_tavor_wrid_find_match(wq, cqe);

	/*
	 * If this is a "successful" completion, then we assert that this
	 * completion must be a "signaled" completion.
	 */
	dapl_os_assert(error || (wre_tmp->wr_signaled_dbd &
	    TAVOR_WRID_ENTRY_SIGNALED));

	/*
	 * If the completion is a "failed" completion, then we save away the
	 * contents of the entry (into the "wre" field passed in) for use
	 * in later CQE processing. Note: We use the
	 * dapli_tavor_wrid_get_wqeaddrsz() function to grab "wqeaddrsz" from
	 * the next entry in the container.
	 * This is required for error processing (where updating these fields
	 * properly is necessary to correct handling of the "error" CQE)
	 */
	if (error && (wre != NULL)) {
		*wre = *wre_tmp;
		wre->wr_wqeaddrsz = dapli_tavor_wrid_get_wqeaddrsz(wq);
	}

	/* Pull out the WRID and return it */
	wrid = wre_tmp->wr_wrid;

	dapl_os_unlock(&wq->wq_wrid_lock->wrl_lock);
	dapl_os_unlock(&cq->cq_wrid_wqhdr_lock);

	return (wrid);
}


/*
 * dapli_tavor_wrid_find_match()
 */
static dapls_tavor_wrid_entry_t *
dapli_tavor_wrid_find_match(dapls_tavor_workq_hdr_t *wq, tavor_hw_cqe_t *cqe)
{
	dapls_tavor_wrid_entry_t	*curr = NULL;
	dapls_tavor_wrid_list_hdr_t	*container;
	uint32_t		wqeaddr_size;
	uint32_t		head, tail, size;
	int			found = 0, last_container;

	/* dapl_os_assert(MUTEX_HELD(&wq->wq_wrid_lock)); */

	/* Pull the "wqeaddrsz" information from the CQE */
	wqeaddr_size = TAVOR_CQE_WQEADDRSZ_GET(cqe);

	/*
	 * Walk the "containers" list(s), find first WR with a matching WQE
	 * addr.  If the current "container" is not the last one on the list,
	 * i.e. not the current one to which we are posting new WRID entries,
	 * then we do not attempt to update the "q_head", "q_tail", and
	 * "q_full" indicators on the main work queue header.  We do, however,
	 * update the "head" and "full" indicators on the individual containers
	 * as we go.  This is imperative because we need to be able to
	 * determine when the current container has been emptied (so that we
	 * can move on to the next container).
	 */
	container = wq->wq_wrid_poll;
	while (container != NULL) {

		/* Is this the last/only "container" on the list */
		last_container = (container != wq->wq_wrid_post) ? 0 : 1;

		/*
		 * First check if we are on an SRQ.  If so, we grab the entry
		 * and break out.  Since SRQ wridlist's are never added to
		 * reaplist, they can only be the last container.
		 */
		if (container->wl_srq_en) {
			dapl_os_assert(last_container == 1);
			curr = dapli_tavor_wrid_find_match_srq(container, cqe);
			break;
		}

		/*
		 * Grab the current "head", "tail" and "size" fields before
		 * walking the list in the current container. Note: the "size"
		 * field here must always be a power-of-2.  The "full"
		 * parameter is checked (and updated) here to distinguish the
		 * "queue full" condition from "queue empty".
		 */
		head = container->wl_head;
		tail = container->wl_tail;
		size = container->wl_size;
		while ((head != tail) || (container->wl_full)) {
			container->wl_full = 0;
			curr = &container->wl_wre[head];
			head = ((head + 1) & (size - 1));
			/*
			 * If the current entry's "wqeaddrsz" matches the one
			 * we're searching for, then this must correspond to
			 * the work request that caused the completion.  Set
			 * the "found" flag and bail out.
			 */
			if (curr->wr_wqeaddrsz == wqeaddr_size) {
				found = 1;
				break;
			}
		}

		/*
		 * If the current container is empty (having reached here the
		 * "head == tail" condition can only mean that the container
		 * is empty), then NULL out the "wrid_old_tail" field (see
		 * tavor_post_send() and tavor_post_recv() for more details)
		 * and (potentially) remove the current container from future
		 * searches.
		 */
		if (head == tail) {
			container->wl_wre_old_tail = NULL;
			/*
			 * If this wasn't the last "container" on the chain,
			 * i.e. the one to which new WRID entries will be
			 * added, then remove it from the list.
			 * Note: we don't "lose" the memory pointed to by this
			 * because we should have already put this container
			 * on the "reapable" list (from where it will later be
			 * pulled).
			 */
			if (!last_container) {
				wq->wq_wrid_poll = container->wl_next;
			}
		}

		/* Update the head index for the container */
		container->wl_head = head;

		/*
		 * If the entry was found in this container, then continue to
		 * bail out.  Else reset the "curr" pointer and move on to the
		 * next container (if there is one).  Note: the only real
		 * reason for setting "curr = NULL" here is so that the ASSERT
		 * below can catch the case where no matching entry was found
		 * on any of the lists.
		 */
		if (found) {
			break;
		} else {
			curr = NULL;
			container = container->wl_next;
		}
	}

	/*
	 * Update work queue header's "head" and "full" conditions to match
	 * the last entry on the container list.  (Note: Only if we're pulling
	 * entries from the last work queue portion of the list, i.e. not from
	 * the previous portions that may be the "reapable" list.)
	 */
	if (last_container) {
		wq->wq_head = wq->wq_wrid_post->wl_head;
		wq->wq_full = wq->wq_wrid_post->wl_full;
	}

	/* Ensure that we've actually found what we were searching for */
	dapl_os_assert(curr != NULL);

	return (curr);
}

/*
 * tavor_wrid_find_match_srq()
 *    Context: Can be called from interrupt or base context.
 */
dapls_tavor_wrid_entry_t *
dapli_tavor_wrid_find_match_srq(dapls_tavor_wrid_list_hdr_t *wl,
    tavor_hw_cqe_t *cqe)
{
	dapls_tavor_wrid_entry_t	*wre;
	uint32_t		wqe_index;
	uint32_t		wqe_addr;
	uint32_t		qsize_msk;
	uint32_t		tail, next_tail;

	/* Grab the WQE addr out of the CQE */
	wqe_addr = TAVOR_CQE_WQEADDRSZ_GET(cqe) & 0xFFFFFFC0;

	/*
	 * Given the 'wqe_addr' just calculated and the srq buf address, we
	 * find the 'wqe_index'.  The 'wre' returned below contains the WRID
	 * that we are looking for.  This indexes into the wre_list for this
	 * specific WQE.
	 */
	wqe_index = TAVOR_SRQ_WQ_INDEX(wl->wl_srq_desc_addr, wqe_addr,
	    wl->wl_srq_wqesz);

	/* ASSERT on impossible wqe_index values */
	dapl_os_assert(wqe_index < wl->wl_size);

	/* Put this WQE back on the free list */

	qsize_msk = wl->wl_size - 1;
	tail	  = wl->wl_freel_tail;

	next_tail = (tail + 1) & qsize_msk;
	wl->wl_freel_entries++;

	dapl_os_assert(wl->wl_freel_entries <= wl->wl_size);

	/* Get the descriptor (IO Address) of the WQE to be built */
	wl->wl_free_list[tail] = wqe_addr;
	wl->wl_freel_tail = next_tail;
	/* Using the index, return the Work Request ID Entry (wre) */
	wre = &wl->wl_wre[wqe_index];

	return (wre);
}

/*
 * dapls_tavor_wrid_cq_reap()
 */
void
dapls_tavor_wrid_cq_reap(ib_cq_handle_t cq)
{
	dapls_tavor_workq_hdr_t	*consume_wqhdr;
	dapls_tavor_wrid_list_hdr_t	*container, *to_free;


	/* dapl_os_assert(MUTEX_HELD(&cq->cq_lock)); */

	/* Lock the list of work queues associated with this CQ */
	dapl_os_lock(&cq->cq_wrid_wqhdr_lock);

	/* Walk the "reapable" list and free up containers */
	container = cq->cq_wrid_reap_head;
	while (container != NULL) {
		to_free	  = container;
		container = container->wl_reap_next;
		/*
		 * If reaping the WRID list containers pulls the last
		 * container from the given work queue header, then we free
		 * the work queue header as well.
		 */
		consume_wqhdr = dapli_tavor_wrid_list_reap(to_free);
		if (consume_wqhdr != NULL) {
			dapli_tavor_cq_wqhdr_remove(cq, consume_wqhdr);
		}
	}

	/* Once finished reaping, we reset the CQ's reap list */
	cq->cq_wrid_reap_head = cq->cq_wrid_reap_tail = NULL;

	dapl_os_unlock(&cq->cq_wrid_wqhdr_lock);
}


/*
 * dapls_tavor_wrid_cq_force_reap()
 */
void
dapls_tavor_wrid_cq_force_reap(ib_cq_handle_t cq)
{
	DAPL_HASH_DATA		curr;
	DAT_RETURN		retval;
	dapls_tavor_workq_hdr_t		*to_free_wqhdr;
	dapls_tavor_wrid_list_hdr_t	*container, *to_free;

	/* dapl_os_assert(MUTEX_HELD(&cq->cq_lock)); */

	/*
	 * The first step is to walk the "reapable" list and free up those
	 * containers.  This is necessary because the containers on the
	 * reapable list are not otherwise connected to the work queue headers
	 * anymore.
	 */
	dapls_tavor_wrid_cq_reap(cq);

	/* Now lock the list of work queues associated with this CQ */
	dapl_os_lock(&cq->cq_wrid_wqhdr_lock);

	/*
	 * Walk the list of work queue headers and free up all the WRID list
	 * containers chained to it.  Note: We don't need to grab the locks
	 * for each of the individual WRID lists here because the only way
	 * things can be added or removed from the list at this point would be
	 * through post a work request to a QP.  But if we've come this far,
	 * then we can be assured that there are no longer any QP associated
	 * with the CQ that we are trying to free.
	 */
	retval = dapls_hash_iterate(cq->cq_wrid_wqhdr_list,
	    DAPL_HASH_ITERATE_INIT, &curr);
	dapl_os_assert(retval == DAT_SUCCESS);

	while (curr != NULL) {
		to_free_wqhdr = (dapls_tavor_workq_hdr_t *)curr;
		container = ((dapls_tavor_workq_hdr_t *)curr)->wq_wrid_poll;
		retval = dapls_hash_iterate(cq->cq_wrid_wqhdr_list,
		    DAPL_HASH_ITERATE_NEXT, &curr);
		dapl_os_assert(retval == DAT_SUCCESS);
		while (container != NULL) {
			to_free	  = container;
			container = container->wl_next;
			/*
			 * If reaping the WRID list containers pulls the last
			 * container from the given work queue header, then
			 * we free the work queue header as well.  Note: we
			 * ignore the return value because we know that the
			 * work queue header should always be freed once the
			 * list of containers has come to an end.
			 */
			(void) dapli_tavor_wrid_list_reap(to_free);
			if (container == NULL) {
				dapli_tavor_cq_wqhdr_remove(cq, to_free_wqhdr);
			}
		}
	}

	dapl_os_lock(&cq->cq_wrid_wqhdr_lock);
}


/*
 * dapli_tavor_wrid_get_list()
 */
static dapls_tavor_wrid_list_hdr_t *
dapli_tavor_wrid_get_list(uint32_t qsize, int wrid_for_srq)
{
	dapls_tavor_wrid_list_hdr_t	*wridlist;
	dapls_tavor_wrid_entry_t	*wl_wre;
	uint32_t			*wl_freel;
	uint32_t			size;
	uint32_t			wl_wre_size;
	uint32_t			wl_freel_size;

	wridlist = NULL;
	wl_wre = NULL;
	wl_freel = NULL;
	size = wl_wre_size = wl_freel_size = 0;
	/*
	 * The WRID list "container" consists of the dapls_tavor_wrid_list_hdr_t
	 * which holds the pointers necessary for maintaining the "reapable"
	 * list, chaining together multiple "containers" old and new, and
	 * tracking the head, tail, size, etc. for each container.  The
	 * "container" also holds all the tavor_wrid_entry_t's, one for
	 * each entry on the corresponding work queue.
	 */

	/*
	 * For wridlist associated with SRQs the wridlock needs to be
	 * allocated and initialized here.
	 */
	size = sizeof (dapls_tavor_wrid_list_hdr_t);
	if (wrid_for_srq) {
		size = size + sizeof (dapls_tavor_wrid_lock_t);
	}
	wridlist = dapl_os_alloc(size);
	if (wridlist == NULL) {
		goto bail;
	}
	if (wrid_for_srq) {
		wridlist->wl_lock = (dapls_tavor_wrid_lock_t *)(
		    (uintptr_t)wridlist + sizeof (dapls_tavor_wrid_list_hdr_t));
		dapl_os_lock_init(&wridlist->wl_lock->wrl_lock);
		wridlist->wl_lock->wrl_on_srq = wrid_for_srq;
	} else {
		wridlist->wl_lock = NULL;
	}
	wl_wre_size = qsize * sizeof (dapls_tavor_wrid_entry_t);
	wl_wre = dapl_os_alloc(wl_wre_size);
	if (wl_wre == NULL) {
		goto bail;
	}
	if (wrid_for_srq) { /* memory for the SRQ free list */
		wl_freel_size = qsize * sizeof (uint32_t);
		wl_freel = dapl_os_alloc(wl_freel_size);
		if (wl_freel == NULL) {
			goto bail;
		}
	}


	/* Complete the "container" initialization */
	wridlist->wl_size = qsize;
	wridlist->wl_full = 0;
	wridlist->wl_head = 0;
	wridlist->wl_tail = 0;
	wridlist->wl_wre = wl_wre;
	wridlist->wl_wre_old_tail  = NULL;
	wridlist->wl_reap_next = NULL;
	wridlist->wl_next  = NULL;
	wridlist->wl_prev  = NULL;
	if (wrid_for_srq) {
		wridlist->wl_srq_en = 1;
		wridlist->wl_free_list = (uint32_t *)wl_freel;
		wridlist->wl_freel_head = 0;
		wridlist->wl_freel_tail = 0;
		wridlist->wl_freel_entries = qsize;
	} else {
		wridlist->wl_srq_en = 0;
		wridlist->wl_free_list = NULL;
		wridlist->wl_freel_head = 0;
		wridlist->wl_freel_tail = 0;
		wridlist->wl_freel_entries = 0;
		wridlist->wl_srq_wqesz = 0;
		wridlist->wl_srq_desc_addr = 0;
	}
	return (wridlist);
bail:
	if (wridlist) {
		if (wrid_for_srq) {
			dapl_os_lock_destroy(&wridlist->wl_lock->wrl_lock);
		}
		dapl_os_free(wridlist, size);
	}
	if (wl_wre) {
		dapl_os_free(wl_wre, wl_wre_size);
	}
	if (wl_freel) {
		dapl_os_free(wl_freel, wl_freel_size);
	}
	return (NULL);
}


/*
 * dapli_tavor_wrid_reaplist_add()
 */
static void
dapli_tavor_wrid_reaplist_add(ib_cq_handle_t cq, dapls_tavor_workq_hdr_t *wq)
{
	/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */

	dapl_os_lock(&wq->wq_wrid_lock->wrl_lock);

	/*
	 * Add the "post" container (the last one on the current chain) to
	 * the CQ's "reapable" list
	 */
	if ((cq->cq_wrid_reap_head == NULL) &&
	    (cq->cq_wrid_reap_tail == NULL)) {
		cq->cq_wrid_reap_head = wq->wq_wrid_post;
		cq->cq_wrid_reap_tail = wq->wq_wrid_post;
	} else {
		cq->cq_wrid_reap_tail->wl_reap_next = wq->wq_wrid_post;
		cq->cq_wrid_reap_tail = wq->wq_wrid_post;
	}

	dapl_os_unlock(&wq->wq_wrid_lock->wrl_lock);
}


/*
 * dapli_tavor_wrid_wqhdr_find()
 */
static dapls_tavor_workq_hdr_t *
dapli_tavor_wrid_wqhdr_find(ib_cq_handle_t cq, uint_t qpn, uint_t send_or_recv)
{
	DAPL_HASH_DATA		curr;
	DAPL_HASH_KEY		key;
	DAT_RETURN		status;

	/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */

	/*
	 * Walk the CQ's work queue list, trying to find a send or recv queue
	 * with the same QP number.  We do this even if we are going to later
	 * create a new entry because it helps us easily find the end of the
	 * list.
	 */
	key = (DAPL_HASH_KEY)(((uint64_t)send_or_recv << 32) | (uint32_t)qpn);

	status = dapls_hash_search(cq->cq_wrid_wqhdr_list, key, &curr);
	if (status == DAT_SUCCESS) {
		return ((dapls_tavor_workq_hdr_t *)curr);
	} else {
		return (NULL);
	}
}




/*
 * dapli_tavor_wrid_get_wqeaddrsz()
 */
static uint32_t
dapli_tavor_wrid_get_wqeaddrsz(dapls_tavor_workq_hdr_t *wq)
{
	dapls_tavor_wrid_entry_t	*wre;
	uint32_t		wqeaddrsz;
	uint32_t		head;

	/*
	 * If the container is empty, then there is no next entry. So just
	 * return zero.  Note: the "head == tail" condition here can only
	 * mean that the container is empty because we have previously pulled
	 * something from the container.
	 *
	 * If the container is not empty, then find the next entry and return
	 * the contents of its "wqeaddrsz" field.
	 */
	if (wq->wq_wrid_poll->wl_head == wq->wq_wrid_poll->wl_tail) {
		wqeaddrsz = 0;
	} else {
		/*
		 * We don't need to calculate the "next" head pointer here
		 * because "head" should already point to the next entry on
		 * the list (since we just pulled something off - in
		 * dapli_tavor_wrid_find_match() - and moved the head index
		 * forward.)
		 */
		head = wq->wq_wrid_poll->wl_head;
		wre = &wq->wq_wrid_poll->wl_wre[head];
		wqeaddrsz = wre->wr_wqeaddrsz;
	}
	return (wqeaddrsz);
}



/*
 * dapli_tavor_wrid_list_reap()
 *    Note: The "wqhdr_list_lock" must be held.
 */
static dapls_tavor_workq_hdr_t *
dapli_tavor_wrid_list_reap(dapls_tavor_wrid_list_hdr_t *wridlist)
{
	dapls_tavor_workq_hdr_t	*wqhdr, *consume_wqhdr = NULL;
	dapls_tavor_wrid_list_hdr_t	*prev, *next;

	/* Get the back pointer to the work queue header (see below) */
	wqhdr = wridlist->wl_wqhdr;
	dapl_os_lock(&wqhdr->wq_wrid_lock->wrl_lock);

	/* Unlink the WRID list "container" from the work queue list */
	prev = wridlist->wl_prev;
	next = wridlist->wl_next;
	if (prev != NULL) {
		prev->wl_next = next;
	}
	if (next != NULL) {
		next->wl_prev = prev;
	}

	/*
	 * If the back pointer to the work queue header shows that it
	 * was pointing to the entry we are about to remove, then the work
	 * queue header is reapable as well.
	 */
	if ((wqhdr->wq_wrid_poll == wridlist) &&
	    (wqhdr->wq_wrid_post == wridlist)) {
		consume_wqhdr = wqhdr;
	}

	/* Be sure to update the "poll" and "post" container pointers */
	if (wqhdr->wq_wrid_poll == wridlist) {
		wqhdr->wq_wrid_poll = next;
	}
	if (wqhdr->wq_wrid_post == wridlist) {
		wqhdr->wq_wrid_post = NULL;
	}

	/*
	 * Calculate the size and free the container, for SRQ wridlist is
	 * freed when srq gets freed
	 */
	if (!wridlist->wl_srq_en) {
		if (wridlist->wl_wre) {
			dapl_os_free(wridlist->wl_wre, wridlist->wl_size *
			    sizeof (dapls_tavor_wrid_entry_t));
		}
		dapl_os_assert(wridlist->wl_free_list == NULL);
		dapl_os_free(wridlist, sizeof (dapls_tavor_wrid_list_hdr_t));
	}

	dapl_os_unlock(&wqhdr->wq_wrid_lock->wrl_lock);

	return (consume_wqhdr);
}

/*
 * dapls_tavor_srq_wrid_init()
 */
DAT_RETURN
dapls_tavor_srq_wrid_init(ib_srq_handle_t srq)
{
	dapls_tavor_wrid_list_hdr_t	*wridlist;
	int i;

	wridlist = dapli_tavor_wrid_get_list(srq->srq_wq_numwqe, 1);


	if (wridlist == NULL) {
		srq->srq_wridlist = NULL;
		return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
	}

	/* initialize the free list with the descriptor addresses */
	wridlist->wl_free_list[0] = srq->srq_wq_desc_addr;
	for (i = 1; i < srq->srq_wq_numwqe; i++) {
		wridlist->wl_free_list[i] = wridlist->wl_free_list[i-1] +
		    srq->srq_wq_wqesz;
	}
	wridlist->wl_srq_wqesz = srq->srq_wq_wqesz;
	wridlist->wl_srq_desc_addr = srq->srq_wq_desc_addr;

	srq->srq_wridlist = wridlist;
	return (DAT_SUCCESS);
}

void
dapls_tavor_srq_wrid_free(ib_srq_handle_t srq)
{
	dapls_tavor_wrid_list_hdr_t	*wridlist;
	size_t				size = 0;

	wridlist = srq->srq_wridlist;
	if (wridlist) {
		dapl_os_assert(wridlist->wl_srq_en == 1);
		if (wridlist->wl_wre) {
			dapl_os_free(wridlist->wl_wre, wridlist->wl_size *
			    sizeof (dapls_tavor_wrid_entry_t));
		}
		if (wridlist->wl_free_list) {
			dapl_os_free(wridlist->wl_free_list, wridlist->wl_size *
			    sizeof (uint32_t));
		}
		if (wridlist->wl_lock) {
			dapl_os_assert(wridlist->wl_lock->wrl_on_srq == 1);
			dapl_os_lock_destroy(&wridlist->wl_lock->wrl_lock);
			size = sizeof (dapls_tavor_wrid_lock_t);
		}
		size = size; /* pacify lint */
		dapl_os_free(wridlist, size +
		    sizeof (dapls_tavor_wrid_list_hdr_t));
		srq->srq_wridlist = NULL;
	}
}


/*
 * dapls_tavor_wrid_init()
 */
DAT_RETURN
dapls_tavor_wrid_init(ib_qp_handle_t qp)
{
	dapls_tavor_workq_hdr_t		*swq;
	dapls_tavor_workq_hdr_t		*rwq;
	dapls_tavor_wrid_list_hdr_t	*s_wridlist;
	dapls_tavor_wrid_list_hdr_t	*r_wridlist;
	uint_t		create_new_swq = 0;
	uint_t		create_new_rwq = 0;

	/*
	 * For each of this QP's Work Queues, make sure we have a (properly
	 * initialized) Work Request ID list attached to the relevant
	 * completion queue.  Grab the CQ lock(s) before manipulating the
	 * lists.
	 */
	dapli_tavor_wrid_wqhdr_lock_both(qp);
	swq = dapli_tavor_wrid_wqhdr_find(qp->qp_sq_cqhdl, qp->qp_num,
	    TAVOR_WR_SEND);
	if (swq == NULL) {
		/* Couldn't find matching work queue header, create it */
		create_new_swq = 1;
		swq = dapli_tavor_wrid_wqhdr_create(qp->qp_sq_cqhdl,
		    qp->qp_num, TAVOR_WR_SEND, 1);
		if (swq == NULL) {
			/*
			 * If we couldn't find/allocate space for the workq
			 * header, then drop the lock(s) and return failure.
			 */
			dapli_tavor_wrid_wqhdr_unlock_both(qp);
			return (DAT_INSUFFICIENT_RESOURCES);
		}
	}
	qp->qp_sq_wqhdr = swq;
	swq->wq_size = qp->qp_sq_numwqe;
	swq->wq_head = 0;
	swq->wq_tail = 0;
	swq->wq_full = 0;

	/*
	 * Allocate space for the dapls_tavor_wrid_entry_t container
	 */
	s_wridlist = dapli_tavor_wrid_get_list(swq->wq_size, 0);
	if (s_wridlist == NULL) {
		/*
		 * If we couldn't allocate space for tracking the WRID
		 * entries, then cleanup the workq header from above (if
		 * necessary, i.e. if we created the workq header).  Then
		 * drop the lock(s) and return failure.
		 */
		if (create_new_swq) {
			dapli_tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl, swq);
		}

		dapli_tavor_wrid_wqhdr_unlock_both(qp);
		return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
	}
	s_wridlist->wl_wqhdr = swq;
	/* Chain the new WRID list container to the workq hdr list */
	dapl_os_lock(&swq->wq_wrid_lock->wrl_lock);
	dapli_tavor_wrid_wqhdr_add(swq, s_wridlist);
	dapl_os_unlock(&swq->wq_wrid_lock->wrl_lock);


	/*
	 * Now we repeat all the above operations for the receive work queue
	 */
	rwq = dapli_tavor_wrid_wqhdr_find(qp->qp_rq_cqhdl, qp->qp_num,
	    TAVOR_WR_RECV);
	if (rwq == NULL) {
		create_new_rwq = 1;
		/* if qp is attached to an SRQ don't need to alloc wrid_lock */
		rwq = dapli_tavor_wrid_wqhdr_create(qp->qp_rq_cqhdl,
		    qp->qp_num, TAVOR_WR_RECV, qp->qp_srq_enabled ? 0 : 1);
		if (rwq == NULL) {
			/*
			 * If we couldn't find/allocate space for the workq
			 * header, then free all the send queue resources we
			 * just allocated and setup (above), drop the lock(s)
			 * and return failure.
			 */
			dapl_os_lock(&swq->wq_wrid_lock->wrl_lock);
			dapli_tavor_wrid_wqhdr_remove(swq, s_wridlist);
			dapl_os_unlock(&swq->wq_wrid_lock->wrl_lock);
			if (create_new_swq) {
				dapli_tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl,
				    swq);
			}

			dapli_tavor_wrid_wqhdr_unlock_both(qp);
			return (DAT_INSUFFICIENT_RESOURCES |
			    DAT_RESOURCE_MEMORY);
		}
	}
	qp->qp_rq_wqhdr = rwq;
	rwq->wq_size = qp->qp_rq_numwqe;
	rwq->wq_head = 0;
	rwq->wq_tail = 0;
	rwq->wq_full = 0;

	/*
	 * Allocate space for the dapls_tavor_wrid_entry_t container
	 * For qp associated with SRQs the SRQ wridlist is used
	 */
	if (qp->qp_srq_enabled) {
		/* Use existing srq_wridlist pointer */
		r_wridlist = qp->qp_srq->srq_wridlist;
		dapl_os_assert(r_wridlist != NULL);
		/* store the wl_lock in the wqhdr */
		rwq->wq_wrid_lock = r_wridlist->wl_lock;
		dapl_os_assert(rwq->wq_wrid_lock != NULL);
	} else {
		/* Allocate memory for the r_wridlist */
		r_wridlist = dapli_tavor_wrid_get_list(rwq->wq_size, 0);
	}
	if (r_wridlist == NULL) {
		/*
		 * If we couldn't allocate space for tracking the WRID
		 * entries, then cleanup all the stuff from above.  Then
		 * drop the lock(s) and return failure.
		 */
		dapl_os_lock(&swq->wq_wrid_lock->wrl_lock);
		dapli_tavor_wrid_wqhdr_remove(swq, s_wridlist);
		dapl_os_unlock(&swq->wq_wrid_lock->wrl_lock);
		if (create_new_swq) {
			dapli_tavor_cq_wqhdr_remove(qp->qp_sq_cqhdl, swq);
		}
		if (create_new_rwq) {
			dapli_tavor_cq_wqhdr_remove(qp->qp_rq_cqhdl, rwq);
		}

		dapli_tavor_wrid_wqhdr_unlock_both(qp);
		return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
	}

	/* For SRQ based QPs r_wridlist does not point to recv wqhdr */
	if (!qp->qp_srq_enabled) {
		r_wridlist->wl_wqhdr = rwq;
	}

	/* Chain the new WRID list "container" to the workq hdr list */
	dapl_os_lock(&rwq->wq_wrid_lock->wrl_lock);
	dapli_tavor_wrid_wqhdr_add(rwq, r_wridlist);
	dapl_os_unlock(&rwq->wq_wrid_lock->wrl_lock);

	dapli_tavor_wrid_wqhdr_unlock_both(qp);

	return (DAT_SUCCESS);
}


/*
 * dapls_tavor_wrid_cleanup()
 */
void
dapls_tavor_wrid_cleanup(DAPL_EP *ep, ib_qp_handle_t qp)
{
	/*
	 * For each of this QP's Work Queues, move the WRID "container" to
	 * the "reapable" list.  Although there may still be unpolled
	 * entries in these containers, it is not a big deal.  We will not
	 * reap the list until either the Poll CQ command detects an empty
	 * condition or the CQ itself is freed.  Grab the CQ lock(s) before
	 * manipulating the lists.
	 */
	dapli_tavor_wrid_wqhdr_lock_both(qp);
	dapli_tavor_wrid_reaplist_add(qp->qp_sq_cqhdl, qp->qp_sq_wqhdr);

	/*
	 * Repeat the above operation for the Recv work queue "container".
	 * However for qps with SRQ we flush the cq entries, remove the
	 * wridlist and wqhdr.
	 * Then drop the CQ lock(s) and return
	 */
	if (qp->qp_srq_enabled) {
		/*
		 * Pull off all (if any) entries for this QP from CQ.  This
		 * only includes entries that have not yet been polled
		 */
		dapl_os_lock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
		DAPL_FLUSH(ep)(qp);

		/* Remove wridlist from WQHDR */
		dapli_tavor_wrid_wqhdr_remove(qp->qp_rq_wqhdr,
		    qp->qp_rq_wqhdr->wq_wrid_post);

		dapl_os_assert(qp->qp_rq_wqhdr->wq_wrid_post == NULL);

		dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);

		/* Free the WQHDR */
		dapli_tavor_cq_wqhdr_remove(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
	} else {
		dapli_tavor_wrid_reaplist_add(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
	}
	dapli_tavor_wrid_wqhdr_unlock_both(qp);
}

/*
 * dapli_tavor_wrid_wqhdr_create()
 */
static dapls_tavor_workq_hdr_t *
dapli_tavor_wrid_wqhdr_create(ib_cq_handle_t cq, uint_t qpn,
    uint_t send_or_recv, uint_t alloc_wrl)
{
	dapls_tavor_workq_hdr_t	*wqhdr_tmp;
	size_t			size, aligned_size;

	/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */

	/*
	 * Allocate space for a work queue header structure and initialize it.
	 * Each work queue header structure includes a "wq_wrid_lock"
	 * which needs to be initialized.
	 *
	 * Note: the address smashing is needed to ensure wq_wrid_lock is
	 * 8-byte aligned, which is not always the case on 32-bit sparc.
	 */
	size = (sizeof (dapls_tavor_workq_hdr_t) + 0x7) & ~0x7;
	aligned_size = size;
	if (alloc_wrl) {
		/* for non-srq wqhdr the lock is allocated with the wqhdr */
		size = size + sizeof (dapls_tavor_wrid_lock_t);
	}
	wqhdr_tmp = dapl_os_alloc(size);
	if (wqhdr_tmp == NULL) {
		return (NULL);
	}
	if (alloc_wrl) {
		wqhdr_tmp->wq_wrid_lock = (dapls_tavor_wrid_lock_t *)
		    (((uintptr_t)wqhdr_tmp + aligned_size) & ~0x7);
		dapl_os_lock_init(&wqhdr_tmp->wq_wrid_lock->wrl_lock);
		/* wrl allocated with wqhdr don't have srq enabled */
		wqhdr_tmp->wq_wrid_lock->wrl_on_srq = 0;
	}

	wqhdr_tmp->wq_qpn	= qpn;
	wqhdr_tmp->wq_send_or_recv = send_or_recv;

	wqhdr_tmp->wq_wrid_poll = NULL;
	wqhdr_tmp->wq_wrid_post = NULL;

	/* Chain the newly allocated work queue header to the CQ's list */
	if (dapli_tavor_cq_wqhdr_add(cq, wqhdr_tmp) != DAT_SUCCESS) {
		if (alloc_wrl) {
			dapl_os_lock_destroy(&wqhdr_tmp->wq_wrid_lock->
			    wrl_lock);
		}
		dapl_os_free(wqhdr_tmp, size);
		wqhdr_tmp = NULL;
	}

	return (wqhdr_tmp);
}

/*
 * dapli_tavor_wrid_wqhdr_add()
 */
static void
dapli_tavor_wrid_wqhdr_add(dapls_tavor_workq_hdr_t *wqhdr,
    dapls_tavor_wrid_list_hdr_t *wridlist)
{
	/* dapl_os_assert(MUTEX_HELD(&wqhdr->wq_wrid_lock)); */

	/* Chain the new WRID list "container" to the work queue list */
	if ((wqhdr->wq_wrid_post == NULL) &&
	    (wqhdr->wq_wrid_poll == NULL)) {
		wqhdr->wq_wrid_poll = wridlist;
		wqhdr->wq_wrid_post = wridlist;
	} else {
		wqhdr->wq_wrid_post->wl_next = wridlist;
		wridlist->wl_prev = wqhdr->wq_wrid_post;
		wqhdr->wq_wrid_post = wridlist;
	}
}


/*
 * dapli_tavor_wrid_wqhdr_remove()
 *    Note: this is only called to remove the most recently added WRID list
 *    container.
 */
static void
dapli_tavor_wrid_wqhdr_remove(dapls_tavor_workq_hdr_t *wqhdr,
    dapls_tavor_wrid_list_hdr_t *wridlist)
{
	dapls_tavor_wrid_list_hdr_t	*prev, *next;

	/* dapl_os_assert(MUTEX_HELD(&wqhdr->wq_wrid_lock)); */

	/* Unlink the WRID list "container" from the work queue list */
	prev = wridlist->wl_prev;
	next = wridlist->wl_next;
	if (prev != NULL) {
		prev->wl_next = next;
	}
	if (next != NULL) {
		next->wl_prev = prev;
	}

	/*
	 * Update any pointers in the work queue hdr that may point to this
	 * WRID list container
	 */
	if (wqhdr->wq_wrid_post == wridlist) {
		wqhdr->wq_wrid_post = prev;
	}
	if (wqhdr->wq_wrid_poll == wridlist) {
		wqhdr->wq_wrid_poll = NULL;
	}
}


/*
 * dapli_tavor_wrid_wqhdr_lock_both()
 */
static void
dapli_tavor_wrid_wqhdr_lock_both(ib_qp_handle_t qp)
{
	ib_cq_handle_t	sq_cq, rq_cq;

	sq_cq = qp->qp_sq_cqhdl;
	rq_cq = qp->qp_rq_cqhdl;

	/*
	 * If both work queues (send and recv) share a completion queue, then
	 * grab the common lock.  If they use different CQs (hence different
	 * "cq_wrid_wqhdr_list" locks), then grab the send one first, then the
	 * receive.  We do this consistently and correctly in
	 * tavor_wrid_wqhdr_unlock_both() below to avoid introducing any kind
	 * of dead lock condition.
	 */
	if (sq_cq == rq_cq) {
		dapl_os_lock(&sq_cq->cq_wrid_wqhdr_lock);
	} else {
		dapl_os_lock(&sq_cq->cq_wrid_wqhdr_lock);
		dapl_os_lock(&rq_cq->cq_wrid_wqhdr_lock);
	}
}

/*
 * dapli_tavor_wrid_wqhdr_unlock_both()
 */
static void
dapli_tavor_wrid_wqhdr_unlock_both(ib_qp_handle_t qp)
{
	ib_cq_handle_t	sq_cq, rq_cq;

	sq_cq = qp->qp_sq_cqhdl;
	rq_cq = qp->qp_rq_cqhdl;

	/*
	 * See tavor_wrid_wqhdr_lock_both() above for more detail
	 */
	if (sq_cq == rq_cq) {
		dapl_os_unlock(&sq_cq->cq_wrid_wqhdr_lock);
	} else {
		dapl_os_unlock(&rq_cq->cq_wrid_wqhdr_lock);
		dapl_os_unlock(&sq_cq->cq_wrid_wqhdr_lock);
	}
}


/*
 * dapli_tavor_cq_wqhdr_add()
 */
static DAT_RETURN
dapli_tavor_cq_wqhdr_add(ib_cq_handle_t cq, dapls_tavor_workq_hdr_t *wqhdr)
{
	DAPL_HASH_KEY		key;

	/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */

	/*
	 * If the CQ's work queue list is empty, then just add it.
	 * Otherwise, chain it to the beginning of the list.
	 */
	key = (DAPL_HASH_KEY)(((uint64_t)wqhdr->wq_send_or_recv << 32) |
	    wqhdr->wq_qpn);

	return (dapls_hash_insert(cq->cq_wrid_wqhdr_list, key, wqhdr));
}


/*
 * dapli_tavor_cq_wqhdr_remove
 */
static void
dapli_tavor_cq_wqhdr_remove(ib_cq_handle_t cq, dapls_tavor_workq_hdr_t *wqhdr)
{
	DAPL_HASH_DATA	curr;
	DAPL_HASH_KEY	key;
	size_t		size = 0;

	/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */

	/* Remove "wqhdr" from the work queue header list on "cq" */

	key = (DAPL_HASH_KEY)(((uint64_t)wqhdr->wq_send_or_recv << 32) |
	    wqhdr->wq_qpn);

	(void) dapls_hash_remove(cq->cq_wrid_wqhdr_list, key,  &curr);

	size = (sizeof (dapls_tavor_workq_hdr_t) + 0x7) & ~0x7;
	if (wqhdr->wq_wrid_lock && (!wqhdr->wq_wrid_lock->wrl_on_srq)) {
		dapl_os_lock_destroy(&wqhdr->wq_wrid_lock->wrl_lock);
		size += sizeof (dapls_tavor_wrid_lock_t);
	}

	/* Free the memory associated with "wqhdr" */
	dapl_os_free(wqhdr, size);
}

/*
 * dapls_tavor_srq_wrid_resize() is called to resize the wridlist
 * associated with SRQS as a result of dat_srq_resize().
 *
 * Returns: DAT_TRUE if successful, otherwise DAT_FALSE
 */
DAT_BOOLEAN
dapls_tavor_srq_wrid_resize(ib_srq_handle_t srq_handle, uint32_t new_size)
{
	dapls_tavor_wrid_list_hdr_t	*wridlist;
	dapls_tavor_wrid_entry_t	*old_wl_wre;
	dapls_tavor_wrid_entry_t	*new_wl_wre;
	uint32_t			*old_wl_freel;
	uint32_t			*new_wl_freel;
	uint32_t			old_size;
	uint32_t			idx;
	uint32_t			prev_idx;
	uint32_t			i;

	wridlist = srq_handle->srq_wridlist;

	if (wridlist == NULL) {
		return (DAT_FALSE);
	}
	dapl_os_assert(wridlist->wl_srq_en);

	dapl_os_lock(&wridlist->wl_lock->wrl_lock);

	old_wl_wre = wridlist->wl_wre;
	old_wl_freel = wridlist->wl_free_list;
	old_size = wridlist->wl_size;

	new_wl_wre = (dapls_tavor_wrid_entry_t *)dapl_os_alloc(new_size *
	    sizeof (dapls_tavor_wrid_entry_t));
	if (new_wl_wre == NULL) {
		goto bail;
	}
	new_wl_freel = dapl_os_alloc(new_size * sizeof (uint32_t));
	if (new_wl_freel == NULL) {
		goto bail;
	}
	/*
	 * we just need to copy the old WREs to the new array. Since the
	 * descriptors are relatively addressed the descriptor to index
	 * mapping doesn't change.
	 */
	(void) dapl_os_memcpy(&new_wl_wre[0], &old_wl_wre[0],
	    old_size * sizeof (dapls_tavor_wrid_entry_t));
	/*
	 * Copy the old free list to the new one
	 */
	idx = wridlist->wl_freel_head;
	for (i = 0; i < wridlist->wl_freel_entries; i++) {
		new_wl_freel[i] = old_wl_freel[idx];
		idx = (idx + 1) % old_size;
	}
	/*
	 * Add the new entries in wl_wre to the new free list
	 */
	idx = wridlist->wl_freel_entries;
	new_wl_freel[idx] = wridlist->wl_srq_desc_addr + old_size *
	    wridlist->wl_srq_wqesz;
	prev_idx = idx;
	idx = (idx + 1) % new_size;
	for (i = 0; i < new_size - old_size - 1; i++) {
		new_wl_freel[idx] = new_wl_freel[prev_idx] +
		    wridlist->wl_srq_wqesz;
		prev_idx = idx;
		idx = (idx + 1) % new_size;
	}
	wridlist->wl_size = new_size;
	wridlist->wl_wre = new_wl_wre;
	wridlist->wl_free_list = new_wl_freel;
	wridlist->wl_freel_head = 0;
	wridlist->wl_freel_tail = idx;
	wridlist->wl_freel_entries = wridlist->wl_freel_entries + new_size -
	    old_size;

	dapl_os_unlock(&wridlist->wl_lock->wrl_lock);

	if (old_wl_wre) {
		dapl_os_free(old_wl_wre, old_size *
		    sizeof (dapls_tavor_wrid_entry_t));
	}
	if (old_wl_freel) {
		dapl_os_free(old_wl_freel, old_size * sizeof (uint32_t));
	}
	return (DAT_TRUE);
bail:
	dapl_os_unlock(&wridlist->wl_lock->wrl_lock);
	if (new_wl_wre) {
		dapl_os_free(new_wl_wre, new_size *
		    sizeof (dapls_tavor_wrid_entry_t));
	}
	return (DAT_FALSE);
}