xref: /illumos-gate/usr/src/uts/common/io/ib/adapters/hermon/hermon_cq.c (revision 6a634c9d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * hermon_cq.c
 *    Hermon Completion Queue Processing Routines
 *
 *    Implements all the routines necessary for allocating, freeing, resizing,
 *    and handling the completion type events that the Hermon hardware can
 *    generate.
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/bitmap.h>
#include <sys/sysmacros.h>

#include <sys/ib/adapters/hermon/hermon.h>

int hermon_should_panic = 0;	/* debugging aid */

#define	hermon_cq_update_ci_doorbell(cq)				\
	/* Build the doorbell record data (low 24 bits only) */		\
	HERMON_UAR_DB_RECORD_WRITE(cq->cq_arm_ci_vdbr,			\
	    cq->cq_consindx & 0x00FFFFFF)

static int hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq,
    uint_t cmd);
#pragma inline(hermon_cq_arm_doorbell)
static void hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr);
#pragma inline(hermon_arm_cq_dbr_init)
static void hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
    hermon_hw_cqe_t *cqe, ibt_wc_t *wc);
static void hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
    hermon_hw_cqe_t *cqe, ibt_wc_t *wc);


/*
 * hermon_cq_alloc()
 *    Context: Can be called only from user or kernel context.
 */
int
hermon_cq_alloc(hermon_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
    ibt_cq_attr_t *cq_attr, uint_t *actual_size, hermon_cqhdl_t *cqhdl,
    uint_t sleepflag)
{
	hermon_rsrc_t		*cqc, *rsrc;
	hermon_umap_db_entry_t	*umapdb;
	hermon_hw_cqc_t		cqc_entry;
	hermon_cqhdl_t		cq;
	ibt_mr_attr_t		mr_attr;
	hermon_mr_options_t	op;
	hermon_pdhdl_t		pd;
	hermon_mrhdl_t		mr;
	hermon_hw_cqe_t		*buf;
	uint64_t		value;
	uint32_t		log_cq_size, uarpg;
	uint_t			cq_is_umap;
	uint32_t		status, flag;
	hermon_cq_sched_t	*cq_schedp;

	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq_attr))

	/*
	 * Determine whether CQ is being allocated for userland access or
	 * whether it is being allocated for kernel access.  If the CQ is
	 * being allocated for userland access, then lookup the UAR
	 * page number for the current process.  Note:  If this is not found
	 * (e.g. if the process has not previously open()'d the Hermon driver),
	 * then an error is returned.
	 */
	cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
	if (cq_is_umap) {
		status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
		    MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
		if (status != DDI_SUCCESS) {
			status = IBT_INVALID_PARAM;
			goto cqalloc_fail;
		}
		uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
	} else {
		uarpg = state->hs_kernel_uar_index;
	}

	/* Use the internal protection domain (PD) for setting up CQs */
	pd = state->hs_pdhdl_internal;

	/* Increment the reference count on the protection domain (PD) */
	hermon_pd_refcnt_inc(pd);

	/*
	 * Allocate an CQ context entry.  This will be filled in with all
	 * the necessary parameters to define the Completion Queue.  And then
	 * ownership will be passed to the hardware in the final step
	 * below.  If we fail here, we must undo the protection domain
	 * reference count.
	 */
	status = hermon_rsrc_alloc(state, HERMON_CQC, 1, sleepflag, &cqc);
	if (status != DDI_SUCCESS) {
		status = IBT_INSUFF_RESOURCE;
		goto cqalloc_fail1;
	}

	/*
	 * Allocate the software structure for tracking the completion queue
	 * (i.e. the Hermon Completion Queue handle).  If we fail here, we must
	 * undo the protection domain reference count and the previous
	 * resource allocation.
	 */
	status = hermon_rsrc_alloc(state, HERMON_CQHDL, 1, sleepflag, &rsrc);
	if (status != DDI_SUCCESS) {
		status = IBT_INSUFF_RESOURCE;
		goto cqalloc_fail2;
	}
	cq = (hermon_cqhdl_t)rsrc->hr_addr;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
	cq->cq_is_umap = cq_is_umap;
	cq->cq_cqnum = cqc->hr_indx;	/* just use index, implicit in Hermon */
	cq->cq_intmod_count = 0;
	cq->cq_intmod_usec = 0;

	/*
	 * If this will be a user-mappable CQ, then allocate an entry for
	 * the "userland resources database".  This will later be added to
	 * the database (after all further CQ operations are successful).
	 * If we fail here, we must undo the reference counts and the
	 * previous resource allocation.
	 */
	if (cq->cq_is_umap) {
		umapdb = hermon_umap_db_alloc(state->hs_instance, cq->cq_cqnum,
		    MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
		if (umapdb == NULL) {
			status = IBT_INSUFF_RESOURCE;
			goto cqalloc_fail3;
		}
	}


	/*
	 * Allocate the doorbell record.  We'll need one for the CQ, handling
	 * both consumer index (SET CI) and the CQ state (CQ ARM).
	 */

	status = hermon_dbr_alloc(state, uarpg, &cq->cq_arm_ci_dbr_acchdl,
	    &cq->cq_arm_ci_vdbr, &cq->cq_arm_ci_pdbr, &cq->cq_dbr_mapoffset);
	if (status != DDI_SUCCESS) {
		status = IBT_INSUFF_RESOURCE;
		goto cqalloc_fail4;
	}

	/*
	 * Calculate the appropriate size for the completion queue.
	 * Note:  All Hermon CQs must be a power-of-2 minus 1 in size.  Also
	 * they may not be any smaller than HERMON_CQ_MIN_SIZE.  This step is
	 * to round the requested size up to the next highest power-of-2
	 */
	cq_attr->cq_size = max(cq_attr->cq_size, HERMON_CQ_MIN_SIZE);
	log_cq_size = highbit(cq_attr->cq_size);

	/*
	 * Next we verify that the rounded-up size is valid (i.e. consistent
	 * with the device limits and/or software-configured limits)
	 */
	if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
		status = IBT_HCA_CQ_EXCEEDED;
		goto cqalloc_fail4a;
	}

	/*
	 * Allocate the memory for Completion Queue.
	 *
	 * Note: Although we use the common queue allocation routine, we
	 * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
	 * kernel system memory) for kernel CQs because it would be
	 * inefficient to have CQs located in DDR memory.  This is primarily
	 * because CQs are read from (by software) more than they are written
	 * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
	 * user-mappable CQs for a similar reason.)
	 * It is also worth noting that, unlike Hermon QP work queues,
	 * completion queues do not have the same strict alignment
	 * requirements.  It is sufficient for the CQ memory to be both
	 * aligned to and bound to addresses which are a multiple of CQE size.
	 */
	cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);

	cq->cq_cqinfo.qa_alloc_align = PAGESIZE;
	cq->cq_cqinfo.qa_bind_align  = PAGESIZE;
	if (cq->cq_is_umap) {
		cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
	} else {
		cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
		hermon_arm_cq_dbr_init(cq->cq_arm_ci_vdbr);
	}
	status = hermon_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
	if (status != DDI_SUCCESS) {
		status = IBT_INSUFF_RESOURCE;
		goto cqalloc_fail4;
	}
	buf = (hermon_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))

	/*
	 * The ownership bit of the CQE's is set by the HW during the process
	 * of transferrring ownership of the CQ (PRM 09.35c, 14.2.1, note D1
	 *
	 */

	/*
	 * Register the memory for the CQ.  The memory for the CQ must
	 * be registered in the Hermon TPT tables.  This gives us the LKey
	 * to specify in the CQ context below.  Note: If this is a user-
	 * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
	 */
	flag = (sleepflag == HERMON_SLEEP) ?  IBT_MR_SLEEP : IBT_MR_NOSLEEP;
	mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
	mr_attr.mr_len	 = cq->cq_cqinfo.qa_size;
	mr_attr.mr_as	 = NULL;
	mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
	op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
	op.mro_bind_dmahdl = cq->cq_cqinfo.qa_dmahdl;
	op.mro_bind_override_addr = 0;
	status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
	    HERMON_CQ_CMPT);
	if (status != DDI_SUCCESS) {
		status = IBT_INSUFF_RESOURCE;
		goto cqalloc_fail5;
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))

	cq->cq_erreqnum = HERMON_CQ_ERREQNUM_GET(state);
	if (cq_attr->cq_flags & IBT_CQ_HID) {
		if (!HERMON_HID_VALID(state, cq_attr->cq_hid)) {
			IBTF_DPRINTF_L2("CQalloc", "bad handler id 0x%x",
			    cq_attr->cq_hid);
			status = IBT_INVALID_PARAM;
			goto cqalloc_fail5;
		}
		cq->cq_eqnum = HERMON_HID_TO_EQNUM(state, cq_attr->cq_hid);
		IBTF_DPRINTF_L2("cqalloc", "hid: eqn %d", cq->cq_eqnum);
	} else {
		cq_schedp = (hermon_cq_sched_t *)cq_attr->cq_sched;
		if (cq_schedp == NULL) {
			cq_schedp = &state->hs_cq_sched_default;
		} else if (cq_schedp != &state->hs_cq_sched_default) {
			int i;
			hermon_cq_sched_t *tmp;

			tmp = state->hs_cq_sched_array;
			for (i = 0; i < state->hs_cq_sched_array_size; i++)
				if (cq_schedp == &tmp[i])
					break;	/* found it */
			if (i >= state->hs_cq_sched_array_size) {
				cmn_err(CE_CONT, "!Invalid cq_sched argument: "
				    "ignored\n");
				cq_schedp = &state->hs_cq_sched_default;
			}
		}
		cq->cq_eqnum = HERMON_HID_TO_EQNUM(state,
		    HERMON_CQSCHED_NEXT_HID(cq_schedp));
		IBTF_DPRINTF_L2("cqalloc", "sched: first-1 %d, len %d, "
		    "eqn %d", cq_schedp->cqs_start_hid - 1,
		    cq_schedp->cqs_len, cq->cq_eqnum);
	}

	/*
	 * Fill in the CQC entry.  This is the final step before passing
	 * ownership of the CQC entry to the Hermon hardware.  We use all of
	 * the information collected/calculated above to fill in the
	 * requisite portions of the CQC.  Note: If this CQ is going to be
	 * used for userland access, then we need to set the UAR page number
	 * appropriately (otherwise it's a "don't care")
	 */
	bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));

	cqc_entry.state		= HERMON_CQ_DISARMED;
	cqc_entry.pg_offs	= cq->cq_cqinfo.qa_pgoffs >> 5;
	cqc_entry.log_cq_sz	= log_cq_size;
	cqc_entry.usr_page	= uarpg;
	cqc_entry.c_eqn		= cq->cq_eqnum;
	cqc_entry.log2_pgsz	= mr->mr_log2_pgsz;
	cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
	cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;
	cqc_entry.dbr_addrh = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 32);
	cqc_entry.dbr_addrl = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 3);

	/*
	 * Write the CQC entry to hardware - we pass ownership of
	 * the entry to the hardware (using the Hermon SW2HW_CQ firmware
	 * command).  Note: In general, this operation shouldn't fail.  But
	 * if it does, we have to undo everything we've done above before
	 * returning error.
	 */
	status = hermon_cmn_ownership_cmd_post(state, SW2HW_CQ, &cqc_entry,
	    sizeof (hermon_hw_cqc_t), cq->cq_cqnum, sleepflag);
	if (status != HERMON_CMD_SUCCESS) {
		cmn_err(CE_CONT, "Hermon: SW2HW_CQ command failed: %08x\n",
		    status);
		if (status == HERMON_CMD_INVALID_STATUS) {
			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
		}
		status = ibc_get_ci_failure(0);
		goto cqalloc_fail6;
	}

	/*
	 * Fill in the rest of the Hermon Completion Queue handle.  Having
	 * successfully transferred ownership of the CQC, we can update the
	 * following fields for use in further operations on the CQ.
	 */
	cq->cq_resize_hdl = 0;
	cq->cq_cqcrsrcp	  = cqc;
	cq->cq_rsrcp	  = rsrc;
	cq->cq_consindx	  = 0;
		/* least restrictive */
	cq->cq_buf	  = buf;
	cq->cq_bufsz	  = (1 << log_cq_size);
	cq->cq_log_cqsz	  = log_cq_size;
	cq->cq_mrhdl	  = mr;
	cq->cq_refcnt	  = 0;
	cq->cq_is_special = 0;
	cq->cq_uarpg	  = uarpg;
	cq->cq_umap_dhp	  = (devmap_cookie_t)NULL;
	avl_create(&cq->cq_wrid_wqhdr_avl_tree, hermon_wrid_workq_compare,
	    sizeof (struct hermon_workq_avl_s),
	    offsetof(struct hermon_workq_avl_s, wqa_link));

	cq->cq_hdlrarg	  = (void *)ibt_cqhdl;

	/*
	 * Put CQ handle in Hermon CQNum-to-CQHdl list.  Then fill in the
	 * "actual_size" and "cqhdl" and return success
	 */
	hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, cq);

	/*
	 * If this is a user-mappable CQ, then we need to insert the previously
	 * allocated entry into the "userland resources database".  This will
	 * allow for later lookup during devmap() (i.e. mmap()) calls.
	 */
	if (cq->cq_is_umap) {
		hermon_umap_db_add(umapdb);
	}

	/*
	 * Fill in the return arguments (if necessary).  This includes the
	 * real completion queue size.
	 */
	if (actual_size != NULL) {
		*actual_size = (1 << log_cq_size) - 1;
	}
	*cqhdl = cq;

	return (DDI_SUCCESS);

/*
 * The following is cleanup for all possible failure cases in this routine
 */
cqalloc_fail6:
	if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
	    sleepflag) != DDI_SUCCESS) {
		HERMON_WARNING(state, "failed to deregister CQ memory");
	}
cqalloc_fail5:
	hermon_queue_free(&cq->cq_cqinfo);
cqalloc_fail4a:
	hermon_dbr_free(state, uarpg, cq->cq_arm_ci_vdbr);
cqalloc_fail4:
	if (cq_is_umap) {
		hermon_umap_db_free(umapdb);
	}
cqalloc_fail3:
	hermon_rsrc_free(state, &rsrc);
cqalloc_fail2:
	hermon_rsrc_free(state, &cqc);
cqalloc_fail1:
	hermon_pd_refcnt_dec(pd);
cqalloc_fail:
	return (status);
}


/*
 * hermon_cq_free()
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
int
hermon_cq_free(hermon_state_t *state, hermon_cqhdl_t *cqhdl, uint_t sleepflag)
{
	hermon_rsrc_t		*cqc, *rsrc;
	hermon_umap_db_entry_t	*umapdb;
	hermon_hw_cqc_t		cqc_entry;
	hermon_pdhdl_t		pd;
	hermon_mrhdl_t		mr;
	hermon_cqhdl_t		cq, resize;
	uint32_t		cqnum;
	uint64_t		value;
	uint_t			maxprot;
	int			status;

	/*
	 * Pull all the necessary information from the Hermon Completion Queue
	 * handle.  This is necessary here because the resource for the
	 * CQ handle is going to be freed up as part of this operation.
	 */
	cq	= *cqhdl;
	mutex_enter(&cq->cq_lock);
	cqc	= cq->cq_cqcrsrcp;
	rsrc	= cq->cq_rsrcp;
	pd	= state->hs_pdhdl_internal;
	mr	= cq->cq_mrhdl;
	cqnum	= cq->cq_cqnum;

	resize = cq->cq_resize_hdl;		/* save the handle for later */

	/*
	 * If there are work queues still associated with the CQ, then return
	 * an error.  Otherwise, we will be holding the CQ lock.
	 */
	if (cq->cq_refcnt != 0) {
		mutex_exit(&cq->cq_lock);
		return (IBT_CQ_BUSY);
	}

	/*
	 * If this was a user-mappable CQ, then we need to remove its entry
	 * from the "userland resources database".  If it is also currently
	 * mmap()'d out to a user process, then we need to call
	 * devmap_devmem_remap() to remap the CQ memory to an invalid mapping.
	 * We also need to invalidate the CQ tracking information for the
	 * user mapping.
	 */
	if (cq->cq_is_umap) {
		status = hermon_umap_db_find(state->hs_instance, cqnum,
		    MLNX_UMAP_CQMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
		    &umapdb);
		if (status != DDI_SUCCESS) {
			mutex_exit(&cq->cq_lock);
			HERMON_WARNING(state, "failed to find in database");
			return (ibc_get_ci_failure(0));
		}
		hermon_umap_db_free(umapdb);
		if (cq->cq_umap_dhp != NULL) {
			maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
			status = devmap_devmem_remap(cq->cq_umap_dhp,
			    state->hs_dip, 0, 0, cq->cq_cqinfo.qa_size,
			    maxprot, DEVMAP_MAPPING_INVALID, NULL);
			if (status != DDI_SUCCESS) {
				mutex_exit(&cq->cq_lock);
				HERMON_WARNING(state, "failed in CQ memory "
				    "devmap_devmem_remap()");
				return (ibc_get_ci_failure(0));
			}
			cq->cq_umap_dhp = (devmap_cookie_t)NULL;
		}
	}

	/*
	 * Put NULL into the Arbel CQNum-to-CQHdl list.  This will allow any
	 * in-progress events to detect that the CQ corresponding to this
	 * number has been freed.
	 */
	hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, NULL);

	mutex_exit(&cq->cq_lock);
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))

	/*
	 * Reclaim CQC entry from hardware (using the Hermon HW2SW_CQ
	 * firmware command).  If the ownership transfer fails for any reason,
	 * then it is an indication that something (either in HW or SW) has
	 * gone seriously wrong.
	 */
	status = hermon_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
	    sizeof (hermon_hw_cqc_t), cqnum, sleepflag);
	if (status != HERMON_CMD_SUCCESS) {
		HERMON_WARNING(state, "failed to reclaim CQC ownership");
		cmn_err(CE_CONT, "Hermon: HW2SW_CQ command failed: %08x\n",
		    status);
		if (status == HERMON_CMD_INVALID_STATUS) {
			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
		}
		return (ibc_get_ci_failure(0));
	}

	/*
	 * From here on, we start reliquishing resources - but check to see
	 * if a resize was in progress - if so, we need to relinquish those
	 * resources as well
	 */


	/*
	 * Deregister the memory for the Completion Queue.  If this fails
	 * for any reason, then it is an indication that something (either
	 * in HW or SW) has gone seriously wrong.  So we print a warning
	 * message and return.
	 */
	status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
	    sleepflag);
	if (status != DDI_SUCCESS) {
		HERMON_WARNING(state, "failed to deregister CQ memory");
		return (ibc_get_ci_failure(0));
	}

	if (resize)	{	/* there was a pointer to a handle */
		mr = resize->cq_mrhdl;	/* reuse the pointer to the region */
		status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
		    sleepflag);
		if (status != DDI_SUCCESS) {
			HERMON_WARNING(state, "failed to deregister resize CQ "
			    "memory");
			return (ibc_get_ci_failure(0));
		}
	}

	/* Free the memory for the CQ */
	hermon_queue_free(&cq->cq_cqinfo);
	if (resize)	{
		hermon_queue_free(&resize->cq_cqinfo);
		/* and the temporary handle */
		kmem_free(resize, sizeof (struct hermon_sw_cq_s));
	}

	/* everything else does not matter for the resize in progress */

	/* Free the dbr */
	hermon_dbr_free(state, cq->cq_uarpg, cq->cq_arm_ci_vdbr);

	/* Free the Hermon Completion Queue handle */
	hermon_rsrc_free(state, &rsrc);

	/* Free up the CQC entry resource */
	hermon_rsrc_free(state, &cqc);

	/* Decrement the reference count on the protection domain (PD) */
	hermon_pd_refcnt_dec(pd);

	/* Set the cqhdl pointer to NULL and return success */
	*cqhdl = NULL;

	return (DDI_SUCCESS);
}


/*
 * hermon_cq_resize()
 *    Context: Can be called only from user or kernel context.
 */
int
hermon_cq_resize(hermon_state_t *state, hermon_cqhdl_t cq, uint_t req_size,
    uint_t *actual_size, uint_t sleepflag)
{
	hermon_hw_cqc_t		cqc_entry;
	hermon_cqhdl_t		resize_hdl;
	hermon_qalloc_info_t	new_cqinfo;
	ibt_mr_attr_t		mr_attr;
	hermon_mr_options_t	op;
	hermon_pdhdl_t		pd;
	hermon_mrhdl_t		mr;
	hermon_hw_cqe_t		*buf;
	uint32_t		new_prod_indx;
	uint_t			log_cq_size;
	int			status, flag;

	if (cq->cq_resize_hdl != 0) {	/* already in process */
		status = IBT_CQ_BUSY;
		goto cqresize_fail;
	}


	/* Use the internal protection domain (PD) for CQs */
	pd = state->hs_pdhdl_internal;

	/*
	 * Calculate the appropriate size for the new resized completion queue.
	 * Note:  All Hermon CQs must be a power-of-2 minus 1 in size.  Also
	 * they may not be any smaller than HERMON_CQ_MIN_SIZE.  This step is
	 * to round the requested size up to the next highest power-of-2
	 */
	req_size = max(req_size, HERMON_CQ_MIN_SIZE);
	log_cq_size = highbit(req_size);

	/*
	 * Next we verify that the rounded-up size is valid (i.e. consistent
	 * with the device limits and/or software-configured limits)
	 */
	if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
		status = IBT_HCA_CQ_EXCEEDED;
		goto cqresize_fail;
	}

	/*
	 * Allocate the memory for newly resized Completion Queue.
	 *
	 * Note: Although we use the common queue allocation routine, we
	 * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
	 * kernel system memory) for kernel CQs because it would be
	 * inefficient to have CQs located in DDR memory.  This is the same
	 * as we do when we first allocate completion queues primarily
	 * because CQs are read from (by software) more than they are written
	 * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
	 * user-mappable CQs for a similar reason.)
	 * It is also worth noting that, unlike Hermon QP work queues,
	 * completion queues do not have the same strict alignment
	 * requirements.  It is sufficient for the CQ memory to be both
	 * aligned to and bound to addresses which are a multiple of CQE size.
	 */

	/* first, alloc the resize_handle */
	resize_hdl = kmem_zalloc(sizeof (struct hermon_sw_cq_s), KM_SLEEP);

	new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
	new_cqinfo.qa_alloc_align = PAGESIZE;
	new_cqinfo.qa_bind_align  = PAGESIZE;
	if (cq->cq_is_umap) {
		new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
	} else {
		new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
	}
	status = hermon_queue_alloc(state, &new_cqinfo, sleepflag);
	if (status != DDI_SUCCESS) {
		/* free the resize handle */
		kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
		status = IBT_INSUFF_RESOURCE;
		goto cqresize_fail;
	}
	buf = (hermon_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))

	/*
	 * No initialization of the cq is needed - the command will do it
	 */

	/*
	 * Register the memory for the CQ.  The memory for the CQ must
	 * be registered in the Hermon TPT tables.  This gives us the LKey
	 * to specify in the CQ context below.
	 */
	flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
	mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
	mr_attr.mr_len	 = new_cqinfo.qa_size;
	mr_attr.mr_as	 = NULL;
	mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
	op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
	op.mro_bind_dmahdl = new_cqinfo.qa_dmahdl;
	op.mro_bind_override_addr = 0;
	status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
	    HERMON_CQ_CMPT);
	if (status != DDI_SUCCESS) {
		hermon_queue_free(&new_cqinfo);
		/* free the resize handle */
		kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
		status = IBT_INSUFF_RESOURCE;
		goto cqresize_fail;
	}
	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))

	/*
	 * Now we grab the CQ lock.  Since we will be updating the actual
	 * CQ location and the producer/consumer indexes, we should hold
	 * the lock.
	 *
	 * We do a ARBEL_NOSLEEP here (and below), though, because we are
	 * holding the "cq_lock" and if we got raised to interrupt level
	 * by priority inversion, we would not want to block in this routine
	 * waiting for success.
	 */
	mutex_enter(&cq->cq_lock);

	/*
	 * Fill in the CQC entry.  For the resize operation this is the
	 * final step before attempting the resize operation on the CQC entry.
	 * We use all of the information collected/calculated above to fill
	 * in the requisite portions of the CQC.
	 */
	bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
	cqc_entry.log_cq_sz	= log_cq_size;
	cqc_entry.pg_offs	= new_cqinfo.qa_pgoffs >> 5;
	cqc_entry.log2_pgsz	= mr->mr_log2_pgsz;
	cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
	cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;

	/*
	 * Write the CQC entry to hardware.  Lastly, we pass ownership of
	 * the entry to the hardware (using the Hermon RESIZE_CQ firmware
	 * command).  Note: In general, this operation shouldn't fail.  But
	 * if it does, we have to undo everything we've done above before
	 * returning error.  Also note that the status returned may indicate
	 * the code to return to the IBTF.
	 */
	status = hermon_resize_cq_cmd_post(state, &cqc_entry, cq->cq_cqnum,
	    &new_prod_indx, HERMON_CMD_NOSLEEP_SPIN);
	if (status != HERMON_CMD_SUCCESS) {
		/* Resize attempt has failed, drop CQ lock and cleanup */
		mutex_exit(&cq->cq_lock);
		if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
		    sleepflag) != DDI_SUCCESS) {
			HERMON_WARNING(state, "failed to deregister CQ memory");
		}
		kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
		hermon_queue_free(&new_cqinfo);
		if (status == HERMON_CMD_BAD_SIZE) {
			return (IBT_CQ_SZ_INSUFFICIENT);
		} else {
			cmn_err(CE_CONT, "Hermon: RESIZE_CQ command failed: "
			    "%08x\n", status);
			if (status == HERMON_CMD_INVALID_STATUS) {
				hermon_fm_ereport(state, HCA_SYS_ERR,
				    HCA_ERR_SRV_LOST);
			}
			return (ibc_get_ci_failure(0));
		}
	}

	/*
	 * For Hermon, we've alloc'd another handle structure and save off the
	 * important things in it. Then, in polling we check to see if there's
	 * a "resizing handle" and if so we look for the "special CQE", opcode
	 * 0x16, that indicates the transition to the new buffer.
	 *
	 * At that point, we'll adjust everything - including dereg and
	 * freeing of the original buffer, updating all the necessary fields
	 * in the cq_hdl, and setting up for the next cqe polling
	 */

	resize_hdl->cq_buf 	= buf;
	resize_hdl->cq_bufsz	= (1 << log_cq_size);
	resize_hdl->cq_mrhdl	= mr;
	resize_hdl->cq_log_cqsz = log_cq_size;

	bcopy(&new_cqinfo, &(resize_hdl->cq_cqinfo),
	    sizeof (struct hermon_qalloc_info_s));

	/* now, save the address in the cq_handle */
	cq->cq_resize_hdl = resize_hdl;

	/*
	 * Drop the CQ lock now.
	 */

	mutex_exit(&cq->cq_lock);
	/*
	 * Fill in the return arguments (if necessary).  This includes the
	 * real new completion queue size.
	 */
	if (actual_size != NULL) {
		*actual_size = (1 << log_cq_size) - 1;
	}

	return (DDI_SUCCESS);

cqresize_fail:
	return (status);
}


/*
 * hermon_cq_modify()
 *    Context: Can be called base context.
 */
/* ARGSUSED */
int
hermon_cq_modify(hermon_state_t *state, hermon_cqhdl_t cq,
    uint_t count, uint_t usec, ibt_cq_handler_id_t hid, uint_t sleepflag)
{
	int	status;
	hermon_hw_cqc_t		cqc_entry;

	mutex_enter(&cq->cq_lock);
	if (count != cq->cq_intmod_count ||
	    usec != cq->cq_intmod_usec) {
		bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
		cqc_entry.cq_max_cnt = count;
		cqc_entry.cq_period = usec;
		status = hermon_modify_cq_cmd_post(state, &cqc_entry,
		    cq->cq_cqnum, MODIFY_MODERATION_CQ, sleepflag);
		if (status != HERMON_CMD_SUCCESS) {
			mutex_exit(&cq->cq_lock);
			cmn_err(CE_CONT, "Hermon: MODIFY_MODERATION_CQ "
			    "command failed: %08x\n", status);
			if (status == HERMON_CMD_INVALID_STATUS) {
				hermon_fm_ereport(state, HCA_SYS_ERR,
				    HCA_ERR_SRV_LOST);
			}
			return (ibc_get_ci_failure(0));
		}
		cq->cq_intmod_count = count;
		cq->cq_intmod_usec = usec;
	}
	if (hid && (hid - 1 != cq->cq_eqnum)) {
		bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
		cqc_entry.c_eqn = HERMON_HID_TO_EQNUM(state, hid);
		status = hermon_modify_cq_cmd_post(state, &cqc_entry,
		    cq->cq_cqnum, MODIFY_EQN, sleepflag);
		if (status != HERMON_CMD_SUCCESS) {
			mutex_exit(&cq->cq_lock);
			cmn_err(CE_CONT, "Hermon: MODIFY_EQN command failed: "
			    "%08x\n", status);
			if (status == HERMON_CMD_INVALID_STATUS) {
				hermon_fm_ereport(state, HCA_SYS_ERR,
				    HCA_ERR_SRV_LOST);
			}
			return (ibc_get_ci_failure(0));
		}
		cq->cq_eqnum = hid - 1;
	}
	mutex_exit(&cq->cq_lock);
	return (DDI_SUCCESS);
}

/*
 * hermon_cq_notify()
 *    Context: Can be called from interrupt or base context.
 */
int
hermon_cq_notify(hermon_state_t *state, hermon_cqhdl_t cq,
    ibt_cq_notify_flags_t flags)
{
	uint_t	cmd;
	ibt_status_t status;

	/* Validate IBT flags and call doorbell routine. */
	if (flags == IBT_NEXT_COMPLETION) {
		cmd = HERMON_CQDB_NOTIFY_CQ;
	} else if (flags == IBT_NEXT_SOLICITED) {
		cmd = HERMON_CQDB_NOTIFY_CQ_SOLICIT;
	} else {
		return (IBT_CQ_NOTIFY_TYPE_INVALID);
	}

	status = hermon_cq_arm_doorbell(state, cq, cmd);
	return (status);
}


/*
 * hermon_cq_poll()
 *    Context: Can be called from interrupt or base context.
 */
int
hermon_cq_poll(hermon_state_t *state, hermon_cqhdl_t cq, ibt_wc_t *wc_p,
    uint_t num_wc, uint_t *num_polled)
{
	hermon_hw_cqe_t	*cqe;
	uint_t		opcode;
	uint32_t	cons_indx, wrap_around_mask, shift, mask;
	uint32_t	polled_cnt, spec_op = 0;
	int		status;

	/*
	 * Check for user-mappable CQ memory.  Note:  We do not allow kernel
	 * clients to poll CQ memory that is accessible directly by the user.
	 * If the CQ memory is user accessible, then return an error.
	 */
	if (cq->cq_is_umap) {
		return (IBT_CQ_HDL_INVALID);
	}

	mutex_enter(&cq->cq_lock);

	/* Get the consumer index */
	cons_indx = cq->cq_consindx;
	shift = cq->cq_log_cqsz;
	mask = cq->cq_bufsz;

	/*
	 * Calculate the wrap around mask.  Note: This operation only works
	 * because all Hermon completion queues have power-of-2 sizes
	 */
	wrap_around_mask = (cq->cq_bufsz - 1);

	/* Calculate the pointer to the first CQ entry */
	cqe = &cq->cq_buf[cons_indx & wrap_around_mask];

	/*
	 * Keep pulling entries from the CQ until we find an entry owned by
	 * the hardware.  As long as there the CQE's owned by SW, process
	 * each entry by calling hermon_cq_cqe_consume() and updating the CQ
	 * consumer index.  Note:  We only update the consumer index if
	 * hermon_cq_cqe_consume() returns HERMON_CQ_SYNC_AND_DB.  Otherwise,
	 * it indicates that we are going to "recycle" the CQE (probably
	 * because it is a error CQE and corresponds to more than one
	 * completion).
	 */
	polled_cnt = 0;
	while (HERMON_CQE_OWNER_IS_SW(cq, cqe, cons_indx, shift, mask)) {
		if (cq->cq_resize_hdl != 0) {	/* in midst of resize */
			/* peek at the opcode */
			opcode = HERMON_CQE_OPCODE_GET(cq, cqe);
			if (opcode == HERMON_CQE_RCV_RESIZE_CODE) {
				hermon_cq_resize_helper(state, cq);

				/* Increment the consumer index */
				cons_indx = (cons_indx + 1);
				spec_op = 1; /* plus one for the limiting CQE */

				wrap_around_mask = (cq->cq_bufsz - 1);

				/* Update the pointer to the next CQ entry */
				cqe = &cq->cq_buf[cons_indx & wrap_around_mask];

				continue;
			}
		}	/* in resizing CQ */

		/*
		 * either resizing and not the special opcode, or
		 * not resizing at all
		 */
		hermon_cq_cqe_consume(state, cq, cqe, &wc_p[polled_cnt++]);

		/* Increment the consumer index */
		cons_indx = (cons_indx + 1);

		/* Update the pointer to the next CQ entry */
		cqe = &cq->cq_buf[cons_indx & wrap_around_mask];

		/*
		 * If we have run out of space to store work completions,
		 * then stop and return the ones we have pulled of the CQ.
		 */
		if (polled_cnt >= num_wc) {
			break;
		}
	}

	/*
	 * Now we only ring the doorbell (to update the consumer index) if
	 * we've actually consumed a CQ entry.
	 */
	if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) {
		/*
		 * Update the consumer index in both the CQ handle and the
		 * doorbell record.
		 */
		cq->cq_consindx = cons_indx;
		hermon_cq_update_ci_doorbell(cq);

	} else if (polled_cnt == 0) {
		if (spec_op != 0) {
			/* if we got the special opcode, update the consindx */
			cq->cq_consindx = cons_indx;
			hermon_cq_update_ci_doorbell(cq);
		}
	}

	mutex_exit(&cq->cq_lock);

	/* Set "num_polled" (if necessary) */
	if (num_polled != NULL) {
		*num_polled = polled_cnt;
	}

	/* Set CQ_EMPTY condition if needed, otherwise return success */
	if (polled_cnt == 0) {
		status = IBT_CQ_EMPTY;
	} else {
		status = DDI_SUCCESS;
	}

	/*
	 * Check if the system is currently panicking.  If it is, then call
	 * the Hermon interrupt service routine.  This step is necessary here
	 * because we might be in a polled I/O mode and without the call to