19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
23*17a2b317SBill Taylor  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor  */
259e39c5baSBill Taylor 
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor  * hermon_cq.c
289e39c5baSBill Taylor  *    Hermon Completion Queue Processing Routines
299e39c5baSBill Taylor  *
309e39c5baSBill Taylor  *    Implements all the routines necessary for allocating, freeing, resizing,
319e39c5baSBill Taylor  *    and handling the completion type events that the Hermon hardware can
329e39c5baSBill Taylor  *    generate.
339e39c5baSBill Taylor  */
349e39c5baSBill Taylor 
359e39c5baSBill Taylor #include <sys/types.h>
369e39c5baSBill Taylor #include <sys/conf.h>
379e39c5baSBill Taylor #include <sys/ddi.h>
389e39c5baSBill Taylor #include <sys/sunddi.h>
399e39c5baSBill Taylor #include <sys/modctl.h>
409e39c5baSBill Taylor #include <sys/bitmap.h>
419e39c5baSBill Taylor #include <sys/sysmacros.h>
429e39c5baSBill Taylor 
439e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
449e39c5baSBill Taylor 
459e39c5baSBill Taylor int hermon_should_panic = 0;	/* debugging aid */
469e39c5baSBill Taylor 
479e39c5baSBill Taylor #define	hermon_cq_update_ci_doorbell(cq)				\
489e39c5baSBill Taylor 	/* Build the doorbell record data (low 24 bits only) */		\
499e39c5baSBill Taylor 	HERMON_UAR_DB_RECORD_WRITE(cq->cq_arm_ci_vdbr,			\
509e39c5baSBill Taylor 	    cq->cq_consindx & 0x00FFFFFF)
519e39c5baSBill Taylor 
529e39c5baSBill Taylor static int hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq,
539e39c5baSBill Taylor     uint_t cmd);
549e39c5baSBill Taylor #pragma inline(hermon_cq_arm_doorbell)
559e39c5baSBill Taylor static void hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr);
569e39c5baSBill Taylor #pragma inline(hermon_arm_cq_dbr_init)
579e39c5baSBill Taylor static void hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
589e39c5baSBill Taylor     hermon_hw_cqe_t *cqe, ibt_wc_t *wc);
599e39c5baSBill Taylor static void hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
609e39c5baSBill Taylor     hermon_hw_cqe_t *cqe, ibt_wc_t *wc);
619e39c5baSBill Taylor 
629e39c5baSBill Taylor 
639e39c5baSBill Taylor /*
649e39c5baSBill Taylor  * hermon_cq_alloc()
659e39c5baSBill Taylor  *    Context: Can be called only from user or kernel context.
669e39c5baSBill Taylor  */
679e39c5baSBill Taylor int
hermon_cq_alloc(hermon_state_t * state,ibt_cq_hdl_t ibt_cqhdl,ibt_cq_attr_t * cq_attr,uint_t * actual_size,hermon_cqhdl_t * cqhdl,uint_t sleepflag)689e39c5baSBill Taylor hermon_cq_alloc(hermon_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
699e39c5baSBill Taylor     ibt_cq_attr_t *cq_attr, uint_t *actual_size, hermon_cqhdl_t *cqhdl,
709e39c5baSBill Taylor     uint_t sleepflag)
719e39c5baSBill Taylor {
729e39c5baSBill Taylor 	hermon_rsrc_t		*cqc, *rsrc;
739e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
749e39c5baSBill Taylor 	hermon_hw_cqc_t		cqc_entry;
759e39c5baSBill Taylor 	hermon_cqhdl_t		cq;
769e39c5baSBill Taylor 	ibt_mr_attr_t		mr_attr;
779e39c5baSBill Taylor 	hermon_mr_options_t	op;
789e39c5baSBill Taylor 	hermon_pdhdl_t		pd;
799e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
809e39c5baSBill Taylor 	hermon_hw_cqe_t		*buf;
819e39c5baSBill Taylor 	uint64_t		value;
829e39c5baSBill Taylor 	uint32_t		log_cq_size, uarpg;
839e39c5baSBill Taylor 	uint_t			cq_is_umap;
849e39c5baSBill Taylor 	uint32_t		status, flag;
85*17a2b317SBill Taylor 	hermon_cq_sched_t	*cq_schedp;
869e39c5baSBill Taylor 
879e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq_attr))
889e39c5baSBill Taylor 
899e39c5baSBill Taylor 	/*
909e39c5baSBill Taylor 	 * Determine whether CQ is being allocated for userland access or
919e39c5baSBill Taylor 	 * whether it is being allocated for kernel access.  If the CQ is
929e39c5baSBill Taylor 	 * being allocated for userland access, then lookup the UAR
939e39c5baSBill Taylor 	 * page number for the current process.  Note:  If this is not found
949e39c5baSBill Taylor 	 * (e.g. if the process has not previously open()'d the Hermon driver),
959e39c5baSBill Taylor 	 * then an error is returned.
969e39c5baSBill Taylor 	 */
979e39c5baSBill Taylor 	cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
989e39c5baSBill Taylor 	if (cq_is_umap) {
999e39c5baSBill Taylor 		status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
1009e39c5baSBill Taylor 		    MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
1019e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
1029e39c5baSBill Taylor 			status = IBT_INVALID_PARAM;
1039e39c5baSBill Taylor 			goto cqalloc_fail;
1049e39c5baSBill Taylor 		}
1059e39c5baSBill Taylor 		uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
1069e39c5baSBill Taylor 	} else {
1079e39c5baSBill Taylor 		uarpg = state->hs_kernel_uar_index;
1089e39c5baSBill Taylor 	}
1099e39c5baSBill Taylor 
1109e39c5baSBill Taylor 	/* Use the internal protection domain (PD) for setting up CQs */
1119e39c5baSBill Taylor 	pd = state->hs_pdhdl_internal;
1129e39c5baSBill Taylor 
1139e39c5baSBill Taylor 	/* Increment the reference count on the protection domain (PD) */
1149e39c5baSBill Taylor 	hermon_pd_refcnt_inc(pd);
1159e39c5baSBill Taylor 
1169e39c5baSBill Taylor 	/*
1179e39c5baSBill Taylor 	 * Allocate an CQ context entry.  This will be filled in with all
1189e39c5baSBill Taylor 	 * the necessary parameters to define the Completion Queue.  And then
1199e39c5baSBill Taylor 	 * ownership will be passed to the hardware in the final step
1209e39c5baSBill Taylor 	 * below.  If we fail here, we must undo the protection domain
1219e39c5baSBill Taylor 	 * reference count.
1229e39c5baSBill Taylor 	 */
1239e39c5baSBill Taylor 	status = hermon_rsrc_alloc(state, HERMON_CQC, 1, sleepflag, &cqc);
1249e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
1259e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
1269e39c5baSBill Taylor 		goto cqalloc_fail1;
1279e39c5baSBill Taylor 	}
1289e39c5baSBill Taylor 
1299e39c5baSBill Taylor 	/*
1309e39c5baSBill Taylor 	 * Allocate the software structure for tracking the completion queue
1319e39c5baSBill Taylor 	 * (i.e. the Hermon Completion Queue handle).  If we fail here, we must
1329e39c5baSBill Taylor 	 * undo the protection domain reference count and the previous
1339e39c5baSBill Taylor 	 * resource allocation.
1349e39c5baSBill Taylor 	 */
1359e39c5baSBill Taylor 	status = hermon_rsrc_alloc(state, HERMON_CQHDL, 1, sleepflag, &rsrc);
1369e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
1379e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
1389e39c5baSBill Taylor 		goto cqalloc_fail2;
1399e39c5baSBill Taylor 	}
1409e39c5baSBill Taylor 	cq = (hermon_cqhdl_t)rsrc->hr_addr;
1419e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
1429e39c5baSBill Taylor 	cq->cq_is_umap = cq_is_umap;
1439e39c5baSBill Taylor 	cq->cq_cqnum = cqc->hr_indx;	/* just use index, implicit in Hermon */
144*17a2b317SBill Taylor 	cq->cq_intmod_count = 0;
145*17a2b317SBill Taylor 	cq->cq_intmod_usec = 0;
1469e39c5baSBill Taylor 
1479e39c5baSBill Taylor 	/*
1489e39c5baSBill Taylor 	 * If this will be a user-mappable CQ, then allocate an entry for
1499e39c5baSBill Taylor 	 * the "userland resources database".  This will later be added to
1509e39c5baSBill Taylor 	 * the database (after all further CQ operations are successful).
1519e39c5baSBill Taylor 	 * If we fail here, we must undo the reference counts and the
1529e39c5baSBill Taylor 	 * previous resource allocation.
1539e39c5baSBill Taylor 	 */
1549e39c5baSBill Taylor 	if (cq->cq_is_umap) {
1559e39c5baSBill Taylor 		umapdb = hermon_umap_db_alloc(state->hs_instance, cq->cq_cqnum,
1569e39c5baSBill Taylor 		    MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
1579e39c5baSBill Taylor 		if (umapdb == NULL) {
1589e39c5baSBill Taylor 			status = IBT_INSUFF_RESOURCE;
1599e39c5baSBill Taylor 			goto cqalloc_fail3;
1609e39c5baSBill Taylor 		}
1619e39c5baSBill Taylor 	}
1629e39c5baSBill Taylor 
1639e39c5baSBill Taylor 
1649e39c5baSBill Taylor 	/*
1659e39c5baSBill Taylor 	 * Allocate the doorbell record.  We'll need one for the CQ, handling
1669e39c5baSBill Taylor 	 * both consumer index (SET CI) and the CQ state (CQ ARM).
1679e39c5baSBill Taylor 	 */
1689e39c5baSBill Taylor 
1699e39c5baSBill Taylor 	status = hermon_dbr_alloc(state, uarpg, &cq->cq_arm_ci_dbr_acchdl,
1709e39c5baSBill Taylor 	    &cq->cq_arm_ci_vdbr, &cq->cq_arm_ci_pdbr, &cq->cq_dbr_mapoffset);
1719e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
1729e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
1739e39c5baSBill Taylor 		goto cqalloc_fail4;
1749e39c5baSBill Taylor 	}
1759e39c5baSBill Taylor 
1769e39c5baSBill Taylor 	/*
1779e39c5baSBill Taylor 	 * Calculate the appropriate size for the completion queue.
1789e39c5baSBill Taylor 	 * Note:  All Hermon CQs must be a power-of-2 minus 1 in size.  Also
1799e39c5baSBill Taylor 	 * they may not be any smaller than HERMON_CQ_MIN_SIZE.  This step is
1809e39c5baSBill Taylor 	 * to round the requested size up to the next highest power-of-2
1819e39c5baSBill Taylor 	 */
1829e39c5baSBill Taylor 	cq_attr->cq_size = max(cq_attr->cq_size, HERMON_CQ_MIN_SIZE);
1839e39c5baSBill Taylor 	log_cq_size = highbit(cq_attr->cq_size);
1849e39c5baSBill Taylor 
1859e39c5baSBill Taylor 	/*
1869e39c5baSBill Taylor 	 * Next we verify that the rounded-up size is valid (i.e. consistent
1879e39c5baSBill Taylor 	 * with the device limits and/or software-configured limits)
1889e39c5baSBill Taylor 	 */
1899e39c5baSBill Taylor 	if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
1909e39c5baSBill Taylor 		status = IBT_HCA_CQ_EXCEEDED;
1919e39c5baSBill Taylor 		goto cqalloc_fail4a;
1929e39c5baSBill Taylor 	}
1939e39c5baSBill Taylor 
1949e39c5baSBill Taylor 	/*
1959e39c5baSBill Taylor 	 * Allocate the memory for Completion Queue.
1969e39c5baSBill Taylor 	 *
1979e39c5baSBill Taylor 	 * Note: Although we use the common queue allocation routine, we
1989e39c5baSBill Taylor 	 * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
1999e39c5baSBill Taylor 	 * kernel system memory) for kernel CQs because it would be
2009e39c5baSBill Taylor 	 * inefficient to have CQs located in DDR memory.  This is primarily
2019e39c5baSBill Taylor 	 * because CQs are read from (by software) more than they are written
2029e39c5baSBill Taylor 	 * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
2039e39c5baSBill Taylor 	 * user-mappable CQs for a similar reason.)
2049e39c5baSBill Taylor 	 * It is also worth noting that, unlike Hermon QP work queues,
2059e39c5baSBill Taylor 	 * completion queues do not have the same strict alignment
2069e39c5baSBill Taylor 	 * requirements.  It is sufficient for the CQ memory to be both
2079e39c5baSBill Taylor 	 * aligned to and bound to addresses which are a multiple of CQE size.
2089e39c5baSBill Taylor 	 */
2099e39c5baSBill Taylor 	cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
2109e39c5baSBill Taylor 
2119e39c5baSBill Taylor 	cq->cq_cqinfo.qa_alloc_align = PAGESIZE;
2129e39c5baSBill Taylor 	cq->cq_cqinfo.qa_bind_align  = PAGESIZE;
2139e39c5baSBill Taylor 	if (cq->cq_is_umap) {
2149e39c5baSBill Taylor 		cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
2159e39c5baSBill Taylor 	} else {
2169e39c5baSBill Taylor 		cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
2179e39c5baSBill Taylor 		hermon_arm_cq_dbr_init(cq->cq_arm_ci_vdbr);
2189e39c5baSBill Taylor 	}
2199e39c5baSBill Taylor 	status = hermon_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
2209e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2219e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
2229e39c5baSBill Taylor 		goto cqalloc_fail4;
2239e39c5baSBill Taylor 	}
2249e39c5baSBill Taylor 	buf = (hermon_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
2259e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
2269e39c5baSBill Taylor 
2279e39c5baSBill Taylor 	/*
2289e39c5baSBill Taylor 	 * The ownership bit of the CQE's is set by the HW during the process
2299e39c5baSBill Taylor 	 * of transferrring ownership of the CQ (PRM 09.35c, 14.2.1, note D1
2309e39c5baSBill Taylor 	 *
2319e39c5baSBill Taylor 	 */
2329e39c5baSBill Taylor 
2339e39c5baSBill Taylor 	/*
2349e39c5baSBill Taylor 	 * Register the memory for the CQ.  The memory for the CQ must
2359e39c5baSBill Taylor 	 * be registered in the Hermon TPT tables.  This gives us the LKey
2369e39c5baSBill Taylor 	 * to specify in the CQ context below.  Note: If this is a user-
2379e39c5baSBill Taylor 	 * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
2389e39c5baSBill Taylor 	 */
2399e39c5baSBill Taylor 	flag = (sleepflag == HERMON_SLEEP) ?  IBT_MR_SLEEP : IBT_MR_NOSLEEP;
2409e39c5baSBill Taylor 	mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
2419e39c5baSBill Taylor 	mr_attr.mr_len	 = cq->cq_cqinfo.qa_size;
2429e39c5baSBill Taylor 	mr_attr.mr_as	 = NULL;
2439e39c5baSBill Taylor 	mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
2449e39c5baSBill Taylor 	op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
2459e39c5baSBill Taylor 	op.mro_bind_dmahdl = cq->cq_cqinfo.qa_dmahdl;
2469e39c5baSBill Taylor 	op.mro_bind_override_addr = 0;
2479e39c5baSBill Taylor 	status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
2489e39c5baSBill Taylor 	    HERMON_CQ_CMPT);
2499e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2509e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
2519e39c5baSBill Taylor 		goto cqalloc_fail5;
2529e39c5baSBill Taylor 	}
2539e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2549e39c5baSBill Taylor 
255*17a2b317SBill Taylor 	cq->cq_erreqnum = HERMON_CQ_ERREQNUM_GET(state);
256*17a2b317SBill Taylor 	if (cq_attr->cq_flags & IBT_CQ_HID) {
257*17a2b317SBill Taylor 		if (!HERMON_HID_VALID(state, cq_attr->cq_hid)) {
258*17a2b317SBill Taylor 			IBTF_DPRINTF_L2("CQalloc", "bad handler id 0x%x",
259*17a2b317SBill Taylor 			    cq_attr->cq_hid);
260*17a2b317SBill Taylor 			status = IBT_INVALID_PARAM;
261*17a2b317SBill Taylor 			goto cqalloc_fail5;
262*17a2b317SBill Taylor 		}
263*17a2b317SBill Taylor 		cq->cq_eqnum = HERMON_HID_TO_EQNUM(state, cq_attr->cq_hid);
264*17a2b317SBill Taylor 		IBTF_DPRINTF_L2("cqalloc", "hid: eqn %d", cq->cq_eqnum);
265*17a2b317SBill Taylor 	} else {
266*17a2b317SBill Taylor 		cq_schedp = (hermon_cq_sched_t *)cq_attr->cq_sched;
267*17a2b317SBill Taylor 		if (cq_schedp == NULL) {
268*17a2b317SBill Taylor 			cq_schedp = &state->hs_cq_sched_default;
269*17a2b317SBill Taylor 		} else if (cq_schedp != &state->hs_cq_sched_default) {
270*17a2b317SBill Taylor 			int i;
271*17a2b317SBill Taylor 			hermon_cq_sched_t *tmp;
272*17a2b317SBill Taylor 
273*17a2b317SBill Taylor 			tmp = state->hs_cq_sched_array;
274*17a2b317SBill Taylor 			for (i = 0; i < state->hs_cq_sched_array_size; i++)
275*17a2b317SBill Taylor 				if (cq_schedp == &tmp[i])
276*17a2b317SBill Taylor 					break;	/* found it */
277*17a2b317SBill Taylor 			if (i >= state->hs_cq_sched_array_size) {
278*17a2b317SBill Taylor 				cmn_err(CE_CONT, "!Invalid cq_sched argument: "
279*17a2b317SBill Taylor 				    "ignored\n");
280*17a2b317SBill Taylor 				cq_schedp = &state->hs_cq_sched_default;
281*17a2b317SBill Taylor 			}
282*17a2b317SBill Taylor 		}
283*17a2b317SBill Taylor 		cq->cq_eqnum = HERMON_HID_TO_EQNUM(state,
284*17a2b317SBill Taylor 		    HERMON_CQSCHED_NEXT_HID(cq_schedp));
285*17a2b317SBill Taylor 		IBTF_DPRINTF_L2("cqalloc", "sched: first-1 %d, len %d, "
286*17a2b317SBill Taylor 		    "eqn %d", cq_schedp->cqs_start_hid - 1,
287*17a2b317SBill Taylor 		    cq_schedp->cqs_len, cq->cq_eqnum);
288*17a2b317SBill Taylor 	}
2899e39c5baSBill Taylor 
2909e39c5baSBill Taylor 	/*
2919e39c5baSBill Taylor 	 * Fill in the CQC entry.  This is the final step before passing
2929e39c5baSBill Taylor 	 * ownership of the CQC entry to the Hermon hardware.  We use all of
2939e39c5baSBill Taylor 	 * the information collected/calculated above to fill in the
2949e39c5baSBill Taylor 	 * requisite portions of the CQC.  Note: If this CQ is going to be
2959e39c5baSBill Taylor 	 * used for userland access, then we need to set the UAR page number
2969e39c5baSBill Taylor 	 * appropriately (otherwise it's a "don't care")
2979e39c5baSBill Taylor 	 */
2989e39c5baSBill Taylor 	bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
2999e39c5baSBill Taylor 
3009e39c5baSBill Taylor 	cqc_entry.state		= HERMON_CQ_DISARMED;
3019e39c5baSBill Taylor 	cqc_entry.pg_offs	= cq->cq_cqinfo.qa_pgoffs >> 5;
3029e39c5baSBill Taylor 	cqc_entry.log_cq_sz	= log_cq_size;
3039e39c5baSBill Taylor 	cqc_entry.usr_page	= uarpg;
3049e39c5baSBill Taylor 	cqc_entry.c_eqn		= cq->cq_eqnum;
3059e39c5baSBill Taylor 	cqc_entry.log2_pgsz	= mr->mr_log2_pgsz;
3069e39c5baSBill Taylor 	cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
3079e39c5baSBill Taylor 	cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;
3089e39c5baSBill Taylor 	cqc_entry.dbr_addrh = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 32);
3099e39c5baSBill Taylor 	cqc_entry.dbr_addrl = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 3);
3109e39c5baSBill Taylor 
3119e39c5baSBill Taylor 	/*
3129e39c5baSBill Taylor 	 * Write the CQC entry to hardware - we pass ownership of
3139e39c5baSBill Taylor 	 * the entry to the hardware (using the Hermon SW2HW_CQ firmware
3149e39c5baSBill Taylor 	 * command).  Note: In general, this operation shouldn't fail.  But
3159e39c5baSBill Taylor 	 * if it does, we have to undo everything we've done above before
3169e39c5baSBill Taylor 	 * returning error.
3179e39c5baSBill Taylor 	 */
3189e39c5baSBill Taylor 	status = hermon_cmn_ownership_cmd_post(state, SW2HW_CQ, &cqc_entry,
3199e39c5baSBill Taylor 	    sizeof (hermon_hw_cqc_t), cq->cq_cqnum, sleepflag);
3209e39c5baSBill Taylor 	if (status != HERMON_CMD_SUCCESS) {
3219e39c5baSBill Taylor 		cmn_err(CE_CONT, "Hermon: SW2HW_CQ command failed: %08x\n",
3229e39c5baSBill Taylor 		    status);
3239e39c5baSBill Taylor 		if (status == HERMON_CMD_INVALID_STATUS) {
3249e39c5baSBill Taylor 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
3259e39c5baSBill Taylor 		}
3269e39c5baSBill Taylor 		status = ibc_get_ci_failure(0);
3279e39c5baSBill Taylor 		goto cqalloc_fail6;
3289e39c5baSBill Taylor 	}
3299e39c5baSBill Taylor 
3309e39c5baSBill Taylor 	/*
3319e39c5baSBill Taylor 	 * Fill in the rest of the Hermon Completion Queue handle.  Having
3329e39c5baSBill Taylor 	 * successfully transferred ownership of the CQC, we can update the
3339e39c5baSBill Taylor 	 * following fields for use in further operations on the CQ.
3349e39c5baSBill Taylor 	 */
3359e39c5baSBill Taylor 	cq->cq_resize_hdl = 0;
3369e39c5baSBill Taylor 	cq->cq_cqcrsrcp	  = cqc;
3379e39c5baSBill Taylor 	cq->cq_rsrcp	  = rsrc;
3389e39c5baSBill Taylor 	cq->cq_consindx	  = 0;
3399e39c5baSBill Taylor 		/* least restrictive */
3409e39c5baSBill Taylor 	cq->cq_buf	  = buf;
3419e39c5baSBill Taylor 	cq->cq_bufsz	  = (1 << log_cq_size);
3429e39c5baSBill Taylor 	cq->cq_log_cqsz	  = log_cq_size;
3439e39c5baSBill Taylor 	cq->cq_mrhdl	  = mr;
3449e39c5baSBill Taylor 	cq->cq_refcnt	  = 0;
3459e39c5baSBill Taylor 	cq->cq_is_special = 0;
3469e39c5baSBill Taylor 	cq->cq_uarpg	  = uarpg;
3479e39c5baSBill Taylor 	cq->cq_umap_dhp	  = (devmap_cookie_t)NULL;
3489e39c5baSBill Taylor 	avl_create(&cq->cq_wrid_wqhdr_avl_tree, hermon_wrid_workq_compare,
3499e39c5baSBill Taylor 	    sizeof (struct hermon_workq_avl_s),
3509e39c5baSBill Taylor 	    offsetof(struct hermon_workq_avl_s, wqa_link));
3519e39c5baSBill Taylor 
3529e39c5baSBill Taylor 	cq->cq_hdlrarg	  = (void *)ibt_cqhdl;
3539e39c5baSBill Taylor 
3549e39c5baSBill Taylor 	/*
3559e39c5baSBill Taylor 	 * Put CQ handle in Hermon CQNum-to-CQHdl list.  Then fill in the
3569e39c5baSBill Taylor 	 * "actual_size" and "cqhdl" and return success
3579e39c5baSBill Taylor 	 */
358*17a2b317SBill Taylor 	hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, cq);
3599e39c5baSBill Taylor 
3609e39c5baSBill Taylor 	/*
3619e39c5baSBill Taylor 	 * If this is a user-mappable CQ, then we need to insert the previously
3629e39c5baSBill Taylor 	 * allocated entry into the "userland resources database".  This will
3639e39c5baSBill Taylor 	 * allow for later lookup during devmap() (i.e. mmap()) calls.
3649e39c5baSBill Taylor 	 */
3659e39c5baSBill Taylor 	if (cq->cq_is_umap) {
3669e39c5baSBill Taylor 		hermon_umap_db_add(umapdb);
3679e39c5baSBill Taylor 	}
3689e39c5baSBill Taylor 
3699e39c5baSBill Taylor 	/*
3709e39c5baSBill Taylor 	 * Fill in the return arguments (if necessary).  This includes the
3719e39c5baSBill Taylor 	 * real completion queue size.
3729e39c5baSBill Taylor 	 */
3739e39c5baSBill Taylor 	if (actual_size != NULL) {
3749e39c5baSBill Taylor 		*actual_size = (1 << log_cq_size) - 1;
3759e39c5baSBill Taylor 	}
3769e39c5baSBill Taylor 	*cqhdl = cq;
3779e39c5baSBill Taylor 
3789e39c5baSBill Taylor 	return (DDI_SUCCESS);
3799e39c5baSBill Taylor 
3809e39c5baSBill Taylor /*
3819e39c5baSBill Taylor  * The following is cleanup for all possible failure cases in this routine
3829e39c5baSBill Taylor  */
3839e39c5baSBill Taylor cqalloc_fail6:
3849e39c5baSBill Taylor 	if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
3859e39c5baSBill Taylor 	    sleepflag) != DDI_SUCCESS) {
3869e39c5baSBill Taylor 		HERMON_WARNING(state, "failed to deregister CQ memory");
3879e39c5baSBill Taylor 	}
3889e39c5baSBill Taylor cqalloc_fail5:
3899e39c5baSBill Taylor 	hermon_queue_free(&cq->cq_cqinfo);
3909e39c5baSBill Taylor cqalloc_fail4a:
3919e39c5baSBill Taylor 	hermon_dbr_free(state, uarpg, cq->cq_arm_ci_vdbr);
3929e39c5baSBill Taylor cqalloc_fail4:
3939e39c5baSBill Taylor 	if (cq_is_umap) {
3949e39c5baSBill Taylor 		hermon_umap_db_free(umapdb);
3959e39c5baSBill Taylor 	}
3969e39c5baSBill Taylor cqalloc_fail3:
3979e39c5baSBill Taylor 	hermon_rsrc_free(state, &rsrc);
3989e39c5baSBill Taylor cqalloc_fail2:
3999e39c5baSBill Taylor 	hermon_rsrc_free(state, &cqc);
4009e39c5baSBill Taylor cqalloc_fail1:
4019e39c5baSBill Taylor 	hermon_pd_refcnt_dec(pd);
4029e39c5baSBill Taylor cqalloc_fail:
4039e39c5baSBill Taylor 	return (status);
4049e39c5baSBill Taylor }
4059e39c5baSBill Taylor 
4069e39c5baSBill Taylor 
4079e39c5baSBill Taylor /*
4089e39c5baSBill Taylor  * hermon_cq_free()
4099e39c5baSBill Taylor  *    Context: Can be called only from user or kernel context.
4109e39c5baSBill Taylor  */
4119e39c5baSBill Taylor /* ARGSUSED */
4129e39c5baSBill Taylor int
hermon_cq_free(hermon_state_t * state,hermon_cqhdl_t * cqhdl,uint_t sleepflag)4139e39c5baSBill Taylor hermon_cq_free(hermon_state_t *state, hermon_cqhdl_t *cqhdl, uint_t sleepflag)
4149e39c5baSBill Taylor {
4159e39c5baSBill Taylor 	hermon_rsrc_t		*cqc, *rsrc;
4169e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
4179e39c5baSBill Taylor 	hermon_hw_cqc_t		cqc_entry;
4189e39c5baSBill Taylor 	hermon_pdhdl_t		pd;
4199e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
4209e39c5baSBill Taylor 	hermon_cqhdl_t		cq, resize;
4219e39c5baSBill Taylor 	uint32_t		cqnum;
4229e39c5baSBill Taylor 	uint64_t		value;
4239e39c5baSBill Taylor 	uint_t			maxprot;
4249e39c5baSBill Taylor 	int			status;
4259e39c5baSBill Taylor 
4269e39c5baSBill Taylor 	/*
4279e39c5baSBill Taylor 	 * Pull all the necessary information from the Hermon Completion Queue
4289e39c5baSBill Taylor 	 * handle.  This is necessary here because the resource for the
4299e39c5baSBill Taylor 	 * CQ handle is going to be freed up as part of this operation.
4309e39c5baSBill Taylor 	 */
4319e39c5baSBill Taylor 	cq	= *cqhdl;
4329e39c5baSBill Taylor 	mutex_enter(&cq->cq_lock);
4339e39c5baSBill Taylor 	cqc	= cq->cq_cqcrsrcp;
4349e39c5baSBill Taylor 	rsrc	= cq->cq_rsrcp;
4359e39c5baSBill Taylor 	pd	= state->hs_pdhdl_internal;
4369e39c5baSBill Taylor 	mr	= cq->cq_mrhdl;
4379e39c5baSBill Taylor 	cqnum	= cq->cq_cqnum;
4389e39c5baSBill Taylor 
4399e39c5baSBill Taylor 	resize = cq->cq_resize_hdl;		/* save the handle for later */
4409e39c5baSBill Taylor 
4419e39c5baSBill Taylor 	/*
4429e39c5baSBill Taylor 	 * If there are work queues still associated with the CQ, then return
4439e39c5baSBill Taylor 	 * an error.  Otherwise, we will be holding the CQ lock.
4449e39c5baSBill Taylor 	 */
4459e39c5baSBill Taylor 	if (cq->cq_refcnt != 0) {
4469e39c5baSBill Taylor 		mutex_exit(&cq->cq_lock);
4479e39c5baSBill Taylor 		return (IBT_CQ_BUSY);
4489e39c5baSBill Taylor 	}
4499e39c5baSBill Taylor 
4509e39c5baSBill Taylor 	/*
4519e39c5baSBill Taylor 	 * If this was a user-mappable CQ, then we need to remove its entry
4529e39c5baSBill Taylor 	 * from the "userland resources database".  If it is also currently
4539e39c5baSBill Taylor 	 * mmap()'d out to a user process, then we need to call
4549e39c5baSBill Taylor 	 * devmap_devmem_remap() to remap the CQ memory to an invalid mapping.
4559e39c5baSBill Taylor 	 * We also need to invalidate the CQ tracking information for the
4569e39c5baSBill Taylor 	 * user mapping.
4579e39c5baSBill Taylor 	 */
4589e39c5baSBill Taylor 	if (cq->cq_is_umap) {
4599e39c5baSBill Taylor 		status = hermon_umap_db_find(state->hs_instance, cqnum,
4609e39c5baSBill Taylor 		    MLNX_UMAP_CQMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
4619e39c5baSBill Taylor 		    &umapdb);
4629e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
4639e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
4649e39c5baSBill Taylor 			HERMON_WARNING(state, "failed to find in database");
4659e39c5baSBill Taylor 			return (ibc_get_ci_failure(0));
4669e39c5baSBill Taylor 		}
4679e39c5baSBill Taylor 		hermon_umap_db_free(umapdb);
4689e39c5baSBill Taylor 		if (cq->cq_umap_dhp != NULL) {
4699e39c5baSBill Taylor 			maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
4709e39c5baSBill Taylor 			status = devmap_devmem_remap(cq->cq_umap_dhp,
4719e39c5baSBill Taylor 			    state->hs_dip, 0, 0, cq->cq_cqinfo.qa_size,
4729e39c5baSBill Taylor 			    maxprot, DEVMAP_MAPPING_INVALID, NULL);
4739e39c5baSBill Taylor 			if (status != DDI_SUCCESS) {
4749e39c5baSBill Taylor 				mutex_exit(&cq->cq_lock);
4759e39c5baSBill Taylor 				HERMON_WARNING(state, "failed in CQ memory "
4769e39c5baSBill Taylor 				    "devmap_devmem_remap()");
4779e39c5baSBill Taylor 				return (ibc_get_ci_failure(0));
4789e39c5baSBill Taylor 			}
4799e39c5baSBill Taylor 			cq->cq_umap_dhp = (devmap_cookie_t)NULL;
4809e39c5baSBill Taylor 		}
4819e39c5baSBill Taylor 	}
4829e39c5baSBill Taylor 
4839e39c5baSBill Taylor 	/*
4849e39c5baSBill Taylor 	 * Put NULL into the Arbel CQNum-to-CQHdl list.  This will allow any
4859e39c5baSBill Taylor 	 * in-progress events to detect that the CQ corresponding to this
4869e39c5baSBill Taylor 	 * number has been freed.
4879e39c5baSBill Taylor 	 */
488*17a2b317SBill Taylor 	hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, NULL);
4899e39c5baSBill Taylor 
4909e39c5baSBill Taylor 	mutex_exit(&cq->cq_lock);
4919e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
4929e39c5baSBill Taylor 
4939e39c5baSBill Taylor 	/*
4949e39c5baSBill Taylor 	 * Reclaim CQC entry from hardware (using the Hermon HW2SW_CQ
4959e39c5baSBill Taylor 	 * firmware command).  If the ownership transfer fails for any reason,
4969e39c5baSBill Taylor 	 * then it is an indication that something (either in HW or SW) has
4979e39c5baSBill Taylor 	 * gone seriously wrong.
4989e39c5baSBill Taylor 	 */
4999e39c5baSBill Taylor 	status = hermon_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
5009e39c5baSBill Taylor 	    sizeof (hermon_hw_cqc_t), cqnum, sleepflag);
5019e39c5baSBill Taylor 	if (status != HERMON_CMD_SUCCESS) {
5029e39c5baSBill Taylor 		HERMON_WARNING(state, "failed to reclaim CQC ownership");
5039e39c5baSBill Taylor 		cmn_err(CE_CONT, "Hermon: HW2SW_CQ command failed: %08x\n",
5049e39c5baSBill Taylor 		    status);
5059e39c5baSBill Taylor 		if (status == HERMON_CMD_INVALID_STATUS) {
5069e39c5baSBill Taylor 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
5079e39c5baSBill Taylor 		}
5089e39c5baSBill Taylor 		return (ibc_get_ci_failure(0));
5099e39c5baSBill Taylor 	}
5109e39c5baSBill Taylor 
5119e39c5baSBill Taylor 	/*
5129e39c5baSBill Taylor 	 * From here on, we start reliquishing resources - but check to see
5139e39c5baSBill Taylor 	 * if a resize was in progress - if so, we need to relinquish those
5149e39c5baSBill Taylor 	 * resources as well
5159e39c5baSBill Taylor 	 */
5169e39c5baSBill Taylor 
5179e39c5baSBill Taylor 
5189e39c5baSBill Taylor 	/*
5199e39c5baSBill Taylor 	 * Deregister the memory for the Completion Queue.  If this fails
5209e39c5baSBill Taylor 	 * for any reason, then it is an indication that something (either
5219e39c5baSBill Taylor 	 * in HW or SW) has gone seriously wrong.  So we print a warning
5229e39c5baSBill Taylor 	 * message and return.
5239e39c5baSBill Taylor 	 */
5249e39c5baSBill Taylor 	status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
5259e39c5baSBill Taylor 	    sleepflag);
5269e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
5279e39c5baSBill Taylor 		HERMON_WARNING(state, "failed to deregister CQ memory");
5289e39c5baSBill Taylor 		return (ibc_get_ci_failure(0));
5299e39c5baSBill Taylor 	}
5309e39c5baSBill Taylor 
5319e39c5baSBill Taylor 	if (resize)	{	/* there was a pointer to a handle */
5329e39c5baSBill Taylor 		mr = resize->cq_mrhdl;	/* reuse the pointer to the region */
5339e39c5baSBill Taylor 		status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
5349e39c5baSBill Taylor 		    sleepflag);
5359e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
5369e39c5baSBill Taylor 			HERMON_WARNING(state, "failed to deregister resize CQ "
5379e39c5baSBill Taylor 			    "memory");
5389e39c5baSBill Taylor 			return (ibc_get_ci_failure(0));
5399e39c5baSBill Taylor 		}
5409e39c5baSBill Taylor 	}
5419e39c5baSBill Taylor 
5429e39c5baSBill Taylor 	/* Free the memory for the CQ */
5439e39c5baSBill Taylor 	hermon_queue_free(&cq->cq_cqinfo);
5449e39c5baSBill Taylor 	if (resize)	{
5459e39c5baSBill Taylor 		hermon_queue_free(&resize->cq_cqinfo);
5469e39c5baSBill Taylor 		/* and the temporary handle */
5479e39c5baSBill Taylor 		kmem_free(resize, sizeof (struct hermon_sw_cq_s));
5489e39c5baSBill Taylor 	}
5499e39c5baSBill Taylor 
5509e39c5baSBill Taylor 	/* everything else does not matter for the resize in progress */
5519e39c5baSBill Taylor 
5529e39c5baSBill Taylor 	/* Free the dbr */
5539e39c5baSBill Taylor 	hermon_dbr_free(state, cq->cq_uarpg, cq->cq_arm_ci_vdbr);
5549e39c5baSBill Taylor 
5559e39c5baSBill Taylor 	/* Free the Hermon Completion Queue handle */
5569e39c5baSBill Taylor 	hermon_rsrc_free(state, &rsrc);
5579e39c5baSBill Taylor 
5589e39c5baSBill Taylor 	/* Free up the CQC entry resource */
5599e39c5baSBill Taylor 	hermon_rsrc_free(state, &cqc);
5609e39c5baSBill Taylor 
5619e39c5baSBill Taylor 	/* Decrement the reference count on the protection domain (PD) */
5629e39c5baSBill Taylor 	hermon_pd_refcnt_dec(pd);
5639e39c5baSBill Taylor 
5649e39c5baSBill Taylor 	/* Set the cqhdl pointer to NULL and return success */
5659e39c5baSBill Taylor 	*cqhdl = NULL;
5669e39c5baSBill Taylor 
5679e39c5baSBill Taylor 	return (DDI_SUCCESS);
5689e39c5baSBill Taylor }
5699e39c5baSBill Taylor 
5709e39c5baSBill Taylor 
5719e39c5baSBill Taylor /*
5729e39c5baSBill Taylor  * hermon_cq_resize()
5739e39c5baSBill Taylor  *    Context: Can be called only from user or kernel context.
5749e39c5baSBill Taylor  */
5759e39c5baSBill Taylor int
hermon_cq_resize(hermon_state_t * state,hermon_cqhdl_t cq,uint_t req_size,uint_t * actual_size,uint_t sleepflag)5769e39c5baSBill Taylor hermon_cq_resize(hermon_state_t *state, hermon_cqhdl_t cq, uint_t req_size,
5779e39c5baSBill Taylor     uint_t *actual_size, uint_t sleepflag)
5789e39c5baSBill Taylor {
5799e39c5baSBill Taylor 	hermon_hw_cqc_t		cqc_entry;
5809e39c5baSBill Taylor 	hermon_cqhdl_t		resize_hdl;
5819e39c5baSBill Taylor 	hermon_qalloc_info_t	new_cqinfo;
5829e39c5baSBill Taylor 	ibt_mr_attr_t		mr_attr;
5839e39c5baSBill Taylor 	hermon_mr_options_t	op;
5849e39c5baSBill Taylor 	hermon_pdhdl_t		pd;
5859e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
5869e39c5baSBill Taylor 	hermon_hw_cqe_t		*buf;
5879e39c5baSBill Taylor 	uint32_t		new_prod_indx;
5889e39c5baSBill Taylor 	uint_t			log_cq_size;
5899e39c5baSBill Taylor 	int			status, flag;
5909e39c5baSBill Taylor 
5919e39c5baSBill Taylor 	if (cq->cq_resize_hdl != 0) {	/* already in process */
5929e39c5baSBill Taylor 		status = IBT_CQ_BUSY;
5939e39c5baSBill Taylor 		goto cqresize_fail;
5949e39c5baSBill Taylor 	}
5959e39c5baSBill Taylor 
5969e39c5baSBill Taylor 
5979e39c5baSBill Taylor 	/* Use the internal protection domain (PD) for CQs */
5989e39c5baSBill Taylor 	pd = state->hs_pdhdl_internal;
5999e39c5baSBill Taylor 
6009e39c5baSBill Taylor 	/*
6019e39c5baSBill Taylor 	 * Calculate the appropriate size for the new resized completion queue.
6029e39c5baSBill Taylor 	 * Note:  All Hermon CQs must be a power-of-2 minus 1 in size.  Also
6039e39c5baSBill Taylor 	 * they may not be any smaller than HERMON_CQ_MIN_SIZE.  This step is
6049e39c5baSBill Taylor 	 * to round the requested size up to the next highest power-of-2
6059e39c5baSBill Taylor 	 */
6069e39c5baSBill Taylor 	req_size = max(req_size, HERMON_CQ_MIN_SIZE);
6079e39c5baSBill Taylor 	log_cq_size = highbit(req_size);
6089e39c5baSBill Taylor 
6099e39c5baSBill Taylor 	/*
6109e39c5baSBill Taylor 	 * Next we verify that the rounded-up size is valid (i.e. consistent
6119e39c5baSBill Taylor 	 * with the device limits and/or software-configured limits)
6129e39c5baSBill Taylor 	 */
6139e39c5baSBill Taylor 	if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
6149e39c5baSBill Taylor 		status = IBT_HCA_CQ_EXCEEDED;
6159e39c5baSBill Taylor 		goto cqresize_fail;
6169e39c5baSBill Taylor 	}
6179e39c5baSBill Taylor 
6189e39c5baSBill Taylor 	/*
6199e39c5baSBill Taylor 	 * Allocate the memory for newly resized Completion Queue.
6209e39c5baSBill Taylor 	 *
6219e39c5baSBill Taylor 	 * Note: Although we use the common queue allocation routine, we
6229e39c5baSBill Taylor 	 * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
6239e39c5baSBill Taylor 	 * kernel system memory) for kernel CQs because it would be
6249e39c5baSBill Taylor 	 * inefficient to have CQs located in DDR memory.  This is the same
6259e39c5baSBill Taylor 	 * as we do when we first allocate completion queues primarily
6269e39c5baSBill Taylor 	 * because CQs are read from (by software) more than they are written
6279e39c5baSBill Taylor 	 * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
6289e39c5baSBill Taylor 	 * user-mappable CQs for a similar reason.)
6299e39c5baSBill Taylor 	 * It is also worth noting that, unlike Hermon QP work queues,
6309e39c5baSBill Taylor 	 * completion queues do not have the same strict alignment
6319e39c5baSBill Taylor 	 * requirements.  It is sufficient for the CQ memory to be both
6329e39c5baSBill Taylor 	 * aligned to and bound to addresses which are a multiple of CQE size.
6339e39c5baSBill Taylor 	 */
6349e39c5baSBill Taylor 
6359e39c5baSBill Taylor 	/* first, alloc the resize_handle */
6369e39c5baSBill Taylor 	resize_hdl = kmem_zalloc(sizeof (struct hermon_sw_cq_s), KM_SLEEP);
6379e39c5baSBill Taylor 
6389e39c5baSBill Taylor 	new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
6399e39c5baSBill Taylor 	new_cqinfo.qa_alloc_align = PAGESIZE;
6409e39c5baSBill Taylor 	new_cqinfo.qa_bind_align  = PAGESIZE;
6419e39c5baSBill Taylor 	if (cq->cq_is_umap) {
6429e39c5baSBill Taylor 		new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
6439e39c5baSBill Taylor 	} else {
6449e39c5baSBill Taylor 		new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
6459e39c5baSBill Taylor 	}
6469e39c5baSBill Taylor 	status = hermon_queue_alloc(state, &new_cqinfo, sleepflag);
6479e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
6489e39c5baSBill Taylor 		/* free the resize handle */
6499e39c5baSBill Taylor 		kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
6509e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
6519e39c5baSBill Taylor 		goto cqresize_fail;
6529e39c5baSBill Taylor 	}
6539e39c5baSBill Taylor 	buf = (hermon_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
6549e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
6559e39c5baSBill Taylor 
6569e39c5baSBill Taylor 	/*
6579e39c5baSBill Taylor 	 * No initialization of the cq is needed - the command will do it
6589e39c5baSBill Taylor 	 */
6599e39c5baSBill Taylor 
6609e39c5baSBill Taylor 	/*
6619e39c5baSBill Taylor 	 * Register the memory for the CQ.  The memory for the CQ must
6629e39c5baSBill Taylor 	 * be registered in the Hermon TPT tables.  This gives us the LKey
6639e39c5baSBill Taylor 	 * to specify in the CQ context below.
6649e39c5baSBill Taylor 	 */
6659e39c5baSBill Taylor 	flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
6669e39c5baSBill Taylor 	mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
6679e39c5baSBill Taylor 	mr_attr.mr_len	 = new_cqinfo.qa_size;
6689e39c5baSBill Taylor 	mr_attr.mr_as	 = NULL;
6699e39c5baSBill Taylor 	mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
6709e39c5baSBill Taylor 	op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
6719e39c5baSBill Taylor 	op.mro_bind_dmahdl = new_cqinfo.qa_dmahdl;
6729e39c5baSBill Taylor 	op.mro_bind_override_addr = 0;
6739e39c5baSBill Taylor 	status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
6749e39c5baSBill Taylor 	    HERMON_CQ_CMPT);
6759e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
6769e39c5baSBill Taylor 		hermon_queue_free(&new_cqinfo);
6779e39c5baSBill Taylor 		/* free the resize handle */
6789e39c5baSBill Taylor 		kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
6799e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
6809e39c5baSBill Taylor 		goto cqresize_fail;
6819e39c5baSBill Taylor 	}
6829e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
6839e39c5baSBill Taylor 
6849e39c5baSBill Taylor 	/*
6859e39c5baSBill Taylor 	 * Now we grab the CQ lock.  Since we will be updating the actual
6869e39c5baSBill Taylor 	 * CQ location and the producer/consumer indexes, we should hold
6879e39c5baSBill Taylor 	 * the lock.
6889e39c5baSBill Taylor 	 *
6899e39c5baSBill Taylor 	 * We do a ARBEL_NOSLEEP here (and below), though, because we are
6909e39c5baSBill Taylor 	 * holding the "cq_lock" and if we got raised to interrupt level
6919e39c5baSBill Taylor 	 * by priority inversion, we would not want to block in this routine
6929e39c5baSBill Taylor 	 * waiting for success.
6939e39c5baSBill Taylor 	 */
6949e39c5baSBill Taylor 	mutex_enter(&cq->cq_lock);
6959e39c5baSBill Taylor 
6969e39c5baSBill Taylor 	/*
6979e39c5baSBill Taylor 	 * Fill in the CQC entry.  For the resize operation this is the
6989e39c5baSBill Taylor 	 * final step before attempting the resize operation on the CQC entry.
6999e39c5baSBill Taylor 	 * We use all of the information collected/calculated above to fill
7009e39c5baSBill Taylor 	 * in the requisite portions of the CQC.
7019e39c5baSBill Taylor 	 */
7029e39c5baSBill Taylor 	bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
7039e39c5baSBill Taylor 	cqc_entry.log_cq_sz	= log_cq_size;
7049e39c5baSBill Taylor 	cqc_entry.pg_offs	= new_cqinfo.qa_pgoffs >> 5;
7059e39c5baSBill Taylor 	cqc_entry.log2_pgsz	= mr->mr_log2_pgsz;
7069e39c5baSBill Taylor 	cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
7079e39c5baSBill Taylor 	cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;
7089e39c5baSBill Taylor 
7099e39c5baSBill Taylor 	/*
7109e39c5baSBill Taylor 	 * Write the CQC entry to hardware.  Lastly, we pass ownership of
7119e39c5baSBill Taylor 	 * the entry to the hardware (using the Hermon RESIZE_CQ firmware
7129e39c5baSBill Taylor 	 * command).  Note: In general, this operation shouldn't fail.  But
7139e39c5baSBill Taylor 	 * if it does, we have to undo everything we've done above before
7149e39c5baSBill Taylor 	 * returning error.  Also note that the status returned may indicate
7159e39c5baSBill Taylor 	 * the code to return to the IBTF.
7169e39c5baSBill Taylor 	 */
7179e39c5baSBill Taylor 	status = hermon_resize_cq_cmd_post(state, &cqc_entry, cq->cq_cqnum,
7189e39c5baSBill Taylor 	    &new_prod_indx, HERMON_CMD_NOSLEEP_SPIN);
7199e39c5baSBill Taylor 	if (status != HERMON_CMD_SUCCESS) {
7209e39c5baSBill Taylor 		/* Resize attempt has failed, drop CQ lock and cleanup */
7219e39c5baSBill Taylor 		mutex_exit(&cq->cq_lock);
7229e39c5baSBill Taylor 		if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
7239e39c5baSBill Taylor 		    sleepflag) != DDI_SUCCESS) {
7249e39c5baSBill Taylor 			HERMON_WARNING(state, "failed to deregister CQ memory");
7259e39c5baSBill Taylor 		}
7269e39c5baSBill Taylor 		kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
7279e39c5baSBill Taylor 		hermon_queue_free(&new_cqinfo);
7289e39c5baSBill Taylor 		if (status == HERMON_CMD_BAD_SIZE) {
7299e39c5baSBill Taylor 			return (IBT_CQ_SZ_INSUFFICIENT);
7309e39c5baSBill Taylor 		} else {
7319e39c5baSBill Taylor 			cmn_err(CE_CONT, "Hermon: RESIZE_CQ command failed: "
7329e39c5baSBill Taylor 			    "%08x\n", status);
7339e39c5baSBill Taylor 			if (status == HERMON_CMD_INVALID_STATUS) {
7349e39c5baSBill Taylor 				hermon_fm_ereport(state, HCA_SYS_ERR,
7359e39c5baSBill Taylor 				    HCA_ERR_SRV_LOST);
7369e39c5baSBill Taylor 			}
7379e39c5baSBill Taylor 			return (ibc_get_ci_failure(0));
7389e39c5baSBill Taylor 		}
7399e39c5baSBill Taylor 	}
7409e39c5baSBill Taylor 
7419e39c5baSBill Taylor 	/*
7429e39c5baSBill Taylor 	 * For Hermon, we've alloc'd another handle structure and save off the
7439e39c5baSBill Taylor 	 * important things in it. Then, in polling we check to see if there's
7449e39c5baSBill Taylor 	 * a "resizing handle" and if so we look for the "special CQE", opcode
7459e39c5baSBill Taylor 	 * 0x16, that indicates the transition to the new buffer.
7469e39c5baSBill Taylor 	 *
7479e39c5baSBill Taylor 	 * At that point, we'll adjust everything - including dereg and
7489e39c5baSBill Taylor 	 * freeing of the original buffer, updating all the necessary fields
7499e39c5baSBill Taylor 	 * in the cq_hdl, and setting up for the next cqe polling
7509e39c5baSBill Taylor 	 */
7519e39c5baSBill Taylor 
7529e39c5baSBill Taylor 	resize_hdl->cq_buf 	= buf;
7539e39c5baSBill Taylor 	resize_hdl->cq_bufsz	= (1 << log_cq_size);
7549e39c5baSBill Taylor 	resize_hdl->cq_mrhdl	= mr;
7559e39c5baSBill Taylor 	resize_hdl->cq_log_cqsz = log_cq_size;
7569e39c5baSBill Taylor 
7579e39c5baSBill Taylor 	bcopy(&new_cqinfo, &(resize_hdl->cq_cqinfo),
7589e39c5baSBill Taylor 	    sizeof (struct hermon_qalloc_info_s));
7599e39c5baSBill Taylor 
7609e39c5baSBill Taylor 	/* now, save the address in the cq_handle */
7619e39c5baSBill Taylor 	cq->cq_resize_hdl = resize_hdl;
7629e39c5baSBill Taylor 
7639e39c5baSBill Taylor 	/*
7649e39c5baSBill Taylor 	 * Drop the CQ lock now.
7659e39c5baSBill Taylor 	 */
7669e39c5baSBill Taylor 
7679e39c5baSBill Taylor 	mutex_exit(&cq->cq_lock);
7689e39c5baSBill Taylor 	/*
7699e39c5baSBill Taylor 	 * Fill in the return arguments (if necessary).  This includes the
7709e39c5baSBill Taylor 	 * real new completion queue size.
7719e39c5baSBill Taylor 	 */
7729e39c5baSBill Taylor 	if (actual_size != NULL) {
7739e39c5baSBill Taylor 		*actual_size = (1 << log_cq_size) - 1;
7749e39c5baSBill Taylor 	}
7759e39c5baSBill Taylor 
7769e39c5baSBill Taylor 	return (DDI_SUCCESS);
7779e39c5baSBill Taylor 
7789e39c5baSBill Taylor cqresize_fail:
7799e39c5baSBill Taylor 	return (status);
7809e39c5baSBill Taylor }
7819e39c5baSBill Taylor 
7829e39c5baSBill Taylor 
7839e39c5baSBill Taylor /*
7849e39c5baSBill Taylor  * hermon_cq_modify()
7859e39c5baSBill Taylor  *    Context: Can be called base context.
7869e39c5baSBill Taylor  */
7879e39c5baSBill Taylor /* ARGSUSED */
7889e39c5baSBill Taylor int
hermon_cq_modify(hermon_state_t * state,hermon_cqhdl_t cq,uint_t count,uint_t usec,ibt_cq_handler_id_t hid,uint_t sleepflag)7899e39c5baSBill Taylor hermon_cq_modify(hermon_state_t *state, hermon_cqhdl_t cq,
7909e39c5baSBill Taylor     uint_t count, uint_t usec, ibt_cq_handler_id_t hid, uint_t sleepflag)
7919e39c5baSBill Taylor {
7929e39c5baSBill Taylor 	int	status;
7939e39c5baSBill Taylor 	hermon_hw_cqc_t		cqc_entry;
7949e39c5baSBill Taylor 
7959e39c5baSBill Taylor 	mutex_enter(&cq->cq_lock);
7969e39c5baSBill Taylor 	if (count != cq->cq_intmod_count ||
7979e39c5baSBill Taylor 	    usec != cq->cq_intmod_usec) {
7989e39c5baSBill Taylor 		bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
7999e39c5baSBill Taylor 		cqc_entry.cq_max_cnt = count;
8009e39c5baSBill Taylor 		cqc_entry.cq_period = usec;
8019e39c5baSBill Taylor 		status = hermon_modify_cq_cmd_post(state, &cqc_entry,
8029e39c5baSBill Taylor 		    cq->cq_cqnum, MODIFY_MODERATION_CQ, sleepflag);
8039e39c5baSBill Taylor 		if (status != HERMON_CMD_SUCCESS) {
8049e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
805*17a2b317SBill Taylor 			cmn_err(CE_CONT, "Hermon: MODIFY_MODERATION_CQ "
806*17a2b317SBill Taylor 			    "command failed: %08x\n", status);
8079e39c5baSBill Taylor 			if (status == HERMON_CMD_INVALID_STATUS) {
8089e39c5baSBill Taylor 				hermon_fm_ereport(state, HCA_SYS_ERR,
8099e39c5baSBill Taylor 				    HCA_ERR_SRV_LOST);
8109e39c5baSBill Taylor 			}
8119e39c5baSBill Taylor 			return (ibc_get_ci_failure(0));
8129e39c5baSBill Taylor 		}
8139e39c5baSBill Taylor 		cq->cq_intmod_count = count;
8149e39c5baSBill Taylor 		cq->cq_intmod_usec = usec;
8159e39c5baSBill Taylor 	}
816*17a2b317SBill Taylor 	if (hid && (hid - 1 != cq->cq_eqnum)) {
817*17a2b317SBill Taylor 		bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
818*17a2b317SBill Taylor 		cqc_entry.c_eqn = HERMON_HID_TO_EQNUM(state, hid);
819*17a2b317SBill Taylor 		status = hermon_modify_cq_cmd_post(state, &cqc_entry,
820*17a2b317SBill Taylor 		    cq->cq_cqnum, MODIFY_EQN, sleepflag);
821*17a2b317SBill Taylor 		if (status != HERMON_CMD_SUCCESS) {
822*17a2b317SBill Taylor 			mutex_exit(&cq->cq_lock);
823*17a2b317SBill Taylor 			cmn_err(CE_CONT, "Hermon: MODIFY_EQN command failed: "
824*17a2b317SBill Taylor 			    "%08x\n", status);
825*17a2b317SBill Taylor 			if (status == HERMON_CMD_INVALID_STATUS) {
826*17a2b317SBill Taylor 				hermon_fm_ereport(state, HCA_SYS_ERR,
827*17a2b317SBill Taylor 				    HCA_ERR_SRV_LOST);
828*17a2b317SBill Taylor 			}
829*17a2b317SBill Taylor 			return (ibc_get_ci_failure(0));
830*17a2b317SBill Taylor 		}
831*17a2b317SBill Taylor 		cq->cq_eqnum = hid - 1;
832*17a2b317SBill Taylor 	}
8339e39c5baSBill Taylor 	mutex_exit(&cq->cq_lock);
8349e39c5baSBill Taylor 	return (DDI_SUCCESS);
8359e39c5baSBill Taylor }
8369e39c5baSBill Taylor 
8379e39c5baSBill Taylor /*
8389e39c5baSBill Taylor  * hermon_cq_notify()
8399e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
8409e39c5baSBill Taylor  */
8419e39c5baSBill Taylor int
hermon_cq_notify(hermon_state_t * state,hermon_cqhdl_t cq,ibt_cq_notify_flags_t flags)8429e39c5baSBill Taylor hermon_cq_notify(hermon_state_t *state, hermon_cqhdl_t cq,
8439e39c5baSBill Taylor     ibt_cq_notify_flags_t flags)
8449e39c5baSBill Taylor {
8459e39c5baSBill Taylor 	uint_t	cmd;
8469e39c5baSBill Taylor 	ibt_status_t status;
8479e39c5baSBill Taylor 
8489e39c5baSBill Taylor 	/* Validate IBT flags and call doorbell routine. */
8499e39c5baSBill Taylor 	if (flags == IBT_NEXT_COMPLETION) {
8509e39c5baSBill Taylor 		cmd = HERMON_CQDB_NOTIFY_CQ;
8519e39c5baSBill Taylor 	} else if (flags == IBT_NEXT_SOLICITED) {
8529e39c5baSBill Taylor 		cmd = HERMON_CQDB_NOTIFY_CQ_SOLICIT;
8539e39c5baSBill Taylor 	} else {
8549e39c5baSBill Taylor 		return (IBT_CQ_NOTIFY_TYPE_INVALID);
8559e39c5baSBill Taylor 	}
8569e39c5baSBill Taylor 
8579e39c5baSBill Taylor 	status = hermon_cq_arm_doorbell(state, cq, cmd);
8589e39c5baSBill Taylor 	return (status);
8599e39c5baSBill Taylor }
8609e39c5baSBill Taylor 
8619e39c5baSBill Taylor 
8629e39c5baSBill Taylor /*
8639e39c5baSBill Taylor  * hermon_cq_poll()
8649e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
8659e39c5baSBill Taylor  */
8669e39c5baSBill Taylor int
hermon_cq_poll(hermon_state_t * state,hermon_cqhdl_t cq,ibt_wc_t * wc_p,uint_t num_wc,uint_t * num_polled)8679e39c5baSBill Taylor hermon_cq_poll(hermon_state_t *state, hermon_cqhdl_t cq, ibt_wc_t *wc_p,
8689e39c5baSBill Taylor     uint_t num_wc, uint_t *num_polled)
8699e39c5baSBill Taylor {
8709e39c5baSBill Taylor 	hermon_hw_cqe_t	*cqe;
8719e39c5baSBill Taylor 	uint_t		opcode;
872*17a2b317SBill Taylor 	uint32_t	cons_indx, wrap_around_mask, shift, mask;
8739e39c5baSBill Taylor 	uint32_t	polled_cnt, spec_op = 0;
8749e39c5baSBill Taylor 	int		status;
8759e39c5baSBill Taylor 
8769e39c5baSBill Taylor 	/*
8779e39c5baSBill Taylor 	 * Check for user-mappable CQ memory.  Note:  We do not allow kernel
8789e39c5baSBill Taylor 	 * clients to poll CQ memory that is accessible directly by the user.
8799e39c5baSBill Taylor 	 * If the CQ memory is user accessible, then return an error.
8809e39c5baSBill Taylor 	 */
8819e39c5baSBill Taylor 	if (cq->cq_is_umap) {
8829e39c5baSBill Taylor 		return (IBT_CQ_HDL_INVALID);
8839e39c5baSBill Taylor 	}
8849e39c5baSBill Taylor 
8859e39c5baSBill Taylor 	mutex_enter(&cq->cq_lock);
8869e39c5baSBill Taylor 
8879e39c5baSBill Taylor 	/* Get the consumer index */
8889e39c5baSBill Taylor 	cons_indx = cq->cq_consindx;
889*17a2b317SBill Taylor 	shift = cq->cq_log_cqsz;
890*17a2b317SBill Taylor 	mask = cq->cq_bufsz;
8919e39c5baSBill Taylor 
8929e39c5baSBill Taylor 	/*
8939e39c5baSBill Taylor 	 * Calculate the wrap around mask.  Note: This operation only works
8949e39c5baSBill Taylor 	 * because all Hermon completion queues have power-of-2 sizes
8959e39c5baSBill Taylor 	 */
8969e39c5baSBill Taylor 	wrap_around_mask = (cq->cq_bufsz - 1);
8979e39c5baSBill Taylor 
8989e39c5baSBill Taylor 	/* Calculate the pointer to the first CQ entry */
8999e39c5baSBill Taylor 	cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9009e39c5baSBill Taylor 
9019e39c5baSBill Taylor 	/*
9029e39c5baSBill Taylor 	 * Keep pulling entries from the CQ until we find an entry owned by
9039e39c5baSBill Taylor 	 * the hardware.  As long as there the CQE's owned by SW, process
9049e39c5baSBill Taylor 	 * each entry by calling hermon_cq_cqe_consume() and updating the CQ
9059e39c5baSBill Taylor 	 * consumer index.  Note:  We only update the consumer index if
9069e39c5baSBill Taylor 	 * hermon_cq_cqe_consume() returns HERMON_CQ_SYNC_AND_DB.  Otherwise,
9079e39c5baSBill Taylor 	 * it indicates that we are going to "recycle" the CQE (probably
9089e39c5baSBill Taylor 	 * because it is a error CQE and corresponds to more than one
9099e39c5baSBill Taylor 	 * completion).
9109e39c5baSBill Taylor 	 */
9119e39c5baSBill Taylor 	polled_cnt = 0;
912*17a2b317SBill Taylor 	while (HERMON_CQE_OWNER_IS_SW(cq, cqe, cons_indx, shift, mask)) {
9139e39c5baSBill Taylor 		if (cq->cq_resize_hdl != 0) {	/* in midst of resize */
9149e39c5baSBill Taylor 			/* peek at the opcode */
9159e39c5baSBill Taylor 			opcode = HERMON_CQE_OPCODE_GET(cq, cqe);
9169e39c5baSBill Taylor 			if (opcode == HERMON_CQE_RCV_RESIZE_CODE) {
9179e39c5baSBill Taylor 				hermon_cq_resize_helper(state, cq);
9189e39c5baSBill Taylor 
9199e39c5baSBill Taylor 				/* Increment the consumer index */
9209e39c5baSBill Taylor 				cons_indx = (cons_indx + 1);
9219e39c5baSBill Taylor 				spec_op = 1; /* plus one for the limiting CQE */
9229e39c5baSBill Taylor 
9239e39c5baSBill Taylor 				wrap_around_mask = (cq->cq_bufsz - 1);
9249e39c5baSBill Taylor 
9259e39c5baSBill Taylor 				/* Update the pointer to the next CQ entry */
9269e39c5baSBill Taylor 				cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9279e39c5baSBill Taylor 
9289e39c5baSBill Taylor 				continue;
9299e39c5baSBill Taylor 			}
9309e39c5baSBill Taylor 		}	/* in resizing CQ */
9319e39c5baSBill Taylor 
9329e39c5baSBill Taylor 		/*
9339e39c5baSBill Taylor 		 * either resizing and not the special opcode, or
9349e39c5baSBill Taylor 		 * not resizing at all
9359e39c5baSBill Taylor 		 */
9369e39c5baSBill Taylor 		hermon_cq_cqe_consume(state, cq, cqe, &wc_p[polled_cnt++]);
9379e39c5baSBill Taylor 
9389e39c5baSBill Taylor 		/* Increment the consumer index */
9399e39c5baSBill Taylor 		cons_indx = (cons_indx + 1);
9409e39c5baSBill Taylor 
9419e39c5baSBill Taylor 		/* Update the pointer to the next CQ entry */
9429e39c5baSBill Taylor 		cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9439e39c5baSBill Taylor 
9449e39c5baSBill Taylor 		/*
9459e39c5baSBill Taylor 		 * If we have run out of space to store work completions,
9469e39c5baSBill Taylor 		 * then stop and return the ones we have pulled of the CQ.
9479e39c5baSBill Taylor 		 */
9489e39c5baSBill Taylor 		if (polled_cnt >= num_wc) {
9499e39c5baSBill Taylor 			break;
9509e39c5baSBill Taylor 		}
9519e39c5baSBill Taylor 	}
9529e39c5baSBill Taylor 
9539e39c5baSBill Taylor 	/*
9549e39c5baSBill Taylor 	 * Now we only ring the doorbell (to update the consumer index) if
955*17a2b317SBill Taylor 	 * we've actually consumed a CQ entry.
9569e39c5baSBill Taylor 	 */
9579e39c5baSBill Taylor 	if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) {
9589e39c5baSBill Taylor 		/*
9599e39c5baSBill Taylor 		 * Update the consumer index in both the CQ handle and the
9609e39c5baSBill Taylor 		 * doorbell record.
9619e39c5baSBill Taylor 		 */
9629e39c5baSBill Taylor 		cq->cq_consindx = cons_indx;
9639e39c5baSBill Taylor 		hermon_cq_update_ci_doorbell(cq);
9649e39c5baSBill Taylor 
9659e39c5baSBill Taylor 	} else if (polled_cnt == 0) {
9669e39c5baSBill Taylor 		if (spec_op != 0) {
9679e39c5baSBill Taylor 			/* if we got the special opcode, update the consindx */
9689e39c5baSBill Taylor 			cq->cq_consindx = cons_indx;
9699e39c5baSBill Taylor 			hermon_cq_update_ci_doorbell(cq);
9709e39c5baSBill Taylor 		}
9719e39c5baSBill Taylor 	}
9729e39c5baSBill Taylor 
9739e39c5baSBill Taylor 	mutex_exit(&cq->cq_lock);
9749e39c5baSBill Taylor 
9759e39c5baSBill Taylor 	/* Set "num_polled" (if necessary) */
9769e39c5baSBill Taylor 	if (num_polled != NULL) {
9779e39c5baSBill Taylor 		*num_polled = polled_cnt;
9789e39c5baSBill Taylor 	}
9799e39c5baSBill Taylor 
9809e39c5baSBill Taylor 	/* Set CQ_EMPTY condition if needed, otherwise return success */
9819e39c5baSBill Taylor 	if (polled_cnt == 0) {
9829e39c5baSBill Taylor 		status = IBT_CQ_EMPTY;
9839e39c5baSBill Taylor 	} else {
9849e39c5baSBill Taylor 		status = DDI_SUCCESS;
9859e39c5baSBill Taylor 	}
9869e39c5baSBill Taylor 
9879e39c5baSBill Taylor 	/*
9889e39c5baSBill Taylor 	 * Check if the system is currently panicking.  If it is, then call
9899e39c5baSBill Taylor 	 * the Hermon interrupt service routine.  This step is necessary here
9909e39c5baSBill Taylor 	 * because we might be in a polled I/O mode and without the call to