19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
23*1ed53a3fSBill Taylor  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
249e39c5baSBill Taylor  * Use is subject to license terms.
259e39c5baSBill Taylor  */
269e39c5baSBill Taylor 
279e39c5baSBill Taylor #include "dapl.h"
289e39c5baSBill Taylor #include "dapl_tavor_hw.h"
299e39c5baSBill Taylor #include "dapl_tavor_wr.h"
309e39c5baSBill Taylor #include "dapl_tavor_ibtf_impl.h"
319e39c5baSBill Taylor 
329e39c5baSBill Taylor #define	HERMON_WQE_SGL_INVALID_LKEY	0x00000100
339e39c5baSBill Taylor #define	HERMON_WQE_SEND_FENCE_MASK	0x40
349e39c5baSBill Taylor #define	HERMON_WQE_NDS_MASK		0x3F
359e39c5baSBill Taylor 
369e39c5baSBill Taylor #define	HERMON_CQDB_NOTIFY_CQ_SOLICIT	(0x1 << 24)
379e39c5baSBill Taylor #define	HERMON_CQDB_NOTIFY_CQ		(0x2 << 24)
389e39c5baSBill Taylor 
399e39c5baSBill Taylor #define	HERMON_CQE_RCV_SEND		0x1
409e39c5baSBill Taylor #define	HERMON_CQE_ERR_OPCODE		0x1E
419e39c5baSBill Taylor #define	HERMON_CQE_RESIZE_OPCODE	0x16
429e39c5baSBill Taylor #define	HERMON_CQE_OPCODE_GET(cqe)	(((uint8_t *)cqe)[31] & 0x1F)
439e39c5baSBill Taylor #define	HERMON_CQE_SENDRECV_GET(cqe)	(((uint8_t *)cqe)[31] & 0x40)
449e39c5baSBill Taylor #define	HERMON_CQE_OWNER_IS_SW(cq, cqe)	((((uint8_t *)cqe)[31] >> 7) == \
459e39c5baSBill Taylor 			((cq->cq_consindx & cq->cq_size) >> cq->cq_log_cqsz))
469e39c5baSBill Taylor 
479e39c5baSBill Taylor #define	HERMON_QP_WQEADDRSZ(wcnt)	((uint32_t)(wcnt << 6))
489e39c5baSBill Taylor 
499e39c5baSBill Taylor #define	HERMON_WQE_SEND_SIGNALED_MASK	0x0000000C00000000ull
509e39c5baSBill Taylor #define	HERMON_WQE_SEND_SOLICIT_MASK	0x0000000200000000ull
519e39c5baSBill Taylor #define	HERMON_WQE_SETCTRL(desc, ctrl)	\
529e39c5baSBill Taylor 	((uint64_t *)(desc))[1] = HTOBE_64(ctrl)
539e39c5baSBill Taylor #define	HERMON_WQE_SETNEXT(desc, nopcode, size, fence)			\
549e39c5baSBill Taylor 	((uint64_t *)(desc))[0] = HTOBE_64((nopcode) | (size) | (fence) | \
559e39c5baSBill Taylor 	(((uint64_t)((uint8_t *)desc)[0] &0x80) << 56))
569e39c5baSBill Taylor #define	HERMON_WQE_BUILD_DATA_SEG(ds, sgl)				\
579e39c5baSBill Taylor {									\
589e39c5baSBill Taylor 	uint64_t		*tmp;					\
599e39c5baSBill Taylor 									\
609e39c5baSBill Taylor 	tmp	= (uint64_t *)(ds);					\
619e39c5baSBill Taylor 	tmp[1]	= HTOBE_64((sgl)->ds_va);				\
629e39c5baSBill Taylor 	((uint32_t *)tmp)[1] = HTOBE_32((sgl)->ds_key);			\
639e39c5baSBill Taylor 	membar_producer();						\
649e39c5baSBill Taylor 	((uint32_t *)tmp)[0] = HTOBE_32((sgl)->ds_len);			\
659e39c5baSBill Taylor }
669e39c5baSBill Taylor 
679e39c5baSBill Taylor 
689e39c5baSBill Taylor /* handy macro, useful because of cq_resize dynamics */
699e39c5baSBill Taylor #define	cq_wrap_around_mask	(cq->cq_size - 1)
709e39c5baSBill Taylor 
719e39c5baSBill Taylor pthread_spinlock_t hermon_bf_lock;
729e39c5baSBill Taylor 
739e39c5baSBill Taylor /*
749e39c5baSBill Taylor  * Function signatures
759e39c5baSBill Taylor  */
769e39c5baSBill Taylor extern uint64_t dapls_tavor_wrid_get_entry(ib_cq_handle_t, tavor_hw_cqe_t *,
779e39c5baSBill Taylor     uint_t, uint_t, dapls_tavor_wrid_entry_t *);
789e39c5baSBill Taylor extern void dapls_tavor_wrid_cq_reap(ib_cq_handle_t);
799e39c5baSBill Taylor extern DAPL_OS_LOCK g_tavor_uar_lock;
809e39c5baSBill Taylor 
819e39c5baSBill Taylor #ifndef	_LP64
829e39c5baSBill Taylor extern void dapls_atomic_assign_64(uint64_t, uint64_t *);
839e39c5baSBill Taylor #endif
849e39c5baSBill Taylor 
859e39c5baSBill Taylor static int dapli_hermon_wqe_send_build(ib_qp_handle_t, ibt_send_wr_t *,
869e39c5baSBill Taylor     uint64_t *, uint_t *);
879e39c5baSBill Taylor static DAT_RETURN dapli_hermon_wqe_recv_build(ib_qp_handle_t, ibt_recv_wr_t *,
889e39c5baSBill Taylor     uint64_t *, uint_t *);
899e39c5baSBill Taylor static int dapli_hermon_cq_cqe_consume(ib_cq_handle_t, uint32_t *, ibt_wc_t *);
909e39c5baSBill Taylor static int dapli_hermon_cq_errcqe_consume(ib_cq_handle_t, uint32_t *,
919e39c5baSBill Taylor     ibt_wc_t *);
929e39c5baSBill Taylor extern void dapli_tavor_wrid_add_entry(dapls_tavor_workq_hdr_t *, uint64_t,
939e39c5baSBill Taylor     uint32_t, uint_t);
949e39c5baSBill Taylor extern void dapli_tavor_wrid_add_entry_srq(ib_srq_handle_t, uint64_t, uint32_t);
959e39c5baSBill Taylor 
969e39c5baSBill Taylor /*
979e39c5baSBill Taylor  * Note: The 64 bit doorbells need to written atomically.
989e39c5baSBill Taylor  * In 32 bit libraries we need to use the special assembly rtn
999e39c5baSBill Taylor  * because compiler generated code splits into 2 word writes
1009e39c5baSBill Taylor  */
1019e39c5baSBill Taylor 
1029e39c5baSBill Taylor /*
1039e39c5baSBill Taylor  * dapli_hermon_cq_doorbell()
1049e39c5baSBill Taylor  * Takes the specified cq cmd and cq number and rings the cq doorbell
1059e39c5baSBill Taylor  */
1069e39c5baSBill Taylor static void
dapli_hermon_cq_doorbell(dapls_hw_uar_t ia_uar,uint32_t cq_cmd,uint32_t cqn,uint32_t cmd_sn,uint32_t cq_param)1079e39c5baSBill Taylor dapli_hermon_cq_doorbell(dapls_hw_uar_t ia_uar, uint32_t cq_cmd, uint32_t cqn,
1089e39c5baSBill Taylor     uint32_t cmd_sn, uint32_t cq_param)
1099e39c5baSBill Taylor {
1109e39c5baSBill Taylor 	uint64_t doorbell;
1119e39c5baSBill Taylor 
1129e39c5baSBill Taylor 	/* Build the doorbell from the parameters */
1139e39c5baSBill Taylor 	doorbell = (cmd_sn | cq_cmd | cqn);
1149e39c5baSBill Taylor 	doorbell = (doorbell << 32) | cq_param;
1159e39c5baSBill Taylor 
1169e39c5baSBill Taylor 	/* Write the doorbell to UAR */
1179e39c5baSBill Taylor #ifdef _LP64
1189e39c5baSBill Taylor 	((tavor_hw_uar_t *)ia_uar)->cq = HTOBE_64(doorbell);
1199e39c5baSBill Taylor 	/* 32 bit version */
1209e39c5baSBill Taylor #elif defined(i386)
1219e39c5baSBill Taylor 	dapl_os_lock(&g_tavor_uar_lock);
1229e39c5baSBill Taylor 	/*
1239e39c5baSBill Taylor 	 * For 32 bit intel we assign the doorbell in the order
1249e39c5baSBill Taylor 	 * prescribed by the Tavor PRM, lower to upper addresses
1259e39c5baSBill Taylor 	 */
1269e39c5baSBill Taylor 	((tavor_hw_uar32_t *)ia_uar)->cq[0] =
1279e39c5baSBill Taylor 	    (uint32_t)HTOBE_32(doorbell >> 32);
1289e39c5baSBill Taylor 	((tavor_hw_uar32_t *)ia_uar)->cq[1] =
1299e39c5baSBill Taylor 	    (uint32_t)HTOBE_32(doorbell & 0x00000000ffffffff);
1309e39c5baSBill Taylor 	dapl_os_unlock(&g_tavor_uar_lock);
1319e39c5baSBill Taylor #else
1329e39c5baSBill Taylor 	dapls_atomic_assign_64(HTOBE_64(doorbell),
1339e39c5baSBill Taylor 	    &((tavor_hw_uar_t *)ia_uar)->cq);
1349e39c5baSBill Taylor #endif
1359e39c5baSBill Taylor }
1369e39c5baSBill Taylor 
1379e39c5baSBill Taylor /*
1389e39c5baSBill Taylor  * dapli_hermon_qp_send_doorbell()
1399e39c5baSBill Taylor  * Takes the specified qp number and rings the send doorbell.
1409e39c5baSBill Taylor  */
1419e39c5baSBill Taylor static void
dapli_hermon_sq_dbreg(dapls_hw_uar_t ia_uar,uint32_t qpn)1429e39c5baSBill Taylor dapli_hermon_sq_dbreg(dapls_hw_uar_t ia_uar, uint32_t qpn)
1439e39c5baSBill Taylor {
1449e39c5baSBill Taylor 	uint64_t doorbell;
1459e39c5baSBill Taylor 
1469e39c5baSBill Taylor 	doorbell = qpn << 8;
1479e39c5baSBill Taylor 
1489e39c5baSBill Taylor 	/* Write the doorbell to UAR */
1499e39c5baSBill Taylor #ifdef _LP64
1509e39c5baSBill Taylor 	((tavor_hw_uar_t *)ia_uar)->send = HTOBE_64(doorbell);
1519e39c5baSBill Taylor #else
1529e39c5baSBill Taylor #if defined(i386)
1539e39c5baSBill Taylor 	dapl_os_lock(&g_tavor_uar_lock);
1549e39c5baSBill Taylor 	/*
1559e39c5baSBill Taylor 	 * For 32 bit intel we assign the doorbell in the order
1569e39c5baSBill Taylor 	 * prescribed by the Tavor PRM, lower to upper addresses
1579e39c5baSBill Taylor 	 */
1589e39c5baSBill Taylor 	((tavor_hw_uar32_t *)ia_uar)->send[0] =
1599e39c5baSBill Taylor 	    (uint32_t)HTOBE_32(doorbell >> 32);
1609e39c5baSBill Taylor 	((tavor_hw_uar32_t *)ia_uar)->send[1] =
1619e39c5baSBill Taylor 	    (uint32_t)HTOBE_32(doorbell & 0x00000000ffffffff);
1629e39c5baSBill Taylor 	dapl_os_unlock(&g_tavor_uar_lock);
1639e39c5baSBill Taylor #else
1649e39c5baSBill Taylor 	dapls_atomic_assign_64(HTOBE_64(doorbell),
1659e39c5baSBill Taylor 	    &((tavor_hw_uar_t *)ia_uar)->send);
1669e39c5baSBill Taylor #endif
1679e39c5baSBill Taylor #endif
1689e39c5baSBill Taylor }
1699e39c5baSBill Taylor 
1709e39c5baSBill Taylor /*
1719e39c5baSBill Taylor  * dapli_hermon_wqe_send_build()
1729e39c5baSBill Taylor  * Constructs a WQE for a given ibt_send_wr_t
1739e39c5baSBill Taylor  */
1749e39c5baSBill Taylor static int
dapli_hermon_wqe_send_build(ib_qp_handle_t qp,ibt_send_wr_t * wr,uint64_t * addr,uint_t * size)1759e39c5baSBill Taylor dapli_hermon_wqe_send_build(ib_qp_handle_t qp, ibt_send_wr_t *wr,
1769e39c5baSBill Taylor     uint64_t *addr, uint_t *size)
1779e39c5baSBill Taylor {
1789e39c5baSBill Taylor 	tavor_hw_snd_wqe_remaddr_t	*rc;
1799e39c5baSBill Taylor 	tavor_hw_snd_wqe_bind_t		*bn;
1809e39c5baSBill Taylor 	tavor_hw_wqe_sgl_t		*ds;
1819e39c5baSBill Taylor 	ibt_wr_ds_t			*sgl;
1829e39c5baSBill Taylor 	uint8_t				*src, *dst, *maxdst;
1839e39c5baSBill Taylor 	uint32_t			nds;
1849e39c5baSBill Taylor 	int				len, thislen, maxlen;
1859e39c5baSBill Taylor 	uint32_t			new_rkey;
1869e39c5baSBill Taylor 	uint32_t			old_rkey;
1879e39c5baSBill Taylor 	int				i, num_ds;
1889e39c5baSBill Taylor 	int				max_inline_bytes = -1;
1899e39c5baSBill Taylor 	uint64_t			ctrl;
1909e39c5baSBill Taylor 	uint64_t			nopcode;
1919e39c5baSBill Taylor 	uint_t				my_size;
1929e39c5baSBill Taylor 
1939e39c5baSBill Taylor 	nds = wr->wr_nds;
1949e39c5baSBill Taylor 	sgl = wr->wr_sgl;
1959e39c5baSBill Taylor 	num_ds = 0;
1969e39c5baSBill Taylor 	ctrl = ((wr->wr_flags & IBT_WR_SEND_SIGNAL) ?
1979e39c5baSBill Taylor 	    HERMON_WQE_SEND_SIGNALED_MASK : 0) |
1989e39c5baSBill Taylor 	    ((wr->wr_flags & IBT_WR_SEND_SOLICIT) ?
1999e39c5baSBill Taylor 	    HERMON_WQE_SEND_SOLICIT_MASK : 0);
2009e39c5baSBill Taylor 
2019e39c5baSBill Taylor 	/*
2029e39c5baSBill Taylor 	 * RC is the only supported transport in UDAPL
2039e39c5baSBill Taylor 	 * For RC requests, we allow "Send", "RDMA Read", "RDMA Write"
2049e39c5baSBill Taylor 	 */
2059e39c5baSBill Taylor 	switch (wr->wr_opcode) {
2069e39c5baSBill Taylor 	case IBT_WRC_SEND:
2079e39c5baSBill Taylor 		/*
2089e39c5baSBill Taylor 		 * If this is a Send request, then all we need is
2099e39c5baSBill Taylor 		 * the Data Segment processing below.
2109e39c5baSBill Taylor 		 * Initialize the information for the Data Segments
2119e39c5baSBill Taylor 		 */
2129e39c5baSBill Taylor 		ds = (tavor_hw_wqe_sgl_t *)((uintptr_t)addr +
2139e39c5baSBill Taylor 		    sizeof (tavor_hw_snd_wqe_nextctrl_t));
2149e39c5baSBill Taylor 		if (qp->qp_sq_inline != 0)
2159e39c5baSBill Taylor 			max_inline_bytes =
2169e39c5baSBill Taylor 			    qp->qp_sq_wqesz - TAVOR_INLINE_HEADER_SIZE_SEND;
2179e39c5baSBill Taylor 		nopcode = TAVOR_WQE_SEND_NOPCODE_SEND;
2189e39c5baSBill Taylor 		break;
2199e39c5baSBill Taylor 	case IBT_WRC_RDMAW:
2209e39c5baSBill Taylor 		if (qp->qp_sq_inline != 0)
2219e39c5baSBill Taylor 			max_inline_bytes =
2229e39c5baSBill Taylor 			    qp->qp_sq_wqesz - TAVOR_INLINE_HEADER_SIZE_RDMAW;
2239e39c5baSBill Taylor 		nopcode = TAVOR_WQE_SEND_NOPCODE_RDMAW;
2249e39c5baSBill Taylor 		/* FALLTHROUGH */
2259e39c5baSBill Taylor 	case IBT_WRC_RDMAR:
2269e39c5baSBill Taylor 		if (wr->wr_opcode == IBT_WRC_RDMAR) {
2279e39c5baSBill Taylor 			if (qp->qp_sq_inline < 0)
2289e39c5baSBill Taylor 				qp->qp_sq_inline = 0;
2299e39c5baSBill Taylor 			nopcode = TAVOR_WQE_SEND_NOPCODE_RDMAR;
2309e39c5baSBill Taylor 		}
2319e39c5baSBill Taylor 		/*
2329e39c5baSBill Taylor 		 * If this is an RDMA Read or RDMA Write request, then fill
2339e39c5baSBill Taylor 		 * in the "Remote Address" header fields.
2349e39c5baSBill Taylor 		 */
2359e39c5baSBill Taylor 		rc = (tavor_hw_snd_wqe_remaddr_t *)((uintptr_t)addr +
2369e39c5baSBill Taylor 		    sizeof (tavor_hw_snd_wqe_nextctrl_t));
2379e39c5baSBill Taylor 
2389e39c5baSBill Taylor 		/*
2399e39c5baSBill Taylor 		 * Build the Remote Address Segment for the WQE, using
2409e39c5baSBill Taylor 		 * the information from the RC work request.
2419e39c5baSBill Taylor 		 */
2429e39c5baSBill Taylor 		TAVOR_WQE_BUILD_REMADDR(rc, &wr->wr.rc.rcwr.rdma);
2439e39c5baSBill Taylor 
2449e39c5baSBill Taylor 		/* Update "ds" for filling in Data Segments (below) */
2459e39c5baSBill Taylor 		ds = (tavor_hw_wqe_sgl_t *)((uintptr_t)rc +
2469e39c5baSBill Taylor 		    sizeof (tavor_hw_snd_wqe_remaddr_t));
2479e39c5baSBill Taylor 		break;
2489e39c5baSBill Taylor 	case IBT_WRC_BIND:
2499e39c5baSBill Taylor 		/*
2509e39c5baSBill Taylor 		 * Generate a new R_key
2519e39c5baSBill Taylor 		 * Increment the upper "unconstrained" bits and need to keep
2529e39c5baSBill Taylor 		 * the lower "constrained" bits the same it represents
2539e39c5baSBill Taylor 		 * the MPT index.
2549e39c5baSBill Taylor 		 */
2559e39c5baSBill Taylor #if 0
2569e39c5baSBill Taylor 	/* XXX - need equiv of "hermon_wr_bind_check(state, wr);" */
2579e39c5baSBill Taylor 	/* XXX - uses hermon_mr_keycalc - what about Sinai vs. Arbel??? */
2589e39c5baSBill Taylor #endif
2599e39c5baSBill Taylor 		old_rkey = wr->wr.rc.rcwr.bind->bind_rkey;
2609e39c5baSBill Taylor 		new_rkey = old_rkey >> 8;	/* index */
2619e39c5baSBill Taylor 		old_rkey = (old_rkey + 1) & 0xff; /* incremented key */
2629e39c5baSBill Taylor 		new_rkey = (new_rkey << 8) | old_rkey;
2639e39c5baSBill Taylor 
2649e39c5baSBill Taylor 		wr->wr.rc.rcwr.bind->bind_rkey_out = new_rkey;
2659e39c5baSBill Taylor 
2669e39c5baSBill Taylor 		bn = (tavor_hw_snd_wqe_bind_t *)((uintptr_t)addr +
2679e39c5baSBill Taylor 		    sizeof (tavor_hw_snd_wqe_nextctrl_t));
2689e39c5baSBill Taylor 
2699e39c5baSBill Taylor 		/*
2709e39c5baSBill Taylor 		 * Build the Bind Memory Window Segments for the WQE,
2719e39c5baSBill Taylor 		 * using the information from the RC Bind memory
2729e39c5baSBill Taylor 		 * window work request.
2739e39c5baSBill Taylor 		 */
2749e39c5baSBill Taylor 		TAVOR_WQE_BUILD_BIND(bn, wr->wr.rc.rcwr.bind);
2759e39c5baSBill Taylor 
2769e39c5baSBill Taylor 		/*
2779e39c5baSBill Taylor 		 * Update the "ds" pointer.  Even though the "bind"
2789e39c5baSBill Taylor 		 * operation requires no SGLs, this is necessary to
2799e39c5baSBill Taylor 		 * facilitate the correct descriptor size calculations
2809e39c5baSBill Taylor 		 * (below).
2819e39c5baSBill Taylor 		 */
2829e39c5baSBill Taylor 		ds = (tavor_hw_wqe_sgl_t *)((uintptr_t)bn +
2839e39c5baSBill Taylor 		    sizeof (tavor_hw_snd_wqe_bind_t));
2849e39c5baSBill Taylor 		nds = 0;
2859e39c5baSBill Taylor 		nopcode = TAVOR_WQE_SEND_NOPCODE_BIND;
2869e39c5baSBill Taylor 		break;
2879e39c5baSBill Taylor 	default:
2889e39c5baSBill Taylor 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
2899e39c5baSBill Taylor 		    "dapli_hermon_wqe_send_build: invalid wr_opcode=%d\n",
2909e39c5baSBill Taylor 		    wr->wr_opcode);
2919e39c5baSBill Taylor 		return (DAT_INTERNAL_ERROR);
2929e39c5baSBill Taylor 	}
2939e39c5baSBill Taylor 
2949e39c5baSBill Taylor 	/*
2959e39c5baSBill Taylor 	 * Now fill in the Data Segments (SGL) for the Send WQE based on
2969e39c5baSBill Taylor 	 * the values setup above (i.e. "sgl", "nds", and the "ds" pointer
2979e39c5baSBill Taylor 	 * Start by checking for a valid number of SGL entries
2989e39c5baSBill Taylor 	 */
2999e39c5baSBill Taylor 	if (nds > qp->qp_sq_sgl) {
3009e39c5baSBill Taylor 		return (DAT_INVALID_PARAMETER);
3019e39c5baSBill Taylor 	}
3029e39c5baSBill Taylor 
3039e39c5baSBill Taylor 	/*
3049e39c5baSBill Taylor 	 * For each SGL in the Send Work Request, fill in the Send WQE's data
3059e39c5baSBill Taylor 	 * segments.  Note: We skip any SGL with zero size because Tavor
3069e39c5baSBill Taylor 	 * hardware cannot handle a zero for "byte_cnt" in the WQE.  Actually
3079e39c5baSBill Taylor 	 * the encoding for zero means a 2GB transfer.  Because of this special
3089e39c5baSBill Taylor 	 * encoding in the hardware, we mask the requested length with
3099e39c5baSBill Taylor 	 * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
3109e39c5baSBill Taylor 	 * zero.)
3119e39c5baSBill Taylor 	 */
3129e39c5baSBill Taylor 	if (max_inline_bytes != -1) {		/* compute total_len */
3139e39c5baSBill Taylor 		len = 0;
3149e39c5baSBill Taylor 		for (i = 0; i < nds; i++)
3159e39c5baSBill Taylor 			len += sgl[i].ds_len;
3169e39c5baSBill Taylor 		if (len == 0)
3179e39c5baSBill Taylor 			max_inline_bytes = -1; /* do not inline */
3189e39c5baSBill Taylor 		else {
3199e39c5baSBill Taylor 			/* need to reduce the length by dword "len" fields */
3209e39c5baSBill Taylor 			max_inline_bytes -= (len / 64) * sizeof (uint32_t);
3219e39c5baSBill Taylor 			if (len > max_inline_bytes)
3229e39c5baSBill Taylor 				max_inline_bytes = -1;	/* too big for inline */
3239e39c5baSBill Taylor 		}
3249e39c5baSBill Taylor 	}
3259e39c5baSBill Taylor 	if (max_inline_bytes != -1) {		/* do "inline" */
3269e39c5baSBill Taylor 
3279e39c5baSBill Taylor 		dst = (uint8_t *)((uint32_t *)ds + 1);
3289e39c5baSBill Taylor 		maxdst = (uint8_t *)(((uintptr_t)dst + 64) & ~(64 - 1));
3299e39c5baSBill Taylor 		maxlen = maxdst - dst;
3309e39c5baSBill Taylor 		thislen = 0;
3319e39c5baSBill Taylor 		i = 0;
3329e39c5baSBill Taylor 		src = (uint8_t *)(uintptr_t)sgl[i].ds_va;
3339e39c5baSBill Taylor 		len = sgl[i].ds_len;
3349e39c5baSBill Taylor 		do {
3359e39c5baSBill Taylor 			/* if this sgl overflows the inline segment */
3369e39c5baSBill Taylor 			if (len > maxlen) {
3379e39c5baSBill Taylor 				if (maxlen) /* might be 0 */
3389e39c5baSBill Taylor 					(void) dapl_os_memcpy(dst,
3399e39c5baSBill Taylor 					    src, maxlen);
3409e39c5baSBill Taylor 				membar_producer();
3419e39c5baSBill Taylor 				*(uint32_t *)ds =
3429e39c5baSBill Taylor 				    HTOBE_32((thislen + maxlen) |
3439e39c5baSBill Taylor 				    TAVOR_WQE_SGL_INLINE_MASK);
3449e39c5baSBill Taylor 				thislen = 0;
3459e39c5baSBill Taylor 				len -= maxlen;
3469e39c5baSBill Taylor 				src += maxlen;
3479e39c5baSBill Taylor 				dst = maxdst + sizeof (uint32_t);
3489e39c5baSBill Taylor 				ds = (tavor_hw_wqe_sgl_t *)(void *)maxdst;
3499e39c5baSBill Taylor 				maxdst += 64;
3509e39c5baSBill Taylor 				maxlen = 64 - sizeof (uint32_t);
3519e39c5baSBill Taylor 			} else { /* this sgl fully fits */
3529e39c5baSBill Taylor 				(void) dapl_os_memcpy(dst,
3539e39c5baSBill Taylor 				    src, len);
3549e39c5baSBill Taylor 				maxlen -= len;  /* room left */
3559e39c5baSBill Taylor 				thislen += len;
3569e39c5baSBill Taylor 				dst += len;
3579e39c5baSBill Taylor 				while (++i < nds)
3589e39c5baSBill Taylor 					if (sgl[i].ds_len)
3599e39c5baSBill Taylor 						break;
3609e39c5baSBill Taylor 				if (i >= nds)
3619e39c5baSBill Taylor 					break;
3629e39c5baSBill Taylor 				src = (uint8_t *)(uintptr_t)sgl[i].ds_va;
3639e39c5baSBill Taylor 				len = sgl[i].ds_len;
3649e39c5baSBill Taylor 			}
3659e39c5baSBill Taylor 		} while (i < nds);
3669e39c5baSBill Taylor 		membar_producer();
3679e39c5baSBill Taylor 		*(uint32_t *)ds = HTOBE_32(thislen |
3689e39c5baSBill Taylor 		    TAVOR_WQE_SGL_INLINE_MASK);
3699e39c5baSBill Taylor 
3709e39c5baSBill Taylor 		/* Return the size of descriptor (in 16-byte chunks) */
3719e39c5baSBill Taylor 		my_size = ((uintptr_t)dst - (uintptr_t)addr + 15) >> 4;
3729e39c5baSBill Taylor 		if (my_size <= (256 >> 4))
3739e39c5baSBill Taylor 			*size = my_size;	/* use Hermon Blueflame */
3749e39c5baSBill Taylor 		else
3759e39c5baSBill Taylor 			*size = 0;
3769e39c5baSBill Taylor 	} else {
3779e39c5baSBill Taylor 		for (i = 0; i < nds; i++) {
3789e39c5baSBill Taylor 			if (sgl[i].ds_len == 0) {
3799e39c5baSBill Taylor 				continue;
3809e39c5baSBill Taylor 			}
3819e39c5baSBill Taylor 
3829e39c5baSBill Taylor 			/*
3839e39c5baSBill Taylor 			 * Fill in the Data Segment(s) for the current WQE,
3849e39c5baSBill Taylor 			 * using the information contained in the
3859e39c5baSBill Taylor 			 * scatter-gather list of the work request.
3869e39c5baSBill Taylor 			 */
3879e39c5baSBill Taylor 			HERMON_WQE_BUILD_DATA_SEG(&ds[num_ds], &sgl[i]);
3889e39c5baSBill Taylor 			num_ds++;
3899e39c5baSBill Taylor 		}
3909e39c5baSBill Taylor 
3919e39c5baSBill Taylor 		/* Return the size of descriptor (in 16-byte chunks) */
3929e39c5baSBill Taylor 		my_size = ((uintptr_t)&ds[num_ds] - (uintptr_t)addr) >> 4;
3939e39c5baSBill Taylor 		*size = 0;	/* do not use Hermon Blueflame */
3949e39c5baSBill Taylor 	}
3959e39c5baSBill Taylor 	HERMON_WQE_SETCTRL(addr, ctrl);
3969e39c5baSBill Taylor 	membar_producer();
3979e39c5baSBill Taylor 	HERMON_WQE_SETNEXT(addr, nopcode << 32, my_size,
3989e39c5baSBill Taylor 	    (wr->wr_flags & IBT_WR_SEND_FENCE) ?
3999e39c5baSBill Taylor 	    HERMON_WQE_SEND_FENCE_MASK : 0);
4009e39c5baSBill Taylor 
4019e39c5baSBill Taylor 	return (DAT_SUCCESS);
4029e39c5baSBill Taylor }
4039e39c5baSBill Taylor 
4049e39c5baSBill Taylor /*
4059e39c5baSBill Taylor  * dapli_hermon_wqe_recv_build()
4069e39c5baSBill Taylor  * Builds the recv WQE for a given ibt_recv_wr_t
4079e39c5baSBill Taylor  */
4089e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_wqe_recv_build(ib_qp_handle_t qp,ibt_recv_wr_t * wr,uint64_t * addr,uint_t * size)4099e39c5baSBill Taylor dapli_hermon_wqe_recv_build(ib_qp_handle_t qp, ibt_recv_wr_t *wr,
4109e39c5baSBill Taylor     uint64_t *addr, uint_t *size)
4119e39c5baSBill Taylor {
4129e39c5baSBill Taylor 	tavor_hw_wqe_sgl_t	*ds;
4139e39c5baSBill Taylor 	int			i;
4149e39c5baSBill Taylor 	int			num_ds;
4159e39c5baSBill Taylor 
4169e39c5baSBill Taylor 	/* Fill in the Data Segments (SGL) for the Recv WQE */
4179e39c5baSBill Taylor 	ds = (tavor_hw_wqe_sgl_t *)addr;
4189e39c5baSBill Taylor 	num_ds = 0;
4199e39c5baSBill Taylor 
4209e39c5baSBill Taylor 	/* Check for valid number of SGL entries */
4219e39c5baSBill Taylor 	if (wr->wr_nds > qp->qp_rq_sgl) {
4229e39c5baSBill Taylor 		return (DAT_INVALID_PARAMETER);
4239e39c5baSBill Taylor 	}
4249e39c5baSBill Taylor 
4259e39c5baSBill Taylor 	/*
4269e39c5baSBill Taylor 	 * For each SGL in the Recv Work Request, fill in the Recv WQE's data
4279e39c5baSBill Taylor 	 * segments.  Note: We skip any SGL with zero size because Tavor
4289e39c5baSBill Taylor 	 * hardware cannot handle a zero for "byte_cnt" in the WQE.  Actually
4299e39c5baSBill Taylor 	 * the encoding for zero means a 2GB transfer.  Because of this special
4309e39c5baSBill Taylor 	 * encoding in the hardware, we mask the requested length with
4319e39c5baSBill Taylor 	 * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
4329e39c5baSBill Taylor 	 * zero.)
4339e39c5baSBill Taylor 	 */
4349e39c5baSBill Taylor 	for (i = 0; i < wr->wr_nds; i++) {
4359e39c5baSBill Taylor 		if (wr->wr_sgl[i].ds_len == 0) {
4369e39c5baSBill Taylor 			continue;
4379e39c5baSBill Taylor 		}
4389e39c5baSBill Taylor 
4399e39c5baSBill Taylor 		/*
4409e39c5baSBill Taylor 		 * Fill in the Data Segment(s) for the receive WQE, using the
4419e39c5baSBill Taylor 		 * information contained in the scatter-gather list of the
4429e39c5baSBill Taylor 		 * work request.
4439e39c5baSBill Taylor 		 */
4449e39c5baSBill Taylor 		TAVOR_WQE_BUILD_DATA_SEG(&ds[num_ds], &wr->wr_sgl[i]);
4459e39c5baSBill Taylor 		num_ds++;
4469e39c5baSBill Taylor 	}
4479e39c5baSBill Taylor 	if (i < qp->qp_rq_sgl) {
4489e39c5baSBill Taylor 		ibt_wr_ds_t sgl;
4499e39c5baSBill Taylor 		sgl.ds_va  = (ib_vaddr_t)0;
4509e39c5baSBill Taylor 		sgl.ds_len = (ib_msglen_t)0;
4519e39c5baSBill Taylor 		sgl.ds_key = (ibt_lkey_t)HERMON_WQE_SGL_INVALID_LKEY;
4529e39c5baSBill Taylor 		TAVOR_WQE_BUILD_DATA_SEG(&ds[num_ds], &sgl);
4539e39c5baSBill Taylor 	}
4549e39c5baSBill Taylor 
4559e39c5baSBill Taylor 	/* Return the size of descriptor (in 16-byte chunks) */
4569e39c5baSBill Taylor 	*size = qp->qp_rq_wqesz >> 4;
4579e39c5baSBill Taylor 
4589e39c5baSBill Taylor 	return (DAT_SUCCESS);
4599e39c5baSBill Taylor }
4609e39c5baSBill Taylor 
4619e39c5baSBill Taylor /*
4629e39c5baSBill Taylor  * dapli_hermon_wqe_srq_build()
4639e39c5baSBill Taylor  * Builds the recv WQE for a given ibt_recv_wr_t
4649e39c5baSBill Taylor  */
4659e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_wqe_srq_build(ib_srq_handle_t srq,ibt_recv_wr_t * wr,uint64_t * addr)4669e39c5baSBill Taylor dapli_hermon_wqe_srq_build(ib_srq_handle_t srq, ibt_recv_wr_t *wr,
4679e39c5baSBill Taylor     uint64_t *addr)
4689e39c5baSBill Taylor {
4699e39c5baSBill Taylor 	tavor_hw_wqe_sgl_t	*ds;
4709e39c5baSBill Taylor 	ibt_wr_ds_t		end_sgl;
4719e39c5baSBill Taylor 	int			i;
4729e39c5baSBill Taylor 	int			num_ds;
4739e39c5baSBill Taylor 
4749e39c5baSBill Taylor 	/* Fill in the Data Segments (SGL) for the Recv WQE */
4759e39c5baSBill Taylor 	ds = (tavor_hw_wqe_sgl_t *)((uintptr_t)addr +
4769e39c5baSBill Taylor 	    sizeof (tavor_hw_rcv_wqe_nextctrl_t));
4779e39c5baSBill Taylor 	num_ds = 0;
4789e39c5baSBill Taylor 
4799e39c5baSBill Taylor 	/* Check for valid number of SGL entries */
4809e39c5baSBill Taylor 	if (wr->wr_nds > srq->srq_wq_sgl) {
4819e39c5baSBill Taylor 		return (DAT_INVALID_PARAMETER);
4829e39c5baSBill Taylor 	}
4839e39c5baSBill Taylor 
4849e39c5baSBill Taylor 	/*
4859e39c5baSBill Taylor 	 * For each SGL in the Recv Work Request, fill in the Recv WQE's data
4869e39c5baSBill Taylor 	 * segments.  Note: We skip any SGL with zero size because Tavor
4879e39c5baSBill Taylor 	 * hardware cannot handle a zero for "byte_cnt" in the WQE.  Actually
4889e39c5baSBill Taylor 	 * the encoding for zero means a 2GB transfer.  Because of this special
4899e39c5baSBill Taylor 	 * encoding in the hardware, we mask the requested length with
4909e39c5baSBill Taylor 	 * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
4919e39c5baSBill Taylor 	 * zero.)
4929e39c5baSBill Taylor 	 */
4939e39c5baSBill Taylor 	for (i = 0; i < wr->wr_nds; i++) {
4949e39c5baSBill Taylor 		if (wr->wr_sgl[i].ds_len == 0) {
4959e39c5baSBill Taylor 			continue;
4969e39c5baSBill Taylor 		}
4979e39c5baSBill Taylor 
4989e39c5baSBill Taylor 		/*
4999e39c5baSBill Taylor 		 * Fill in the Data Segment(s) for the receive WQE, using the
5009e39c5baSBill Taylor 		 * information contained in the scatter-gather list of the
5019e39c5baSBill Taylor 		 * work request.
5029e39c5baSBill Taylor 		 */
5039e39c5baSBill Taylor 		TAVOR_WQE_BUILD_DATA_SEG(&ds[num_ds], &wr->wr_sgl[i]);
5049e39c5baSBill Taylor 		num_ds++;
5059e39c5baSBill Taylor 	}
5069e39c5baSBill Taylor 
5079e39c5baSBill Taylor 	/*
5089e39c5baSBill Taylor 	 * For SRQ, if the number of data segments is less than the maximum
5099e39c5baSBill Taylor 	 * specified at alloc, then we have to fill in a special "key" entry in
5109e39c5baSBill Taylor 	 * the sgl entry after the last valid one in this post request.  We do
5119e39c5baSBill Taylor 	 * that here.
5129e39c5baSBill Taylor 	 */
5139e39c5baSBill Taylor 	if (num_ds < srq->srq_wq_sgl) {
5149e39c5baSBill Taylor 		end_sgl.ds_va  = (ib_vaddr_t)0;
5159e39c5baSBill Taylor 		end_sgl.ds_len = (ib_msglen_t)0;
5169e39c5baSBill Taylor 		end_sgl.ds_key = (ibt_lkey_t)HERMON_WQE_SGL_INVALID_LKEY;
5179e39c5baSBill Taylor 		TAVOR_WQE_BUILD_DATA_SEG(&ds[num_ds], &end_sgl);
5189e39c5baSBill Taylor 	}
5199e39c5baSBill Taylor 
5209e39c5baSBill Taylor 	return (DAT_SUCCESS);
5219e39c5baSBill Taylor }
5229e39c5baSBill Taylor 
5239e39c5baSBill Taylor /*
5249e39c5baSBill Taylor  * dapli_hermon_cq_peek()
5259e39c5baSBill Taylor  * Peeks into a given CQ to check if there are any events that can be
5269e39c5baSBill Taylor  * polled. It returns the number of CQEs that can be polled.
5279e39c5baSBill Taylor  */
5289e39c5baSBill Taylor static void
dapli_hermon_cq_peek(ib_cq_handle_t cq,int * num_cqe)5299e39c5baSBill Taylor dapli_hermon_cq_peek(ib_cq_handle_t cq, int *num_cqe)
5309e39c5baSBill Taylor {
5319e39c5baSBill Taylor 	uint32_t		*cqe;
5329e39c5baSBill Taylor 	uint32_t		imm_eth_pkey_cred;
5339e39c5baSBill Taylor 	uint32_t		cons_indx;
5349e39c5baSBill Taylor 	int			polled_cnt;
5359e39c5baSBill Taylor 	uint_t			doorbell_cnt;
5369e39c5baSBill Taylor 	uint_t			opcode;
5379e39c5baSBill Taylor 
5389e39c5baSBill Taylor 	/* Get the consumer index */
5399e39c5baSBill Taylor 	cons_indx = cq->cq_consindx & cq_wrap_around_mask;
5409e39c5baSBill Taylor 
5419e39c5baSBill Taylor 	/* Calculate the pointer to the first CQ entry */
5429e39c5baSBill Taylor 	cqe = (uint32_t *)&cq->cq_addr[cons_indx];
5439e39c5baSBill Taylor 
5449e39c5baSBill Taylor 	/*
5459e39c5baSBill Taylor 	 * Count entries in the CQ until we find an entry owned by
5469e39c5baSBill Taylor 	 * the hardware.
5479e39c5baSBill Taylor 	 */
5489e39c5baSBill Taylor 	polled_cnt = 0;
5499e39c5baSBill Taylor 	while (HERMON_CQE_OWNER_IS_SW(cq, cqe)) {
5509e39c5baSBill Taylor 		opcode = HERMON_CQE_OPCODE_GET(cqe);
5519e39c5baSBill Taylor 		/* Error CQE map to multiple work completions */
5529e39c5baSBill Taylor 		if (opcode == HERMON_CQE_ERR_OPCODE) {
5539e39c5baSBill Taylor 			imm_eth_pkey_cred =
5549e39c5baSBill Taylor 			    TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cqe);
5559e39c5baSBill Taylor 			doorbell_cnt =
5569e39c5baSBill Taylor 			    imm_eth_pkey_cred & TAVOR_CQE_ERR_DBDCNT_MASK;
5579e39c5baSBill Taylor 			polled_cnt += (doorbell_cnt + 1);
5589e39c5baSBill Taylor 		} else {
5599e39c5baSBill Taylor 			polled_cnt++;
5609e39c5baSBill Taylor 		}
5619e39c5baSBill Taylor 		/* Increment the consumer index */
5629e39c5baSBill Taylor 		cons_indx = (cons_indx + 1) & cq_wrap_around_mask;
5639e39c5baSBill Taylor 
5649e39c5baSBill Taylor 		/* Update the pointer to the next CQ entry */
5659e39c5baSBill Taylor 		cqe = (uint32_t *)&cq->cq_addr[cons_indx];
5669e39c5baSBill Taylor 	}
5679e39c5baSBill Taylor 
5689e39c5baSBill Taylor 	*num_cqe = polled_cnt;
5699e39c5baSBill Taylor }
5709e39c5baSBill Taylor 
5719e39c5baSBill Taylor #define	dapli_hermon_cq_update_ci(cq, dbp) \
5729e39c5baSBill Taylor 	(dbp)[0] = HTOBE_32(cq->cq_consindx & 0xFFFFFF)
5739e39c5baSBill Taylor 
5749e39c5baSBill Taylor /*
5759e39c5baSBill Taylor  * dapli_hermon_cq_resize_helper()
5769e39c5baSBill Taylor  * This routine switches from the pre-cq_resize buffer to the new buffer.
5779e39c5baSBill Taylor  */
5789e39c5baSBill Taylor static int
dapli_hermon_cq_resize_helper(ib_cq_handle_t cq)5799e39c5baSBill Taylor dapli_hermon_cq_resize_helper(ib_cq_handle_t cq)
5809e39c5baSBill Taylor {
5819e39c5baSBill Taylor 	int i;
5829e39c5baSBill Taylor 
5839e39c5baSBill Taylor 	if ((cq->cq_resize_addr == 0) ||
5849e39c5baSBill Taylor 	    (munmap((char *)cq->cq_addr, cq->cq_map_len) < 0)) {
5859e39c5baSBill Taylor 		dapl_dbg_log(DAPL_DBG_TYPE_ERR, "cq_resize_helper: "
5869e39c5baSBill Taylor 		    "munmap(%p:0x%llx) failed(%d)\n", cq->cq_addr,
5879e39c5baSBill Taylor 		    cq->cq_map_len, errno);
5889e39c5baSBill Taylor 		return (1);	/* FAILED */
5899e39c5baSBill Taylor 	}
5909e39c5baSBill Taylor 	cq->cq_addr		= cq->cq_resize_addr;
5919e39c5baSBill Taylor 	cq->cq_map_offset	= cq->cq_resize_map_offset;
5929e39c5baSBill Taylor 	cq->cq_map_len		= cq->cq_resize_map_len;
5939e39c5baSBill Taylor 	cq->cq_size		= cq->cq_resize_size;
5949e39c5baSBill Taylor 	cq->cq_cqesz		= cq->cq_resize_cqesz;
5959e39c5baSBill Taylor 	cq->cq_resize_addr	= 0;
5969e39c5baSBill Taylor 	cq->cq_resize_map_offset = 0;
5979e39c5baSBill Taylor 	cq->cq_resize_map_len	= 0;
5989e39c5baSBill Taylor 	cq->cq_resize_size	= 0;
5999e39c5baSBill Taylor 	cq->cq_resize_cqesz	= 0;
6009e39c5baSBill Taylor 	for (i = 0; (1 << i) < cq->cq_size; i++)
6019e39c5baSBill Taylor 		;
6029e39c5baSBill Taylor 	cq->cq_log_cqsz = i;
6039e39c5baSBill Taylor 
6049e39c5baSBill Taylor 	cq->cq_consindx++;	/* consume the RESIZE cqe */
6059e39c5baSBill Taylor 
6069e39c5baSBill Taylor 	return (0);	/* SUCCESS */
6079e39c5baSBill Taylor }
6089e39c5baSBill Taylor 
6099e39c5baSBill Taylor /*
6109e39c5baSBill Taylor  * dapli_hermon_cq_poll()
6119e39c5baSBill Taylor  * This routine polls CQEs out of a CQ and puts them into the ibt_wc_t
6129e39c5baSBill Taylor  * array that is passed in.
6139e39c5baSBill Taylor  */
6149e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_cq_poll(ib_cq_handle_t cq,ibt_wc_t * wc_p,uint_t num_wc,uint_t * num_polled)6159e39c5baSBill Taylor dapli_hermon_cq_poll(ib_cq_handle_t cq, ibt_wc_t *wc_p, uint_t num_wc,
6169e39c5baSBill Taylor     uint_t *num_polled)
6179e39c5baSBill Taylor {
6189e39c5baSBill Taylor 	uint32_t		*cqe;
6199e39c5baSBill Taylor 	uint32_t		cons_indx;
6209e39c5baSBill Taylor 	uint32_t		polled_cnt;
6219e39c5baSBill Taylor 	DAT_RETURN		dat_status;
6229e39c5baSBill Taylor 	int			status;
6239e39c5baSBill Taylor 
6249e39c5baSBill Taylor 	/* Get the consumer index */
6259e39c5baSBill Taylor 	cons_indx = cq->cq_consindx & cq_wrap_around_mask;
6269e39c5baSBill Taylor 
6279e39c5baSBill Taylor 	/* Calculate the pointer to the first CQ entry */
6289e39c5baSBill Taylor 	cqe = (uint32_t *)&cq->cq_addr[cons_indx];
6299e39c5baSBill Taylor 
6309e39c5baSBill Taylor 	/*
6319e39c5baSBill Taylor 	 * Keep pulling entries from the CQ until we find an entry owned by
6329e39c5baSBill Taylor 	 * the hardware.  As long as there the CQE's owned by SW, process
6339e39c5baSBill Taylor 	 * each entry by calling dapli_hermon_cq_cqe_consume() and updating the
6349e39c5baSBill Taylor 	 * CQ consumer index.  Note:  We only update the consumer index if
6359e39c5baSBill Taylor 	 * dapli_hermon_cq_cqe_consume() returns TAVOR_CQ_SYNC_AND_DB.
6369e39c5baSBill Taylor 	 * Otherwise, it indicates that we are going to "recycle" the CQE
6379e39c5baSBill Taylor 	 * (probably because it is a error CQE and corresponds to more than one
6389e39c5baSBill Taylor 	 * completion).
6399e39c5baSBill Taylor 	 */
6409e39c5baSBill Taylor 	polled_cnt = 0;
6419e39c5baSBill Taylor 	while (HERMON_CQE_OWNER_IS_SW(cq, cqe)) {
6429e39c5baSBill Taylor 		if (HERMON_CQE_OPCODE_GET(cqe) == HERMON_CQE_RESIZE_OPCODE) {
6439e39c5baSBill Taylor 			if (dapli_hermon_cq_resize_helper(cq))
6449e39c5baSBill Taylor 				return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
6459e39c5baSBill Taylor 			cons_indx = cq->cq_consindx & cq_wrap_around_mask;
6469e39c5baSBill Taylor 			cqe = (uint32_t *)&cq->cq_addr[cons_indx];
6479e39c5baSBill Taylor 			continue;
6489e39c5baSBill Taylor 		}
6499e39c5baSBill Taylor 		status = dapli_hermon_cq_cqe_consume(cq, cqe,
6509e39c5baSBill Taylor 		    &wc_p[polled_cnt++]);
6519e39c5baSBill Taylor 		if (status == TAVOR_CQ_SYNC_AND_DB) {
6529e39c5baSBill Taylor 			/* Reset to hardware ownership is implicit in Hermon */
6539e39c5baSBill Taylor 			cq->cq_consindx++;	/* incr the total counter */
6549e39c5baSBill Taylor 
6559e39c5baSBill Taylor 			/* Increment the consumer index */
6569e39c5baSBill Taylor 			cons_indx = (cons_indx + 1) & cq_wrap_around_mask;
6579e39c5baSBill Taylor 
6589e39c5baSBill Taylor 			/* Update the pointer to the next CQ entry */
6599e39c5baSBill Taylor 			cqe = (uint32_t *)&cq->cq_addr[cons_indx];
6609e39c5baSBill Taylor 		}
6619e39c5baSBill Taylor 
6629e39c5baSBill Taylor 		/*
6639e39c5baSBill Taylor 		 * If we have run out of space to store work completions,
6649e39c5baSBill Taylor 		 * then stop and return the ones we have pulled of the CQ.
6659e39c5baSBill Taylor 		 */
6669e39c5baSBill Taylor 		if (polled_cnt >= num_wc) {
6679e39c5baSBill Taylor 			break;
6689e39c5baSBill Taylor 		}
6699e39c5baSBill Taylor 	}
6709e39c5baSBill Taylor 
6719e39c5baSBill Taylor 	dat_status = DAT_SUCCESS;
6729e39c5baSBill Taylor 	/*
6739e39c5baSBill Taylor 	 * Now we only ring the doorbell (to update the consumer index) if
6749e39c5baSBill Taylor 	 * we've actually consumed a CQ entry.  If we have, for example,
6759e39c5baSBill Taylor 	 * pulled from a CQE that we are still in the process of "recycling"
6769e39c5baSBill Taylor 	 * for error purposes, then we would not update the consumer index.
6779e39c5baSBill Taylor 	 */
6789e39c5baSBill Taylor 	if (polled_cnt != 0) {
6799e39c5baSBill Taylor 		/*
6809e39c5baSBill Taylor 		 * Update the consumer index in both the CQ handle and the
6819e39c5baSBill Taylor 		 * doorbell record.
6829e39c5baSBill Taylor 		 */
6839e39c5baSBill Taylor 		dapli_hermon_cq_update_ci(cq, cq->cq_poll_dbp);
6849e39c5baSBill Taylor 	} else if (polled_cnt == 0) {
6859e39c5baSBill Taylor 		/*
6869e39c5baSBill Taylor 		 * If the CQ is empty, we can try to free up some of the WRID
6879e39c5baSBill Taylor 		 * list containers.
6889e39c5baSBill Taylor 		 */
6899e39c5baSBill Taylor 		if (cq->cq_wrid_reap_head)	/* look before leaping */
6909e39c5baSBill Taylor 			dapls_tavor_wrid_cq_reap(cq);
6919e39c5baSBill Taylor 		dat_status = DAT_ERROR(DAT_QUEUE_EMPTY, 0);
6929e39c5baSBill Taylor 	}
6939e39c5baSBill Taylor 
6949e39c5baSBill Taylor 	if (num_polled != NULL) {
6959e39c5baSBill Taylor 		*num_polled = polled_cnt;
6969e39c5baSBill Taylor 	}
6979e39c5baSBill Taylor 
6989e39c5baSBill Taylor 	return (dat_status);
6999e39c5baSBill Taylor }
7009e39c5baSBill Taylor 
7019e39c5baSBill Taylor /*
7029e39c5baSBill Taylor  * dapli_hermon_cq_poll_one()
7039e39c5baSBill Taylor  * This routine polls one CQE out of a CQ and puts ot into the ibt_wc_t
7049e39c5baSBill Taylor  * that is passed in.  See above for more comments/details.
7059e39c5baSBill Taylor  */
7069e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_cq_poll_one(ib_cq_handle_t cq,ibt_wc_t * wc_p)7079e39c5baSBill Taylor dapli_hermon_cq_poll_one(ib_cq_handle_t cq, ibt_wc_t *wc_p)
7089e39c5baSBill Taylor {
7099e39c5baSBill Taylor 	uint32_t		*cqe;
7109e39c5baSBill Taylor 	uint32_t		cons_indx;
7119e39c5baSBill Taylor 	DAT_RETURN		dat_status;
7129e39c5baSBill Taylor 	int			status;
7139e39c5baSBill Taylor 
7149e39c5baSBill Taylor start_over:
7159e39c5baSBill Taylor 	/* Get the consumer index */
7169e39c5baSBill Taylor 	cons_indx = cq->cq_consindx & cq_wrap_around_mask;
7179e39c5baSBill Taylor 
7189e39c5baSBill Taylor 	/* Calculate the pointer to the first CQ entry */
7199e39c5baSBill Taylor 	cqe = (uint32_t *)&cq->cq_addr[cons_indx];
7209e39c5baSBill Taylor 
7219e39c5baSBill Taylor 	/*
7229e39c5baSBill Taylor 	 * Keep pulling entries from the CQ until we find an entry owned by
7239e39c5baSBill Taylor 	 * the hardware.  As long as there the CQE's owned by SW, process
7249e39c5baSBill Taylor 	 * each entry by calling dapli_hermon_cq_cqe_consume() and updating the
7259e39c5baSBill Taylor 	 * CQ consumer index.  Note:  We only update the consumer index if
7269e39c5baSBill Taylor 	 * dapli_hermon_cq_cqe_consume() returns TAVOR_CQ_SYNC_AND_DB.
7279e39c5baSBill Taylor 	 * Otherwise, it indicates that we are going to "recycle" the CQE
7289e39c5baSBill Taylor 	 * (probably because it is a error CQE and corresponds to more than one
7299e39c5baSBill Taylor 	 * completion).
7309e39c5baSBill Taylor 	 */
7319e39c5baSBill Taylor 	if (HERMON_CQE_OWNER_IS_SW(cq, cqe)) {
7329e39c5baSBill Taylor 		if (HERMON_CQE_OPCODE_GET(cqe) == HERMON_CQE_RESIZE_OPCODE) {
7339e39c5baSBill Taylor 			if (dapli_hermon_cq_resize_helper(cq))
7349e39c5baSBill Taylor 				return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
7359e39c5baSBill Taylor 			goto start_over;
7369e39c5baSBill Taylor 		}
7379e39c5baSBill Taylor 		status = dapli_hermon_cq_cqe_consume(cq, cqe, wc_p);
7389e39c5baSBill Taylor 		if (status == TAVOR_CQ_SYNC_AND_DB) {
7399e39c5baSBill Taylor 			/* Reset to hardware ownership is implicit in Hermon */
7409e39c5baSBill Taylor 
7419e39c5baSBill Taylor 			/* Increment the consumer index */
7429e39c5baSBill Taylor 			cq->cq_consindx++;
7439e39c5baSBill Taylor 			dapli_hermon_cq_update_ci(cq, cq->cq_poll_dbp);
7449e39c5baSBill Taylor 		}
7459e39c5baSBill Taylor 		dat_status = DAT_SUCCESS;
7469e39c5baSBill Taylor 	} else {
7479e39c5baSBill Taylor 		if (cq->cq_wrid_reap_head)	/* look before leaping */
7489e39c5baSBill Taylor 			dapls_tavor_wrid_cq_reap(cq);
7499e39c5baSBill Taylor 		dat_status = DAT_ERROR(DAT_QUEUE_EMPTY, 0);
7509e39c5baSBill Taylor 	}
7519e39c5baSBill Taylor 	return (dat_status);
7529e39c5baSBill Taylor }
7539e39c5baSBill Taylor 
7549e39c5baSBill Taylor /*
7559e39c5baSBill Taylor  * dapli_hermon_cq_cqe_consume()
7569e39c5baSBill Taylor  * Converts a given CQE into a ibt_wc_t object
7579e39c5baSBill Taylor  */
7589e39c5baSBill Taylor static int
dapli_hermon_cq_cqe_consume(ib_cq_handle_t cqhdl,uint32_t * cqe,ibt_wc_t * wc)7599e39c5baSBill Taylor dapli_hermon_cq_cqe_consume(ib_cq_handle_t cqhdl, uint32_t *cqe,
7609e39c5baSBill Taylor     ibt_wc_t *wc)
7619e39c5baSBill Taylor {
7629e39c5baSBill Taylor 	uint_t		flags;
7639e39c5baSBill Taylor 	uint_t		type;
7649e39c5baSBill Taylor 	uint_t		opcode;
7659e39c5baSBill Taylor 	int		status;
7669e39c5baSBill Taylor 
7679e39c5baSBill Taylor 	/*
7689e39c5baSBill Taylor 	 * Determine if this is an "error" CQE by examining "opcode".  If it
7699e39c5baSBill Taylor 	 * is an error CQE, then call dapli_hermon_cq_errcqe_consume() and
7709e39c5baSBill Taylor 	 * return whatever status it returns.  Otherwise, this is a successful
7719e39c5baSBill Taylor 	 * completion.
7729e39c5baSBill Taylor 	 */
7739e39c5baSBill Taylor 	opcode = HERMON_CQE_OPCODE_GET(cqe);
7749e39c5baSBill Taylor 	if (opcode == HERMON_CQE_ERR_OPCODE) {
7759e39c5baSBill Taylor 		status = dapli_hermon_cq_errcqe_consume(cqhdl, cqe, wc);
7769e39c5baSBill Taylor 		return (status);
7779e39c5baSBill Taylor 	}
7789e39c5baSBill Taylor 	TAVOR_CQE_WQEADDRSZ_SET(cqe, (HTOBE_32(cqe[6]) >> 10) &
7799e39c5baSBill Taylor 	    ~HERMON_WQE_NDS_MASK);
7809e39c5baSBill Taylor 
7819e39c5baSBill Taylor 	/*
7829e39c5baSBill Taylor 	 * Fetch the Work Request ID using the information in the CQE.
7839e39c5baSBill Taylor 	 * See tavor_wr.c for more details.
7849e39c5baSBill Taylor 	 */
7859e39c5baSBill Taylor 	wc->wc_id = dapls_tavor_wrid_get_entry(cqhdl, (tavor_hw_cqe_t *)cqe,
7869e39c5baSBill Taylor 	    HERMON_CQE_SENDRECV_GET(cqe) >> 6, 0, NULL);
7879e39c5baSBill Taylor 	wc->wc_qpn = TAVOR_CQE_QPNUM_GET(cqe);
7889e39c5baSBill Taylor 
7899e39c5baSBill Taylor 	/*
7909e39c5baSBill Taylor 	 * Parse the CQE opcode to determine completion type.  This will set
7919e39c5baSBill Taylor 	 * not only the type of the completion, but also any flags that might
7929e39c5baSBill Taylor 	 * be associated with it (e.g. whether immediate data is present).
7939e39c5baSBill Taylor 	 */
7949e39c5baSBill Taylor 	flags = IBT_WC_NO_FLAGS;
7959e39c5baSBill Taylor 	if (HERMON_CQE_SENDRECV_GET(cqe) != TAVOR_COMPLETION_RECV) {
7969e39c5baSBill Taylor 
7979e39c5baSBill Taylor 		/*
7989e39c5baSBill Taylor 		 * Send CQE
7999e39c5baSBill Taylor 		 *
8009e39c5baSBill Taylor 		 * The following opcodes will not be generated in uDAPL
8019e39c5baSBill Taylor 		 * case TAVOR_CQE_SND_RDMAWR_IMM:
8029e39c5baSBill Taylor 		 * case TAVOR_CQE_SND_SEND_IMM:
8039e39c5baSBill Taylor 		 * case TAVOR_CQE_SND_ATOMIC_CS:
8049e39c5baSBill Taylor 		 * case TAVOR_CQE_SND_ATOMIC_FA:
8059e39c5baSBill Taylor 		 */
8069e39c5baSBill Taylor 		switch (opcode) {
8079e39c5baSBill Taylor 		case TAVOR_CQE_SND_RDMAWR:
8089e39c5baSBill Taylor 			type = IBT_WRC_RDMAW;
8099e39c5baSBill Taylor 			break;
8109e39c5baSBill Taylor 
8119e39c5baSBill Taylor 		case TAVOR_CQE_SND_SEND:
8129e39c5baSBill Taylor 			type = IBT_WRC_SEND;
8139e39c5baSBill Taylor 			break;
8149e39c5baSBill Taylor 
8159e39c5baSBill Taylor 		case TAVOR_CQE_SND_RDMARD:
8169e39c5baSBill Taylor 			type = IBT_WRC_RDMAR;
8179e39c5baSBill Taylor 			wc->wc_bytes_xfer = TAVOR_CQE_BYTECNT_GET(cqe);
8189e39c5baSBill Taylor 			break;
8199e39c5baSBill Taylor 
8209e39c5baSBill Taylor 		case TAVOR_CQE_SND_BIND_MW:
8219e39c5baSBill Taylor 			type = IBT_WRC_BIND;
8229e39c5baSBill Taylor 			break;
8239e39c5baSBill Taylor 
8249e39c5baSBill Taylor 		default:
8259e39c5baSBill Taylor 			wc->wc_status = IBT_WC_LOCAL_CHAN_OP_ERR;
8269e39c5baSBill Taylor 			return (TAVOR_CQ_SYNC_AND_DB);
8279e39c5baSBill Taylor 		}
8289e39c5baSBill Taylor 	} else {
8299e39c5baSBill Taylor 
8309e39c5baSBill Taylor 		/*
8319e39c5baSBill Taylor 		 * Receive CQE
8329e39c5baSBill Taylor 		 *
8339e39c5baSBill Taylor 		 * The following opcodes will not be generated in uDAPL
8349e39c5baSBill Taylor 		 *
8359e39c5baSBill Taylor 		 * case TAVOR_CQE_RCV_RECV_IMM:
8369e39c5baSBill Taylor 		 * case TAVOR_CQE_RCV_RECV_IMM2:
8379e39c5baSBill Taylor 		 * case TAVOR_CQE_RCV_RDMAWR_IMM:
8389e39c5baSBill Taylor 		 * case TAVOR_CQE_RCV_RDMAWR_IMM2:
8399e39c5baSBill Taylor 		 */
8409e39c5baSBill Taylor 		switch (opcode) {
8419e39c5baSBill Taylor 		case HERMON_CQE_RCV_SEND:
8429e39c5baSBill Taylor 			type = IBT_WRC_RECV;
8439e39c5baSBill Taylor 			wc->wc_bytes_xfer = TAVOR_CQE_BYTECNT_GET(cqe);
8449e39c5baSBill Taylor 			break;
8459e39c5baSBill Taylor 		default:
8469e39c5baSBill Taylor 			wc->wc_status = IBT_WC_LOCAL_CHAN_OP_ERR;
8479e39c5baSBill Taylor 			return (TAVOR_CQ_SYNC_AND_DB);
8489e39c5baSBill Taylor 		}
8499e39c5baSBill Taylor 	}
8509e39c5baSBill Taylor 	wc->wc_type = type;
8519e39c5baSBill Taylor 	wc->wc_flags = flags;
8529e39c5baSBill Taylor 	/* If we got here, completion status must be success */
8539e39c5baSBill Taylor 	wc->wc_status = IBT_WC_SUCCESS;
8549e39c5baSBill Taylor 
8559e39c5baSBill Taylor 	return (TAVOR_CQ_SYNC_AND_DB);
8569e39c5baSBill Taylor }
8579e39c5baSBill Taylor 
8589e39c5baSBill Taylor /*
8599e39c5baSBill Taylor  * dapli_hermon_cq_errcqe_consume()
8609e39c5baSBill Taylor  */
8619e39c5baSBill Taylor static int
dapli_hermon_cq_errcqe_consume(ib_cq_handle_t cqhdl,uint32_t * cqe,ibt_wc_t * wc)8629e39c5baSBill Taylor dapli_hermon_cq_errcqe_consume(ib_cq_handle_t cqhdl, uint32_t *cqe,
8639e39c5baSBill Taylor     ibt_wc_t *wc)
8649e39c5baSBill Taylor {
8659e39c5baSBill Taylor 	dapls_tavor_wrid_entry_t	wre;
8669e39c5baSBill Taylor 	uint_t			status;
8679e39c5baSBill Taylor 	uint_t			send_or_recv;
8689e39c5baSBill Taylor 
8699e39c5baSBill Taylor 	dapl_dbg_log(DAPL_DBG_TYPE_EVD, "errcqe_consume:cqe.eth=%x, wqe=%x\n",
8709e39c5baSBill Taylor 	    TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cqe),
8719e39c5baSBill Taylor 	    TAVOR_CQE_WQEADDRSZ_GET(cqe));
8729e39c5baSBill Taylor 
8739e39c5baSBill Taylor 	status = ((uint8_t *)cqe)[0x1B];
8749e39c5baSBill Taylor 	TAVOR_CQE_WQEADDRSZ_SET(cqe, (HTOBE_32(cqe[6]) >> 10) &
8759e39c5baSBill Taylor 	    ~HERMON_WQE_NDS_MASK);
8769e39c5baSBill Taylor 	if (HERMON_CQE_SENDRECV_GET(cqe) == 0) {
8779e39c5baSBill Taylor 		send_or_recv = 0;
8789e39c5baSBill Taylor 	} else {
8799e39c5baSBill Taylor 		send_or_recv = 1;
8809e39c5baSBill Taylor 	}
8819e39c5baSBill Taylor 
8829e39c5baSBill Taylor 	/*
8839e39c5baSBill Taylor 	 * Fetch the Work Request ID using the information in the CQE.
8849e39c5baSBill Taylor 	 * See tavor_wr.c for more details.
8859e39c5baSBill Taylor 	 */
8869e39c5baSBill Taylor 	wc->wc_id = dapls_tavor_wrid_get_entry(cqhdl, (tavor_hw_cqe_t *)cqe,
8879e39c5baSBill Taylor 	    send_or_recv, 1, &wre);
8889e39c5baSBill Taylor 	wc->wc_qpn = TAVOR_CQE_QPNUM_GET(cqe);
8899e39c5baSBill Taylor 
8909e39c5baSBill Taylor 	/*
8919e39c5baSBill Taylor 	 * Parse the CQE opcode to determine completion type.  We know that
8929e39c5baSBill Taylor 	 * the CQE is an error completion, so we extract only the completion
8939e39c5baSBill Taylor 	 * status here.
8949e39c5baSBill Taylor 	 */
8959e39c5baSBill Taylor 	switch (status) {
8969e39c5baSBill Taylor 	case TAVOR_CQE_LOC_LEN_ERR:
8979e39c5baSBill Taylor 		status = IBT_WC_LOCAL_LEN_ERR;
8989e39c5baSBill Taylor 		break;
8999e39c5baSBill Taylor 
9009e39c5baSBill Taylor 	case TAVOR_CQE_LOC_OP_ERR:
9019e39c5baSBill Taylor 		status = IBT_WC_LOCAL_CHAN_OP_ERR;
9029e39c5baSBill Taylor 		break;
9039e39c5baSBill Taylor 
9049e39c5baSBill Taylor 	case TAVOR_CQE_LOC_PROT_ERR:
9059e39c5baSBill Taylor 		status = IBT_WC_LOCAL_PROTECT_ERR;
9069e39c5baSBill Taylor 		break;
9079e39c5baSBill Taylor 
9089e39c5baSBill Taylor 	case TAVOR_CQE_WR_FLUSHED_ERR:
9099e39c5baSBill Taylor 		status = IBT_WC_WR_FLUSHED_ERR;
9109e39c5baSBill Taylor 		break;
9119e39c5baSBill Taylor 
9129e39c5baSBill Taylor 	case TAVOR_CQE_MW_BIND_ERR:
9139e39c5baSBill Taylor 		status = IBT_WC_MEM_WIN_BIND_ERR;
9149e39c5baSBill Taylor 		break;
9159e39c5baSBill Taylor 
9169e39c5baSBill Taylor 	case TAVOR_CQE_BAD_RESPONSE_ERR:
9179e39c5baSBill Taylor 		status = IBT_WC_BAD_RESPONSE_ERR;
9189e39c5baSBill Taylor 		break;
9199e39c5baSBill Taylor 
9209e39c5baSBill Taylor 	case TAVOR_CQE_LOCAL_ACCESS_ERR:
9219e39c5baSBill Taylor 		status = IBT_WC_LOCAL_ACCESS_ERR;
9229e39c5baSBill Taylor 		break;
9239e39c5baSBill Taylor 
9249e39c5baSBill Taylor 	case TAVOR_CQE_REM_INV_REQ_ERR:
9259e39c5baSBill Taylor 		status = IBT_WC_REMOTE_INVALID_REQ_ERR;
9269e39c5baSBill Taylor 		break;
9279e39c5baSBill Taylor 
9289e39c5baSBill Taylor 	case TAVOR_CQE_REM_ACC_ERR:
9299e39c5baSBill Taylor 		status = IBT_WC_REMOTE_ACCESS_ERR;
9309e39c5baSBill Taylor 		break;
9319e39c5baSBill Taylor 
9329e39c5baSBill Taylor 	case TAVOR_CQE_REM_OP_ERR:
9339e39c5baSBill Taylor 		status = IBT_WC_REMOTE_OP_ERR;
9349e39c5baSBill Taylor 		break;
9359e39c5baSBill Taylor 
9369e39c5baSBill Taylor 	case TAVOR_CQE_TRANS_TO_ERR:
9379e39c5baSBill Taylor 		status = IBT_WC_TRANS_TIMEOUT_ERR;
9389e39c5baSBill Taylor 		break;
9399e39c5baSBill Taylor 
9409e39c5baSBill Taylor 	case TAVOR_CQE_RNRNAK_TO_ERR:
9419e39c5baSBill Taylor 		status = IBT_WC_RNR_NAK_TIMEOUT_ERR;
9429e39c5baSBill Taylor 		break;
9439e39c5baSBill Taylor 
9449e39c5baSBill Taylor 	/*
9459e39c5baSBill Taylor 	 * The following error codes are not supported in the Tavor driver
9469e39c5baSBill Taylor 	 * as they relate only to Reliable Datagram completion statuses:
9479e39c5baSBill Taylor 	 *    case TAVOR_CQE_LOCAL_RDD_VIO_ERR:
9489e39c5baSBill Taylor 	 *    case TAVOR_CQE_REM_INV_RD_REQ_ERR:
9499e39c5baSBill Taylor 	 *    case TAVOR_CQE_EEC_REM_ABORTED_ERR:
9509e39c5baSBill Taylor 	 *    case TAVOR_CQE_INV_EEC_NUM_ERR:
9519e39c5baSBill Taylor 	 *    case TAVOR_CQE_INV_EEC_STATE_ERR:
9529e39c5baSBill Taylor 	 *    case TAVOR_CQE_LOC_EEC_ERR:
9539e39c5baSBill Taylor 	 */
9549e39c5baSBill Taylor 
9559e39c5baSBill Taylor 	default:
9569e39c5baSBill Taylor 		status = IBT_WC_LOCAL_CHAN_OP_ERR;
9579e39c5baSBill Taylor 		break;
9589e39c5baSBill Taylor 	}
9599e39c5baSBill Taylor 	wc->wc_status = status;
9609e39c5baSBill Taylor 	wc->wc_type = 0;
9619e39c5baSBill Taylor 
9629e39c5baSBill Taylor 	/*
9639e39c5baSBill Taylor 	 * Consume the CQE
9649e39c5baSBill Taylor 	 *    Return status to indicate that doorbell and sync may be
9659e39c5baSBill Taylor 	 *    necessary.
9669e39c5baSBill Taylor 	 */
9679e39c5baSBill Taylor 	return (TAVOR_CQ_SYNC_AND_DB);
9689e39c5baSBill Taylor }
9699e39c5baSBill Taylor 
9709e39c5baSBill Taylor /*
9719e39c5baSBill Taylor  * dapli_hermon_cq_notify()
9729e39c5baSBill Taylor  * This function is used for arming the CQ by ringing the CQ doorbell.
9739e39c5baSBill Taylor  *
9749e39c5baSBill Taylor  * Note: there is something very subtle here.  This code assumes a very
9759e39c5baSBill Taylor  * specific behavior of the kernel driver.  The cmd_sn field of the
9769e39c5baSBill Taylor  * arm_dbr is updated by the kernel driver whenever a notification
9779e39c5baSBill Taylor  * event for the cq is received.  This code extracts the cmd_sn field
9789e39c5baSBill Taylor  * from the arm_dbr to know the right value to use.  The arm_dbr is
9799e39c5baSBill Taylor  * always updated atomically so that neither the kernel driver nor this
9809e39c5baSBill Taylor  * will get confused about what the other is doing.
9819e39c5baSBill Taylor  *
9829e39c5baSBill Taylor  * Note: param is not used here.  It is necessary for arming a CQ for
9839e39c5baSBill Taylor  * N completions (param is N), but no uDAPL API supports this for now.
9849e39c5baSBill Taylor  * Thus, we declare ARGSUSED to make lint happy.
9859e39c5baSBill Taylor  */
9869e39c5baSBill Taylor /*ARGSUSED*/
9879e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_cq_notify(ib_cq_handle_t cq,int flags,uint32_t param)9889e39c5baSBill Taylor dapli_hermon_cq_notify(ib_cq_handle_t cq, int flags, uint32_t param)
9899e39c5baSBill Taylor {
9909e39c5baSBill Taylor 	uint32_t	cqnum;
9919e39c5baSBill Taylor 	uint32_t	*target;
9929e39c5baSBill Taylor 	uint32_t	old_cmd, cmp, new, tmp, cmd_sn;
9939e39c5baSBill Taylor 
9949e39c5baSBill Taylor 	/*
9959e39c5baSBill Taylor 	 * Determine if we are trying to get the next completion or the next
9969e39c5baSBill Taylor 	 * "solicited" completion.  Then hit the appropriate doorbell.
9979e39c5baSBill Taylor 	 */
9989e39c5baSBill Taylor 	cqnum = cq->cq_num;
9999e39c5baSBill Taylor 	target = cq->cq_arm_dbp;
10009e39c5baSBill Taylor retry:
10019e39c5baSBill Taylor 	cmp = *target;
10029e39c5baSBill Taylor 	tmp = HTOBE_32(cmp);
10039e39c5baSBill Taylor 	old_cmd = tmp & (0x7 << 24);
10049e39c5baSBill Taylor 	cmd_sn = tmp & (0x3 << 28);
10059e39c5baSBill Taylor 
10069e39c5baSBill Taylor 	if (flags == IB_NOTIFY_ON_NEXT_COMP) {
10079e39c5baSBill Taylor 		if (old_cmd != HERMON_CQDB_NOTIFY_CQ) {
10089e39c5baSBill Taylor 			new = HTOBE_32(cmd_sn | HERMON_CQDB_NOTIFY_CQ |
10099e39c5baSBill Taylor 			    (cq->cq_consindx & 0xFFFFFF));
10109e39c5baSBill Taylor 			tmp = atomic_cas_32(target, cmp, new);
10119e39c5baSBill Taylor 			if (tmp != cmp)
10129e39c5baSBill Taylor 				goto retry;
10139e39c5baSBill Taylor 			dapli_hermon_cq_doorbell(cq->cq_iauar,
10149e39c5baSBill Taylor 			    HERMON_CQDB_NOTIFY_CQ, cqnum,
10159e39c5baSBill Taylor 			    cmd_sn, cq->cq_consindx);
10169e39c5baSBill Taylor 		} /* else it's already armed */
10179e39c5baSBill Taylor 	} else if (flags == IB_NOTIFY_ON_NEXT_SOLICITED) {
10189e39c5baSBill Taylor 		if (old_cmd != HERMON_CQDB_NOTIFY_CQ &&
10199e39c5baSBill Taylor 		    old_cmd != HERMON_CQDB_NOTIFY_CQ_SOLICIT) {
10209e39c5baSBill Taylor 			new = HTOBE_32(cmd_sn | HERMON_CQDB_NOTIFY_CQ_SOLICIT |
10219e39c5baSBill Taylor 			    (cq->cq_consindx & 0xFFFFFF));
10229e39c5baSBill Taylor 			tmp = atomic_cas_32(target, cmp, new);
10239e39c5baSBill Taylor 			if (tmp != cmp)
10249e39c5baSBill Taylor 				goto retry;
10259e39c5baSBill Taylor 			dapli_hermon_cq_doorbell(cq->cq_iauar,
10269e39c5baSBill Taylor 			    HERMON_CQDB_NOTIFY_CQ_SOLICIT, cqnum,
10279e39c5baSBill Taylor 			    cmd_sn, cq->cq_consindx);
10289e39c5baSBill Taylor 		} /* else it's already armed */
10299e39c5baSBill Taylor 	} else {
10309e39c5baSBill Taylor 		return (DAT_INVALID_PARAMETER);
10319e39c5baSBill Taylor 	}
10329e39c5baSBill Taylor 
10339e39c5baSBill Taylor 	return (DAT_SUCCESS);
10349e39c5baSBill Taylor }
10359e39c5baSBill Taylor 
10369e39c5baSBill Taylor /*
10379e39c5baSBill Taylor  * Since uDAPL posts 1 wqe per request, we
10389e39c5baSBill Taylor  * only need to do stores for the last one.
10399e39c5baSBill Taylor  */
10409e39c5baSBill Taylor static void
dapli_hermon_wqe_headroom(ib_qp_handle_t qp,uint32_t start)10419e39c5baSBill Taylor dapli_hermon_wqe_headroom(ib_qp_handle_t qp, uint32_t start)
10429e39c5baSBill Taylor {
10439e39c5baSBill Taylor 	uint32_t *wqe_start, *wqe_top, *wqe_base, qsize, invalue;
10449e39c5baSBill Taylor 	int hdrmwqes, wqesizebytes, sectperwqe, i, j;
10459e39c5baSBill Taylor 
10469e39c5baSBill Taylor 	qsize = qp->qp_sq_numwqe;
10479e39c5baSBill Taylor 	wqesizebytes = qp->qp_sq_wqesz;
10489e39c5baSBill Taylor 	sectperwqe = wqesizebytes >> 6;
10499e39c5baSBill Taylor 	hdrmwqes = qp->qp_sq_headroom;
10509e39c5baSBill Taylor 	wqe_base = (uint32_t *)TAVOR_QP_SQ_ENTRY(qp, 0);
10519e39c5baSBill Taylor 	wqe_top = (uint32_t *)TAVOR_QP_SQ_ENTRY(qp, qsize);
10529e39c5baSBill Taylor 	wqe_start = (uint32_t *)TAVOR_QP_SQ_ENTRY(qp, start);
10539e39c5baSBill Taylor 
10549e39c5baSBill Taylor 	for (i = 0; i < hdrmwqes - 1; i++) {
10559e39c5baSBill Taylor 		wqe_start += sectperwqe * 16;
10569e39c5baSBill Taylor 		if (wqe_start == wqe_top)
10579e39c5baSBill Taylor 			wqe_start = wqe_base;
10589e39c5baSBill Taylor 	}
10599e39c5baSBill Taylor 	invalue = HTOBE_32(*wqe_start);
10609e39c5baSBill Taylor 	invalue |= 0x7FFFFFFF;
10619e39c5baSBill Taylor 	*wqe_start = HTOBE_32(invalue);
10629e39c5baSBill Taylor 	wqe_start += 16;
10639e39c5baSBill Taylor 	for (j = 1; j < sectperwqe; j++) {
10649e39c5baSBill Taylor 		*wqe_start = 0xFFFFFFFF;
10659e39c5baSBill Taylor 		wqe_start += 16;
10669e39c5baSBill Taylor 	}
10679e39c5baSBill Taylor }
10689e39c5baSBill Taylor 
10699e39c5baSBill Taylor /*
10709e39c5baSBill Taylor  * dapli_hermon_post_send()
10719e39c5baSBill Taylor  */
10729e39c5baSBill Taylor /* ARGSUSED */
10739e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_post_send(DAPL_EP * ep,ibt_send_wr_t * wr,boolean_t ns)10749e39c5baSBill Taylor dapli_hermon_post_send(DAPL_EP *ep, ibt_send_wr_t *wr, boolean_t ns)
10759e39c5baSBill Taylor {
10769e39c5baSBill Taylor 	dapls_tavor_wrid_list_hdr_t	*wridlist;
10779e39c5baSBill Taylor 	dapls_tavor_wrid_entry_t	*wre_last;
10789e39c5baSBill Taylor 	uint64_t			*desc;
10799e39c5baSBill Taylor 	uint64_t			*wqe_addr;
10809e39c5baSBill Taylor 	uint32_t			desc_sz;
10819e39c5baSBill Taylor 	uint32_t			wqeaddrsz, signaled_dbd;
10829e39c5baSBill Taylor 	uint32_t			head, tail, next_tail, qsize_msk;
10839e39c5baSBill Taylor 	int				status;
10849e39c5baSBill Taylor 	ib_qp_handle_t			qp;
10859e39c5baSBill Taylor 
10869e39c5baSBill Taylor 	if ((ep->qp_state == IBT_STATE_RESET) ||
10879e39c5baSBill Taylor 	    (ep->qp_state == IBT_STATE_INIT) ||
10889e39c5baSBill Taylor 	    (ep->qp_state == IBT_STATE_RTR)) {
10899e39c5baSBill Taylor 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
10909e39c5baSBill Taylor 		    "post_send: invalid qp_state %d\n", ep->qp_state);
10919e39c5baSBill Taylor 		return (DAT_INVALID_STATE);
10929e39c5baSBill Taylor 	}
10939e39c5baSBill Taylor 
10949e39c5baSBill Taylor 	qp = ep->qp_handle;
10959e39c5baSBill Taylor 
10969e39c5baSBill Taylor 	/* Grab the lock for the WRID list */
10979e39c5baSBill Taylor 	dapl_os_lock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
10989e39c5baSBill Taylor 	wridlist  = qp->qp_sq_wqhdr->wq_wrid_post;
10999e39c5baSBill Taylor 
11009e39c5baSBill Taylor 	/* Save away some initial QP state */
11019e39c5baSBill Taylor 	qsize_msk = qp->qp_sq_wqhdr->wq_size - 1;
11029e39c5baSBill Taylor 	tail	  = qp->qp_sq_wqhdr->wq_tail;
11039e39c5baSBill Taylor 	head	  = qp->qp_sq_wqhdr->wq_head;
11049e39c5baSBill Taylor 
11059e39c5baSBill Taylor 	/*
11069e39c5baSBill Taylor 	 * Check for "queue full" condition.  If the queue is already full,
11079e39c5baSBill Taylor 	 * then no more WQEs can be posted, return an error
11089e39c5baSBill Taylor 	 */
11099e39c5baSBill Taylor 	if (qp->qp_sq_wqhdr->wq_full != 0) {
11109e39c5baSBill Taylor 		dapl_os_unlock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
11119e39c5baSBill Taylor 		return (DAT_INSUFFICIENT_RESOURCES);
11129e39c5baSBill Taylor 	}
11139e39c5baSBill Taylor 
11149e39c5baSBill Taylor 	/*
11159e39c5baSBill Taylor 	 * Increment the "tail index" and check for "queue full" condition.
11169e39c5baSBill Taylor 	 * If we detect that the current work request is going to fill the
11179e39c5baSBill Taylor 	 * work queue, then we mark this condition and continue.
11189e39c5baSBill Taylor 	 */
11199e39c5baSBill Taylor 	next_tail = (tail + 1) & qsize_msk;
11209e39c5baSBill Taylor 	if (next_tail == head) {
11219e39c5baSBill Taylor 		qp->qp_sq_wqhdr->wq_full = 1;
11229e39c5baSBill Taylor 	}
11239e39c5baSBill Taylor 
11249e39c5baSBill Taylor 	/*
11259e39c5baSBill Taylor 	 * Get the user virtual address of the location where the next
11269e39c5baSBill Taylor 	 * Send WQE should be built
11279e39c5baSBill Taylor 	 */
11289e39c5baSBill Taylor 	wqe_addr = TAVOR_QP_SQ_ENTRY(qp, tail);
11299e39c5baSBill Taylor 
11309e39c5baSBill Taylor 	/*
11319e39c5baSBill Taylor 	 * Call tavor_wqe_send_build() to build the WQE at the given address.
11329e39c5baSBill Taylor 	 * This routine uses the information in the ibt_send_wr_t and
11339e39c5baSBill Taylor 	 * returns the size of the WQE when it returns.
11349e39c5baSBill Taylor 	 */
11359e39c5baSBill Taylor 	status = dapli_hermon_wqe_send_build(qp, wr, wqe_addr, &desc_sz);
11369e39c5baSBill Taylor 	if (status != DAT_SUCCESS) {
11379e39c5baSBill Taylor 		dapl_os_unlock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
11389e39c5baSBill Taylor 		return (status);
11399e39c5baSBill Taylor 	}
11409e39c5baSBill Taylor 
11419e39c5baSBill Taylor 	/*
11429e39c5baSBill Taylor 	 * Get the descriptor (io address) corresponding to the location
11439e39c5baSBill Taylor 	 * Send WQE was built.
11449e39c5baSBill Taylor 	 */
11459e39c5baSBill Taylor 	desc = TAVOR_QP_SQ_ENTRY(qp, tail);
11469e39c5baSBill Taylor 
11479e39c5baSBill Taylor 	/*
11489e39c5baSBill Taylor 	 * Add a WRID entry to the WRID list.  Need to calculate the
11499e39c5baSBill Taylor 	 * "wqeaddr" to pass to dapli_tavor_wrid_add_entry().
11509e39c5baSBill Taylor 	 * signaled_dbd is still calculated, but ignored.
11519e39c5baSBill Taylor 	 */
11529e39c5baSBill Taylor 	wqeaddrsz = HERMON_QP_WQEADDRSZ(qp->qp_sq_counter);
11539e39c5baSBill Taylor 
11549e39c5baSBill Taylor 	if (wr->wr_flags & IBT_WR_SEND_SIGNAL) {
11559e39c5baSBill Taylor 		signaled_dbd = TAVOR_WRID_ENTRY_SIGNALED;
11569e39c5baSBill Taylor 	}
11579e39c5baSBill Taylor 
11589e39c5baSBill Taylor 	dapli_tavor_wrid_add_entry(qp->qp_sq_wqhdr, wr->wr_id, wqeaddrsz,
11599e39c5baSBill Taylor 	    signaled_dbd);
11609e39c5baSBill Taylor 
11619e39c5baSBill Taylor 	dapli_hermon_wqe_headroom(qp, next_tail);
11629e39c5baSBill Taylor 	*(uint8_t *)desc ^= 0x80;	/* set owner bit */
11639e39c5baSBill Taylor 
11649e39c5baSBill Taylor 	/*
11659e39c5baSBill Taylor 	 * Now if the WRID tail entry is non-NULL, then this
11669e39c5baSBill Taylor 	 * represents the entry to which we are chaining the
11679e39c5baSBill Taylor 	 * new entries.  Since we are going to ring the
11689e39c5baSBill Taylor 	 * doorbell for this WQE, we want set its "dbd" bit.
11699e39c5baSBill Taylor 	 *
11709e39c5baSBill Taylor 	 * On the other hand, if the tail is NULL, even though
11719e39c5baSBill Taylor 	 * we will have rung the doorbell for the previous WQE
11729e39c5baSBill Taylor 	 * (for the hardware's sake) it is irrelevant to our
11739e39c5baSBill Taylor 	 * purposes (for tracking WRIDs) because we know the
11749e39c5baSBill Taylor 	 * request must have already completed.
11759e39c5baSBill Taylor 	 */
11769e39c5baSBill Taylor 	wre_last = wridlist->wl_wre_old_tail;
11779e39c5baSBill Taylor 	if (wre_last != NULL) {
11789e39c5baSBill Taylor 		wre_last->wr_signaled_dbd |= TAVOR_WRID_ENTRY_DOORBELLED;
11799e39c5baSBill Taylor 	}
11809e39c5baSBill Taylor 
11819e39c5baSBill Taylor 	/* Update some of the state in the QP */
11829e39c5baSBill Taylor 	qp->qp_sq_lastwqeaddr	 = wqe_addr;
11839e39c5baSBill Taylor 	qp->qp_sq_wqhdr->wq_tail = next_tail;
11849e39c5baSBill Taylor 
11859e39c5baSBill Taylor 	if (desc_sz && qp->qp_ia_bf != NULL) {	/* use Hermon Blueflame */
11869e39c5baSBill Taylor 		uint64_t *bf_dest, *src64;
11879e39c5baSBill Taylor 		uint8_t *src8;
11889e39c5baSBill Taylor 		int i;
11899e39c5baSBill Taylor 
11909e39c5baSBill Taylor 		(void) pthread_spin_lock(&hermon_bf_lock);
11919e39c5baSBill Taylor 
11929e39c5baSBill Taylor 		src8 = (uint8_t *)desc;
11939e39c5baSBill Taylor 		src8[1] = (uint8_t)(qp->qp_sq_counter >> 8);
11949e39c5baSBill Taylor 		src8[2] = (uint8_t)qp->qp_sq_counter;
11959e39c5baSBill Taylor 		src8[4] = (uint8_t)(qp->qp_num >> 16);
11969e39c5baSBill Taylor 		src8[5] = (uint8_t)(qp->qp_num >> 8);
11979e39c5baSBill Taylor 		src8[6] = (uint8_t)qp->qp_num;
11989e39c5baSBill Taylor 
11999e39c5baSBill Taylor 		src64 = (uint64_t *)desc;
12009e39c5baSBill Taylor 		bf_dest = (uint64_t *)((uintptr_t)qp->qp_ia_bf +
12019e39c5baSBill Taylor 		    *qp->qp_ia_bf_toggle);
12029e39c5baSBill Taylor 		*qp->qp_ia_bf_toggle ^= 256;	/* 2 256-byte buffers */
1203*1ed53a3fSBill Taylor 		for (i = 0; i < desc_sz * 2; i += 8) {
12049e39c5baSBill Taylor 			bf_dest[i] = src64[i];
12059e39c5baSBill Taylor 			bf_dest[i + 1] = src64[i + 1];
1206*1ed53a3fSBill Taylor 			bf_dest[i + 2] = src64[i + 2];
1207*1ed53a3fSBill Taylor 			bf_dest[i + 3] = src64[i + 3];
1208*1ed53a3fSBill Taylor 			bf_dest[i + 4] = src64[i + 4];
1209*1ed53a3fSBill Taylor 			bf_dest[i + 5] = src64[i + 5];
1210*1ed53a3fSBill Taylor 			bf_dest[i + 6] = src64[i + 6];
1211*1ed53a3fSBill Taylor 			bf_dest[i + 7] = src64[i + 7];
12129e39c5baSBill Taylor 		}
12139e39c5baSBill Taylor 		(void) pthread_spin_unlock(&hermon_bf_lock);
12149e39c5baSBill Taylor 	} else {
12159e39c5baSBill Taylor 		/* Ring the doorbell */
12169e39c5baSBill Taylor 		dapli_hermon_sq_dbreg(qp->qp_iauar, qp->qp_num);
12179e39c5baSBill Taylor 	}
12189e39c5baSBill Taylor 	qp->qp_sq_counter++;
12199e39c5baSBill Taylor 
12209e39c5baSBill Taylor 	dapl_os_unlock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
12219e39c5baSBill Taylor 
12229e39c5baSBill Taylor 	return (DAT_SUCCESS);
12239e39c5baSBill Taylor }
12249e39c5baSBill Taylor 
12259e39c5baSBill Taylor /*
12269e39c5baSBill Taylor  * dapli_hermon_post_recv()
12279e39c5baSBill Taylor  */
12289e39c5baSBill Taylor /* ARGSUSED */
12299e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_post_recv(DAPL_EP * ep,ibt_recv_wr_t * wr,boolean_t ns)12309e39c5baSBill Taylor dapli_hermon_post_recv(DAPL_EP	*ep, ibt_recv_wr_t *wr, boolean_t ns)
12319e39c5baSBill Taylor {
12329e39c5baSBill Taylor 	dapls_tavor_wrid_list_hdr_t	*wridlist;
12339e39c5baSBill Taylor 	dapls_tavor_wrid_entry_t	*wre_last;
12349e39c5baSBill Taylor 	ib_qp_handle_t			qp;
12359e39c5baSBill Taylor 	DAT_RETURN			status;
12369e39c5baSBill Taylor 	uint64_t			*wqe_addr;
12379e39c5baSBill Taylor 	uint32_t			desc_sz;
12389e39c5baSBill Taylor 	uint32_t			wqeaddrsz;
12399e39c5baSBill Taylor 	uint32_t			head, tail, next_tail, qsize_msk;
12409e39c5baSBill Taylor 
12419e39c5baSBill Taylor 	if (ep->qp_state == IBT_STATE_RESET) {
12429e39c5baSBill Taylor 		dapl_dbg_log(DAPL_DBG_TYPE_ERR,
12439e39c5baSBill Taylor 		    "post_recv: invalid qp_state %d\n", ep->qp_state);
12449e39c5baSBill Taylor 		return (DAT_INVALID_STATE);
12459e39c5baSBill Taylor 	}
12469e39c5baSBill Taylor 	qp = ep->qp_handle;
12479e39c5baSBill Taylor 
12489e39c5baSBill Taylor 	/* Grab the lock for the WRID list */
12499e39c5baSBill Taylor 	dapl_os_lock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
12509e39c5baSBill Taylor 	wridlist  = qp->qp_rq_wqhdr->wq_wrid_post;
12519e39c5baSBill Taylor 
12529e39c5baSBill Taylor 	/* Save away some initial QP state */
12539e39c5baSBill Taylor 	qsize_msk = qp->qp_rq_wqhdr->wq_size - 1;
12549e39c5baSBill Taylor 	tail	  = qp->qp_rq_wqhdr->wq_tail;
12559e39c5baSBill Taylor 	head	  = qp->qp_rq_wqhdr->wq_head;
12569e39c5baSBill Taylor 
12579e39c5baSBill Taylor 	/*
12589e39c5baSBill Taylor 	 * For the ibt_recv_wr_t passed in, parse the request and build a
12599e39c5baSBill Taylor 	 * Recv WQE. Link the WQE with the previous WQE and ring the
12609e39c5baSBill Taylor 	 * door bell.
12619e39c5baSBill Taylor 	 */
12629e39c5baSBill Taylor 
12639e39c5baSBill Taylor 	/*
12649e39c5baSBill Taylor 	 * Check for "queue full" condition.  If the queue is already full,
12659e39c5baSBill Taylor 	 * then no more WQEs can be posted. So return an error.
12669e39c5baSBill Taylor 	 */
12679e39c5baSBill Taylor 	if (qp->qp_rq_wqhdr->wq_full != 0) {
12689e39c5baSBill Taylor 		dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
12699e39c5baSBill Taylor 		return (DAT_INSUFFICIENT_RESOURCES);
12709e39c5baSBill Taylor 	}
12719e39c5baSBill Taylor 
12729e39c5baSBill Taylor 	/*
12739e39c5baSBill Taylor 	 * Increment the "tail index" and check for "queue
12749e39c5baSBill Taylor 	 * full" condition.  If we detect that the current
12759e39c5baSBill Taylor 	 * work request is going to fill the work queue, then
12769e39c5baSBill Taylor 	 * we mark this condition and continue.
12779e39c5baSBill Taylor 	 */
12789e39c5baSBill Taylor 	next_tail = (tail + 1) & qsize_msk;
12799e39c5baSBill Taylor 	if (next_tail == head) {
12809e39c5baSBill Taylor 		qp->qp_rq_wqhdr->wq_full = 1;
12819e39c5baSBill Taylor 	}
12829e39c5baSBill Taylor 
12839e39c5baSBill Taylor 	/* The user virtual address of the WQE to be built */
12849e39c5baSBill Taylor 	wqe_addr = TAVOR_QP_RQ_ENTRY(qp, tail);
12859e39c5baSBill Taylor 
12869e39c5baSBill Taylor 	/*
12879e39c5baSBill Taylor 	 * Call tavor_wqe_recv_build() to build the WQE at the given
12889e39c5baSBill Taylor 	 * address. This routine uses the information in the
12899e39c5baSBill Taylor 	 * ibt_recv_wr_t and returns the size of the WQE.
12909e39c5baSBill Taylor 	 */
12919e39c5baSBill Taylor 	status = dapli_hermon_wqe_recv_build(qp, wr, wqe_addr, &desc_sz);
12929e39c5baSBill Taylor 	if (status != DAT_SUCCESS) {
12939e39c5baSBill Taylor 		dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
12949e39c5baSBill Taylor 		return (DAT_INTERNAL_ERROR);
12959e39c5baSBill Taylor 	}
12969e39c5baSBill Taylor 
12979e39c5baSBill Taylor 	/*
12989e39c5baSBill Taylor 	 * Add a WRID entry to the WRID list.  Need to calculate the
12999e39c5baSBill Taylor 	 * "wqeaddr" and "signaled_dbd" values to pass to
13009e39c5baSBill Taylor 	 * dapli_tavor_wrid_add_entry().
13019e39c5baSBill Taylor 	 * Note: all Recv WQEs are essentially "signaled"
13029e39c5baSBill Taylor 	 */
13039e39c5baSBill Taylor 	wqeaddrsz = HERMON_QP_WQEADDRSZ(qp->qp_rq_counter);
13049e39c5baSBill Taylor 	dapli_tavor_wrid_add_entry(qp->qp_rq_wqhdr, wr->wr_id, wqeaddrsz,
13059e39c5baSBill Taylor 	    (uint32_t)TAVOR_WRID_ENTRY_SIGNALED);
13069e39c5baSBill Taylor 
13079e39c5baSBill Taylor 	/*
13089e39c5baSBill Taylor 	 * Now if the WRID tail entry is non-NULL, then this
13099e39c5baSBill Taylor 	 * represents the entry to which we are chaining the
13109e39c5baSBill Taylor 	 * new entries.  Since we are going to ring the
13119e39c5baSBill Taylor 	 * doorbell for this WQE, we want set its "dbd" bit.
13129e39c5baSBill Taylor 	 *
13139e39c5baSBill Taylor 	 * On the other hand, if the tail is NULL, even though
13149e39c5baSBill Taylor 	 * we will have rung the doorbell for the previous WQE
13159e39c5baSBill Taylor 	 * (for the hardware's sake) it is irrelevant to our
13169e39c5baSBill Taylor 	 * purposes (for tracking WRIDs) because we know the
13179e39c5baSBill Taylor 	 * request must have already completed.
13189e39c5baSBill Taylor 	 */
13199e39c5baSBill Taylor 	wre_last = wridlist->wl_wre_old_tail;
13209e39c5baSBill Taylor 	if (wre_last != NULL) {
13219e39c5baSBill Taylor 		wre_last->wr_signaled_dbd |= TAVOR_WRID_ENTRY_DOORBELLED;
13229e39c5baSBill Taylor 	}
13239e39c5baSBill Taylor 
13249e39c5baSBill Taylor 	/* Update some of the state in the QP */
13259e39c5baSBill Taylor 	qp->qp_rq_lastwqeaddr	 = wqe_addr;
13269e39c5baSBill Taylor 	qp->qp_rq_wqhdr->wq_tail = next_tail;
13279e39c5baSBill Taylor 
13289e39c5baSBill Taylor 	/* Update the doorbell record */
13299e39c5baSBill Taylor 	qp->qp_rq_counter++;
13309e39c5baSBill Taylor 	(qp->qp_rq_dbp)[0] = HTOBE_32(qp->qp_rq_counter);
13319e39c5baSBill Taylor 
13329e39c5baSBill Taylor 	dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
13339e39c5baSBill Taylor 
13349e39c5baSBill Taylor 	return (DAT_SUCCESS);
13359e39c5baSBill Taylor }
13369e39c5baSBill Taylor 
13379e39c5baSBill Taylor /*
13389e39c5baSBill Taylor  * dapli_hermon_post_srq()
13399e39c5baSBill Taylor  */
13409e39c5baSBill Taylor /* ARGSUSED */
13419e39c5baSBill Taylor static DAT_RETURN
dapli_hermon_post_srq(DAPL_SRQ * srqp,ibt_recv_wr_t * wr,boolean_t ns)13429e39c5baSBill Taylor dapli_hermon_post_srq(DAPL_SRQ *srqp, ibt_recv_wr_t *wr, boolean_t ns)
13439e39c5baSBill Taylor {
13449e39c5baSBill Taylor 	ib_srq_handle_t			srq;
13459e39c5baSBill Taylor 	DAT_RETURN			status;
13469e39c5baSBill Taylor 	uint32_t			desc;
13479e39c5baSBill Taylor 	uint64_t			*wqe_addr;
13489e39c5baSBill Taylor 	uint32_t			head, next_head, qsize_msk;
13499e39c5baSBill Taylor 	uint32_t			wqe_index;
13509e39c5baSBill Taylor 
13519e39c5baSBill Taylor 
13529e39c5baSBill Taylor 	srq = srqp->srq_handle;
13539e39c5baSBill Taylor 
13549e39c5baSBill Taylor 	/* Grab the lock for the WRID list */
13559e39c5baSBill Taylor 	dapl_os_lock(&srq->srq_wridlist->wl_lock->wrl_lock);
13569e39c5baSBill Taylor 
13579e39c5baSBill Taylor 	/*
13589e39c5baSBill Taylor 	 * For the ibt_recv_wr_t passed in, parse the request and build a
13599e39c5baSBill Taylor 	 * Recv WQE. Link the WQE with the previous WQE and ring the
13609e39c5baSBill Taylor 	 * door bell.
13619e39c5baSBill Taylor 	 */
13629e39c5baSBill Taylor 
13639e39c5baSBill Taylor 	/*
13649e39c5baSBill Taylor 	 * Check for "queue full" condition.  If the queue is already full,
13659e39c5baSBill Taylor 	 * ie. there are no free entries, then no more WQEs can be posted.
13669e39c5baSBill Taylor 	 * So return an error.
13679e39c5baSBill Taylor 	 */
13689e39c5baSBill Taylor 	if (srq->srq_wridlist->wl_freel_entries == 0) {
13699e39c5baSBill Taylor 		dapl_os_unlock(&srq->srq_wridlist->wl_lock->wrl_lock);
13709e39c5baSBill Taylor 		return (DAT_INSUFFICIENT_RESOURCES);
13719e39c5baSBill Taylor 	}
13729e39c5baSBill Taylor 
13739e39c5baSBill Taylor 	/* Save away some initial SRQ state */
13749e39c5baSBill Taylor 	qsize_msk = srq->srq_wridlist->wl_size - 1;
13759e39c5baSBill Taylor 	head	  = srq->srq_wridlist->wl_freel_head;
13769e39c5baSBill Taylor 
13779e39c5baSBill Taylor 	next_head = (head + 1) & qsize_msk;
13789e39c5baSBill Taylor 
13799e39c5baSBill Taylor 	/* Get the descriptor (IO Address) of the WQE to be built */
13809e39c5baSBill Taylor 	desc = srq->srq_wridlist->wl_free_list[head];
13819e39c5baSBill Taylor 
13829e39c5baSBill Taylor 	wqe_index = TAVOR_SRQ_WQ_INDEX(srq->srq_wq_desc_addr, desc,
13839e39c5baSBill Taylor 	    srq->srq_wq_wqesz);
13849e39c5baSBill Taylor 
13859e39c5baSBill Taylor 	/* The user virtual address of the WQE to be built */
13869e39c5baSBill Taylor 	wqe_addr = TAVOR_SRQ_WQ_ENTRY(srq, wqe_index);
13879e39c5baSBill Taylor 
13889e39c5baSBill Taylor 	/*
13899e39c5baSBill Taylor 	 * Call dapli_hermon_wqe_srq_build() to build the WQE at the given
13909e39c5baSBill Taylor 	 * address. This routine uses the information in the
13919e39c5baSBill Taylor 	 * ibt_recv_wr_t and returns the size of the WQE.
13929e39c5baSBill Taylor 	 */
13939e39c5baSBill Taylor 	status = dapli_hermon_wqe_srq_build(srq, wr, wqe_addr);
13949e39c5baSBill Taylor 	if (status != DAT_SUCCESS) {
13959e39c5baSBill Taylor 		dapl_os_unlock(&srq->srq_wridlist->wl_lock->wrl_lock);
13969e39c5baSBill Taylor 		return (status);
13979e39c5baSBill Taylor 	}
13989e39c5baSBill Taylor 
13999e39c5baSBill Taylor 	/*
14009e39c5baSBill Taylor 	 * Add a WRID entry to the WRID list.
14019e39c5baSBill Taylor 	 */
14029e39c5baSBill Taylor 	dapli_tavor_wrid_add_entry_srq(srq, wr->wr_id, wqe_index);
14039e39c5baSBill Taylor 
14049e39c5baSBill Taylor #if 0
14059e39c5baSBill Taylor 	if (srq->srq_wq_lastwqeindex == -1) {
14069e39c5baSBill Taylor 		last_wqe_addr = NULL;
14079e39c5baSBill Taylor 	} else {
14089e39c5baSBill Taylor 		last_wqe_addr = TAVOR_SRQ_WQ_ENTRY(srq,
14099e39c5baSBill Taylor 		    srq->srq_wq_lastwqeindex);
14109e39c5baSBill Taylor 	}
14119e39c5baSBill Taylor 	/*
14129e39c5baSBill Taylor 	 * Now link the chain to the old chain (if there was one)
14139e39c5baSBill Taylor 	 * and update the wqe_counter in the doorbell record.
14149e39c5baSBill Taylor 	 */
14159e39c5baSBill Taylor XXX
14169e39c5baSBill Taylor 	dapli_tavor_wqe_srq_linknext(wqe_addr, ns, desc, last_wqe_addr);
14179e39c5baSBill Taylor #endif
14189e39c5baSBill Taylor 
14199e39c5baSBill Taylor 	/* Update some of the state in the SRQ */
14209e39c5baSBill Taylor 	srq->srq_wq_lastwqeindex	 = wqe_index;
14219e39c5baSBill Taylor 	srq->srq_wridlist->wl_freel_head = next_head;
14229e39c5baSBill Taylor 	srq->srq_wridlist->wl_freel_entries--;
14239e39c5baSBill Taylor 	dapl_os_assert(srq->srq_wridlist->wl_freel_entries <=
14249e39c5baSBill Taylor 	    srq->srq_wridlist->wl_size);
14259e39c5baSBill Taylor 
14269e39c5baSBill Taylor 	/* Update the doorbell record */
14279e39c5baSBill Taylor 	srq->srq_counter++;
14289e39c5baSBill Taylor 	(srq->srq_dbp)[0] = HTOBE_32(srq->srq_counter);
14299e39c5baSBill Taylor 
14309e39c5baSBill Taylor 	dapl_os_unlock(&srq->srq_wridlist->wl_lock->wrl_lock);
14319e39c5baSBill Taylor 
14329e39c5baSBill Taylor 	return (DAT_SUCCESS);
14339e39c5baSBill Taylor }
14349e39c5baSBill Taylor 
14359e39c5baSBill Taylor /*
14369e39c5baSBill Taylor  * dapli_hermon_cq_srq_entries_flush()
14379e39c5baSBill Taylor  */
14389e39c5baSBill Taylor static void
dapli_hermon_cq_srq_entries_flush(ib_qp_handle_t qp)14399e39c5baSBill Taylor dapli_hermon_cq_srq_entries_flush(ib_qp_handle_t qp)
14409e39c5baSBill Taylor {
14419e39c5baSBill Taylor 	ib_cq_handle_t		cq;
14429e39c5baSBill Taylor 	dapls_tavor_workq_hdr_t	*wqhdr;
14439e39c5baSBill Taylor 	tavor_hw_cqe_t		*cqe;
14449e39c5baSBill Taylor 	tavor_hw_cqe_t		*next_cqe;
14459e39c5baSBill Taylor 	uint32_t		cons_indx, tail_cons_indx;
14469e39c5baSBill Taylor 	uint32_t		new_indx, check_indx, indx;
14479e39c5baSBill Taylor 	int			cqe_qpnum, cqe_type;
14489e39c5baSBill Taylor 	int			outstanding_cqes, removed_cqes;
14499e39c5baSBill Taylor 	int			i;
14509e39c5baSBill Taylor 
14519e39c5baSBill Taylor 	/* ASSERT(MUTEX_HELD(&qp->qp_rq_cqhdl->cq_lock)); */
14529e39c5baSBill Taylor 
14539e39c5baSBill Taylor 	cq = qp->qp_rq_cqhdl;
14549e39c5baSBill Taylor 	wqhdr = qp->qp_rq_wqhdr;
14559e39c5baSBill Taylor 
14569e39c5baSBill Taylor 	dapl_os_assert(wqhdr->wq_wrid_post != NULL);
14579e39c5baSBill Taylor 	dapl_os_assert(wqhdr->wq_wrid_post->wl_srq_en != 0);
14589e39c5baSBill Taylor 
14599e39c5baSBill Taylor 	/* Get the consumer index */
14609e39c5baSBill Taylor 	cons_indx = cq->cq_consindx;
14619e39c5baSBill Taylor 
14629e39c5baSBill Taylor 	/* Calculate the pointer to the first CQ entry */
14639e39c5baSBill Taylor 	cqe = &cq->cq_addr[cons_indx];
14649e39c5baSBill Taylor 
14659e39c5baSBill Taylor 	/*
14669e39c5baSBill Taylor 	 * Loop through the CQ looking for entries owned by software.  If an
14679e39c5baSBill Taylor 	 * entry is owned by software then we increment an 'outstanding_cqes'
14689e39c5baSBill Taylor 	 * count to know how many entries total we have on our CQ.  We use this
14699e39c5baSBill Taylor 	 * value further down to know how many entries to loop through looking
14709e39c5baSBill Taylor 	 * for our same QP number.
14719e39c5baSBill Taylor 	 */
14729e39c5baSBill Taylor 	outstanding_cqes = 0;
14739e39c5baSBill Taylor 	tail_cons_indx = cons_indx;
14749e39c5baSBill Taylor 	while (TAVOR_CQE_OWNER_IS_SW(cqe)) {
14759e39c5baSBill Taylor 		/* increment total cqes count */
14769e39c5baSBill Taylor 		outstanding_cqes++;
14779e39c5baSBill Taylor 
14789e39c5baSBill Taylor 		/* increment the consumer index */
14799e39c5baSBill Taylor 		tail_cons_indx = (tail_cons_indx + 1) & cq_wrap_around_mask;
14809e39c5baSBill Taylor 
14819e39c5baSBill Taylor 		/* update the pointer to the next cq entry */
14829e39c5baSBill Taylor 		cqe = &cq->cq_addr[tail_cons_indx];
14839e39c5baSBill Taylor 	}
14849e39c5baSBill Taylor 
14859e39c5baSBill Taylor 	/*
14869e39c5baSBill Taylor 	 * Using the 'tail_cons_indx' that was just set, we now know how many
14879e39c5baSBill Taylor 	 * total CQEs possible there are.  Set the 'check_indx' and the
14889e39c5baSBill Taylor 	 * 'new_indx' to the last entry identified by 'tail_cons_indx'
14899e39c5baSBill Taylor 	 */
14909e39c5baSBill Taylor 	check_indx = new_indx = (tail_cons_indx - 1) & cq_wrap_around_mask;
14919e39c5baSBill Taylor 
14929e39c5baSBill Taylor 	for (i = 0; i < outstanding_cqes; i++) {
14939e39c5baSBill Taylor 		cqe = &cq->cq_addr[check_indx];
14949e39c5baSBill Taylor 
14959e39c5baSBill Taylor 		/* Grab QP number from CQE */
14969e39c5baSBill Taylor 		cqe_qpnum = TAVOR_CQE_QPNUM_GET(cqe);
14979e39c5baSBill Taylor 		cqe_type = HERMON_CQE_SENDRECV_GET(cqe);
14989e39c5baSBill Taylor 
14999e39c5baSBill Taylor 		/*
15009e39c5baSBill Taylor 		 * If the QP number is the same in the CQE as the QP that we
15019e39c5baSBill Taylor 		 * have on this SRQ, then we must free up the entry off the
15029e39c5baSBill Taylor 		 * SRQ.  We also make sure that the completion type is of the
15039e39c5baSBill Taylor 		 * 'TAVOR_COMPLETION_RECV' type.  So any send completions on
15049e39c5baSBill Taylor 		 * this CQ will be left as-is.  The handling of returning
15059e39c5baSBill Taylor 		 * entries back to HW ownership happens further down.
15069e39c5baSBill Taylor 		 */
15079e39c5baSBill Taylor 		if (cqe_qpnum == qp->qp_num &&
15089e39c5baSBill Taylor 		    cqe_type == TAVOR_COMPLETION_RECV) {
15099e39c5baSBill Taylor 			/* Add back to SRQ free list */
15109e39c5baSBill Taylor 			(void) dapli_tavor_wrid_find_match_srq(
15119e39c5baSBill Taylor 			    wqhdr->wq_wrid_post, cqe);
15129e39c5baSBill Taylor 		} else {
15139e39c5baSBill Taylor 			/* Do Copy */
15149e39c5baSBill Taylor 			if (check_indx != new_indx) {
15159e39c5baSBill Taylor 				next_cqe = &cq->cq_addr[new_indx];
15169e39c5baSBill Taylor 				/*
15179e39c5baSBill Taylor 				 * Copy the CQE into the "next_cqe"
15189e39c5baSBill Taylor 				 * pointer.
15199e39c5baSBill Taylor 				 */
15209e39c5baSBill Taylor 				(void) dapl_os_memcpy(next_cqe, cqe,
15219e39c5baSBill Taylor 				    sizeof (tavor_hw_cqe_t));
15229e39c5baSBill Taylor 			}
15239e39c5baSBill Taylor 			new_indx = (new_indx - 1) & cq_wrap_around_mask;
15249e39c5baSBill Taylor 		}
15259e39c5baSBill Taylor 		/* Move index to next CQE to check */
15269e39c5baSBill Taylor 		check_indx = (check_indx - 1) & cq_wrap_around_mask;
15279e39c5baSBill Taylor 	}
15289e39c5baSBill Taylor 
15299e39c5baSBill Taylor 	/* Initialize removed cqes count */
15309e39c5baSBill Taylor 	removed_cqes = 0;
15319e39c5baSBill Taylor 
15329e39c5baSBill Taylor 	/* If an entry was removed */
15339e39c5baSBill Taylor 	if (check_indx != new_indx) {
15349e39c5baSBill Taylor 
15359e39c5baSBill Taylor 		/*
15369e39c5baSBill Taylor 		 * Set current pointer back to the beginning consumer index.
15379e39c5baSBill Taylor 		 * At this point, all unclaimed entries have been copied to the
15389e39c5baSBill Taylor 		 * index specified by 'new_indx'.  This 'new_indx' will be used
15399e39c5baSBill Taylor 		 * as the new consumer index after we mark all freed entries as
15409e39c5baSBill Taylor 		 * having HW ownership.  We do that here.
15419e39c5baSBill Taylor 		 */
15429e39c5baSBill Taylor 
15439e39c5baSBill Taylor 		/* Loop through all entries until we reach our new pointer */
15449e39c5baSBill Taylor 		for (indx = cons_indx; indx <= new_indx;
15459e39c5baSBill Taylor 		    indx = (indx + 1) & cq_wrap_around_mask) {
15469e39c5baSBill Taylor 			removed_cqes++;
15479e39c5baSBill Taylor 			cqe = &cq->cq_addr[indx];
15489e39c5baSBill Taylor 
15499e39c5baSBill Taylor 			/* Reset entry to hardware ownership */
15509e39c5baSBill Taylor 			TAVOR_CQE_OWNER_SET_HW(cqe);
15519e39c5baSBill Taylor 		}
15529e39c5baSBill Taylor 	}
15539e39c5baSBill Taylor 
15549e39c5baSBill Taylor 	/*
15559e39c5baSBill Taylor 	 * Update consumer index to be the 'new_indx'.  This moves it past all
15569e39c5baSBill Taylor 	 * removed entries.  Because 'new_indx' is pointing to the last
15579e39c5baSBill Taylor 	 * previously valid SW owned entry, we add 1 to point the cons_indx to
15589e39c5baSBill Taylor 	 * the first HW owned entry.
15599e39c5baSBill Taylor 	 */
15609e39c5baSBill Taylor 	cons_indx = (new_indx + 1) & cq_wrap_around_mask;
15619e39c5baSBill Taylor 
15629e39c5baSBill Taylor 	/*
15639e39c5baSBill Taylor 	 * Now we only ring the doorbell (to update the consumer index) if
15649e39c5baSBill Taylor 	 * we've actually consumed a CQ entry.  If we found no QP number
15659e39c5baSBill Taylor 	 * matches above, then we would not have removed anything.  So only if
15669e39c5baSBill Taylor 	 * something was removed do we ring the doorbell.
15679e39c5baSBill Taylor 	 */
15689e39c5baSBill Taylor 	if ((removed_cqes != 0) && (cq->cq_consindx != cons_indx)) {
15699e39c5baSBill Taylor 		/*
15709e39c5baSBill Taylor 		 * Update the consumer index in both the CQ handle and the
15719e39c5baSBill Taylor 		 * doorbell record.
15729e39c5baSBill Taylor 		 */
15739e39c5baSBill Taylor 		cq->cq_consindx = cons_indx;
15749e39c5baSBill Taylor 		dapli_hermon_cq_update_ci(cq, cq->cq_poll_dbp);
15759e39c5baSBill Taylor 	}
15769e39c5baSBill Taylor }
15779e39c5baSBill Taylor 
15789e39c5baSBill Taylor static void
dapli_hermon_rq_prelink(caddr_t first,uint32_t desc_off,uint32_t wqesz,uint32_t numwqe,uint32_t nds)15799e39c5baSBill Taylor dapli_hermon_rq_prelink(caddr_t first, uint32_t desc_off, uint32_t wqesz,
15809e39c5baSBill Taylor     uint32_t numwqe, uint32_t nds)
15819e39c5baSBill Taylor {
15829e39c5baSBill Taylor 	int i;
15839e39c5baSBill Taylor 	uint32_t *p = (uint32_t *)(uintptr_t)first;
15849e39c5baSBill Taylor 	uint32_t off = desc_off;
15859e39c5baSBill Taylor 	uint32_t pincr = wqesz / sizeof (uint32_t);
15869e39c5baSBill Taylor 	ibt_wr_ds_t sgl;
15879e39c5baSBill Taylor 
15889e39c5baSBill Taylor 	sgl.ds_va = (ib_vaddr_t)0;
15899e39c5baSBill Taylor 	sgl.ds_key = HERMON_WQE_SGL_INVALID_LKEY;
15909e39c5baSBill Taylor 	sgl.ds_len = (ib_msglen_t)0;
15919e39c5baSBill Taylor 
15929e39c5baSBill Taylor 	for (i = 0; i < numwqe - 1; i++, p += pincr) {
15939e39c5baSBill Taylor 		off += wqesz;
15949e39c5baSBill Taylor 		p[0] = HTOBE_32(off);	/* link curr to next */
15959e39c5baSBill Taylor 		p[1] = nds;		/* nds is 0 for SRQ */
15969e39c5baSBill Taylor 		TAVOR_WQE_BUILD_DATA_SEG((void *)&p[2], &sgl);
15979e39c5baSBill Taylor 	}
15989e39c5baSBill Taylor 	p[0] = HTOBE_32(desc_off); /* link last to first */
15999e39c5baSBill Taylor 	p[1] = nds;
16009e39c5baSBill Taylor 	TAVOR_WQE_BUILD_DATA_SEG((void *)&p[2], &sgl);
16019e39c5baSBill Taylor }
16029e39c5baSBill Taylor 
16039e39c5baSBill Taylor static void
dapli_hermon_sq_init(caddr_t first,uint32_t wqesz,uint32_t numwqe)16049e39c5baSBill Taylor dapli_hermon_sq_init(caddr_t first, uint32_t wqesz, uint32_t numwqe)
16059e39c5baSBill Taylor {
16069e39c5baSBill Taylor 	int i, j;
16079e39c5baSBill Taylor 	uint64_t *wqe = (uint64_t *)(uintptr_t)first;
16089e39c5baSBill Taylor 
16099e39c5baSBill Taylor 	for (i = 0; i < numwqe; i++) {
16109e39c5baSBill Taylor 		for (j = 0; j < wqesz; j += 64, wqe += 8)
16119e39c5baSBill Taylor 			*(uint32_t *)wqe = 0xFFFFFFFF;
16129e39c5baSBill Taylor 	}
16139e39c5baSBill Taylor }
16149e39c5baSBill Taylor 
16159e39c5baSBill Taylor static void
dapli_hermon_qp_init(ib_qp_handle_t qp)16169e39c5baSBill Taylor dapli_hermon_qp_init(ib_qp_handle_t qp)
16179e39c5baSBill Taylor {
16189e39c5baSBill Taylor 	dapli_hermon_sq_init(qp->qp_sq_buf, qp->qp_sq_wqesz, qp->qp_sq_numwqe);
16199e39c5baSBill Taylor 	qp->qp_rq_counter = 0;
16209e39c5baSBill Taylor 	qp->qp_sq_counter = 0;
16219e39c5baSBill Taylor }
16229e39c5baSBill Taylor 
16239e39c5baSBill Taylor static void
dapli_hermon_cq_init(ib_cq_handle_t cq)16249e39c5baSBill Taylor dapli_hermon_cq_init(ib_cq_handle_t cq)
16259e39c5baSBill Taylor {
16269e39c5baSBill Taylor 	uint32_t i;
16279e39c5baSBill Taylor 
16289e39c5baSBill Taylor 	(cq->cq_arm_dbp)[0] = HTOBE_32(1 << 28);
16299e39c5baSBill Taylor 	for (i = 0; (1 << i) < cq->cq_size; i++)
16309e39c5baSBill Taylor 		;
16319e39c5baSBill Taylor 	cq->cq_log_cqsz = i;
16329e39c5baSBill Taylor 	cq->cq_consindx = 0;
16339e39c5baSBill Taylor 
16349e39c5baSBill Taylor 	/* cq_resize -- needs testing */
16359e39c5baSBill Taylor }
16369e39c5baSBill Taylor 
16379e39c5baSBill Taylor static void
dapli_hermon_srq_init(ib_srq_handle_t srq)16389e39c5baSBill Taylor dapli_hermon_srq_init(ib_srq_handle_t srq)
16399e39c5baSBill Taylor {
16409e39c5baSBill Taylor 	/* pre-link the whole shared receive queue */
16419e39c5baSBill Taylor 	dapli_hermon_rq_prelink(srq->srq_addr, srq->srq_wq_desc_addr,
16429e39c5baSBill Taylor 	    srq->srq_wq_wqesz, srq->srq_wq_numwqe, 0);
16439e39c5baSBill Taylor 	srq->srq_counter = 0;
16449e39c5baSBill Taylor 
16459e39c5baSBill Taylor 	/* needs testing */
16469e39c5baSBill Taylor }
16479e39c5baSBill Taylor 
16489e39c5baSBill Taylor void
dapls_init_funcs_hermon(DAPL_HCA * hca_ptr)16499e39c5baSBill Taylor dapls_init_funcs_hermon(DAPL_HCA *hca_ptr)
16509e39c5baSBill Taylor {
16519e39c5baSBill Taylor 	hca_ptr->post_send = dapli_hermon_post_send;
16529e39c5baSBill Taylor 	hca_ptr->post_recv = dapli_hermon_post_recv;
16539e39c5baSBill Taylor 	hca_ptr->post_srq = dapli_hermon_post_srq;
16549e39c5baSBill Taylor 	hca_ptr->cq_peek = dapli_hermon_cq_peek;
16559e39c5baSBill Taylor 	hca_ptr->cq_poll = dapli_hermon_cq_poll;
16569e39c5baSBill Taylor 	hca_ptr->cq_poll_one = dapli_hermon_cq_poll_one;
16579e39c5baSBill Taylor 	hca_ptr->cq_notify = dapli_hermon_cq_notify;
16589e39c5baSBill Taylor 	hca_ptr->srq_flush = dapli_hermon_cq_srq_entries_flush;
16599e39c5baSBill Taylor 	hca_ptr->qp_init = dapli_hermon_qp_init;
16609e39c5baSBill Taylor 	hca_ptr->cq_init = dapli_hermon_cq_init;
16619e39c5baSBill Taylor 	hca_ptr->srq_init = dapli_hermon_srq_init;
16629e39c5baSBill Taylor 	hca_ptr->hermon_resize_cq = 1;
16639e39c5baSBill Taylor 
16649e39c5baSBill Taylor 	(void) pthread_spin_init(&hermon_bf_lock, 0);
16659e39c5baSBill Taylor }
1666