19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
23*17a2b317SBill Taylor  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor  */
259e39c5baSBill Taylor 
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor  * hermon_wr.c
289e39c5baSBill Taylor  *    Hermon Work Request Processing Routines
299e39c5baSBill Taylor  *
309e39c5baSBill Taylor  *    Implements all the routines necessary to provide the PostSend(),
319e39c5baSBill Taylor  *    PostRecv() and PostSRQ() verbs.  Also contains all the code
329e39c5baSBill Taylor  *    necessary to implement the Hermon WRID tracking mechanism.
339e39c5baSBill Taylor  */
349e39c5baSBill Taylor 
359e39c5baSBill Taylor #include <sys/types.h>
369e39c5baSBill Taylor #include <sys/conf.h>
379e39c5baSBill Taylor #include <sys/ddi.h>
389e39c5baSBill Taylor #include <sys/sunddi.h>
399e39c5baSBill Taylor #include <sys/modctl.h>
409e39c5baSBill Taylor #include <sys/avl.h>
419e39c5baSBill Taylor 
429e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
439e39c5baSBill Taylor 
449e39c5baSBill Taylor static uint32_t hermon_wr_get_immediate(ibt_send_wr_t *wr);
459e39c5baSBill Taylor static int hermon_wr_bind_check(hermon_state_t *state, ibt_send_wr_t *wr);
469e39c5baSBill Taylor static int hermon_wqe_send_build(hermon_state_t *state, hermon_qphdl_t qp,
479e39c5baSBill Taylor     ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
489e39c5baSBill Taylor static int hermon_wqe_mlx_build(hermon_state_t *state, hermon_qphdl_t qp,
499e39c5baSBill Taylor     ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
509e39c5baSBill Taylor static void hermon_wqe_headroom(uint_t from, hermon_qphdl_t qp);
519e39c5baSBill Taylor static int hermon_wqe_recv_build(hermon_state_t *state, hermon_qphdl_t qp,
529e39c5baSBill Taylor     ibt_recv_wr_t *wr, uint64_t *desc);
539e39c5baSBill Taylor static int hermon_wqe_srq_build(hermon_state_t *state, hermon_srqhdl_t srq,
549e39c5baSBill Taylor     ibt_recv_wr_t *wr, uint64_t *desc);
559e39c5baSBill Taylor static hermon_workq_avl_t *hermon_wrid_wqavl_find(hermon_cqhdl_t cq, uint_t qpn,
569e39c5baSBill Taylor     uint_t send_or_recv);
579e39c5baSBill Taylor static void hermon_cq_workq_add(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl);
589e39c5baSBill Taylor static void hermon_cq_workq_remove(hermon_cqhdl_t cq,
599e39c5baSBill Taylor     hermon_workq_avl_t *wqavl);
609e39c5baSBill Taylor 
619e39c5baSBill Taylor static	ibt_wr_ds_t	null_sgl = { 0, 0x00000100, 0 };
629e39c5baSBill Taylor 
63c7facc54SBill Taylor /*
64c7facc54SBill Taylor  * Add ability to try to debug RDMA_READ/RDMA_WRITE failures.
65c7facc54SBill Taylor  *
66c7facc54SBill Taylor  *      0x1 - print rkey used during post_send
67c7facc54SBill Taylor  *      0x2 - print sgls used during post_send
68c7facc54SBill Taylor  *	0x4 - print FMR comings and goings
69c7facc54SBill Taylor  */
70c7facc54SBill Taylor int hermon_rdma_debug = 0x0;
71c7facc54SBill Taylor 
729e39c5baSBill Taylor static int
hermon_post_send_ud(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint_t num_wr,uint_t * num_posted)739e39c5baSBill Taylor hermon_post_send_ud(hermon_state_t *state, hermon_qphdl_t qp,
749e39c5baSBill Taylor     ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
759e39c5baSBill Taylor {
769e39c5baSBill Taylor 	hermon_hw_snd_wqe_ud_t		*ud;
779e39c5baSBill Taylor 	hermon_workq_hdr_t		*wq;
789e39c5baSBill Taylor 	hermon_ahhdl_t			ah;
79*17a2b317SBill Taylor 	ibt_wr_rfci_send_t		*rfci;
80*17a2b317SBill Taylor 	ibt_wr_init_send_t		*is;
819e39c5baSBill Taylor 	ibt_ud_dest_t			*dest;
829e39c5baSBill Taylor 	uint64_t			*desc;
839e39c5baSBill Taylor 	uint32_t			desc_sz;
849e39c5baSBill Taylor 	uint32_t			signaled_dbd, solicited;
859e39c5baSBill Taylor 	uint32_t			head, tail, next_tail, qsize_msk;
869e39c5baSBill Taylor 	uint32_t			hdrmwqes;
879e39c5baSBill Taylor 	uint32_t			nopcode, fence, immed_data = 0;
889c865d64SRajkumar Sivaprakasam 	hermon_hw_wqe_sgl_t		*ds, *old_ds;
899e39c5baSBill Taylor 	ibt_wr_ds_t			*sgl;
90*17a2b317SBill Taylor 	int				nds;
919e39c5baSBill Taylor 	int				i, j, last_ds, num_ds, status;
929e39c5baSBill Taylor 	uint32_t			*wqe_start;
939e39c5baSBill Taylor 	int				sectperwqe;
949e39c5baSBill Taylor 	uint_t				posted_cnt = 0;
95*17a2b317SBill Taylor 	int				total_len, strong_order, fc_bits, cksum;
96*17a2b317SBill Taylor 
979e39c5baSBill Taylor 
989e39c5baSBill Taylor 	/* initialize the FMA retry loop */
999e39c5baSBill Taylor 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
1009e39c5baSBill Taylor 
1019e39c5baSBill Taylor 	ASSERT(MUTEX_HELD(&qp->qp_sq_lock));
1029e39c5baSBill Taylor 	_NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock))
1039e39c5baSBill Taylor 
1049e39c5baSBill Taylor 	/* Grab the lock for the WRID list */
1059e39c5baSBill Taylor 	membar_consumer();
1069e39c5baSBill Taylor 
1079e39c5baSBill Taylor 	/* Save away some initial QP state */
1089e39c5baSBill Taylor 	wq = qp->qp_sq_wqhdr;
1099e39c5baSBill Taylor 	qsize_msk = wq->wq_mask;
1109e39c5baSBill Taylor 	hdrmwqes  = qp->qp_sq_hdrmwqes;		/* in WQEs  */
1119e39c5baSBill Taylor 	sectperwqe = 1 << (qp->qp_sq_log_wqesz - 2);
1129e39c5baSBill Taylor 
1139e39c5baSBill Taylor 	tail	  = wq->wq_tail;
1149e39c5baSBill Taylor 	head	  = wq->wq_head;
1159e39c5baSBill Taylor 	status	  = DDI_SUCCESS;
1169e39c5baSBill Taylor 
1179e39c5baSBill Taylor post_next:
1189e39c5baSBill Taylor 	/*
1199e39c5baSBill Taylor 	 * Check for "queue full" condition.  If the queue
1209e39c5baSBill Taylor 	 * is already full, then no more WQEs can be posted.
1219e39c5baSBill Taylor 	 * So break out, ring a doorbell (if necessary) and
1229e39c5baSBill Taylor 	 * return an error
1239e39c5baSBill Taylor 	 */
1249e39c5baSBill Taylor 	if (wq->wq_full != 0) {
1259e39c5baSBill Taylor 		status = IBT_QP_FULL;
1269e39c5baSBill Taylor 		goto done;
1279e39c5baSBill Taylor 	}
1289e39c5baSBill Taylor 
1299e39c5baSBill Taylor 	next_tail = (tail + 1) & qsize_msk;
1309e39c5baSBill Taylor 	if (((tail + hdrmwqes) & qsize_msk) == head) {
1319e39c5baSBill Taylor 		wq->wq_full = 1;
1329e39c5baSBill Taylor 	}
1339e39c5baSBill Taylor 
1349e39c5baSBill Taylor 	desc = HERMON_QP_SQ_ENTRY(qp, tail);
1359e39c5baSBill Taylor 
1369e39c5baSBill Taylor 	nds = wr->wr_nds;
1379e39c5baSBill Taylor 	sgl = wr->wr_sgl;
1389e39c5baSBill Taylor 	num_ds = 0;
139*17a2b317SBill Taylor 	strong_order = 0;
140*17a2b317SBill Taylor 	fc_bits = 0;
141*17a2b317SBill Taylor 	cksum = 0;
1429e39c5baSBill Taylor 
1439e39c5baSBill Taylor 	/*
1449e39c5baSBill Taylor 	 * Build a Send or Send_LSO WQE
1459e39c5baSBill Taylor 	 */
146*17a2b317SBill Taylor 	switch (wr->wr_opcode) {
147*17a2b317SBill Taylor 	case IBT_WRC_SEND_LSO:
148*17a2b317SBill Taylor 		if (wr->wr_trans != IBT_UD_SRV) {
149*17a2b317SBill Taylor 			status = IBT_QP_SRV_TYPE_INVALID;
150*17a2b317SBill Taylor 			goto done;
151*17a2b317SBill Taylor 		}
1529e39c5baSBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_LSO;
153*17a2b317SBill Taylor 		if (wr->wr_flags & IBT_WR_SEND_CKSUM)
154*17a2b317SBill Taylor 			cksum = 0x30;
1559c865d64SRajkumar Sivaprakasam 		if (wr->wr.ud_lso.lso_hdr_sz > 60) {
1569c865d64SRajkumar Sivaprakasam 			nopcode |= (1 << 6);	/* ReRead bit must be set */
1579c865d64SRajkumar Sivaprakasam 		}
1589e39c5baSBill Taylor 		dest = wr->wr.ud_lso.lso_ud_dest;
1599e39c5baSBill Taylor 		ah = (hermon_ahhdl_t)dest->ud_ah;
1609e39c5baSBill Taylor 		if (ah == NULL) {
1619e39c5baSBill Taylor 			status = IBT_AH_HDL_INVALID;
1629e39c5baSBill Taylor 			goto done;
1639e39c5baSBill Taylor 		}
164*17a2b317SBill Taylor 		ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
165*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
166*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud +
167*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ud_t));
1689e39c5baSBill Taylor 		HERMON_WQE_BUILD_UD(qp, ud, ah, dest);
1699e39c5baSBill Taylor 
1709e39c5baSBill Taylor 		total_len = (4 + 0xf + wr->wr.ud_lso.lso_hdr_sz) & ~0xf;
1719e39c5baSBill Taylor 		if ((uintptr_t)ds + total_len + (nds * 16) >
1729e39c5baSBill Taylor 		    (uintptr_t)desc + (1 << qp->qp_sq_log_wqesz)) {
1739e39c5baSBill Taylor 			status = IBT_QP_SGL_LEN_INVALID;
1749e39c5baSBill Taylor 			goto done;
1759e39c5baSBill Taylor 		}
1769e39c5baSBill Taylor 		old_ds = ds;
1779c865d64SRajkumar Sivaprakasam 		bcopy(wr->wr.ud_lso.lso_hdr, (uint32_t *)old_ds + 1,
1789e39c5baSBill Taylor 		    wr->wr.ud_lso.lso_hdr_sz);
1799c865d64SRajkumar Sivaprakasam 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + total_len);
1809c865d64SRajkumar Sivaprakasam 		i = 0;
181*17a2b317SBill Taylor 		break;
182*17a2b317SBill Taylor 
183*17a2b317SBill Taylor 	case IBT_WRC_SEND:
184*17a2b317SBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_SEND;
185*17a2b317SBill Taylor 		if (qp->qp_serv_type == HERMON_QP_UD) {
186*17a2b317SBill Taylor 			if (wr->wr_trans != IBT_UD_SRV) {
187*17a2b317SBill Taylor 				status = IBT_QP_SRV_TYPE_INVALID;
188*17a2b317SBill Taylor 				goto done;
189*17a2b317SBill Taylor 			}
190*17a2b317SBill Taylor 			if (wr->wr_flags & IBT_WR_SEND_CKSUM)
191*17a2b317SBill Taylor 				cksum = 0x30;
192*17a2b317SBill Taylor 			dest = wr->wr.ud.udwr_dest;
193*17a2b317SBill Taylor 		} else if (qp->qp_serv_type == HERMON_QP_RFCI) {
194*17a2b317SBill Taylor 			if (wr->wr_trans != IBT_RFCI_SRV) {
195*17a2b317SBill Taylor 				status = IBT_QP_SRV_TYPE_INVALID;
196*17a2b317SBill Taylor 				goto done;
197*17a2b317SBill Taylor 			}
198*17a2b317SBill Taylor 			rfci = &wr->wr.fc.rfci_send;
199*17a2b317SBill Taylor 			if ((wr->wr_flags & IBT_WR_SEND_FC_CRC) != 0) {
200*17a2b317SBill Taylor 				nopcode |= (rfci->rfci_eof << 16);
201*17a2b317SBill Taylor 				fc_bits = 0x40;	/* set FCRC */
202*17a2b317SBill Taylor 			}
203*17a2b317SBill Taylor 			dest = rfci->rfci_dest;
204*17a2b317SBill Taylor 		} else {
205*17a2b317SBill Taylor 			status = IBT_QP_OP_TYPE_INVALID;
206*17a2b317SBill Taylor 			goto done;
207*17a2b317SBill Taylor 		}
2089e39c5baSBill Taylor 		if (wr->wr_flags & IBT_WR_SEND_IMMED) {
209*17a2b317SBill Taylor 			/* "|=" changes 0xa to 0xb without touching FCEOF */
210*17a2b317SBill Taylor 			nopcode |= HERMON_WQE_SEND_NOPCODE_SENDI;
2119e39c5baSBill Taylor 			immed_data = wr->wr.ud.udwr_immed;
2129e39c5baSBill Taylor 		}
2139e39c5baSBill Taylor 		ah = (hermon_ahhdl_t)dest->ud_ah;
2149e39c5baSBill Taylor 		if (ah == NULL) {
2159e39c5baSBill Taylor 			status = IBT_AH_HDL_INVALID;
2169e39c5baSBill Taylor 			goto done;
2179e39c5baSBill Taylor 		}
218*17a2b317SBill Taylor 		ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
219*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
220*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud +
221*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ud_t));
2229e39c5baSBill Taylor 		HERMON_WQE_BUILD_UD(qp, ud, ah, dest);
2239e39c5baSBill Taylor 		i = 0;
224*17a2b317SBill Taylor 		break;
225*17a2b317SBill Taylor 
226*17a2b317SBill Taylor 	case IBT_WRC_INIT_SEND_FCMD:
227*17a2b317SBill Taylor 		if (qp->qp_serv_type != HERMON_QP_FCMND) {
228*17a2b317SBill Taylor 			status = IBT_QP_OP_TYPE_INVALID;
229*17a2b317SBill Taylor 			goto done;
230*17a2b317SBill Taylor 		}
231*17a2b317SBill Taylor 		if (wr->wr_trans != IBT_FCMD_SRV) {
232*17a2b317SBill Taylor 			status = IBT_QP_SRV_TYPE_INVALID;
233*17a2b317SBill Taylor 			goto done;
234*17a2b317SBill Taylor 		}
235*17a2b317SBill Taylor 		nopcode = HERMON_WQE_FCP_OPCODE_INIT_AND_SEND;
236*17a2b317SBill Taylor 		is = wr->wr.fc.fc_is;
237*17a2b317SBill Taylor 		dest = is->is_ctl.fc_dest;
238*17a2b317SBill Taylor 		ah = (hermon_ahhdl_t)dest->ud_ah;
239*17a2b317SBill Taylor 		if (ah == NULL) {
240*17a2b317SBill Taylor 			status = IBT_AH_HDL_INVALID;
241*17a2b317SBill Taylor 			goto done;
242*17a2b317SBill Taylor 		}
243*17a2b317SBill Taylor 		ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
244*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
245*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud +
246*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ud_t));
247*17a2b317SBill Taylor 		HERMON_WQE_BUILD_UD(qp, ud, ah, dest);
248*17a2b317SBill Taylor 		old_ds = ds;
249*17a2b317SBill Taylor 		/* move ds beyond the FCP-3 Init Segment */
250*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + 0x10);
251*17a2b317SBill Taylor 		i = 0;
252*17a2b317SBill Taylor 		break;
253*17a2b317SBill Taylor 
254*17a2b317SBill Taylor 	case IBT_WRC_FAST_REG_PMR:
255*17a2b317SBill Taylor 	{
256*17a2b317SBill Taylor 		hermon_hw_snd_wqe_frwr_t	*frwr;
257*17a2b317SBill Taylor 
258*17a2b317SBill Taylor 		if (qp->qp_serv_type != HERMON_QP_FCMND) {
259*17a2b317SBill Taylor 			status = IBT_QP_OP_TYPE_INVALID;
260*17a2b317SBill Taylor 			goto done;
261*17a2b317SBill Taylor 		}
262*17a2b317SBill Taylor 		if (wr->wr_trans != IBT_FCMD_SRV) {
263*17a2b317SBill Taylor 			status = IBT_QP_SRV_TYPE_INVALID;
264*17a2b317SBill Taylor 			goto done;
265*17a2b317SBill Taylor 		}
266*17a2b317SBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_FRWR;
267*17a2b317SBill Taylor 		frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc +
268*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
269*17a2b317SBill Taylor 		HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.fc.reg_pmr);
270*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)frwr +
271*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_frwr_t));
272*17a2b317SBill Taylor 		nds = 0;
273*17a2b317SBill Taylor 		strong_order = 0x80;
274*17a2b317SBill Taylor 		break;
275*17a2b317SBill Taylor 	}
276*17a2b317SBill Taylor 
277*17a2b317SBill Taylor #if 0
278*17a2b317SBill Taylor 	/* firmware does not support this */
279*17a2b317SBill Taylor 	case IBT_WRC_LOCAL_INVALIDATE:
280*17a2b317SBill Taylor 	{
281*17a2b317SBill Taylor 		hermon_hw_snd_wqe_local_inv_t	*li;
282*17a2b317SBill Taylor 
283*17a2b317SBill Taylor 		if (qp->qp_serv_type != HERMON_QP_FCMND) {
284*17a2b317SBill Taylor 			status = IBT_QP_OP_TYPE_INVALID;
285*17a2b317SBill Taylor 			goto done;
286*17a2b317SBill Taylor 		}
287*17a2b317SBill Taylor 		if (wr->wr_trans != IBT_FCMD_SRV) {
288*17a2b317SBill Taylor 			status = IBT_QP_SRV_TYPE_INVALID;
289*17a2b317SBill Taylor 			goto done;
290*17a2b317SBill Taylor 		}
291*17a2b317SBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_LCL_INV;
292*17a2b317SBill Taylor 		li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc +
293*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
294*17a2b317SBill Taylor 		HERMON_WQE_BUILD_LI(qp, li, wr->wr.fc.li);
295*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)li +
296*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_local_inv_t));
297*17a2b317SBill Taylor 		nds = 0;
298*17a2b317SBill Taylor 		strong_order = 0x80;
299*17a2b317SBill Taylor 		break;
300*17a2b317SBill Taylor 	}
301*17a2b317SBill Taylor #endif
302*17a2b317SBill Taylor 	default:
3039e39c5baSBill Taylor 		status = IBT_QP_OP_TYPE_INVALID;
3049e39c5baSBill Taylor 		goto done;
3059e39c5baSBill Taylor 	}
3069e39c5baSBill Taylor 
3079e39c5baSBill Taylor 	if (nds > qp->qp_sq_sgl) {
3089e39c5baSBill Taylor 		status = IBT_QP_SGL_LEN_INVALID;
3099e39c5baSBill Taylor 		goto done;
3109e39c5baSBill Taylor 	}
3119e39c5baSBill Taylor 	for (last_ds = num_ds, j = i; j < nds; j++) {
3129e39c5baSBill Taylor 		if (sgl[j].ds_len != 0)
3139e39c5baSBill Taylor 			last_ds++;	/* real last ds of wqe to fill */
3149e39c5baSBill Taylor 	}
3159e39c5baSBill Taylor 	desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4;
3169e39c5baSBill Taylor 	for (j = nds; --j >= i; ) {
3179e39c5baSBill Taylor 		if (sgl[j].ds_len == 0) {
3189e39c5baSBill Taylor 			continue;
3199e39c5baSBill Taylor 		}
3209e39c5baSBill Taylor 
3219e39c5baSBill Taylor 		/*
3229e39c5baSBill Taylor 		 * Fill in the Data Segment(s) for the current WQE, using the
3239e39c5baSBill Taylor 		 * information contained in the scatter-gather list of the
3249e39c5baSBill Taylor 		 * work request.
3259e39c5baSBill Taylor 		 */
3269e39c5baSBill Taylor 		last_ds--;
3279e39c5baSBill Taylor 		HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[j]);
3289e39c5baSBill Taylor 	}
3299e39c5baSBill Taylor 
3309c865d64SRajkumar Sivaprakasam 	membar_producer();
3319c865d64SRajkumar Sivaprakasam 
3329c865d64SRajkumar Sivaprakasam 	if (wr->wr_opcode == IBT_WRC_SEND_LSO) {
3339c865d64SRajkumar Sivaprakasam 		HERMON_WQE_BUILD_LSO(qp, old_ds, wr->wr.ud_lso.lso_mss,
3349c865d64SRajkumar Sivaprakasam 		    wr->wr.ud_lso.lso_hdr_sz);
335*17a2b317SBill Taylor 	} else if (wr->wr_opcode == IBT_WRC_INIT_SEND_FCMD) {
336*17a2b317SBill Taylor 		/* This sits in the STAMP, so must be set after setting SGL */
337*17a2b317SBill Taylor 		HERMON_WQE_BUILD_FCP3_INIT(old_ds, is->is_ctl.fc_frame_ctrl,
338*17a2b317SBill Taylor 		    is->is_cs_priority, is->is_tx_seq_id, is->is_fc_mtu,
339*17a2b317SBill Taylor 		    is->is_dest_id, is->is_op, is->is_rem_exch,
340*17a2b317SBill Taylor 		    is->is_exch_qp_idx);
341*17a2b317SBill Taylor 
342*17a2b317SBill Taylor 		/* The following will be used in HERMON_WQE_SET_CTRL_SEGMENT */
343*17a2b317SBill Taylor 		/* SIT bit in FCP-3 ctrl segment */
344*17a2b317SBill Taylor 		desc_sz |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_SIT) ? 0x80 : 0;
345*17a2b317SBill Taylor 		/* LS bit in FCP-3 ctrl segment */
346*17a2b317SBill Taylor 		fc_bits |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_LAST_SEQ) ?
347*17a2b317SBill Taylor 		    0x10000 : 0;
348*17a2b317SBill Taylor 		fc_bits |= ((is->is_ctl.fc_routing_ctrl & 0xF) << 20) |
349*17a2b317SBill Taylor 		    (is->is_ctl.fc_seq_id << 24);
350*17a2b317SBill Taylor 		immed_data = is->is_ctl.fc_parameter;
3519c865d64SRajkumar Sivaprakasam 	}
3529c865d64SRajkumar Sivaprakasam 
3539e39c5baSBill Taylor 	fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
3549e39c5baSBill Taylor 
3559e39c5baSBill Taylor 	signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
356*17a2b317SBill Taylor 	    (wr->wr_flags & IBT_WR_SEND_SIGNAL)) ? 0xC : 0;
3579e39c5baSBill Taylor 
358*17a2b317SBill Taylor 	solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0;
3599e39c5baSBill Taylor 
3609e39c5baSBill Taylor 	HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data,
361*17a2b317SBill Taylor 	    solicited, signaled_dbd, cksum, qp, strong_order, fc_bits);
3629e39c5baSBill Taylor 
3639e39c5baSBill Taylor 	wq->wq_wrid[tail] = wr->wr_id;
3649e39c5baSBill Taylor 
3659e39c5baSBill Taylor 	tail = next_tail;
3669e39c5baSBill Taylor 
3679e39c5baSBill Taylor 	/* Update some of the state in the QP */
3689e39c5baSBill Taylor 	wq->wq_tail = tail;
3699e39c5baSBill Taylor 
3709e39c5baSBill Taylor 	membar_producer();
3719e39c5baSBill Taylor 
3729e39c5baSBill Taylor 	/* Now set the ownership bit and opcode (first dword). */
3739e39c5baSBill Taylor 	HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode);
3749e39c5baSBill Taylor 
3759e39c5baSBill Taylor 	posted_cnt++;
3769e39c5baSBill Taylor 	if (--num_wr > 0) {
3779e39c5baSBill Taylor 		/* do the invalidate of the headroom */
3789e39c5baSBill Taylor 		wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
3799e39c5baSBill Taylor 		    (tail + hdrmwqes) & qsize_msk);
3809e39c5baSBill Taylor 		for (i = 16; i < sectperwqe; i += 16) {
3819e39c5baSBill Taylor 			wqe_start[i] = 0xFFFFFFFF;
3829e39c5baSBill Taylor 		}
3839e39c5baSBill Taylor 
3849e39c5baSBill Taylor 		wr++;
3859e39c5baSBill Taylor 		goto post_next;
3869e39c5baSBill Taylor 	}
3879e39c5baSBill Taylor done:
3889e39c5baSBill Taylor 	if (posted_cnt != 0) {
3899e39c5baSBill Taylor 		ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
3909e39c5baSBill Taylor 
3919e39c5baSBill Taylor 		membar_producer();
3929e39c5baSBill Taylor 
3939e39c5baSBill Taylor 		/* the FMA retry loop starts for Hermon doorbell register. */
3949e39c5baSBill Taylor 		hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
3959e39c5baSBill Taylor 		    fm_status, fm_test_num);
3969e39c5baSBill Taylor 
3979e39c5baSBill Taylor 		HERMON_UAR_DOORBELL(state, uarhdl,
3989e39c5baSBill Taylor 		    (uint64_t *)(void *)&state->hs_uar->send,
3999e39c5baSBill Taylor 		    (uint64_t)qp->qp_ring);
4009e39c5baSBill Taylor 
4019e39c5baSBill Taylor 		/* the FMA retry loop ends. */
4029e39c5baSBill Taylor 		hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
4039e39c5baSBill Taylor 		    fm_status, fm_test_num);
4049e39c5baSBill Taylor 
4059e39c5baSBill Taylor 		/* do the invalidate of the headroom */
4069e39c5baSBill Taylor 		wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
4079e39c5baSBill Taylor 		    (tail + hdrmwqes) & qsize_msk);
4089e39c5baSBill Taylor 		for (i = 16; i < sectperwqe; i += 16) {
4099e39c5baSBill Taylor 			wqe_start[i] = 0xFFFFFFFF;
4109e39c5baSBill Taylor 		}
4119e39c5baSBill Taylor 	}
4129e39c5baSBill Taylor 	if (num_posted != NULL)
4139e39c5baSBill Taylor 		*num_posted = posted_cnt;
4149e39c5baSBill Taylor 
4159e39c5baSBill Taylor 	mutex_exit(&qp->qp_sq_lock);
4169e39c5baSBill Taylor 
4179e39c5baSBill Taylor 	return (status);
4189e39c5baSBill Taylor 
4199e39c5baSBill Taylor pio_error:
4209e39c5baSBill Taylor 	mutex_exit(&qp->qp_sq_lock);
4219e39c5baSBill Taylor 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
4229e39c5baSBill Taylor 	return (ibc_get_ci_failure(0));
4239e39c5baSBill Taylor }
4249e39c5baSBill Taylor 
4259e39c5baSBill Taylor static int
hermon_post_send_rc(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint_t num_wr,uint_t * num_posted)4269e39c5baSBill Taylor hermon_post_send_rc(hermon_state_t *state, hermon_qphdl_t qp,
4279e39c5baSBill Taylor     ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
4289e39c5baSBill Taylor {
4299e39c5baSBill Taylor 	uint64_t			*desc;
4309e39c5baSBill Taylor 	hermon_workq_hdr_t		*wq;
4319e39c5baSBill Taylor 	uint32_t			desc_sz;
4329e39c5baSBill Taylor 	uint32_t			signaled_dbd, solicited;
4339e39c5baSBill Taylor 	uint32_t			head, tail, next_tail, qsize_msk;
4349e39c5baSBill Taylor 	uint32_t			hdrmwqes;
4359e39c5baSBill Taylor 	int				status;
4369e39c5baSBill Taylor 	uint32_t			nopcode, fence, immed_data = 0;
4379e39c5baSBill Taylor 	hermon_hw_snd_wqe_remaddr_t	*rc;
4389e39c5baSBill Taylor 	hermon_hw_snd_wqe_atomic_t	*at;
4399e39c5baSBill Taylor 	hermon_hw_snd_wqe_bind_t	*bn;
440*17a2b317SBill Taylor 	hermon_hw_snd_wqe_frwr_t	*frwr;
441*17a2b317SBill Taylor 	hermon_hw_snd_wqe_local_inv_t	*li;
4429e39c5baSBill Taylor 	hermon_hw_wqe_sgl_t		*ds;
4439e39c5baSBill Taylor 	ibt_wr_ds_t			*sgl;
444*17a2b317SBill Taylor 	int				nds;
4459e39c5baSBill Taylor 	int				i, last_ds, num_ds;
4469e39c5baSBill Taylor 	uint32_t			*wqe_start;
4479e39c5baSBill Taylor 	int				sectperwqe;
4489e39c5baSBill Taylor 	uint_t				posted_cnt = 0;
449*17a2b317SBill Taylor 	int				strong_order;
450c7facc54SBill Taylor 	int				print_rdma;
451c7facc54SBill Taylor 	int				rlen;
452c7facc54SBill Taylor 	uint32_t			rkey;
453c7facc54SBill Taylor 	uint64_t			raddr;
4549e39c5baSBill Taylor 
4559e39c5baSBill Taylor 	/* initialize the FMA retry loop */
4569e39c5baSBill Taylor 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
4579e39c5baSBill Taylor 
4589e39c5baSBill Taylor 	ASSERT(MUTEX_HELD(&qp->qp_sq_lock));
4599e39c5baSBill Taylor 	_NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock))
4609e39c5baSBill Taylor 
4619e39c5baSBill Taylor 	/* Save away some initial QP state */
4629e39c5baSBill Taylor 	wq = qp->qp_sq_wqhdr;
4639e39c5baSBill Taylor 	qsize_msk = wq->wq_mask;
4649e39c5baSBill Taylor 	hdrmwqes  = qp->qp_sq_hdrmwqes;		/* in WQEs  */
4659e39c5baSBill Taylor 	sectperwqe = 1 << (qp->qp_sq_log_wqesz - 2);
4669e39c5baSBill Taylor 
4679e39c5baSBill Taylor 	tail	  = wq->wq_tail;
4689e39c5baSBill Taylor 	head	  = wq->wq_head;
4699e39c5baSBill Taylor 	status	  = DDI_SUCCESS;
4709e39c5baSBill Taylor 
4719e39c5baSBill Taylor post_next:
472c7facc54SBill Taylor 	print_rdma = 0;
473c7facc54SBill Taylor 	rlen = 0;
474*17a2b317SBill Taylor 	strong_order = 0;
475c7facc54SBill Taylor 
4769e39c5baSBill Taylor 	/*
4779e39c5baSBill Taylor 	 * Check for "queue full" condition.  If the queue
4789e39c5baSBill Taylor 	 * is already full, then no more WQEs can be posted.
4799e39c5baSBill Taylor 	 * So break out, ring a doorbell (if necessary) and
4809e39c5baSBill Taylor 	 * return an error
4819e39c5baSBill Taylor 	 */
4829e39c5baSBill Taylor 	if (wq->wq_full != 0) {
4839e39c5baSBill Taylor 		status = IBT_QP_FULL;
4849e39c5baSBill Taylor 		goto done;
4859e39c5baSBill Taylor 	}
4869e39c5baSBill Taylor 	next_tail = (tail + 1) & qsize_msk;
4879e39c5baSBill Taylor 	if (((tail + hdrmwqes) & qsize_msk) == head) {
4889e39c5baSBill Taylor 		wq->wq_full = 1;
4899e39c5baSBill Taylor 	}
4909e39c5baSBill Taylor 
4919e39c5baSBill Taylor 	desc = HERMON_QP_SQ_ENTRY(qp, tail);
4929e39c5baSBill Taylor 
4939e39c5baSBill Taylor 	ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc +
4949e39c5baSBill Taylor 	    sizeof (hermon_hw_snd_wqe_ctrl_t));
4959e39c5baSBill Taylor 	nds = wr->wr_nds;
4969e39c5baSBill Taylor 	sgl = wr->wr_sgl;
4979e39c5baSBill Taylor 	num_ds = 0;
498*17a2b317SBill Taylor 	if (wr->wr_trans != IBT_RC_SRV) {
499*17a2b317SBill Taylor 		status = IBT_QP_SRV_TYPE_INVALID;
500*17a2b317SBill Taylor 		goto done;
501*17a2b317SBill Taylor 	}
5029e39c5baSBill Taylor 
5039e39c5baSBill Taylor 	/*
5049e39c5baSBill Taylor 	 * Validate the operation type.  For RC requests, we allow
5059e39c5baSBill Taylor 	 * "Send", "RDMA Read", "RDMA Write", various "Atomic"
5069e39c5baSBill Taylor 	 * operations, and memory window "Bind"
5079e39c5baSBill Taylor 	 */
5089e39c5baSBill Taylor 	switch (wr->wr_opcode) {
5099e39c5baSBill Taylor 	default:
5109e39c5baSBill Taylor 		status = IBT_QP_OP_TYPE_INVALID;
5119e39c5baSBill Taylor 		goto done;
5129e39c5baSBill Taylor 
5139e39c5baSBill Taylor 	case IBT_WRC_SEND:
514*17a2b317SBill Taylor 		if (wr->wr_flags & IBT_WR_SEND_REMOTE_INVAL) {
515*17a2b317SBill Taylor 			nopcode = HERMON_WQE_SEND_NOPCODE_SND_INV;
516*17a2b317SBill Taylor 			immed_data = wr->wr.rc.rcwr.send_inval;
517*17a2b317SBill Taylor 		} else if (wr->wr_flags & IBT_WR_SEND_IMMED) {
5189e39c5baSBill Taylor 			nopcode = HERMON_WQE_SEND_NOPCODE_SENDI;
5199e39c5baSBill Taylor 			immed_data = wr->wr.rc.rcwr.send_immed;
5209e39c5baSBill Taylor 		} else {
5219e39c5baSBill Taylor 			nopcode = HERMON_WQE_SEND_NOPCODE_SEND;
5229e39c5baSBill Taylor 		}
5239e39c5baSBill Taylor 		break;
5249e39c5baSBill Taylor 
5259e39c5baSBill Taylor 	/*
5269e39c5baSBill Taylor 	 * If this is an RDMA Read or RDMA Write request, then fill
5279e39c5baSBill Taylor 	 * in the "Remote Address" header fields.
5289e39c5baSBill Taylor 	 */
5299e39c5baSBill Taylor 	case IBT_WRC_RDMAW:
5309e39c5baSBill Taylor 		if (wr->wr_flags & IBT_WR_SEND_IMMED) {
5319e39c5baSBill Taylor 			nopcode = HERMON_WQE_SEND_NOPCODE_RDMAWI;
5329e39c5baSBill Taylor 			immed_data = wr->wr.rc.rcwr.rdma.rdma_immed;
5339e39c5baSBill Taylor 		} else {
5349e39c5baSBill Taylor 			nopcode = HERMON_WQE_SEND_NOPCODE_RDMAW;
5359e39c5baSBill Taylor 		}
5369e39c5baSBill Taylor 		/* FALLTHROUGH */
5379e39c5baSBill Taylor 	case IBT_WRC_RDMAR:
5389e39c5baSBill Taylor 		if (wr->wr_opcode == IBT_WRC_RDMAR)
5399e39c5baSBill Taylor 			nopcode = HERMON_WQE_SEND_NOPCODE_RDMAR;
5409e39c5baSBill Taylor 		rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
5419e39c5baSBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
5429e39c5baSBill Taylor 
5439e39c5baSBill Taylor 		/*
5449e39c5baSBill Taylor 		 * Build the Remote Address Segment for the WQE, using
5459e39c5baSBill Taylor 		 * the information from the RC work request.
5469e39c5baSBill Taylor 		 */
5479e39c5baSBill Taylor 		HERMON_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma);
5489e39c5baSBill Taylor 
549c7facc54SBill Taylor 		if (hermon_rdma_debug) {
550c7facc54SBill Taylor 			print_rdma = hermon_rdma_debug;
551c7facc54SBill Taylor 			rkey = wr->wr.rc.rcwr.rdma.rdma_rkey;
552c7facc54SBill Taylor 			raddr = wr->wr.rc.rcwr.rdma.rdma_raddr;
553c7facc54SBill Taylor 		}
554c7facc54SBill Taylor 
5559e39c5baSBill Taylor 		/* Update "ds" for filling in Data Segments (below) */
5569e39c5baSBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)rc +
5579e39c5baSBill Taylor 		    sizeof (hermon_hw_snd_wqe_remaddr_t));
5589e39c5baSBill Taylor 		break;
5599e39c5baSBill Taylor 
5609e39c5baSBill Taylor 	/*
5619e39c5baSBill Taylor 	 * If this is one of the Atomic type operations (i.e
5629e39c5baSBill Taylor 	 * Compare-Swap or Fetch-Add), then fill in both the "Remote
5639e39c5baSBill Taylor 	 * Address" header fields and the "Atomic" header fields.
5649e39c5baSBill Taylor 	 */
5659e39c5baSBill Taylor 	case IBT_WRC_CSWAP:
5669e39c5baSBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_ATMCS;
5679e39c5baSBill Taylor 		/* FALLTHROUGH */
5689e39c5baSBill Taylor 	case IBT_WRC_FADD:
5699e39c5baSBill Taylor 		if (wr->wr_opcode == IBT_WRC_FADD)
5709e39c5baSBill Taylor 			nopcode = HERMON_WQE_SEND_NOPCODE_ATMFA;
5719e39c5baSBill Taylor 		rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
5729e39c5baSBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
5739e39c5baSBill Taylor 		at = (hermon_hw_snd_wqe_atomic_t *)((uintptr_t)rc +
5749e39c5baSBill Taylor 		    sizeof (hermon_hw_snd_wqe_remaddr_t));
5759e39c5baSBill Taylor 
5769e39c5baSBill Taylor 		/*
5779e39c5baSBill Taylor 		 * Build the Remote Address and Atomic Segments for
5789e39c5baSBill Taylor 		 * the WQE, using the information from the RC Atomic
5799e39c5baSBill Taylor 		 * work request.
5809e39c5baSBill Taylor 		 */
5819e39c5baSBill Taylor 		HERMON_WQE_BUILD_RC_ATOMIC_REMADDR(qp, rc, wr);
5829e39c5baSBill Taylor 		HERMON_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic);
5839e39c5baSBill Taylor 
5849e39c5baSBill Taylor 		/* Update "ds" for filling in Data Segments (below) */
5859e39c5baSBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)at +
5869e39c5baSBill Taylor 		    sizeof (hermon_hw_snd_wqe_atomic_t));
5879e39c5baSBill Taylor 
5889e39c5baSBill Taylor 		/*
5899e39c5baSBill Taylor 		 * Update "nds" and "sgl" because Atomic requests have
5909e39c5baSBill Taylor 		 * only a single Data Segment.
5919e39c5baSBill Taylor 		 */
5929e39c5baSBill Taylor 		nds = 1;
5939e39c5baSBill Taylor 		sgl = wr->wr_sgl;
5949e39c5baSBill Taylor 		break;
5959e39c5baSBill Taylor 
5969e39c5baSBill Taylor 	/*
5979e39c5baSBill Taylor 	 * If this is memory window Bind operation, then we call the
5989e39c5baSBill Taylor 	 * hermon_wr_bind_check() routine to validate the request and
5999e39c5baSBill Taylor 	 * to generate the updated RKey.  If this is successful, then
6009e39c5baSBill Taylor 	 * we fill in the WQE's "Bind" header fields.
6019e39c5baSBill Taylor 	 */
6029e39c5baSBill Taylor 	case IBT_WRC_BIND:
6039e39c5baSBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_BIND;
6049e39c5baSBill Taylor 		status = hermon_wr_bind_check(state, wr);
6059e39c5baSBill Taylor 		if (status != DDI_SUCCESS)
6069e39c5baSBill Taylor 			goto done;
6079e39c5baSBill Taylor 
6089e39c5baSBill Taylor 		bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc +
6099e39c5baSBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
6109e39c5baSBill Taylor 
6119e39c5baSBill Taylor 		/*
6129e39c5baSBill Taylor 		 * Build the Bind Memory Window Segments for the WQE,
6139e39c5baSBill Taylor 		 * using the information from the RC Bind memory
6149e39c5baSBill Taylor 		 * window work request.
6159e39c5baSBill Taylor 		 */
6169e39c5baSBill Taylor 		HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind);
6179e39c5baSBill Taylor 
6189e39c5baSBill Taylor 		/*
6199e39c5baSBill Taylor 		 * Update the "ds" pointer.  Even though the "bind"
6209e39c5baSBill Taylor 		 * operation requires no SGLs, this is necessary to
6219e39c5baSBill Taylor 		 * facilitate the correct descriptor size calculations
6229e39c5baSBill Taylor 		 * (below).
6239e39c5baSBill Taylor 		 */
6249e39c5baSBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)bn +
6259e39c5baSBill Taylor 		    sizeof (hermon_hw_snd_wqe_bind_t));
6269e39c5baSBill Taylor 		nds = 0;
627*17a2b317SBill Taylor 		break;
628*17a2b317SBill Taylor 
629*17a2b317SBill Taylor 	case IBT_WRC_FAST_REG_PMR:
630*17a2b317SBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_FRWR;
631*17a2b317SBill Taylor 		frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc +
632*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
633*17a2b317SBill Taylor 		HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.rc.rcwr.reg_pmr);
634*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)frwr +
635*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_frwr_t));
636*17a2b317SBill Taylor 		nds = 0;
637*17a2b317SBill Taylor 		strong_order = 0x80;
638*17a2b317SBill Taylor 		break;
639*17a2b317SBill Taylor 
640*17a2b317SBill Taylor 	case IBT_WRC_LOCAL_INVALIDATE:
641*17a2b317SBill Taylor 		nopcode = HERMON_WQE_SEND_NOPCODE_LCL_INV;
642*17a2b317SBill Taylor 		li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc +
643*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_ctrl_t));
644*17a2b317SBill Taylor 		HERMON_WQE_BUILD_LI(qp, li, wr->wr.rc.rcwr.li);
645*17a2b317SBill Taylor 		ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)li +
646*17a2b317SBill Taylor 		    sizeof (hermon_hw_snd_wqe_local_inv_t));
647*17a2b317SBill Taylor 		nds = 0;
648*17a2b317SBill Taylor 		strong_order = 0x80;
649*17a2b317SBill Taylor 		break;
6509e39c5baSBill Taylor 	}
6519e39c5baSBill Taylor 
6529e39c5baSBill Taylor 	/*
6539e39c5baSBill Taylor 	 * Now fill in the Data Segments (SGL) for the Send WQE based
6549e39c5baSBill Taylor 	 * on the values setup above (i.e. "sgl", "nds", and the "ds"
6559e39c5baSBill Taylor 	 * pointer. Start by checking for a valid number of SGL entries
6569e39c5baSBill Taylor 	 */
6579e39c5baSBill Taylor 	if (nds > qp->qp_sq_sgl) {
6589e39c5baSBill Taylor 		status = IBT_QP_SGL_LEN_INVALID;
6599e39c5baSBill Taylor 		goto done;
6609e39c5baSBill Taylor 	}
6619e39c5baSBill Taylor 
6629e39c5baSBill Taylor 	for (last_ds = num_ds, i = 0; i < nds; i++) {
6639e39c5baSBill Taylor 		if (sgl[i].ds_len != 0)
6649e39c5baSBill Taylor 			last_ds++;	/* real last ds of wqe to fill */
6659e39c5baSBill Taylor 	}
6669e39c5baSBill Taylor 	desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4;
6679e39c5baSBill Taylor 	for (i = nds; --i >= 0; ) {
6689e39c5baSBill Taylor 		if (sgl[i].ds_len == 0) {
6699e39c5baSBill Taylor 			continue;
6709e39c5baSBill Taylor 		}
671c7facc54SBill Taylor 		rlen += sgl[i].ds_len;
672c7facc54SBill Taylor 		if (print_rdma & 0x2)
673c7facc54SBill Taylor 			IBTF_DPRINTF_L2("rdma", "post: [%d]: laddr %llx  "
674c7facc54SBill Taylor 			    "llen %x", i, sgl[i].ds_va, sgl[i].ds_len);
6759e39c5baSBill Taylor 
6769e39c5baSBill Taylor 		/*
6779e39c5baSBill Taylor 		 * Fill in the Data Segment(s) for the current WQE, using the
6789e39c5baSBill Taylor 		 * information contained in the scatter-gather list of the
6799e39c5baSBill Taylor 		 * work request.
6809e39c5baSBill Taylor 		 */
6819e39c5baSBill Taylor 		last_ds--;
6829e39c5baSBill Taylor 		HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[i]);
6839e39c5baSBill Taylor 	}
684*17a2b317SBill Taylor 	/* ensure RDMA READ does not exceed HCA limit */
685*17a2b317SBill Taylor 	if ((wr->wr_opcode == IBT_WRC_RDMAR) && (desc_sz >
686*17a2b317SBill Taylor 	    state->hs_ibtfinfo.hca_attr->hca_conn_rdma_read_sgl_sz + 2)) {
687*17a2b317SBill Taylor 		status = IBT_QP_SGL_LEN_INVALID;
688*17a2b317SBill Taylor 		goto done;
689*17a2b317SBill Taylor 	}
6909e39c5baSBill Taylor 
691c7facc54SBill Taylor 	if (print_rdma & 0x1) {
692c7facc54SBill Taylor 		IBTF_DPRINTF_L2("rdma", "post: indx %x  rkey %x  raddr %llx  "
693c7facc54SBill Taylor 		    "total len %x", tail, rkey, raddr, rlen);
694c7facc54SBill Taylor 	}
695c7facc54SBill Taylor 
6969e39c5baSBill Taylor 	fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
6979e39c5baSBill Taylor 
6989e39c5baSBill Taylor 	signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
699*17a2b317SBill Taylor 	    (wr->wr_flags & IBT_WR_SEND_SIGNAL)) ? 0xC : 0;
7009e39c5baSBill Taylor 
701*17a2b317SBill Taylor 	solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0;
7029e39c5baSBill Taylor 
7039e39c5baSBill Taylor 	HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data, solicited,
704*17a2b317SBill Taylor 	    signaled_dbd, 0, qp, strong_order, 0);
7059e39c5baSBill Taylor 
7069e39c5baSBill Taylor 	wq->wq_wrid[tail] = wr->wr_id;
7079e39c5baSBill Taylor 
7089e39c5baSBill Taylor 	tail = next_tail;
7099e39c5baSBill Taylor 
7109e39c5baSBill Taylor 	/* Update some of the state in the QP */
7119e39c5baSBill Taylor 	wq->wq_tail = tail;
7129e39c5baSBill Taylor 
7139e39c5baSBill Taylor 	membar_producer();
7149e39c5baSBill Taylor 
7159e39c5baSBill Taylor 	/* Now set the ownership bit of the first one in the chain. */
7169e39c5baSBill Taylor 	HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode);
7179e39c5baSBill Taylor 
7189e39c5baSBill Taylor 	posted_cnt++;
7199e39c5baSBill Taylor 	if (--num_wr > 0) {
7209e39c5baSBill Taylor 		/* do the invalidate of the headroom */
7219e39c5baSBill Taylor 		wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
7229e39c5baSBill Taylor 		    (tail + hdrmwqes) & qsize_msk);
7239e39c5baSBill Taylor 		for (i = 16; i < sectperwqe; i += 16) {
7249e39c5baSBill Taylor 			wqe_start[i] = 0xFFFFFFFF;
7259e39c5baSBill Taylor 		}
7269e39c5baSBill Taylor 
7279e39c5baSBill Taylor 		wr++;
7289e39c5baSBill Taylor 		goto post_next;
7299e39c5baSBill Taylor 	}
7309e39c5baSBill Taylor done:
7319e39c5baSBill Taylor 
7329e39c5baSBill Taylor 	if (posted_cnt != 0) {
7339e39c5baSBill Taylor 		ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
7349e39c5baSBill Taylor 
7359e39c5baSBill Taylor 		membar_producer();
7369e39c5baSBill Taylor 
7379e39c5baSBill Taylor 		/* the FMA retry loop starts for Hermon doorbell register. */
7389e39c5baSBill Taylor 		hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
7399e39c5baSBill Taylor 		    fm_status, fm_test_num);
7409e39c5baSBill Taylor 
7419e39c5baSBill Taylor 		/* Ring the doorbell */
7429e39c5baSBill Taylor 		HERMON_UAR_DOORBELL(state, uarhdl,
7439e39c5baSBill Taylor 		    (uint64_t *)(void *)&state->hs_uar->send,
7449e39c5baSBill Taylor 		    (uint64_t)qp->qp_ring);
7459e39c5baSBill Taylor 
7469e39c5baSBill Taylor 		/* the FMA retry loop ends. */
7479e39c5baSBill Taylor 		hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
7489e39c5baSBill Taylor 		    fm_status, fm_test_num);
7499e39c5baSBill Taylor 
7509e39c5baSBill Taylor 		/* do the invalidate of the headroom */
7519e39c5baSBill Taylor 		wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
7529e39c5baSBill Taylor 		    (tail + hdrmwqes) & qsize_msk);
7539e39c5baSBill Taylor 		for (i = 16; i < sectperwqe; i += 16) {
7549e39c5baSBill Taylor 			wqe_start[i] = 0xFFFFFFFF;
7559e39c5baSBill Taylor 		}
7569e39c5baSBill Taylor 	}
7579e39c5baSBill Taylor 	/*
7589e39c5baSBill Taylor 	 * Update the "num_posted" return value (if necessary).
7599e39c5baSBill Taylor 	 * Then drop the locks and return success.
7609e39c5baSBill Taylor 	 */
7619e39c5baSBill Taylor 	if (num_posted != NULL) {
7629e39c5baSBill Taylor 		*num_posted = posted_cnt;
7639e39c5baSBill Taylor 	}
7649e39c5baSBill Taylor 
7659e39c5baSBill Taylor 	mutex_exit(&qp->qp_sq_lock);
7669e39c5baSBill Taylor 	return (status);
7679e39c5baSBill Taylor 
7689e39c5baSBill Taylor pio_error:
7699e39c5baSBill Taylor 	mutex_exit(&qp->qp_sq_lock);
7709e39c5baSBill Taylor 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
7719e39c5baSBill Taylor 	return (ibc_get_ci_failure(0));
7729e39c5baSBill Taylor }
7739e39c5baSBill Taylor 
7749e39c5baSBill Taylor /*
7759e39c5baSBill Taylor  * hermon_post_send()
7769e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
7779e39c5baSBill Taylor  */
7789e39c5baSBill Taylor int
hermon_post_send(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint_t num_wr,uint_t * num_posted)7799e39c5baSBill Taylor hermon_post_send(hermon_state_t *state, hermon_qphdl_t qp,
7809e39c5baSBill Taylor     ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
7819e39c5baSBill Taylor {
7829e39c5baSBill Taylor 	ibt_send_wr_t 			*curr_wr;
7839e39c5baSBill Taylor 	hermon_workq_hdr_t		*wq;
7849e39c5baSBill Taylor 	hermon_ahhdl_t			ah;
7859e39c5baSBill Taylor 	uint64_t			*desc, *prev;
7869e39c5baSBill Taylor 	uint32_t			desc_sz;
7879e39c5baSBill Taylor 	uint32_t			signaled_dbd, solicited;
7889e39c5baSBill Taylor 	uint32_t			head, tail, next_tail, qsize_msk;
7899e39c5baSBill Taylor 	uint32_t			hdrmwqes;
7909e39c5baSBill Taylor 	uint_t				currindx, wrindx, numremain;
7919e39c5baSBill Taylor 	uint_t				chainlen;
7929e39c5baSBill Taylor 	uint_t				posted_cnt, maxstat;
7939e39c5baSBill Taylor 	uint_t				total_posted;
7949e39c5baSBill Taylor 	int				status;
7959e39c5baSBill Taylor 	uint32_t			nopcode, fence, immed_data = 0;
7969e39c5baSBill Taylor 	uint32_t			prev_nopcode;
797*17a2b317SBill Taylor 	uint_t				qp_state;
7989e39c5baSBill Taylor 
7999e39c5baSBill Taylor 	/* initialize the FMA retry loop */
8009e39c5baSBill Taylor 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
8019e39c5baSBill Taylor 
8029e39c5baSBill Taylor 	/*
8039e39c5baSBill Taylor 	 * Check for user-mappable QP memory.  Note:  We do not allow kernel
8049e39c5baSBill Taylor 	 * clients to post to QP memory that is accessible directly by the
8059e39c5baSBill Taylor 	 * user.  If the QP memory is user accessible, then return an error.
8069e39c5baSBill Taylor 	 */
807*17a2b317SBill Taylor 	if (qp->qp_alloc_flags & IBT_QP_USER_MAP) {
8089e39c5baSBill Taylor 		return (IBT_QP_HDL_INVALID);
8099e39c5baSBill Taylor 	}
8109e39c5baSBill Taylor 
811*17a2b317SBill Taylor 	mutex_enter(&qp->qp_sq_lock);
8129e39c5baSBill Taylor 
8139e39c5baSBill Taylor 	/*
8149e39c5baSBill Taylor 	 * Check QP state.  Can not post Send requests from the "Reset",
8159e39c5baSBill Taylor 	 * "Init", or "RTR" states
8169e39c5baSBill Taylor 	 */
817*17a2b317SBill Taylor 	qp_state = qp->qp_state_for_post_send;
818*17a2b317SBill Taylor 	if ((qp_state == HERMON_QP_RESET) ||
819*17a2b317SBill Taylor 	    (qp_state == HERMON_QP_INIT) ||
820*17a2b317SBill Taylor 	    (qp_state == HERMON_QP_RTR)) {
821*17a2b317SBill Taylor 		mutex_exit(&qp->qp_sq_lock);
8229e39c5baSBill Taylor 		return (IBT_QP_STATE_INVALID);
8239e39c5baSBill Taylor 	}
8249e39c5baSBill Taylor 
8259e39c5baSBill Taylor 	if (qp->qp_is_special)
8269e39c5baSBill Taylor 		goto post_many;
8279e39c5baSBill Taylor 
8289e39c5baSBill Taylor 	/* Use these optimized functions most of the time */
829*17a2b317SBill Taylor 	if (qp->qp_type == IBT_UD_RQP) {
8309e39c5baSBill Taylor 		return (hermon_post_send_ud(state, qp, wr, num_wr, num_posted));
831c7facc54SBill Taylor 	}
8329e39c5baSBill Taylor 
833c7facc54SBill Taylor 	if (qp->qp_serv_type == HERMON_QP_RC) {
8349e39c5baSBill Taylor 		return (hermon_post_send_rc(state, qp, wr, num_wr, num_posted));
835c7facc54SBill Taylor 	}
8369e39c5baSBill Taylor 
8379e39c5baSBill Taylor 	if (qp->qp_serv_type == HERMON_QP_UC)
8389e39c5baSBill Taylor 		goto post_many;
8399e39c5baSBill Taylor 
8409e39c5baSBill Taylor 	mutex_exit(&qp->qp_sq_lock);
8419e39c5baSBill Taylor 	return (IBT_QP_SRV_TYPE_INVALID);
8429e39c5baSBill Taylor 
8439e39c5baSBill Taylor post_many:
8449e39c5baSBill Taylor 	/* general loop for non-optimized posting */
8459e39c5baSBill Taylor 
8469e39c5baSBill Taylor 	/* Save away some initial QP state */
8479e39c5baSBill Taylor 	wq = qp->qp_sq_wqhdr;
8489e39c5baSBill Taylor 	qsize_msk = wq->wq_mask;
8499e39c5baSBill Taylor 	tail	  = wq->wq_tail;
8509e39c5baSBill Taylor 	head	  = wq->wq_head;
8519e39c5baSBill Taylor 	hdrmwqes  = qp->qp_sq_hdrmwqes;		/* in WQEs  */
8529e39c5baSBill Taylor 
8539e39c5baSBill Taylor 	/* Initialize posted_cnt */
8549e39c5baSBill Taylor 	posted_cnt = 0;
8559e39c5baSBill Taylor 	total_posted = 0;
8569e39c5baSBill Taylor 
8579e39c5baSBill Taylor 	/*
8589e39c5baSBill Taylor 	 * For each ibt_send_wr_t in the wr[] list passed in, parse the
8599e39c5baSBill Taylor 	 * request and build a Send WQE.  NOTE:  Because we are potentially
8609e39c5baSBill Taylor 	 * building a chain of WQEs to post, we want to build them all first,
8619e39c5baSBill Taylor 	 * and set the valid (HW Ownership) bit on all but the first.
8629e39c5baSBill Taylor 	 * However, we do not want to validate the first one until the
8639e39c5baSBill Taylor 	 * entire chain of WQEs has been built.  Then in the final
8649e39c5baSBill Taylor 	 * we set the valid bit in the first, flush if needed, and as a last
8659e39c5baSBill Taylor 	 * step ring the appropriate doorbell.  NOTE: the doorbell ring may
8669e39c5baSBill Taylor 	 * NOT be needed if the HCA is already processing, but the doorbell
8679e39c5baSBill Taylor 	 * ring will be done regardless. NOTE ALSO:  It is possible for
8689e39c5baSBill Taylor 	 * more Work Requests to be posted than the HW will support at one
8699e39c5baSBill Taylor 	 * shot.  If this happens, we need to be able to post and ring
8709e39c5baSBill Taylor 	 * several chains here until the the entire request is complete.
8719e39c5baSBill Taylor 	 * NOTE ALSO:  the term "chain" is used to differentiate it from
8729e39c5baSBill Taylor 	 * Work Request List passed in; and because that's the terminology