19e39c5baSBill Taylor /*
29e39c5baSBill Taylor * CDDL HEADER START
39e39c5baSBill Taylor *
49e39c5baSBill Taylor * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor *
89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor * See the License for the specific language governing permissions
119e39c5baSBill Taylor * and limitations under the License.
129e39c5baSBill Taylor *
139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor *
199e39c5baSBill Taylor * CDDL HEADER END
209e39c5baSBill Taylor */
219e39c5baSBill Taylor
229e39c5baSBill Taylor /*
23*17a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor */
259e39c5baSBill Taylor
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor * hermon_wr.c
289e39c5baSBill Taylor * Hermon Work Request Processing Routines
299e39c5baSBill Taylor *
309e39c5baSBill Taylor * Implements all the routines necessary to provide the PostSend(),
319e39c5baSBill Taylor * PostRecv() and PostSRQ() verbs. Also contains all the code
329e39c5baSBill Taylor * necessary to implement the Hermon WRID tracking mechanism.
339e39c5baSBill Taylor */
349e39c5baSBill Taylor
359e39c5baSBill Taylor #include <sys/types.h>
369e39c5baSBill Taylor #include <sys/conf.h>
379e39c5baSBill Taylor #include <sys/ddi.h>
389e39c5baSBill Taylor #include <sys/sunddi.h>
399e39c5baSBill Taylor #include <sys/modctl.h>
409e39c5baSBill Taylor #include <sys/avl.h>
419e39c5baSBill Taylor
429e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
439e39c5baSBill Taylor
449e39c5baSBill Taylor static uint32_t hermon_wr_get_immediate(ibt_send_wr_t *wr);
459e39c5baSBill Taylor static int hermon_wr_bind_check(hermon_state_t *state, ibt_send_wr_t *wr);
469e39c5baSBill Taylor static int hermon_wqe_send_build(hermon_state_t *state, hermon_qphdl_t qp,
479e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
489e39c5baSBill Taylor static int hermon_wqe_mlx_build(hermon_state_t *state, hermon_qphdl_t qp,
499e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
509e39c5baSBill Taylor static void hermon_wqe_headroom(uint_t from, hermon_qphdl_t qp);
519e39c5baSBill Taylor static int hermon_wqe_recv_build(hermon_state_t *state, hermon_qphdl_t qp,
529e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc);
539e39c5baSBill Taylor static int hermon_wqe_srq_build(hermon_state_t *state, hermon_srqhdl_t srq,
549e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc);
559e39c5baSBill Taylor static hermon_workq_avl_t *hermon_wrid_wqavl_find(hermon_cqhdl_t cq, uint_t qpn,
569e39c5baSBill Taylor uint_t send_or_recv);
579e39c5baSBill Taylor static void hermon_cq_workq_add(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl);
589e39c5baSBill Taylor static void hermon_cq_workq_remove(hermon_cqhdl_t cq,
599e39c5baSBill Taylor hermon_workq_avl_t *wqavl);
609e39c5baSBill Taylor
619e39c5baSBill Taylor static ibt_wr_ds_t null_sgl = { 0, 0x00000100, 0 };
629e39c5baSBill Taylor
63c7facc54SBill Taylor /*
64c7facc54SBill Taylor * Add ability to try to debug RDMA_READ/RDMA_WRITE failures.
65c7facc54SBill Taylor *
66c7facc54SBill Taylor * 0x1 - print rkey used during post_send
67c7facc54SBill Taylor * 0x2 - print sgls used during post_send
68c7facc54SBill Taylor * 0x4 - print FMR comings and goings
69c7facc54SBill Taylor */
70c7facc54SBill Taylor int hermon_rdma_debug = 0x0;
71c7facc54SBill Taylor
729e39c5baSBill Taylor static int
hermon_post_send_ud(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint_t num_wr,uint_t * num_posted)739e39c5baSBill Taylor hermon_post_send_ud(hermon_state_t *state, hermon_qphdl_t qp,
749e39c5baSBill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
759e39c5baSBill Taylor {
769e39c5baSBill Taylor hermon_hw_snd_wqe_ud_t *ud;
779e39c5baSBill Taylor hermon_workq_hdr_t *wq;
789e39c5baSBill Taylor hermon_ahhdl_t ah;
79*17a2b317SBill Taylor ibt_wr_rfci_send_t *rfci;
80*17a2b317SBill Taylor ibt_wr_init_send_t *is;
819e39c5baSBill Taylor ibt_ud_dest_t *dest;
829e39c5baSBill Taylor uint64_t *desc;
839e39c5baSBill Taylor uint32_t desc_sz;
849e39c5baSBill Taylor uint32_t signaled_dbd, solicited;
859e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk;
869e39c5baSBill Taylor uint32_t hdrmwqes;
879e39c5baSBill Taylor uint32_t nopcode, fence, immed_data = 0;
889c865d64SRajkumar Sivaprakasam hermon_hw_wqe_sgl_t *ds, *old_ds;
899e39c5baSBill Taylor ibt_wr_ds_t *sgl;
90*17a2b317SBill Taylor int nds;
919e39c5baSBill Taylor int i, j, last_ds, num_ds, status;
929e39c5baSBill Taylor uint32_t *wqe_start;
939e39c5baSBill Taylor int sectperwqe;
949e39c5baSBill Taylor uint_t posted_cnt = 0;
95*17a2b317SBill Taylor int total_len, strong_order, fc_bits, cksum;
96*17a2b317SBill Taylor
979e39c5baSBill Taylor
989e39c5baSBill Taylor /* initialize the FMA retry loop */
999e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
1009e39c5baSBill Taylor
1019e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock));
1029e39c5baSBill Taylor _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock))
1039e39c5baSBill Taylor
1049e39c5baSBill Taylor /* Grab the lock for the WRID list */
1059e39c5baSBill Taylor membar_consumer();
1069e39c5baSBill Taylor
1079e39c5baSBill Taylor /* Save away some initial QP state */
1089e39c5baSBill Taylor wq = qp->qp_sq_wqhdr;
1099e39c5baSBill Taylor qsize_msk = wq->wq_mask;
1109e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes; /* in WQEs */
1119e39c5baSBill Taylor sectperwqe = 1 << (qp->qp_sq_log_wqesz - 2);
1129e39c5baSBill Taylor
1139e39c5baSBill Taylor tail = wq->wq_tail;
1149e39c5baSBill Taylor head = wq->wq_head;
1159e39c5baSBill Taylor status = DDI_SUCCESS;
1169e39c5baSBill Taylor
1179e39c5baSBill Taylor post_next:
1189e39c5baSBill Taylor /*
1199e39c5baSBill Taylor * Check for "queue full" condition. If the queue
1209e39c5baSBill Taylor * is already full, then no more WQEs can be posted.
1219e39c5baSBill Taylor * So break out, ring a doorbell (if necessary) and
1229e39c5baSBill Taylor * return an error
1239e39c5baSBill Taylor */
1249e39c5baSBill Taylor if (wq->wq_full != 0) {
1259e39c5baSBill Taylor status = IBT_QP_FULL;
1269e39c5baSBill Taylor goto done;
1279e39c5baSBill Taylor }
1289e39c5baSBill Taylor
1299e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk;
1309e39c5baSBill Taylor if (((tail + hdrmwqes) & qsize_msk) == head) {
1319e39c5baSBill Taylor wq->wq_full = 1;
1329e39c5baSBill Taylor }
1339e39c5baSBill Taylor
1349e39c5baSBill Taylor desc = HERMON_QP_SQ_ENTRY(qp, tail);
1359e39c5baSBill Taylor
1369e39c5baSBill Taylor nds = wr->wr_nds;
1379e39c5baSBill Taylor sgl = wr->wr_sgl;
1389e39c5baSBill Taylor num_ds = 0;
139*17a2b317SBill Taylor strong_order = 0;
140*17a2b317SBill Taylor fc_bits = 0;
141*17a2b317SBill Taylor cksum = 0;
1429e39c5baSBill Taylor
1439e39c5baSBill Taylor /*
1449e39c5baSBill Taylor * Build a Send or Send_LSO WQE
1459e39c5baSBill Taylor */
146*17a2b317SBill Taylor switch (wr->wr_opcode) {
147*17a2b317SBill Taylor case IBT_WRC_SEND_LSO:
148*17a2b317SBill Taylor if (wr->wr_trans != IBT_UD_SRV) {
149*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID;
150*17a2b317SBill Taylor goto done;
151*17a2b317SBill Taylor }
1529e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LSO;
153*17a2b317SBill Taylor if (wr->wr_flags & IBT_WR_SEND_CKSUM)
154*17a2b317SBill Taylor cksum = 0x30;
1559c865d64SRajkumar Sivaprakasam if (wr->wr.ud_lso.lso_hdr_sz > 60) {
1569c865d64SRajkumar Sivaprakasam nopcode |= (1 << 6); /* ReRead bit must be set */
1579c865d64SRajkumar Sivaprakasam }
1589e39c5baSBill Taylor dest = wr->wr.ud_lso.lso_ud_dest;
1599e39c5baSBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah;
1609e39c5baSBill Taylor if (ah == NULL) {
1619e39c5baSBill Taylor status = IBT_AH_HDL_INVALID;
1629e39c5baSBill Taylor goto done;
1639e39c5baSBill Taylor }
164*17a2b317SBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
165*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
166*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud +
167*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ud_t));
1689e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, dest);
1699e39c5baSBill Taylor
1709e39c5baSBill Taylor total_len = (4 + 0xf + wr->wr.ud_lso.lso_hdr_sz) & ~0xf;
1719e39c5baSBill Taylor if ((uintptr_t)ds + total_len + (nds * 16) >
1729e39c5baSBill Taylor (uintptr_t)desc + (1 << qp->qp_sq_log_wqesz)) {
1739e39c5baSBill Taylor status = IBT_QP_SGL_LEN_INVALID;
1749e39c5baSBill Taylor goto done;
1759e39c5baSBill Taylor }
1769e39c5baSBill Taylor old_ds = ds;
1779c865d64SRajkumar Sivaprakasam bcopy(wr->wr.ud_lso.lso_hdr, (uint32_t *)old_ds + 1,
1789e39c5baSBill Taylor wr->wr.ud_lso.lso_hdr_sz);
1799c865d64SRajkumar Sivaprakasam ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + total_len);
1809c865d64SRajkumar Sivaprakasam i = 0;
181*17a2b317SBill Taylor break;
182*17a2b317SBill Taylor
183*17a2b317SBill Taylor case IBT_WRC_SEND:
184*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SEND;
185*17a2b317SBill Taylor if (qp->qp_serv_type == HERMON_QP_UD) {
186*17a2b317SBill Taylor if (wr->wr_trans != IBT_UD_SRV) {
187*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID;
188*17a2b317SBill Taylor goto done;
189*17a2b317SBill Taylor }
190*17a2b317SBill Taylor if (wr->wr_flags & IBT_WR_SEND_CKSUM)
191*17a2b317SBill Taylor cksum = 0x30;
192*17a2b317SBill Taylor dest = wr->wr.ud.udwr_dest;
193*17a2b317SBill Taylor } else if (qp->qp_serv_type == HERMON_QP_RFCI) {
194*17a2b317SBill Taylor if (wr->wr_trans != IBT_RFCI_SRV) {
195*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID;
196*17a2b317SBill Taylor goto done;
197*17a2b317SBill Taylor }
198*17a2b317SBill Taylor rfci = &wr->wr.fc.rfci_send;
199*17a2b317SBill Taylor if ((wr->wr_flags & IBT_WR_SEND_FC_CRC) != 0) {
200*17a2b317SBill Taylor nopcode |= (rfci->rfci_eof << 16);
201*17a2b317SBill Taylor fc_bits = 0x40; /* set FCRC */
202*17a2b317SBill Taylor }
203*17a2b317SBill Taylor dest = rfci->rfci_dest;
204*17a2b317SBill Taylor } else {
205*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID;
206*17a2b317SBill Taylor goto done;
207*17a2b317SBill Taylor }
2089e39c5baSBill Taylor if (wr->wr_flags & IBT_WR_SEND_IMMED) {
209*17a2b317SBill Taylor /* "|=" changes 0xa to 0xb without touching FCEOF */
210*17a2b317SBill Taylor nopcode |= HERMON_WQE_SEND_NOPCODE_SENDI;
2119e39c5baSBill Taylor immed_data = wr->wr.ud.udwr_immed;
2129e39c5baSBill Taylor }
2139e39c5baSBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah;
2149e39c5baSBill Taylor if (ah == NULL) {
2159e39c5baSBill Taylor status = IBT_AH_HDL_INVALID;
2169e39c5baSBill Taylor goto done;
2179e39c5baSBill Taylor }
218*17a2b317SBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
219*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
220*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud +
221*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ud_t));
2229e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, dest);
2239e39c5baSBill Taylor i = 0;
224*17a2b317SBill Taylor break;
225*17a2b317SBill Taylor
226*17a2b317SBill Taylor case IBT_WRC_INIT_SEND_FCMD:
227*17a2b317SBill Taylor if (qp->qp_serv_type != HERMON_QP_FCMND) {
228*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID;
229*17a2b317SBill Taylor goto done;
230*17a2b317SBill Taylor }
231*17a2b317SBill Taylor if (wr->wr_trans != IBT_FCMD_SRV) {
232*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID;
233*17a2b317SBill Taylor goto done;
234*17a2b317SBill Taylor }
235*17a2b317SBill Taylor nopcode = HERMON_WQE_FCP_OPCODE_INIT_AND_SEND;
236*17a2b317SBill Taylor is = wr->wr.fc.fc_is;
237*17a2b317SBill Taylor dest = is->is_ctl.fc_dest;
238*17a2b317SBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah;
239*17a2b317SBill Taylor if (ah == NULL) {
240*17a2b317SBill Taylor status = IBT_AH_HDL_INVALID;
241*17a2b317SBill Taylor goto done;
242*17a2b317SBill Taylor }
243*17a2b317SBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
244*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
245*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud +
246*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ud_t));
247*17a2b317SBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, dest);
248*17a2b317SBill Taylor old_ds = ds;
249*17a2b317SBill Taylor /* move ds beyond the FCP-3 Init Segment */
250*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + 0x10);
251*17a2b317SBill Taylor i = 0;
252*17a2b317SBill Taylor break;
253*17a2b317SBill Taylor
254*17a2b317SBill Taylor case IBT_WRC_FAST_REG_PMR:
255*17a2b317SBill Taylor {
256*17a2b317SBill Taylor hermon_hw_snd_wqe_frwr_t *frwr;
257*17a2b317SBill Taylor
258*17a2b317SBill Taylor if (qp->qp_serv_type != HERMON_QP_FCMND) {
259*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID;
260*17a2b317SBill Taylor goto done;
261*17a2b317SBill Taylor }
262*17a2b317SBill Taylor if (wr->wr_trans != IBT_FCMD_SRV) {
263*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID;
264*17a2b317SBill Taylor goto done;
265*17a2b317SBill Taylor }
266*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_FRWR;
267*17a2b317SBill Taylor frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc +
268*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
269*17a2b317SBill Taylor HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.fc.reg_pmr);
270*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)frwr +
271*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_frwr_t));
272*17a2b317SBill Taylor nds = 0;
273*17a2b317SBill Taylor strong_order = 0x80;
274*17a2b317SBill Taylor break;
275*17a2b317SBill Taylor }
276*17a2b317SBill Taylor
277*17a2b317SBill Taylor #if 0
278*17a2b317SBill Taylor /* firmware does not support this */
279*17a2b317SBill Taylor case IBT_WRC_LOCAL_INVALIDATE:
280*17a2b317SBill Taylor {
281*17a2b317SBill Taylor hermon_hw_snd_wqe_local_inv_t *li;
282*17a2b317SBill Taylor
283*17a2b317SBill Taylor if (qp->qp_serv_type != HERMON_QP_FCMND) {
284*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID;
285*17a2b317SBill Taylor goto done;
286*17a2b317SBill Taylor }
287*17a2b317SBill Taylor if (wr->wr_trans != IBT_FCMD_SRV) {
288*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID;
289*17a2b317SBill Taylor goto done;
290*17a2b317SBill Taylor }
291*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LCL_INV;
292*17a2b317SBill Taylor li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc +
293*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
294*17a2b317SBill Taylor HERMON_WQE_BUILD_LI(qp, li, wr->wr.fc.li);
295*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)li +
296*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_local_inv_t));
297*17a2b317SBill Taylor nds = 0;
298*17a2b317SBill Taylor strong_order = 0x80;
299*17a2b317SBill Taylor break;
300*17a2b317SBill Taylor }
301*17a2b317SBill Taylor #endif
302*17a2b317SBill Taylor default:
3039e39c5baSBill Taylor status = IBT_QP_OP_TYPE_INVALID;
3049e39c5baSBill Taylor goto done;
3059e39c5baSBill Taylor }
3069e39c5baSBill Taylor
3079e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) {
3089e39c5baSBill Taylor status = IBT_QP_SGL_LEN_INVALID;
3099e39c5baSBill Taylor goto done;
3109e39c5baSBill Taylor }
3119e39c5baSBill Taylor for (last_ds = num_ds, j = i; j < nds; j++) {
3129e39c5baSBill Taylor if (sgl[j].ds_len != 0)
3139e39c5baSBill Taylor last_ds++; /* real last ds of wqe to fill */
3149e39c5baSBill Taylor }
3159e39c5baSBill Taylor desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4;
3169e39c5baSBill Taylor for (j = nds; --j >= i; ) {
3179e39c5baSBill Taylor if (sgl[j].ds_len == 0) {
3189e39c5baSBill Taylor continue;
3199e39c5baSBill Taylor }
3209e39c5baSBill Taylor
3219e39c5baSBill Taylor /*
3229e39c5baSBill Taylor * Fill in the Data Segment(s) for the current WQE, using the
3239e39c5baSBill Taylor * information contained in the scatter-gather list of the
3249e39c5baSBill Taylor * work request.
3259e39c5baSBill Taylor */
3269e39c5baSBill Taylor last_ds--;
3279e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[j]);
3289e39c5baSBill Taylor }
3299e39c5baSBill Taylor
3309c865d64SRajkumar Sivaprakasam membar_producer();
3319c865d64SRajkumar Sivaprakasam
3329c865d64SRajkumar Sivaprakasam if (wr->wr_opcode == IBT_WRC_SEND_LSO) {
3339c865d64SRajkumar Sivaprakasam HERMON_WQE_BUILD_LSO(qp, old_ds, wr->wr.ud_lso.lso_mss,
3349c865d64SRajkumar Sivaprakasam wr->wr.ud_lso.lso_hdr_sz);
335*17a2b317SBill Taylor } else if (wr->wr_opcode == IBT_WRC_INIT_SEND_FCMD) {
336*17a2b317SBill Taylor /* This sits in the STAMP, so must be set after setting SGL */
337*17a2b317SBill Taylor HERMON_WQE_BUILD_FCP3_INIT(old_ds, is->is_ctl.fc_frame_ctrl,
338*17a2b317SBill Taylor is->is_cs_priority, is->is_tx_seq_id, is->is_fc_mtu,
339*17a2b317SBill Taylor is->is_dest_id, is->is_op, is->is_rem_exch,
340*17a2b317SBill Taylor is->is_exch_qp_idx);
341*17a2b317SBill Taylor
342*17a2b317SBill Taylor /* The following will be used in HERMON_WQE_SET_CTRL_SEGMENT */
343*17a2b317SBill Taylor /* SIT bit in FCP-3 ctrl segment */
344*17a2b317SBill Taylor desc_sz |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_SIT) ? 0x80 : 0;
345*17a2b317SBill Taylor /* LS bit in FCP-3 ctrl segment */
346*17a2b317SBill Taylor fc_bits |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_LAST_SEQ) ?
347*17a2b317SBill Taylor 0x10000 : 0;
348*17a2b317SBill Taylor fc_bits |= ((is->is_ctl.fc_routing_ctrl & 0xF) << 20) |
349*17a2b317SBill Taylor (is->is_ctl.fc_seq_id << 24);
350*17a2b317SBill Taylor immed_data = is->is_ctl.fc_parameter;
3519c865d64SRajkumar Sivaprakasam }
3529c865d64SRajkumar Sivaprakasam
3539e39c5baSBill Taylor fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
3549e39c5baSBill Taylor
3559e39c5baSBill Taylor signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
356*17a2b317SBill Taylor (wr->wr_flags & IBT_WR_SEND_SIGNAL)) ? 0xC : 0;
3579e39c5baSBill Taylor
358*17a2b317SBill Taylor solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0;
3599e39c5baSBill Taylor
3609e39c5baSBill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data,
361*17a2b317SBill Taylor solicited, signaled_dbd, cksum, qp, strong_order, fc_bits);
3629e39c5baSBill Taylor
3639e39c5baSBill Taylor wq->wq_wrid[tail] = wr->wr_id;
3649e39c5baSBill Taylor
3659e39c5baSBill Taylor tail = next_tail;
3669e39c5baSBill Taylor
3679e39c5baSBill Taylor /* Update some of the state in the QP */
3689e39c5baSBill Taylor wq->wq_tail = tail;
3699e39c5baSBill Taylor
3709e39c5baSBill Taylor membar_producer();
3719e39c5baSBill Taylor
3729e39c5baSBill Taylor /* Now set the ownership bit and opcode (first dword). */
3739e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode);
3749e39c5baSBill Taylor
3759e39c5baSBill Taylor posted_cnt++;
3769e39c5baSBill Taylor if (--num_wr > 0) {
3779e39c5baSBill Taylor /* do the invalidate of the headroom */
3789e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
3799e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk);
3809e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) {
3819e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF;
3829e39c5baSBill Taylor }
3839e39c5baSBill Taylor
3849e39c5baSBill Taylor wr++;
3859e39c5baSBill Taylor goto post_next;
3869e39c5baSBill Taylor }
3879e39c5baSBill Taylor done:
3889e39c5baSBill Taylor if (posted_cnt != 0) {
3899e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
3909e39c5baSBill Taylor
3919e39c5baSBill Taylor membar_producer();
3929e39c5baSBill Taylor
3939e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell register. */
3949e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
3959e39c5baSBill Taylor fm_status, fm_test_num);
3969e39c5baSBill Taylor
3979e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl,
3989e39c5baSBill Taylor (uint64_t *)(void *)&state->hs_uar->send,
3999e39c5baSBill Taylor (uint64_t)qp->qp_ring);
4009e39c5baSBill Taylor
4019e39c5baSBill Taylor /* the FMA retry loop ends. */
4029e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
4039e39c5baSBill Taylor fm_status, fm_test_num);
4049e39c5baSBill Taylor
4059e39c5baSBill Taylor /* do the invalidate of the headroom */
4069e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
4079e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk);
4089e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) {
4099e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF;
4109e39c5baSBill Taylor }
4119e39c5baSBill Taylor }
4129e39c5baSBill Taylor if (num_posted != NULL)
4139e39c5baSBill Taylor *num_posted = posted_cnt;
4149e39c5baSBill Taylor
4159e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock);
4169e39c5baSBill Taylor
4179e39c5baSBill Taylor return (status);
4189e39c5baSBill Taylor
4199e39c5baSBill Taylor pio_error:
4209e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock);
4219e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
4229e39c5baSBill Taylor return (ibc_get_ci_failure(0));
4239e39c5baSBill Taylor }
4249e39c5baSBill Taylor
4259e39c5baSBill Taylor static int
hermon_post_send_rc(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint_t num_wr,uint_t * num_posted)4269e39c5baSBill Taylor hermon_post_send_rc(hermon_state_t *state, hermon_qphdl_t qp,
4279e39c5baSBill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
4289e39c5baSBill Taylor {
4299e39c5baSBill Taylor uint64_t *desc;
4309e39c5baSBill Taylor hermon_workq_hdr_t *wq;
4319e39c5baSBill Taylor uint32_t desc_sz;
4329e39c5baSBill Taylor uint32_t signaled_dbd, solicited;
4339e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk;
4349e39c5baSBill Taylor uint32_t hdrmwqes;
4359e39c5baSBill Taylor int status;
4369e39c5baSBill Taylor uint32_t nopcode, fence, immed_data = 0;
4379e39c5baSBill Taylor hermon_hw_snd_wqe_remaddr_t *rc;
4389e39c5baSBill Taylor hermon_hw_snd_wqe_atomic_t *at;
4399e39c5baSBill Taylor hermon_hw_snd_wqe_bind_t *bn;
440*17a2b317SBill Taylor hermon_hw_snd_wqe_frwr_t *frwr;
441*17a2b317SBill Taylor hermon_hw_snd_wqe_local_inv_t *li;
4429e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds;
4439e39c5baSBill Taylor ibt_wr_ds_t *sgl;
444*17a2b317SBill Taylor int nds;
4459e39c5baSBill Taylor int i, last_ds, num_ds;
4469e39c5baSBill Taylor uint32_t *wqe_start;
4479e39c5baSBill Taylor int sectperwqe;
4489e39c5baSBill Taylor uint_t posted_cnt = 0;
449*17a2b317SBill Taylor int strong_order;
450c7facc54SBill Taylor int print_rdma;
451c7facc54SBill Taylor int rlen;
452c7facc54SBill Taylor uint32_t rkey;
453c7facc54SBill Taylor uint64_t raddr;
4549e39c5baSBill Taylor
4559e39c5baSBill Taylor /* initialize the FMA retry loop */
4569e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
4579e39c5baSBill Taylor
4589e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock));
4599e39c5baSBill Taylor _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock))
4609e39c5baSBill Taylor
4619e39c5baSBill Taylor /* Save away some initial QP state */
4629e39c5baSBill Taylor wq = qp->qp_sq_wqhdr;
4639e39c5baSBill Taylor qsize_msk = wq->wq_mask;
4649e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes; /* in WQEs */
4659e39c5baSBill Taylor sectperwqe = 1 << (qp->qp_sq_log_wqesz - 2);
4669e39c5baSBill Taylor
4679e39c5baSBill Taylor tail = wq->wq_tail;
4689e39c5baSBill Taylor head = wq->wq_head;
4699e39c5baSBill Taylor status = DDI_SUCCESS;
4709e39c5baSBill Taylor
4719e39c5baSBill Taylor post_next:
472c7facc54SBill Taylor print_rdma = 0;
473c7facc54SBill Taylor rlen = 0;
474*17a2b317SBill Taylor strong_order = 0;
475c7facc54SBill Taylor
4769e39c5baSBill Taylor /*
4779e39c5baSBill Taylor * Check for "queue full" condition. If the queue
4789e39c5baSBill Taylor * is already full, then no more WQEs can be posted.
4799e39c5baSBill Taylor * So break out, ring a doorbell (if necessary) and
4809e39c5baSBill Taylor * return an error
4819e39c5baSBill Taylor */
4829e39c5baSBill Taylor if (wq->wq_full != 0) {
4839e39c5baSBill Taylor status = IBT_QP_FULL;
4849e39c5baSBill Taylor goto done;
4859e39c5baSBill Taylor }
4869e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk;
4879e39c5baSBill Taylor if (((tail + hdrmwqes) & qsize_msk) == head) {
4889e39c5baSBill Taylor wq->wq_full = 1;
4899e39c5baSBill Taylor }
4909e39c5baSBill Taylor
4919e39c5baSBill Taylor desc = HERMON_QP_SQ_ENTRY(qp, tail);
4929e39c5baSBill Taylor
4939e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc +
4949e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
4959e39c5baSBill Taylor nds = wr->wr_nds;
4969e39c5baSBill Taylor sgl = wr->wr_sgl;
4979e39c5baSBill Taylor num_ds = 0;
498*17a2b317SBill Taylor if (wr->wr_trans != IBT_RC_SRV) {
499*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID;
500*17a2b317SBill Taylor goto done;
501*17a2b317SBill Taylor }
5029e39c5baSBill Taylor
5039e39c5baSBill Taylor /*
5049e39c5baSBill Taylor * Validate the operation type. For RC requests, we allow
5059e39c5baSBill Taylor * "Send", "RDMA Read", "RDMA Write", various "Atomic"
5069e39c5baSBill Taylor * operations, and memory window "Bind"
5079e39c5baSBill Taylor */
5089e39c5baSBill Taylor switch (wr->wr_opcode) {
5099e39c5baSBill Taylor default:
5109e39c5baSBill Taylor status = IBT_QP_OP_TYPE_INVALID;
5119e39c5baSBill Taylor goto done;
5129e39c5baSBill Taylor
5139e39c5baSBill Taylor case IBT_WRC_SEND:
514*17a2b317SBill Taylor if (wr->wr_flags & IBT_WR_SEND_REMOTE_INVAL) {
515*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SND_INV;
516*17a2b317SBill Taylor immed_data = wr->wr.rc.rcwr.send_inval;
517*17a2b317SBill Taylor } else if (wr->wr_flags & IBT_WR_SEND_IMMED) {
5189e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SENDI;
5199e39c5baSBill Taylor immed_data = wr->wr.rc.rcwr.send_immed;
5209e39c5baSBill Taylor } else {
5219e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SEND;
5229e39c5baSBill Taylor }
5239e39c5baSBill Taylor break;
5249e39c5baSBill Taylor
5259e39c5baSBill Taylor /*
5269e39c5baSBill Taylor * If this is an RDMA Read or RDMA Write request, then fill
5279e39c5baSBill Taylor * in the "Remote Address" header fields.
5289e39c5baSBill Taylor */
5299e39c5baSBill Taylor case IBT_WRC_RDMAW:
5309e39c5baSBill Taylor if (wr->wr_flags & IBT_WR_SEND_IMMED) {
5319e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAWI;
5329e39c5baSBill Taylor immed_data = wr->wr.rc.rcwr.rdma.rdma_immed;
5339e39c5baSBill Taylor } else {
5349e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAW;
5359e39c5baSBill Taylor }
5369e39c5baSBill Taylor /* FALLTHROUGH */
5379e39c5baSBill Taylor case IBT_WRC_RDMAR:
5389e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_RDMAR)
5399e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAR;
5409e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
5419e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
5429e39c5baSBill Taylor
5439e39c5baSBill Taylor /*
5449e39c5baSBill Taylor * Build the Remote Address Segment for the WQE, using
5459e39c5baSBill Taylor * the information from the RC work request.
5469e39c5baSBill Taylor */
5479e39c5baSBill Taylor HERMON_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma);
5489e39c5baSBill Taylor
549c7facc54SBill Taylor if (hermon_rdma_debug) {
550c7facc54SBill Taylor print_rdma = hermon_rdma_debug;
551c7facc54SBill Taylor rkey = wr->wr.rc.rcwr.rdma.rdma_rkey;
552c7facc54SBill Taylor raddr = wr->wr.rc.rcwr.rdma.rdma_raddr;
553c7facc54SBill Taylor }
554c7facc54SBill Taylor
5559e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */
5569e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)rc +
5579e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t));
5589e39c5baSBill Taylor break;
5599e39c5baSBill Taylor
5609e39c5baSBill Taylor /*
5619e39c5baSBill Taylor * If this is one of the Atomic type operations (i.e
5629e39c5baSBill Taylor * Compare-Swap or Fetch-Add), then fill in both the "Remote
5639e39c5baSBill Taylor * Address" header fields and the "Atomic" header fields.
5649e39c5baSBill Taylor */
5659e39c5baSBill Taylor case IBT_WRC_CSWAP:
5669e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMCS;
5679e39c5baSBill Taylor /* FALLTHROUGH */
5689e39c5baSBill Taylor case IBT_WRC_FADD:
5699e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_FADD)
5709e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMFA;
5719e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
5729e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
5739e39c5baSBill Taylor at = (hermon_hw_snd_wqe_atomic_t *)((uintptr_t)rc +
5749e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t));
5759e39c5baSBill Taylor
5769e39c5baSBill Taylor /*
5779e39c5baSBill Taylor * Build the Remote Address and Atomic Segments for
5789e39c5baSBill Taylor * the WQE, using the information from the RC Atomic
5799e39c5baSBill Taylor * work request.
5809e39c5baSBill Taylor */
5819e39c5baSBill Taylor HERMON_WQE_BUILD_RC_ATOMIC_REMADDR(qp, rc, wr);
5829e39c5baSBill Taylor HERMON_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic);
5839e39c5baSBill Taylor
5849e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */
5859e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)at +
5869e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_atomic_t));
5879e39c5baSBill Taylor
5889e39c5baSBill Taylor /*
5899e39c5baSBill Taylor * Update "nds" and "sgl" because Atomic requests have
5909e39c5baSBill Taylor * only a single Data Segment.
5919e39c5baSBill Taylor */
5929e39c5baSBill Taylor nds = 1;
5939e39c5baSBill Taylor sgl = wr->wr_sgl;
5949e39c5baSBill Taylor break;
5959e39c5baSBill Taylor
5969e39c5baSBill Taylor /*
5979e39c5baSBill Taylor * If this is memory window Bind operation, then we call the
5989e39c5baSBill Taylor * hermon_wr_bind_check() routine to validate the request and
5999e39c5baSBill Taylor * to generate the updated RKey. If this is successful, then
6009e39c5baSBill Taylor * we fill in the WQE's "Bind" header fields.
6019e39c5baSBill Taylor */
6029e39c5baSBill Taylor case IBT_WRC_BIND:
6039e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_BIND;
6049e39c5baSBill Taylor status = hermon_wr_bind_check(state, wr);
6059e39c5baSBill Taylor if (status != DDI_SUCCESS)
6069e39c5baSBill Taylor goto done;
6079e39c5baSBill Taylor
6089e39c5baSBill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc +
6099e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
6109e39c5baSBill Taylor
6119e39c5baSBill Taylor /*
6129e39c5baSBill Taylor * Build the Bind Memory Window Segments for the WQE,
6139e39c5baSBill Taylor * using the information from the RC Bind memory
6149e39c5baSBill Taylor * window work request.
6159e39c5baSBill Taylor */
6169e39c5baSBill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind);
6179e39c5baSBill Taylor
6189e39c5baSBill Taylor /*
6199e39c5baSBill Taylor * Update the "ds" pointer. Even though the "bind"
6209e39c5baSBill Taylor * operation requires no SGLs, this is necessary to
6219e39c5baSBill Taylor * facilitate the correct descriptor size calculations
6229e39c5baSBill Taylor * (below).
6239e39c5baSBill Taylor */
6249e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)bn +
6259e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_bind_t));
6269e39c5baSBill Taylor nds = 0;
627*17a2b317SBill Taylor break;
628*17a2b317SBill Taylor
629*17a2b317SBill Taylor case IBT_WRC_FAST_REG_PMR:
630*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_FRWR;
631*17a2b317SBill Taylor frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc +
632*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
633*17a2b317SBill Taylor HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.rc.rcwr.reg_pmr);
634*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)frwr +
635*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_frwr_t));
636*17a2b317SBill Taylor nds = 0;
637*17a2b317SBill Taylor strong_order = 0x80;
638*17a2b317SBill Taylor break;
639*17a2b317SBill Taylor
640*17a2b317SBill Taylor case IBT_WRC_LOCAL_INVALIDATE:
641*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LCL_INV;
642*17a2b317SBill Taylor li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc +
643*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
644*17a2b317SBill Taylor HERMON_WQE_BUILD_LI(qp, li, wr->wr.rc.rcwr.li);
645*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)li +
646*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_local_inv_t));
647*17a2b317SBill Taylor nds = 0;
648*17a2b317SBill Taylor strong_order = 0x80;
649*17a2b317SBill Taylor break;
6509e39c5baSBill Taylor }
6519e39c5baSBill Taylor
6529e39c5baSBill Taylor /*
6539e39c5baSBill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based
6549e39c5baSBill Taylor * on the values setup above (i.e. "sgl", "nds", and the "ds"
6559e39c5baSBill Taylor * pointer. Start by checking for a valid number of SGL entries
6569e39c5baSBill Taylor */
6579e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) {
6589e39c5baSBill Taylor status = IBT_QP_SGL_LEN_INVALID;
6599e39c5baSBill Taylor goto done;
6609e39c5baSBill Taylor }
6619e39c5baSBill Taylor
6629e39c5baSBill Taylor for (last_ds = num_ds, i = 0; i < nds; i++) {
6639e39c5baSBill Taylor if (sgl[i].ds_len != 0)
6649e39c5baSBill Taylor last_ds++; /* real last ds of wqe to fill */
6659e39c5baSBill Taylor }
6669e39c5baSBill Taylor desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4;
6679e39c5baSBill Taylor for (i = nds; --i >= 0; ) {
6689e39c5baSBill Taylor if (sgl[i].ds_len == 0) {
6699e39c5baSBill Taylor continue;
6709e39c5baSBill Taylor }
671c7facc54SBill Taylor rlen += sgl[i].ds_len;
672c7facc54SBill Taylor if (print_rdma & 0x2)
673c7facc54SBill Taylor IBTF_DPRINTF_L2("rdma", "post: [%d]: laddr %llx "
674c7facc54SBill Taylor "llen %x", i, sgl[i].ds_va, sgl[i].ds_len);
6759e39c5baSBill Taylor
6769e39c5baSBill Taylor /*
6779e39c5baSBill Taylor * Fill in the Data Segment(s) for the current WQE, using the
6789e39c5baSBill Taylor * information contained in the scatter-gather list of the
6799e39c5baSBill Taylor * work request.
6809e39c5baSBill Taylor */
6819e39c5baSBill Taylor last_ds--;
6829e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[i]);
6839e39c5baSBill Taylor }
684*17a2b317SBill Taylor /* ensure RDMA READ does not exceed HCA limit */
685*17a2b317SBill Taylor if ((wr->wr_opcode == IBT_WRC_RDMAR) && (desc_sz >
686*17a2b317SBill Taylor state->hs_ibtfinfo.hca_attr->hca_conn_rdma_read_sgl_sz + 2)) {
687*17a2b317SBill Taylor status = IBT_QP_SGL_LEN_INVALID;
688*17a2b317SBill Taylor goto done;
689*17a2b317SBill Taylor }
6909e39c5baSBill Taylor
691c7facc54SBill Taylor if (print_rdma & 0x1) {
692c7facc54SBill Taylor IBTF_DPRINTF_L2("rdma", "post: indx %x rkey %x raddr %llx "
693c7facc54SBill Taylor "total len %x", tail, rkey, raddr, rlen);
694c7facc54SBill Taylor }
695c7facc54SBill Taylor
6969e39c5baSBill Taylor fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
6979e39c5baSBill Taylor
6989e39c5baSBill Taylor signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
699*17a2b317SBill Taylor (wr->wr_flags & IBT_WR_SEND_SIGNAL)) ? 0xC : 0;
7009e39c5baSBill Taylor
701*17a2b317SBill Taylor solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0;
7029e39c5baSBill Taylor
7039e39c5baSBill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data, solicited,
704*17a2b317SBill Taylor signaled_dbd, 0, qp, strong_order, 0);
7059e39c5baSBill Taylor
7069e39c5baSBill Taylor wq->wq_wrid[tail] = wr->wr_id;
7079e39c5baSBill Taylor
7089e39c5baSBill Taylor tail = next_tail;
7099e39c5baSBill Taylor
7109e39c5baSBill Taylor /* Update some of the state in the QP */
7119e39c5baSBill Taylor wq->wq_tail = tail;
7129e39c5baSBill Taylor
7139e39c5baSBill Taylor membar_producer();
7149e39c5baSBill Taylor
7159e39c5baSBill Taylor /* Now set the ownership bit of the first one in the chain. */
7169e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode);
7179e39c5baSBill Taylor
7189e39c5baSBill Taylor posted_cnt++;
7199e39c5baSBill Taylor if (--num_wr > 0) {
7209e39c5baSBill Taylor /* do the invalidate of the headroom */
7219e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
7229e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk);
7239e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) {
7249e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF;
7259e39c5baSBill Taylor }
7269e39c5baSBill Taylor
7279e39c5baSBill Taylor wr++;
7289e39c5baSBill Taylor goto post_next;
7299e39c5baSBill Taylor }
7309e39c5baSBill Taylor done:
7319e39c5baSBill Taylor
7329e39c5baSBill Taylor if (posted_cnt != 0) {
7339e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
7349e39c5baSBill Taylor
7359e39c5baSBill Taylor membar_producer();
7369e39c5baSBill Taylor
7379e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell register. */
7389e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
7399e39c5baSBill Taylor fm_status, fm_test_num);
7409e39c5baSBill Taylor
7419e39c5baSBill Taylor /* Ring the doorbell */
7429e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl,
7439e39c5baSBill Taylor (uint64_t *)(void *)&state->hs_uar->send,
7449e39c5baSBill Taylor (uint64_t)qp->qp_ring);
7459e39c5baSBill Taylor
7469e39c5baSBill Taylor /* the FMA retry loop ends. */
7479e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
7489e39c5baSBill Taylor fm_status, fm_test_num);
7499e39c5baSBill Taylor
7509e39c5baSBill Taylor /* do the invalidate of the headroom */
7519e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp,
7529e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk);
7539e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) {
7549e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF;
7559e39c5baSBill Taylor }
7569e39c5baSBill Taylor }
7579e39c5baSBill Taylor /*
7589e39c5baSBill Taylor * Update the "num_posted" return value (if necessary).
7599e39c5baSBill Taylor * Then drop the locks and return success.
7609e39c5baSBill Taylor */
7619e39c5baSBill Taylor if (num_posted != NULL) {
7629e39c5baSBill Taylor *num_posted = posted_cnt;
7639e39c5baSBill Taylor }
7649e39c5baSBill Taylor
7659e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock);
7669e39c5baSBill Taylor return (status);
7679e39c5baSBill Taylor
7689e39c5baSBill Taylor pio_error:
7699e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock);
7709e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
7719e39c5baSBill Taylor return (ibc_get_ci_failure(0));
7729e39c5baSBill Taylor }
7739e39c5baSBill Taylor
7749e39c5baSBill Taylor /*
7759e39c5baSBill Taylor * hermon_post_send()
7769e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
7779e39c5baSBill Taylor */
7789e39c5baSBill Taylor int
hermon_post_send(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint_t num_wr,uint_t * num_posted)7799e39c5baSBill Taylor hermon_post_send(hermon_state_t *state, hermon_qphdl_t qp,
7809e39c5baSBill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
7819e39c5baSBill Taylor {
7829e39c5baSBill Taylor ibt_send_wr_t *curr_wr;
7839e39c5baSBill Taylor hermon_workq_hdr_t *wq;
7849e39c5baSBill Taylor hermon_ahhdl_t ah;
7859e39c5baSBill Taylor uint64_t *desc, *prev;
7869e39c5baSBill Taylor uint32_t desc_sz;
7879e39c5baSBill Taylor uint32_t signaled_dbd, solicited;
7889e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk;
7899e39c5baSBill Taylor uint32_t hdrmwqes;
7909e39c5baSBill Taylor uint_t currindx, wrindx, numremain;
7919e39c5baSBill Taylor uint_t chainlen;
7929e39c5baSBill Taylor uint_t posted_cnt, maxstat;
7939e39c5baSBill Taylor uint_t total_posted;
7949e39c5baSBill Taylor int status;
7959e39c5baSBill Taylor uint32_t nopcode, fence, immed_data = 0;
7969e39c5baSBill Taylor uint32_t prev_nopcode;
797*17a2b317SBill Taylor uint_t qp_state;
7989e39c5baSBill Taylor
7999e39c5baSBill Taylor /* initialize the FMA retry loop */
8009e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
8019e39c5baSBill Taylor
8029e39c5baSBill Taylor /*
8039e39c5baSBill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
8049e39c5baSBill Taylor * clients to post to QP memory that is accessible directly by the
8059e39c5baSBill Taylor * user. If the QP memory is user accessible, then return an error.
8069e39c5baSBill Taylor */
807*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP) {
8089e39c5baSBill Taylor return (IBT_QP_HDL_INVALID);
8099e39c5baSBill Taylor }
8109e39c5baSBill Taylor
811*17a2b317SBill Taylor mutex_enter(&qp->qp_sq_lock);
8129e39c5baSBill Taylor
8139e39c5baSBill Taylor /*
8149e39c5baSBill Taylor * Check QP state. Can not post Send requests from the "Reset",
8159e39c5baSBill Taylor * "Init", or "RTR" states
8169e39c5baSBill Taylor */
817*17a2b317SBill Taylor qp_state = qp->qp_state_for_post_send;
818*17a2b317SBill Taylor if ((qp_state == HERMON_QP_RESET) ||
819*17a2b317SBill Taylor (qp_state == HERMON_QP_INIT) ||
820*17a2b317SBill Taylor (qp_state == HERMON_QP_RTR)) {
821*17a2b317SBill Taylor mutex_exit(&qp->qp_sq_lock);
8229e39c5baSBill Taylor return (IBT_QP_STATE_INVALID);
8239e39c5baSBill Taylor }
8249e39c5baSBill Taylor
8259e39c5baSBill Taylor if (qp->qp_is_special)
8269e39c5baSBill Taylor goto post_many;
8279e39c5baSBill Taylor
8289e39c5baSBill Taylor /* Use these optimized functions most of the time */
829*17a2b317SBill Taylor if (qp->qp_type == IBT_UD_RQP) {
8309e39c5baSBill Taylor return (hermon_post_send_ud(state, qp, wr, num_wr, num_posted));
831c7facc54SBill Taylor }
8329e39c5baSBill Taylor
833c7facc54SBill Taylor if (qp->qp_serv_type == HERMON_QP_RC) {
8349e39c5baSBill Taylor return (hermon_post_send_rc(state, qp, wr, num_wr, num_posted));
835c7facc54SBill Taylor }
8369e39c5baSBill Taylor
8379e39c5baSBill Taylor if (qp->qp_serv_type == HERMON_QP_UC)
8389e39c5baSBill Taylor goto post_many;
8399e39c5baSBill Taylor
8409e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock);
8419e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID);
8429e39c5baSBill Taylor
8439e39c5baSBill Taylor post_many:
8449e39c5baSBill Taylor /* general loop for non-optimized posting */
8459e39c5baSBill Taylor
8469e39c5baSBill Taylor /* Save away some initial QP state */
8479e39c5baSBill Taylor wq = qp->qp_sq_wqhdr;
8489e39c5baSBill Taylor qsize_msk = wq->wq_mask;
8499e39c5baSBill Taylor tail = wq->wq_tail;
8509e39c5baSBill Taylor head = wq->wq_head;
8519e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes; /* in WQEs */
8529e39c5baSBill Taylor
8539e39c5baSBill Taylor /* Initialize posted_cnt */
8549e39c5baSBill Taylor posted_cnt = 0;
8559e39c5baSBill Taylor total_posted = 0;
8569e39c5baSBill Taylor
8579e39c5baSBill Taylor /*
8589e39c5baSBill Taylor * For each ibt_send_wr_t in the wr[] list passed in, parse the
8599e39c5baSBill Taylor * request and build a Send WQE. NOTE: Because we are potentially
8609e39c5baSBill Taylor * building a chain of WQEs to post, we want to build them all first,
8619e39c5baSBill Taylor * and set the valid (HW Ownership) bit on all but the first.
8629e39c5baSBill Taylor * However, we do not want to validate the first one until the
8639e39c5baSBill Taylor * entire chain of WQEs has been built. Then in the final
8649e39c5baSBill Taylor * we set the valid bit in the first, flush if needed, and as a last
8659e39c5baSBill Taylor * step ring the appropriate doorbell. NOTE: the doorbell ring may
8669e39c5baSBill Taylor * NOT be needed if the HCA is already processing, but the doorbell
8679e39c5baSBill Taylor * ring will be done regardless. NOTE ALSO: It is possible for
8689e39c5baSBill Taylor * more Work Requests to be posted than the HW will support at one
8699e39c5baSBill Taylor * shot. If this happens, we need to be able to post and ring
8709e39c5baSBill Taylor * several chains here until the the entire request is complete.
8719e39c5baSBill Taylor * NOTE ALSO: the term "chain" is used to differentiate it from
8729e39c5baSBill Taylor * Work Request List passed in; and because that's the terminology
8739e39c5baSBill Taylor * from the previous generations of HCA - but the WQEs are not, in fact
8749e39c5baSBill Taylor * chained together for Hermon
8759e39c5baSBill Taylor */
8769e39c5baSBill Taylor
8779e39c5baSBill Taylor wrindx = 0;
8789e39c5baSBill Taylor numremain = num_wr;
8799e39c5baSBill Taylor status = DDI_SUCCESS;
8809e39c5baSBill Taylor while ((wrindx < num_wr) && (status == DDI_SUCCESS)) {
8819e39c5baSBill Taylor /*
8829e39c5baSBill Taylor * For the first WQE on a new chain we need "prev" to point
8839e39c5baSBill Taylor * to the current descriptor.
8849e39c5baSBill Taylor */
8859e39c5baSBill Taylor prev = HERMON_QP_SQ_ENTRY(qp, tail);
8869e39c5baSBill Taylor
8879e39c5baSBill Taylor /*
8889e39c5baSBill Taylor * Break the request up into lists that are less than or
8899e39c5baSBill Taylor * equal to the maximum number of WQEs that can be posted
8909e39c5baSBill Taylor * per doorbell ring - 256 currently
8919e39c5baSBill Taylor */
8929e39c5baSBill Taylor chainlen = (numremain > HERMON_QP_MAXDESC_PER_DB) ?
8939e39c5baSBill Taylor HERMON_QP_MAXDESC_PER_DB : numremain;
8949e39c5baSBill Taylor numremain -= chainlen;
8959e39c5baSBill Taylor
8969e39c5baSBill Taylor for (currindx = 0; currindx < chainlen; currindx++, wrindx++) {
8979e39c5baSBill Taylor /*
8989e39c5baSBill Taylor * Check for "queue full" condition. If the queue
8999e39c5baSBill Taylor * is already full, then no more WQEs can be posted.
9009e39c5baSBill Taylor * So break out, ring a doorbell (if necessary) and
9019e39c5baSBill Taylor * return an error
9029e39c5baSBill Taylor */
9039e39c5baSBill Taylor if (wq->wq_full != 0) {
9049e39c5baSBill Taylor status = IBT_QP_FULL;
9059e39c5baSBill Taylor break;
9069e39c5baSBill Taylor }
9079e39c5baSBill Taylor
9089e39c5baSBill Taylor /*
9099e39c5baSBill Taylor * Increment the "tail index". Check for "queue
9109e39c5baSBill Taylor * full" condition incl. headroom. If we detect that
9119e39c5baSBill Taylor * the current work request is going to fill the work
9129e39c5baSBill Taylor * queue, then we mark this condition and continue.
9139e39c5baSBill Taylor * Don't need >=, because going one-by-one we have to
9149e39c5baSBill Taylor * hit it exactly sooner or later
9159e39c5baSBill Taylor */
9169e39c5baSBill Taylor
9179e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk;
9189e39c5baSBill Taylor if (((tail + hdrmwqes) & qsize_msk) == head) {
9199e39c5baSBill Taylor wq->wq_full = 1;
9209e39c5baSBill Taylor }
9219e39c5baSBill Taylor
9229e39c5baSBill Taylor /*
9239e39c5baSBill Taylor * Get the address of the location where the next
9249e39c5baSBill Taylor * Send WQE should be built
9259e39c5baSBill Taylor */
9269e39c5baSBill Taylor desc = HERMON_QP_SQ_ENTRY(qp, tail);
9279e39c5baSBill Taylor /*
9289e39c5baSBill Taylor * Call hermon_wqe_send_build() to build the WQE
9299e39c5baSBill Taylor * at the given address. This routine uses the
9309e39c5baSBill Taylor * information in the ibt_send_wr_t list (wr[]) and
9319e39c5baSBill Taylor * returns the size of the WQE when it returns.
9329e39c5baSBill Taylor */
9339e39c5baSBill Taylor status = hermon_wqe_send_build(state, qp,
9349e39c5baSBill Taylor &wr[wrindx], desc, &desc_sz);
9359e39c5baSBill Taylor if (status != DDI_SUCCESS) {
9369e39c5baSBill Taylor break;
9379e39c5baSBill Taylor }
9389e39c5baSBill Taylor
9399e39c5baSBill Taylor /*
9409e39c5baSBill Taylor * Now, build the Ctrl Segment based on
9419e39c5baSBill Taylor * what was just done
9429e39c5baSBill Taylor */
9439e39c5baSBill Taylor curr_wr = &wr[wrindx];
9449e39c5baSBill Taylor
9459e39c5baSBill Taylor switch (curr_wr->wr_opcode) {
9469e39c5baSBill Taylor case IBT_WRC_RDMAW:
9479e39c5baSBill Taylor if (curr_wr->wr_flags & IBT_WR_SEND_IMMED) {
9489e39c5baSBill Taylor nopcode =
9499e39c5baSBill Taylor HERMON_WQE_SEND_NOPCODE_RDMAWI;
9509e39c5baSBill Taylor immed_data =
9519e39c5baSBill Taylor hermon_wr_get_immediate(curr_wr);
9529e39c5baSBill Taylor } else {
9539e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAW;
9549e39c5baSBill Taylor }
9559e39c5baSBill Taylor break;
9569e39c5baSBill Taylor
9579e39c5baSBill Taylor case IBT_WRC_SEND:
9589e39c5baSBill Taylor if (curr_wr->wr_flags & IBT_WR_SEND_IMMED) {
9599e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SENDI;
9609e39c5baSBill Taylor immed_data =
9619e39c5baSBill Taylor hermon_wr_get_immediate(curr_wr);
9629e39c5baSBill Taylor } else {
9639e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SEND;
9649e39c5baSBill Taylor }
9659e39c5baSBill Taylor break;
9669e39c5baSBill Taylor
9679e39c5baSBill Taylor case IBT_WRC_SEND_LSO:
9689e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LSO;
9699e39c5baSBill Taylor break;
9709e39c5baSBill Taylor
9719e39c5baSBill Taylor case IBT_WRC_RDMAR:
9729e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAR;
9739e39c5baSBill Taylor break;
9749e39c5baSBill Taylor
9759e39c5baSBill Taylor case IBT_WRC_CSWAP:
9769e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMCS;
9779e39c5baSBill Taylor break;
9789e39c5baSBill Taylor
9799e39c5baSBill Taylor case IBT_WRC_FADD:
9809e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMFA;
9819e39c5baSBill Taylor break;
9829e39c5baSBill Taylor
9839e39c5baSBill Taylor case IBT_WRC_BIND:
9849e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_BIND;
9859e39c5baSBill Taylor break;
9869e39c5baSBill Taylor }
9879e39c5baSBill Taylor
9889e39c5baSBill Taylor fence = (curr_wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
9899e39c5baSBill Taylor
9909e39c5baSBill Taylor /*
9919e39c5baSBill Taylor * now, build up the control segment, leaving the
9929e39c5baSBill Taylor * owner bit as it is
9939e39c5baSBill Taylor */
9949e39c5baSBill Taylor
9959e39c5baSBill Taylor if ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
9969e39c5baSBill Taylor (curr_wr->wr_flags & IBT_WR_SEND_SIGNAL)) {
997*17a2b317SBill Taylor signaled_dbd = 0xC;
9989e39c5baSBill Taylor } else {
9999e39c5baSBill Taylor signaled_dbd = 0;
10009e39c5baSBill Taylor }
10019e39c5baSBill Taylor if (curr_wr->wr_flags & IBT_WR_SEND_SOLICIT)
1002*17a2b317SBill Taylor solicited = 0x2;
10039e39c5baSBill Taylor else
10049e39c5baSBill Taylor solicited = 0;
10059e39c5baSBill Taylor
10069e39c5baSBill Taylor if (qp->qp_is_special) {
1007949b58c7SBill Taylor /* Ensure correctness, set the ReRead bit */
1008949b58c7SBill Taylor nopcode |= (1 << 6);
10099e39c5baSBill Taylor ah = (hermon_ahhdl_t)
10109e39c5baSBill Taylor curr_wr->wr.ud.udwr_dest->ud_ah;
10119e39c5baSBill Taylor mutex_enter(&ah->ah_lock);
10129e39c5baSBill Taylor maxstat = ah->ah_udav->max_stat_rate;
10139e39c5baSBill Taylor HERMON_WQE_SET_MLX_CTRL_SEGMENT(desc, desc_sz,
10149e39c5baSBill Taylor signaled_dbd, maxstat, ah->ah_udav->rlid,
10159e39c5baSBill Taylor qp, ah->ah_udav->sl);
10169e39c5baSBill Taylor mutex_exit(&ah->ah_lock);
10179e39c5baSBill Taylor } else {
10189e39c5baSBill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz,
10199e39c5baSBill Taylor fence, immed_data, solicited,
1020*17a2b317SBill Taylor signaled_dbd, 0, qp, 0, 0);
10219e39c5baSBill Taylor }
10229e39c5baSBill Taylor wq->wq_wrid[tail] = curr_wr->wr_id;
10239e39c5baSBill Taylor
10249e39c5baSBill Taylor /*
10259e39c5baSBill Taylor * If this is not the first descriptor on the current
10269e39c5baSBill Taylor * chain, then set the ownership bit.
10279e39c5baSBill Taylor */
10289e39c5baSBill Taylor if (currindx != 0) { /* not the first */
10299e39c5baSBill Taylor membar_producer();
10309e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp,
10319e39c5baSBill Taylor (uint32_t *)desc, nopcode);
10329e39c5baSBill Taylor } else
10339e39c5baSBill Taylor prev_nopcode = nopcode;
10349e39c5baSBill Taylor
10359e39c5baSBill Taylor /*
10369e39c5baSBill Taylor * Update the current "tail index" and increment
10379e39c5baSBill Taylor * "posted_cnt"
10389e39c5baSBill Taylor */
10399e39c5baSBill Taylor tail = next_tail;
10409e39c5baSBill Taylor posted_cnt++;
10419e39c5baSBill Taylor }
10429e39c5baSBill Taylor
10439e39c5baSBill Taylor /*
10449e39c5baSBill Taylor * If we reach here and there are one or more WQEs which have
10459e39c5baSBill Taylor * been successfully built as a chain, we have to finish up
10469e39c5baSBill Taylor * and prepare them for writing to the HW
10479e39c5baSBill Taylor * The steps are:
10489e39c5baSBill Taylor * 1. do the headroom fixup
10499e39c5baSBill Taylor * 2. add in the size of the headroom for the sync
10509e39c5baSBill Taylor * 3. write the owner bit for the first WQE
10519e39c5baSBill Taylor * 4. sync them
10529e39c5baSBill Taylor * 5. fix up the structures
10539e39c5baSBill Taylor * 6. hit the doorbell in UAR
10549e39c5baSBill Taylor */
10559e39c5baSBill Taylor if (posted_cnt != 0) {
10569e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
10579e39c5baSBill Taylor
10589e39c5baSBill Taylor /* do the invalidate of the headroom */
10599e39c5baSBill Taylor
10609e39c5baSBill Taylor hermon_wqe_headroom(tail, qp);
10619e39c5baSBill Taylor
10629e39c5baSBill Taylor /* Update some of the state in the QP */
10639e39c5baSBill Taylor wq->wq_tail = tail;
10649e39c5baSBill Taylor total_posted += posted_cnt;
10659e39c5baSBill Taylor posted_cnt = 0;
10669e39c5baSBill Taylor
10679e39c5baSBill Taylor membar_producer();
10689e39c5baSBill Taylor
10699e39c5baSBill Taylor /*
10709e39c5baSBill Taylor * Now set the ownership bit of the first
10719e39c5baSBill Taylor * one in the chain
10729e39c5baSBill Taylor */
10739e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)prev,
10749e39c5baSBill Taylor prev_nopcode);
10759e39c5baSBill Taylor
10769e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell. */
10779e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
10789e39c5baSBill Taylor fm_status, fm_test);
10799e39c5baSBill Taylor
10809e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl,
10819e39c5baSBill Taylor (uint64_t *)(void *)&state->hs_uar->send,
10829e39c5baSBill Taylor (uint64_t)qp->qp_ring);
10839e39c5baSBill Taylor
10849e39c5baSBill Taylor /* the FMA retry loop ends. */
10859e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
10869e39c5baSBill Taylor fm_status, fm_test);
10879e39c5baSBill Taylor }
10889e39c5baSBill Taylor }
10899e39c5baSBill Taylor
10909e39c5baSBill Taylor /*
10919e39c5baSBill Taylor * Update the "num_posted" return value (if necessary).
10929e39c5baSBill Taylor * Then drop the locks and return success.
10939e39c5baSBill Taylor */
10949e39c5baSBill Taylor if (num_posted != NULL) {
10959e39c5baSBill Taylor *num_posted = total_posted;
10969e39c5baSBill Taylor }
10979e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock);
10989e39c5baSBill Taylor return (status);
10999e39c5baSBill Taylor
11009e39c5baSBill Taylor pio_error:
11019e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock);
11029e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
11039e39c5baSBill Taylor return (ibc_get_ci_failure(0));
11049e39c5baSBill Taylor }
11059e39c5baSBill Taylor
11069e39c5baSBill Taylor
11079e39c5baSBill Taylor /*
11089e39c5baSBill Taylor * hermon_post_recv()
11099e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
11109e39c5baSBill Taylor */
11119e39c5baSBill Taylor int
hermon_post_recv(hermon_state_t * state,hermon_qphdl_t qp,ibt_recv_wr_t * wr,uint_t num_wr,uint_t * num_posted)11129e39c5baSBill Taylor hermon_post_recv(hermon_state_t *state, hermon_qphdl_t qp,
11139e39c5baSBill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted)
11149e39c5baSBill Taylor {
11159e39c5baSBill Taylor uint64_t *desc;
11169e39c5baSBill Taylor hermon_workq_hdr_t *wq;
11179e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk;
11189e39c5baSBill Taylor uint_t wrindx;
11199e39c5baSBill Taylor uint_t posted_cnt;
11209e39c5baSBill Taylor int status;
11219e39c5baSBill Taylor
11229e39c5baSBill Taylor /*
11239e39c5baSBill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
11249e39c5baSBill Taylor * clients to post to QP memory that is accessible directly by the
11259e39c5baSBill Taylor * user. If the QP memory is user accessible, then return an error.
11269e39c5baSBill Taylor */
1127*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP) {
11289e39c5baSBill Taylor return (IBT_QP_HDL_INVALID);
11299e39c5baSBill Taylor }
11309e39c5baSBill Taylor
11319e39c5baSBill Taylor /* Initialize posted_cnt */
11329e39c5baSBill Taylor posted_cnt = 0;
11339e39c5baSBill Taylor
11349e39c5baSBill Taylor mutex_enter(&qp->qp_lock);
11359e39c5baSBill Taylor
11369e39c5baSBill Taylor /*
11379e39c5baSBill Taylor * Check if QP is associated with an SRQ
11389e39c5baSBill Taylor */
1139*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
11409e39c5baSBill Taylor mutex_exit(&qp->qp_lock);
11419e39c5baSBill Taylor return (IBT_SRQ_IN_USE);
11429e39c5baSBill Taylor }
11439e39c5baSBill Taylor
11449e39c5baSBill Taylor /*
11459e39c5baSBill Taylor * Check QP state. Can not post Recv requests from the "Reset" state
11469e39c5baSBill Taylor */
11479e39c5baSBill Taylor if (qp->qp_state == HERMON_QP_RESET) {
11489e39c5baSBill Taylor mutex_exit(&qp->qp_lock);
11499e39c5baSBill Taylor return (IBT_QP_STATE_INVALID);
11509e39c5baSBill Taylor }
11519e39c5baSBill Taylor
11529e39c5baSBill Taylor /* Check that work request transport type is valid */
1153*17a2b317SBill Taylor if ((qp->qp_type != IBT_UD_RQP) &&
11549e39c5baSBill Taylor (qp->qp_serv_type != HERMON_QP_RC) &&
11559e39c5baSBill Taylor (qp->qp_serv_type != HERMON_QP_UC)) {
11569e39c5baSBill Taylor mutex_exit(&qp->qp_lock);
11579e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID);
11589e39c5baSBill Taylor }
11599e39c5baSBill Taylor
11609e39c5baSBill Taylor /*
11619e39c5baSBill Taylor * Grab the lock for the WRID list, i.e., membar_consumer().
11629e39c5baSBill Taylor * This is not needed because the mutex_enter() above has
11639e39c5baSBill Taylor * the same effect.
11649e39c5baSBill Taylor */
11659e39c5baSBill Taylor
11669e39c5baSBill Taylor /* Save away some initial QP state */
11679e39c5baSBill Taylor wq = qp->qp_rq_wqhdr;
11689e39c5baSBill Taylor qsize_msk = wq->wq_mask;
11699e39c5baSBill Taylor tail = wq->wq_tail;
11709e39c5baSBill Taylor head = wq->wq_head;
11719e39c5baSBill Taylor
11729e39c5baSBill Taylor wrindx = 0;
11739e39c5baSBill Taylor status = DDI_SUCCESS;
11749e39c5baSBill Taylor
11759e39c5baSBill Taylor for (wrindx = 0; wrindx < num_wr; wrindx++) {
11769e39c5baSBill Taylor if (wq->wq_full != 0) {
11779e39c5baSBill Taylor status = IBT_QP_FULL;
11789e39c5baSBill Taylor break;
11799e39c5baSBill Taylor }
11809e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk;
11819e39c5baSBill Taylor if (next_tail == head) {
11829e39c5baSBill Taylor wq->wq_full = 1;
11839e39c5baSBill Taylor }
11849e39c5baSBill Taylor desc = HERMON_QP_RQ_ENTRY(qp, tail);
11859e39c5baSBill Taylor status = hermon_wqe_recv_build(state, qp, &wr[wrindx], desc);
11869e39c5baSBill Taylor if (status != DDI_SUCCESS) {
11879e39c5baSBill Taylor break;
11889e39c5baSBill Taylor }
11899e39c5baSBill Taylor
11909e39c5baSBill Taylor wq->wq_wrid[tail] = wr[wrindx].wr_id;
11919e39c5baSBill Taylor qp->qp_rq_wqecntr++;
11929e39c5baSBill Taylor
11939e39c5baSBill Taylor tail = next_tail;
11949e39c5baSBill Taylor posted_cnt++;
11959e39c5baSBill Taylor }
11969e39c5baSBill Taylor
11979e39c5baSBill Taylor if (posted_cnt != 0) {
11989e39c5baSBill Taylor
11999e39c5baSBill Taylor wq->wq_tail = tail;
12009e39c5baSBill Taylor
12019e39c5baSBill Taylor membar_producer(); /* ensure wrids are visible */
12029e39c5baSBill Taylor
12039e39c5baSBill Taylor /* Update the doorbell record w/ wqecntr */
12049e39c5baSBill Taylor HERMON_UAR_DB_RECORD_WRITE(qp->qp_rq_vdbr,
12059e39c5baSBill Taylor qp->qp_rq_wqecntr & 0xFFFF);
12069e39c5baSBill Taylor }
12079e39c5baSBill Taylor
12089e39c5baSBill Taylor if (num_posted != NULL) {
12099e39c5baSBill Taylor *num_posted = posted_cnt;
12109e39c5baSBill Taylor }
12119e39c5baSBill Taylor
12129e39c5baSBill Taylor
1213*17a2b317SBill Taylor mutex_exit(&qp->qp_lock);
12149e39c5baSBill Taylor return (status);
12159e39c5baSBill Taylor }
12169e39c5baSBill Taylor
12179e39c5baSBill Taylor /*
12189e39c5baSBill Taylor * hermon_post_srq()
12199e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
12209e39c5baSBill Taylor */
12219e39c5baSBill Taylor int
hermon_post_srq(hermon_state_t * state,hermon_srqhdl_t srq,ibt_recv_wr_t * wr,uint_t num_wr,uint_t * num_posted)12229e39c5baSBill Taylor hermon_post_srq(hermon_state_t *state, hermon_srqhdl_t srq,
12239e39c5baSBill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted)
12249e39c5baSBill Taylor {
12259e39c5baSBill Taylor uint64_t *desc;
12269e39c5baSBill Taylor hermon_workq_hdr_t *wq;
12279e39c5baSBill Taylor uint_t indx, wrindx;
12289e39c5baSBill Taylor uint_t posted_cnt;
12299e39c5baSBill Taylor int status;
12309e39c5baSBill Taylor
12319e39c5baSBill Taylor mutex_enter(&srq->srq_lock);
12329e39c5baSBill Taylor
12339e39c5baSBill Taylor /*
12349e39c5baSBill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
12359e39c5baSBill Taylor * clients to post to QP memory that is accessible directly by the
12369e39c5baSBill Taylor * user. If the QP memory is user accessible, then return an error.
12379e39c5baSBill Taylor */
12389e39c5baSBill Taylor if (srq->srq_is_umap) {
12399e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
12409e39c5baSBill Taylor return (IBT_SRQ_HDL_INVALID);
12419e39c5baSBill Taylor }
12429e39c5baSBill Taylor
12439e39c5baSBill Taylor /*
12449e39c5baSBill Taylor * Check SRQ state. Can not post Recv requests when SRQ is in error
12459e39c5baSBill Taylor */
12469e39c5baSBill Taylor if (srq->srq_state == HERMON_SRQ_STATE_ERROR) {
12479e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
12489e39c5baSBill Taylor return (IBT_QP_STATE_INVALID);
12499e39c5baSBill Taylor }
12509e39c5baSBill Taylor
12519e39c5baSBill Taylor status = DDI_SUCCESS;
12529e39c5baSBill Taylor posted_cnt = 0;
12539e39c5baSBill Taylor wq = srq->srq_wq_wqhdr;
12549e39c5baSBill Taylor indx = wq->wq_head;
12559e39c5baSBill Taylor
12569e39c5baSBill Taylor for (wrindx = 0; wrindx < num_wr; wrindx++) {
12579e39c5baSBill Taylor
12589e39c5baSBill Taylor if (indx == wq->wq_tail) {
12599e39c5baSBill Taylor status = IBT_QP_FULL;
12609e39c5baSBill Taylor break;
12619e39c5baSBill Taylor }
12629e39c5baSBill Taylor desc = HERMON_SRQ_WQE_ADDR(srq, indx);
12639e39c5baSBill Taylor
12649e39c5baSBill Taylor wq->wq_wrid[indx] = wr[wrindx].wr_id;
12659e39c5baSBill Taylor
12669e39c5baSBill Taylor status = hermon_wqe_srq_build(state, srq, &wr[wrindx], desc);
12679e39c5baSBill Taylor if (status != DDI_SUCCESS) {
12689e39c5baSBill Taylor break;
12699e39c5baSBill Taylor }
12709e39c5baSBill Taylor
12719e39c5baSBill Taylor posted_cnt++;
12729e39c5baSBill Taylor indx = htons(((uint16_t *)desc)[1]);
12739e39c5baSBill Taylor wq->wq_head = indx;
12749e39c5baSBill Taylor }
12759e39c5baSBill Taylor
12769e39c5baSBill Taylor if (posted_cnt != 0) {
12779e39c5baSBill Taylor
12789e39c5baSBill Taylor srq->srq_wq_wqecntr += posted_cnt;
12799e39c5baSBill Taylor
12809e39c5baSBill Taylor membar_producer(); /* ensure wrids are visible */
12819e39c5baSBill Taylor
12829e39c5baSBill Taylor /* Ring the doorbell w/ wqecntr */
12839e39c5baSBill Taylor HERMON_UAR_DB_RECORD_WRITE(srq->srq_wq_vdbr,
12849e39c5baSBill Taylor srq->srq_wq_wqecntr & 0xFFFF);
12859e39c5baSBill Taylor }
12869e39c5baSBill Taylor
12879e39c5baSBill Taylor if (num_posted != NULL) {
12889e39c5baSBill Taylor *num_posted = posted_cnt;
12899e39c5baSBill Taylor }
12909e39c5baSBill Taylor
12919e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
12929e39c5baSBill Taylor return (status);
12939e39c5baSBill Taylor }
12949e39c5baSBill Taylor
12959e39c5baSBill Taylor
12969e39c5baSBill Taylor /*
12979e39c5baSBill Taylor * hermon_wqe_send_build()
12989e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
12999e39c5baSBill Taylor */
13009e39c5baSBill Taylor static int
hermon_wqe_send_build(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint64_t * desc,uint_t * size)13019e39c5baSBill Taylor hermon_wqe_send_build(hermon_state_t *state, hermon_qphdl_t qp,
13029e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size)
13039e39c5baSBill Taylor {
13049e39c5baSBill Taylor hermon_hw_snd_wqe_ud_t *ud;
13059e39c5baSBill Taylor hermon_hw_snd_wqe_remaddr_t *rc;
13069e39c5baSBill Taylor hermon_hw_snd_wqe_atomic_t *at;
13079e39c5baSBill Taylor hermon_hw_snd_wqe_remaddr_t *uc;
13089e39c5baSBill Taylor hermon_hw_snd_wqe_bind_t *bn;
13099e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds, *old_ds;
13109e39c5baSBill Taylor ibt_ud_dest_t *dest;
13119e39c5baSBill Taylor ibt_wr_ds_t *sgl;
13129e39c5baSBill Taylor hermon_ahhdl_t ah;
13139e39c5baSBill Taylor uint32_t nds;
13149e39c5baSBill Taylor int i, j, last_ds, num_ds, status;
13159e39c5baSBill Taylor int tmpsize;
13169e39c5baSBill Taylor
13179e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock));
13189e39c5baSBill Taylor
13199e39c5baSBill Taylor /* Initialize the information for the Data Segments */
13209e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc +
13219e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
13229e39c5baSBill Taylor nds = wr->wr_nds;
13239e39c5baSBill Taylor sgl = wr->wr_sgl;
13249e39c5baSBill Taylor num_ds = 0;
13259e39c5baSBill Taylor i = 0;
13269e39c5baSBill Taylor
13279e39c5baSBill Taylor /*
13289e39c5baSBill Taylor * Build a Send WQE depends first and foremost on the transport
13299e39c5baSBill Taylor * type of Work Request (i.e. UD, RC, or UC)
13309e39c5baSBill Taylor */
13319e39c5baSBill Taylor switch (wr->wr_trans) {
13329e39c5baSBill Taylor case IBT_UD_SRV:
13339e39c5baSBill Taylor /* Ensure that work request transport type matches QP type */
13349e39c5baSBill Taylor if (qp->qp_serv_type != HERMON_QP_UD) {
13359e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID);
13369e39c5baSBill Taylor }
13379e39c5baSBill Taylor
13389e39c5baSBill Taylor /*
13399e39c5baSBill Taylor * Validate the operation type. For UD requests, only the
13409e39c5baSBill Taylor * "Send" and "Send LSO" operations are valid.
13419e39c5baSBill Taylor */
13429e39c5baSBill Taylor if (wr->wr_opcode != IBT_WRC_SEND &&
13439e39c5baSBill Taylor wr->wr_opcode != IBT_WRC_SEND_LSO) {
13449e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID);
13459e39c5baSBill Taylor }
13469e39c5baSBill Taylor
13479e39c5baSBill Taylor /*
13489e39c5baSBill Taylor * If this is a Special QP (QP0 or QP1), then we need to
13499e39c5baSBill Taylor * build MLX WQEs instead. So jump to hermon_wqe_mlx_build()
13509e39c5baSBill Taylor * and return whatever status it returns
13519e39c5baSBill Taylor */
13529e39c5baSBill Taylor if (qp->qp_is_special) {
13539e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND_LSO) {
13549e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID);
13559e39c5baSBill Taylor }
13569e39c5baSBill Taylor status = hermon_wqe_mlx_build(state, qp,
13579e39c5baSBill Taylor wr, desc, size);
13589e39c5baSBill Taylor return (status);
13599e39c5baSBill Taylor }
13609e39c5baSBill Taylor
13619e39c5baSBill Taylor /*
13629e39c5baSBill Taylor * Otherwise, if this is a normal UD Send request, then fill
13639e39c5baSBill Taylor * all the fields in the Hermon UD header for the WQE. Note:
13649e39c5baSBill Taylor * to do this we'll need to extract some information from the
13659e39c5baSBill Taylor * Address Handle passed with the work request.
13669e39c5baSBill Taylor */
13679e39c5baSBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
13689e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
13699e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) {
13709e39c5baSBill Taylor dest = wr->wr.ud.udwr_dest;
13719e39c5baSBill Taylor } else {
13729e39c5baSBill Taylor dest = wr->wr.ud_lso.lso_ud_dest;
13739e39c5baSBill Taylor }
13749e39c5baSBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah;
13759e39c5baSBill Taylor if (ah == NULL) {
13769e39c5baSBill Taylor return (IBT_AH_HDL_INVALID);
13779e39c5baSBill Taylor }
13789e39c5baSBill Taylor
13799e39c5baSBill Taylor /*
13809e39c5baSBill Taylor * Build the Unreliable Datagram Segment for the WQE, using
13819e39c5baSBill Taylor * the information from the address handle and the work
13829e39c5baSBill Taylor * request.
13839e39c5baSBill Taylor */
13849e39c5baSBill Taylor /* mutex_enter(&ah->ah_lock); */
13859e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) {
13869e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, wr->wr.ud.udwr_dest);
13879e39c5baSBill Taylor } else { /* IBT_WRC_SEND_LSO */
13889e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah,
13899e39c5baSBill Taylor wr->wr.ud_lso.lso_ud_dest);
13909e39c5baSBill Taylor }
13919e39c5baSBill Taylor /* mutex_exit(&ah->ah_lock); */
13929e39c5baSBill Taylor
13939e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */
13949e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud +
13959e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ud_t));
13969e39c5baSBill Taylor
13979e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND_LSO) {
13989e39c5baSBill Taylor int total_len;
13999e39c5baSBill Taylor
14009e39c5baSBill Taylor total_len = (4 + 0xf + wr->wr.ud_lso.lso_hdr_sz) & ~0xf;
14019e39c5baSBill Taylor if ((uintptr_t)ds + total_len + (nds * 16) >
14029e39c5baSBill Taylor (uintptr_t)desc + (1 << qp->qp_sq_log_wqesz))
14039e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID);
14049e39c5baSBill Taylor
14059e39c5baSBill Taylor bcopy(wr->wr.ud_lso.lso_hdr, (uint32_t *)ds + 1,
14069e39c5baSBill Taylor wr->wr.ud_lso.lso_hdr_sz);
14079e39c5baSBill Taylor old_ds = ds;
14089e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + total_len);
14099e39c5baSBill Taylor for (; i < nds; i++) {
14109e39c5baSBill Taylor if (sgl[i].ds_len == 0)
14119e39c5baSBill Taylor continue;
14129e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[num_ds],
14139e39c5baSBill Taylor &sgl[i]);
14149e39c5baSBill Taylor num_ds++;
14159e39c5baSBill Taylor i++;
14169e39c5baSBill Taylor break;
14179e39c5baSBill Taylor }
14189e39c5baSBill Taylor membar_producer();
14199e39c5baSBill Taylor HERMON_WQE_BUILD_LSO(qp, old_ds, wr->wr.ud_lso.lso_mss,
14209e39c5baSBill Taylor wr->wr.ud_lso.lso_hdr_sz);
14219e39c5baSBill Taylor }
14229e39c5baSBill Taylor
14239e39c5baSBill Taylor break;
14249e39c5baSBill Taylor
14259e39c5baSBill Taylor case IBT_RC_SRV:
14269e39c5baSBill Taylor /* Ensure that work request transport type matches QP type */
14279e39c5baSBill Taylor if (qp->qp_serv_type != HERMON_QP_RC) {
14289e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID);
14299e39c5baSBill Taylor }
14309e39c5baSBill Taylor
14319e39c5baSBill Taylor /*
14329e39c5baSBill Taylor * Validate the operation type. For RC requests, we allow
14339e39c5baSBill Taylor * "Send", "RDMA Read", "RDMA Write", various "Atomic"
14349e39c5baSBill Taylor * operations, and memory window "Bind"
14359e39c5baSBill Taylor */
14369e39c5baSBill Taylor if ((wr->wr_opcode != IBT_WRC_SEND) &&
14379e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_RDMAR) &&
14389e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_RDMAW) &&
14399e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_CSWAP) &&
14409e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_FADD) &&
14419e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_BIND)) {
14429e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID);
14439e39c5baSBill Taylor }
14449e39c5baSBill Taylor
14459e39c5baSBill Taylor /*
14469e39c5baSBill Taylor * If this is a Send request, then all we need to do is break
14479e39c5baSBill Taylor * out and here and begin the Data Segment processing below
14489e39c5baSBill Taylor */
14499e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) {
14509e39c5baSBill Taylor break;
14519e39c5baSBill Taylor }
14529e39c5baSBill Taylor
14539e39c5baSBill Taylor /*
14549e39c5baSBill Taylor * If this is an RDMA Read or RDMA Write request, then fill
14559e39c5baSBill Taylor * in the "Remote Address" header fields.
14569e39c5baSBill Taylor */
14579e39c5baSBill Taylor if ((wr->wr_opcode == IBT_WRC_RDMAR) ||
14589e39c5baSBill Taylor (wr->wr_opcode == IBT_WRC_RDMAW)) {
14599e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
14609e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
14619e39c5baSBill Taylor
14629e39c5baSBill Taylor /*
14639e39c5baSBill Taylor * Build the Remote Address Segment for the WQE, using
14649e39c5baSBill Taylor * the information from the RC work request.
14659e39c5baSBill Taylor */
14669e39c5baSBill Taylor HERMON_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma);
14679e39c5baSBill Taylor
14689e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */
14699e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)rc +
14709e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t));
14719e39c5baSBill Taylor break;
14729e39c5baSBill Taylor }
14739e39c5baSBill Taylor
14749e39c5baSBill Taylor /*
14759e39c5baSBill Taylor * If this is one of the Atomic type operations (i.e
14769e39c5baSBill Taylor * Compare-Swap or Fetch-Add), then fill in both the "Remote
14779e39c5baSBill Taylor * Address" header fields and the "Atomic" header fields.
14789e39c5baSBill Taylor */
14799e39c5baSBill Taylor if ((wr->wr_opcode == IBT_WRC_CSWAP) ||
14809e39c5baSBill Taylor (wr->wr_opcode == IBT_WRC_FADD)) {
14819e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
14829e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
14839e39c5baSBill Taylor at = (hermon_hw_snd_wqe_atomic_t *)((uintptr_t)rc +
14849e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t));
14859e39c5baSBill Taylor
14869e39c5baSBill Taylor /*
14879e39c5baSBill Taylor * Build the Remote Address and Atomic Segments for
14889e39c5baSBill Taylor * the WQE, using the information from the RC Atomic
14899e39c5baSBill Taylor * work request.
14909e39c5baSBill Taylor */
14919e39c5baSBill Taylor HERMON_WQE_BUILD_RC_ATOMIC_REMADDR(qp, rc, wr);
14929e39c5baSBill Taylor HERMON_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic);
14939e39c5baSBill Taylor
14949e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */
14959e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)at +
14969e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_atomic_t));
14979e39c5baSBill Taylor
14989e39c5baSBill Taylor /*
14999e39c5baSBill Taylor * Update "nds" and "sgl" because Atomic requests have
15009e39c5baSBill Taylor * only a single Data Segment (and they are encoded
15019e39c5baSBill Taylor * somewhat differently in the work request.
15029e39c5baSBill Taylor */
15039e39c5baSBill Taylor nds = 1;
15049e39c5baSBill Taylor sgl = wr->wr_sgl;
15059e39c5baSBill Taylor break;
15069e39c5baSBill Taylor }
15079e39c5baSBill Taylor
15089e39c5baSBill Taylor /*
15099e39c5baSBill Taylor * If this is memory window Bind operation, then we call the
15109e39c5baSBill Taylor * hermon_wr_bind_check() routine to validate the request and
15119e39c5baSBill Taylor * to generate the updated RKey. If this is successful, then
15129e39c5baSBill Taylor * we fill in the WQE's "Bind" header fields.
15139e39c5baSBill Taylor */
15149e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_BIND) {
15159e39c5baSBill Taylor status = hermon_wr_bind_check(state, wr);
15169e39c5baSBill Taylor if (status != DDI_SUCCESS) {
15179e39c5baSBill Taylor return (status);
15189e39c5baSBill Taylor }
15199e39c5baSBill Taylor
15209e39c5baSBill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc +
15219e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
15229e39c5baSBill Taylor
15239e39c5baSBill Taylor /*
15249e39c5baSBill Taylor * Build the Bind Memory Window Segments for the WQE,
15259e39c5baSBill Taylor * using the information from the RC Bind memory
15269e39c5baSBill Taylor * window work request.
15279e39c5baSBill Taylor */
15289e39c5baSBill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind);
15299e39c5baSBill Taylor
15309e39c5baSBill Taylor /*
15319e39c5baSBill Taylor * Update the "ds" pointer. Even though the "bind"
15329e39c5baSBill Taylor * operation requires no SGLs, this is necessary to
15339e39c5baSBill Taylor * facilitate the correct descriptor size calculations
15349e39c5baSBill Taylor * (below).
15359e39c5baSBill Taylor */
15369e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)bn +
15379e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_bind_t));
15389e39c5baSBill Taylor nds = 0;
15399e39c5baSBill Taylor }
15409e39c5baSBill Taylor break;
15419e39c5baSBill Taylor
15429e39c5baSBill Taylor case IBT_UC_SRV:
15439e39c5baSBill Taylor /* Ensure that work request transport type matches QP type */
15449e39c5baSBill Taylor if (qp->qp_serv_type != HERMON_QP_UC) {
15459e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID);
15469e39c5baSBill Taylor }
15479e39c5baSBill Taylor
15489e39c5baSBill Taylor /*
15499e39c5baSBill Taylor * Validate the operation type. For UC requests, we only
15509e39c5baSBill Taylor * allow "Send", "RDMA Write", and memory window "Bind".
15519e39c5baSBill Taylor * Note: Unlike RC, UC does not allow "RDMA Read" or "Atomic"
15529e39c5baSBill Taylor * operations
15539e39c5baSBill Taylor */
15549e39c5baSBill Taylor if ((wr->wr_opcode != IBT_WRC_SEND) &&
15559e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_RDMAW) &&
15569e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_BIND)) {
15579e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID);
15589e39c5baSBill Taylor }
15599e39c5baSBill Taylor
15609e39c5baSBill Taylor /*
15619e39c5baSBill Taylor * If this is a Send request, then all we need to do is break
15629e39c5baSBill Taylor * out and here and begin the Data Segment processing below
15639e39c5baSBill Taylor */
15649e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) {
15659e39c5baSBill Taylor break;
15669e39c5baSBill Taylor }
15679e39c5baSBill Taylor
15689e39c5baSBill Taylor /*
15699e39c5baSBill Taylor * If this is an RDMA Write request, then fill in the "Remote
15709e39c5baSBill Taylor * Address" header fields.
15719e39c5baSBill Taylor */
15729e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_RDMAW) {
15739e39c5baSBill Taylor uc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
15749e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
15759e39c5baSBill Taylor
15769e39c5baSBill Taylor /*
15779e39c5baSBill Taylor * Build the Remote Address Segment for the WQE, using
15789e39c5baSBill Taylor * the information from the UC work request.
15799e39c5baSBill Taylor */
15809e39c5baSBill Taylor HERMON_WQE_BUILD_REMADDR(qp, uc, &wr->wr.uc.ucwr.rdma);
15819e39c5baSBill Taylor
15829e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */
15839e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)uc +
15849e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t));
15859e39c5baSBill Taylor break;
15869e39c5baSBill Taylor }
15879e39c5baSBill Taylor
15889e39c5baSBill Taylor /*
15899e39c5baSBill Taylor * If this is memory window Bind operation, then we call the
15909e39c5baSBill Taylor * hermon_wr_bind_check() routine to validate the request and
15919e39c5baSBill Taylor * to generate the updated RKey. If this is successful, then
15929e39c5baSBill Taylor * we fill in the WQE's "Bind" header fields.
15939e39c5baSBill Taylor */
15949e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_BIND) {
15959e39c5baSBill Taylor status = hermon_wr_bind_check(state, wr);
15969e39c5baSBill Taylor if (status != DDI_SUCCESS) {
15979e39c5baSBill Taylor return (status);
15989e39c5baSBill Taylor }
15999e39c5baSBill Taylor
16009e39c5baSBill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc +
16019e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t));
16029e39c5baSBill Taylor
16039e39c5baSBill Taylor /*
16049e39c5baSBill Taylor * Build the Bind Memory Window Segments for the WQE,
16059e39c5baSBill Taylor * using the information from the UC Bind memory
16069e39c5baSBill Taylor * window work request.
16079e39c5baSBill Taylor */
16089e39c5baSBill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.uc.ucwr.bind);
16099e39c5baSBill Taylor
16109e39c5baSBill Taylor /*
16119e39c5baSBill Taylor * Update the "ds" pointer. Even though the "bind"
16129e39c5baSBill Taylor * operation requires no SGLs, this is necessary to
16139e39c5baSBill Taylor * facilitate the correct descriptor size calculations
16149e39c5baSBill Taylor * (below).
16159e39c5baSBill Taylor */
16169e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)bn +
16179e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_bind_t));
16189e39c5baSBill Taylor nds = 0;
16199e39c5baSBill Taylor }
16209e39c5baSBill Taylor break;
16219e39c5baSBill Taylor
16229e39c5baSBill Taylor default:
16239e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID);
16249e39c5baSBill Taylor }
16259e39c5baSBill Taylor
16269e39c5baSBill Taylor /*
16279e39c5baSBill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based on
16289e39c5baSBill Taylor * the values setup above (i.e. "sgl", "nds", and the "ds" pointer
16299e39c5baSBill Taylor * Start by checking for a valid number of SGL entries
16309e39c5baSBill Taylor */
16319e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) {
16329e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID);
16339e39c5baSBill Taylor }
16349e39c5baSBill Taylor
16359e39c5baSBill Taylor /*
16369e39c5baSBill Taylor * For each SGL in the Send Work Request, fill in the Send WQE's data
16379e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon
16389e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
16399e39c5baSBill Taylor * the encoding for zero means a 2GB transfer.
16409e39c5baSBill Taylor */
16419e39c5baSBill Taylor for (last_ds = num_ds, j = i; j < nds; j++) {
16429e39c5baSBill Taylor if (sgl[j].ds_len != 0)
16439e39c5baSBill Taylor last_ds++; /* real last ds of wqe to fill */
16449e39c5baSBill Taylor }
16459e39c5baSBill Taylor
16469e39c5baSBill Taylor /*
16479e39c5baSBill Taylor * Return the size of descriptor (in 16-byte chunks)
16489e39c5baSBill Taylor * For Hermon, we want them (for now) to be on stride size
16499e39c5baSBill Taylor * boundaries, which was implicit in Tavor/Arbel
16509e39c5baSBill Taylor *
16519e39c5baSBill Taylor */
16529e39c5baSBill Taylor tmpsize = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc);
16539e39c5baSBill Taylor
16549e39c5baSBill Taylor *size = tmpsize >> 0x4;
16559e39c5baSBill Taylor
16569e39c5baSBill Taylor for (j = nds; --j >= i; ) {
16579e39c5baSBill Taylor if (sgl[j].ds_len == 0) {
16589e39c5baSBill Taylor continue;
16599e39c5baSBill Taylor }
16609e39c5baSBill Taylor
16619e39c5baSBill Taylor /*
16629e39c5baSBill Taylor * Fill in the Data Segment(s) for the current WQE, using the
16639e39c5baSBill Taylor * information contained in the scatter-gather list of the
16649e39c5baSBill Taylor * work request.
16659e39c5baSBill Taylor */
16669e39c5baSBill Taylor last_ds--;
16679e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[j]);
16689e39c5baSBill Taylor }
16699e39c5baSBill Taylor
16709e39c5baSBill Taylor return (DDI_SUCCESS);
16719e39c5baSBill Taylor }
16729e39c5baSBill Taylor
16739e39c5baSBill Taylor
16749e39c5baSBill Taylor
16759e39c5baSBill Taylor /*
16769e39c5baSBill Taylor * hermon_wqe_mlx_build()
16779e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
16789e39c5baSBill Taylor */
16799e39c5baSBill Taylor static int
hermon_wqe_mlx_build(hermon_state_t * state,hermon_qphdl_t qp,ibt_send_wr_t * wr,uint64_t * desc,uint_t * size)16809e39c5baSBill Taylor hermon_wqe_mlx_build(hermon_state_t *state, hermon_qphdl_t qp,
16819e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size)
16829e39c5baSBill Taylor {
16839e39c5baSBill Taylor hermon_ahhdl_t ah;
16849e39c5baSBill Taylor hermon_hw_udav_t *udav;
16859e39c5baSBill Taylor ib_lrh_hdr_t *lrh;
16869e39c5baSBill Taylor ib_grh_t *grh;
16879e39c5baSBill Taylor ib_bth_hdr_t *bth;
16889e39c5baSBill Taylor ib_deth_hdr_t *deth;
16899e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds;
16909e39c5baSBill Taylor ibt_wr_ds_t *sgl;
16919e39c5baSBill Taylor uint8_t *mgmtclass, *hpoint, *hcount;
16929e39c5baSBill Taylor uint32_t nds, offset, pktlen;
16939e39c5baSBill Taylor uint32_t desc_sz;
16949e39c5baSBill Taylor int i, num_ds;
16959e39c5baSBill Taylor int tmpsize;
16969e39c5baSBill Taylor
16979e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock));
16989e39c5baSBill Taylor
16999e39c5baSBill Taylor /* Initialize the information for the Data Segments */
17009e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc +
17019e39c5baSBill Taylor sizeof (hermon_hw_mlx_wqe_nextctrl_t));
17029e39c5baSBill Taylor
17039e39c5baSBill Taylor /*
17049e39c5baSBill Taylor * Pull the address handle from the work request. The UDAV will
17059e39c5baSBill Taylor * be used to answer some questions about the request.
17069e39c5baSBill Taylor */
17079e39c5baSBill Taylor ah = (hermon_ahhdl_t)wr->wr.ud.udwr_dest->ud_ah;
17089e39c5baSBill Taylor if (ah == NULL) {
17099e39c5baSBill Taylor return (IBT_AH_HDL_INVALID);
17109e39c5baSBill Taylor }
17119e39c5baSBill Taylor mutex_enter(&ah->ah_lock);
17129e39c5baSBill Taylor udav = ah->ah_udav;
17139e39c5baSBill Taylor
17149e39c5baSBill Taylor /*
17159e39c5baSBill Taylor * If the request is for QP1 and the destination LID is equal to
17169e39c5baSBill Taylor * the Permissive LID, then return an error. This combination is
17179e39c5baSBill Taylor * not allowed
17189e39c5baSBill Taylor */
17199e39c5baSBill Taylor if ((udav->rlid == IB_LID_PERMISSIVE) &&
17209e39c5baSBill Taylor (qp->qp_is_special == HERMON_QP_GSI)) {
17219e39c5baSBill Taylor mutex_exit(&ah->ah_lock);
17229e39c5baSBill Taylor return (IBT_AH_HDL_INVALID);
17239e39c5baSBill Taylor }
17249e39c5baSBill Taylor
17259e39c5baSBill Taylor /*
17269e39c5baSBill Taylor * Calculate the size of the packet headers, including the GRH
17279e39c5baSBill Taylor * (if necessary)
17289e39c5baSBill Taylor */
17299e39c5baSBill Taylor desc_sz = sizeof (ib_lrh_hdr_t) + sizeof (ib_bth_hdr_t) +
17309e39c5baSBill Taylor sizeof (ib_deth_hdr_t);
17319e39c5baSBill Taylor if (udav->grh) {
17329e39c5baSBill Taylor desc_sz += sizeof (ib_grh_t);
17339e39c5baSBill Taylor }
17349e39c5baSBill Taylor
17359e39c5baSBill Taylor /*
17369e39c5baSBill Taylor * Begin to build the first "inline" data segment for the packet
17379e39c5baSBill Taylor * headers. Note: By specifying "inline" we can build the contents
17389e39c5baSBill Taylor * of the MAD packet headers directly into the work queue (as part
17399e39c5baSBill Taylor * descriptor). This has the advantage of both speeding things up
17409e39c5baSBill Taylor * and of not requiring the driver to allocate/register any additional
17419e39c5baSBill Taylor * memory for the packet headers.
17429e39c5baSBill Taylor */
17439e39c5baSBill Taylor HERMON_WQE_BUILD_INLINE(qp, &ds[0], desc_sz);
17449e39c5baSBill Taylor desc_sz += 4;
17459e39c5baSBill Taylor
17469e39c5baSBill Taylor /*
17479e39c5baSBill Taylor * Build Local Route Header (LRH)
17489e39c5baSBill Taylor * We start here by building the LRH into a temporary location.
17499e39c5baSBill Taylor * When we have finished we copy the LRH data into the descriptor.
17509e39c5baSBill Taylor *
17519e39c5baSBill Taylor * Notice that the VL values are hardcoded. This is not a problem
17529e39c5baSBill Taylor * because VL15 is decided later based on the value in the MLX
17539e39c5baSBill Taylor * transport "next/ctrl" header (see the "vl15" bit below), and it
17549e39c5baSBill Taylor * is otherwise (meaning for QP1) chosen from the SL-to-VL table
17559e39c5baSBill Taylor * values. This rule does not hold for loopback packets however
17569e39c5baSBill Taylor * (all of which bypass the SL-to-VL tables) and it is the reason
17579e39c5baSBill Taylor * that non-QP0 MADs are setup with VL hardcoded to zero below.
17589e39c5baSBill Taylor *
17599e39c5baSBill Taylor * Notice also that Source LID is hardcoded to the Permissive LID
17609e39c5baSBill Taylor * (0xFFFF). This is also not a problem because if the Destination
17619e39c5baSBill Taylor * LID is not the Permissive LID, then the "slr" value in the MLX
17629e39c5baSBill Taylor * transport "next/ctrl" header will be set to zero and the hardware
17639e39c5baSBill Taylor * will pull the LID from value in the port.
17649e39c5baSBill Taylor */
17659e39c5baSBill Taylor lrh = (ib_lrh_hdr_t *)((uintptr_t)&ds[0] + 4);
17669e39c5baSBill Taylor pktlen = (desc_sz + 0x100) >> 2;
17679e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_LRH(lrh, qp, udav, pktlen);
17689e39c5baSBill Taylor
17699e39c5baSBill Taylor /*
17709e39c5baSBill Taylor * Build Global Route Header (GRH)
17719e39c5baSBill Taylor * This is only built if necessary as defined by the "grh" bit in
17729e39c5baSBill Taylor * the address vector. Note: We also calculate the offset to the
17739e39c5baSBill Taylor * next header (BTH) based on whether or not the "grh" bit is set.
17749e39c5baSBill Taylor */
17759e39c5baSBill Taylor if (udav->grh) {
17769e39c5baSBill Taylor /*
17779e39c5baSBill Taylor * If the request is for QP0, then return an error. The
17789e39c5baSBill Taylor * combination of global routine (GRH) and QP0 is not allowed.
17799e39c5baSBill Taylor */
17809e39c5baSBill Taylor if (qp->qp_is_special == HERMON_QP_SMI) {
17819e39c5baSBill Taylor mutex_exit(&ah->ah_lock);
17829e39c5baSBill Taylor return (IBT_AH_HDL_INVALID);
17839e39c5baSBill Taylor }
17849e39c5baSBill Taylor grh = (ib_grh_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t));
17859e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_GRH(state, grh, qp, udav, pktlen);
17869e39c5baSBill Taylor
17879e39c5baSBill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)grh + sizeof (ib_grh_t));
17889e39c5baSBill Taylor } else {
17899e39c5baSBill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t));
17909e39c5baSBill Taylor }
17919e39c5baSBill Taylor mutex_exit(&ah->ah_lock);
17929e39c5baSBill Taylor
17939e39c5baSBill Taylor
17949e39c5baSBill Taylor /*
17959e39c5baSBill Taylor * Build Base Transport Header (BTH)
17969e39c5baSBill Taylor * Notice that the M, PadCnt, and TVer fields are all set
17979e39c5baSBill Taylor * to zero implicitly. This is true for all Management Datagrams
17989e39c5baSBill Taylor * MADs whether GSI are SMI.
17999e39c5baSBill Taylor */
18009e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_BTH(state, bth, qp, wr);
18019e39c5baSBill Taylor
18029e39c5baSBill Taylor /*
18039e39c5baSBill Taylor * Build Datagram Extended Transport Header (DETH)
18049e39c5baSBill Taylor */
18059e39c5baSBill Taylor deth = (ib_deth_hdr_t *)((uintptr_t)bth + sizeof (ib_bth_hdr_t));
18069e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_DETH(deth, qp);
18079e39c5baSBill Taylor
18089e39c5baSBill Taylor /* Ensure that the Data Segment is aligned on a 16-byte boundary */
18099e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)deth + sizeof (ib_deth_hdr_t));
18109e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)(((uintptr_t)ds + 0xF) & ~0xF);
18119e39c5baSBill Taylor nds = wr->wr_nds;
18129e39c5baSBill Taylor sgl = wr->wr_sgl;
18139e39c5baSBill Taylor num_ds = 0;
18149e39c5baSBill Taylor
18159e39c5baSBill Taylor /*
18169e39c5baSBill Taylor * Now fill in the Data Segments (SGL) for the MLX WQE based on the
18179e39c5baSBill Taylor * values set up above (i.e. "sgl", "nds", and the "ds" pointer
18189e39c5baSBill Taylor * Start by checking for a valid number of SGL entries
18199e39c5baSBill Taylor */
18209e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) {
18219e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID);
18229e39c5baSBill Taylor }
18239e39c5baSBill Taylor
18249e39c5baSBill Taylor /*
18259e39c5baSBill Taylor * For each SGL in the Send Work Request, fill in the MLX WQE's data
18269e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon
18279e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
18289e39c5baSBill Taylor * the encoding for zero means a 2GB transfer. Because of this special
18299e39c5baSBill Taylor * encoding in the hardware, we mask the requested length with
18309e39c5baSBill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
18319e39c5baSBill Taylor * zero.)
18329e39c5baSBill Taylor */
18339e39c5baSBill Taylor mgmtclass = hpoint = hcount = NULL;
18349e39c5baSBill Taylor offset = 0;
18359e39c5baSBill Taylor for (i = 0; i < nds; i++) {
18369e39c5baSBill Taylor if (sgl[i].ds_len == 0) {
18379e39c5baSBill Taylor continue;
18389e39c5baSBill Taylor }
18399e39c5baSBill Taylor
18409e39c5baSBill Taylor /*
18419e39c5baSBill Taylor * Fill in the Data Segment(s) for the MLX send WQE, using
18429e39c5baSBill Taylor * the information contained in the scatter-gather list of
18439e39c5baSBill Taylor * the work request.
18449e39c5baSBill Taylor */
18459e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[num_ds], &sgl[i]);
18469e39c5baSBill Taylor
18479e39c5baSBill Taylor /*
18489e39c5baSBill Taylor * Search through the contents of all MADs posted to QP0 to
18499e39c5baSBill Taylor * initialize pointers to the places where Directed Route "hop
18509e39c5baSBill Taylor * pointer", "hop count", and "mgmtclass" would be. Hermon
18519e39c5baSBill Taylor * needs these updated (i.e. incremented or decremented, as
18529e39c5baSBill Taylor * necessary) by software.
18539e39c5baSBill Taylor */
18549e39c5baSBill Taylor if (qp->qp_is_special == HERMON_QP_SMI) {
18559e39c5baSBill Taylor
18569e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_GET_MGMTCLASS(mgmtclass,
18579e39c5baSBill Taylor offset, sgl[i].ds_va, sgl[i].ds_len);
18589e39c5baSBill Taylor
18599e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_GET_HOPPOINTER(hpoint,
18609e39c5baSBill Taylor offset, sgl[i].ds_va, sgl[i].ds_len);
18619e39c5baSBill Taylor
18629e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_GET_HOPCOUNT(hcount,
18639e39c5baSBill Taylor offset, sgl[i].ds_va, sgl[i].ds_len);
18649e39c5baSBill Taylor
18659e39c5baSBill Taylor offset += sgl[i].ds_len;
18669e39c5baSBill Taylor }
18679e39c5baSBill Taylor num_ds++;
18689e39c5baSBill Taylor }
18699e39c5baSBill Taylor
18709e39c5baSBill Taylor /*
18719e39c5baSBill Taylor * Hermon's Directed Route MADs need to have the "hop pointer"
18729e39c5baSBill Taylor * incremented/decremented (as necessary) depending on whether it is
18739e39c5baSBill Taylor * currently less than or greater than the "hop count" (i.e. whether
18749e39c5baSBill Taylor * the MAD is a request or a response.)
18759e39c5baSBill Taylor */
18769e39c5baSBill Taylor if (qp->qp_is_special == HERMON_QP_SMI) {
18779e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_DO_HOPPOINTER_MODIFY(*mgmtclass,
18789e39c5baSBill Taylor *hpoint, *hcount);
18799e39c5baSBill Taylor }
18809e39c5baSBill Taylor
18819e39c5baSBill Taylor /*
18829e39c5baSBill Taylor * Now fill in the ICRC Data Segment. This data segment is inlined
18839e39c5baSBill Taylor * just like the packets headers above, but it is only four bytes and
18849e39c5baSBill Taylor * set to zero (to indicate that we wish the hardware to generate ICRC.
18859e39c5baSBill Taylor */
18869e39c5baSBill Taylor HERMON_WQE_BUILD_INLINE_ICRC(qp, &ds[num_ds], 4, 0);
18879e39c5baSBill Taylor num_ds++;
18889e39c5baSBill Taylor
18899e39c5baSBill Taylor /*
18909e39c5baSBill Taylor * Return the size of descriptor (in 16-byte chunks)
18919e39c5baSBill Taylor * For Hermon, we want them (for now) to be on stride size
18929e39c5baSBill Taylor * boundaries, which was implicit in Tavor/Arbel
18939e39c5baSBill Taylor */
18949e39c5baSBill Taylor tmpsize = ((uintptr_t)&ds[num_ds] - (uintptr_t)desc);
18959e39c5baSBill Taylor
18969e39c5baSBill Taylor *size = tmpsize >> 0x04;
18979e39c5baSBill Taylor
18989e39c5baSBill Taylor return (DDI_SUCCESS);
18999e39c5baSBill Taylor }
19009e39c5baSBill Taylor
19019e39c5baSBill Taylor
19029e39c5baSBill Taylor
19039e39c5baSBill Taylor /*
19049e39c5baSBill Taylor * hermon_wqe_recv_build()
19059e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
19069e39c5baSBill Taylor */
19079e39c5baSBill Taylor /* ARGSUSED */
19089e39c5baSBill Taylor static int
hermon_wqe_recv_build(hermon_state_t * state,hermon_qphdl_t qp,ibt_recv_wr_t * wr,uint64_t * desc)19099e39c5baSBill Taylor hermon_wqe_recv_build(hermon_state_t *state, hermon_qphdl_t qp,
19109e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc)
19119e39c5baSBill Taylor {
19129e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds;
19139e39c5baSBill Taylor int i, num_ds;
19149e39c5baSBill Taylor
1915*17a2b317SBill Taylor ASSERT(MUTEX_HELD(&qp->qp_lock));
19169e39c5baSBill Taylor
19179e39c5baSBill Taylor /*
19189e39c5baSBill Taylor * Fill in the Data Segments (SGL) for the Recv WQE - don't
19199e39c5baSBill Taylor * need to have a reserved for the ctrl, there is none on the
19209e39c5baSBill Taylor * recv queue for hermon, but will need to put an invalid
19219e39c5baSBill Taylor * (null) scatter pointer per PRM
19229e39c5baSBill Taylor */
19239e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)(uintptr_t)desc;
19249e39c5baSBill Taylor num_ds = 0;
19259e39c5baSBill Taylor
19269e39c5baSBill Taylor /* Check for valid number of SGL entries */
19279e39c5baSBill Taylor if (wr->wr_nds > qp->qp_rq_sgl) {
19289e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID);
19299e39c5baSBill Taylor }
19309e39c5baSBill Taylor
19319e39c5baSBill Taylor /*
19329e39c5baSBill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
19339e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon
19349e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
19359e39c5baSBill Taylor * the encoding for zero means a 2GB transfer. Because of this special
19369e39c5baSBill Taylor * encoding in the hardware, we mask the requested length with
19379e39c5baSBill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
19389e39c5baSBill Taylor * zero.)
19399e39c5baSBill Taylor */
19409e39c5baSBill Taylor for (i = 0; i < wr->wr_nds; i++) {
19419e39c5baSBill Taylor if (wr->wr_sgl[i].ds_len == 0) {
19429e39c5baSBill Taylor continue;
19439e39c5baSBill Taylor }
19449e39c5baSBill Taylor
19459e39c5baSBill Taylor /*
19469e39c5baSBill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
19479e39c5baSBill Taylor * information contained in the scatter-gather list of the
19489e39c5baSBill Taylor * work request.
19499e39c5baSBill Taylor */
19509e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &wr->wr_sgl[i]);
19519e39c5baSBill Taylor num_ds++;
19529e39c5baSBill Taylor }
19539e39c5baSBill Taylor
19549e39c5baSBill Taylor /* put the null sgl pointer as well if needed */
19559e39c5baSBill Taylor if (num_ds < qp->qp_rq_sgl) {
19569e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &null_sgl);
19579e39c5baSBill Taylor }
19589e39c5baSBill Taylor
19599e39c5baSBill Taylor return (DDI_SUCCESS);
19609e39c5baSBill Taylor }
19619e39c5baSBill Taylor
19629e39c5baSBill Taylor
19639e39c5baSBill Taylor
19649e39c5baSBill Taylor /*
19659e39c5baSBill Taylor * hermon_wqe_srq_build()
19669e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
19679e39c5baSBill Taylor */
19689e39c5baSBill Taylor /* ARGSUSED */
19699e39c5baSBill Taylor static int
hermon_wqe_srq_build(hermon_state_t * state,hermon_srqhdl_t srq,ibt_recv_wr_t * wr,uint64_t * desc)19709e39c5baSBill Taylor hermon_wqe_srq_build(hermon_state_t *state, hermon_srqhdl_t srq,
19719e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc)
19729e39c5baSBill Taylor {
19739e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds;
19749e39c5baSBill Taylor int i, num_ds;
19759e39c5baSBill Taylor
19769e39c5baSBill Taylor ASSERT(MUTEX_HELD(&srq->srq_lock));
19779e39c5baSBill Taylor
19789e39c5baSBill Taylor /* Fill in the Data Segments (SGL) for the Recv WQE */
19799e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc +
19809e39c5baSBill Taylor sizeof (hermon_hw_srq_wqe_next_t));
19819e39c5baSBill Taylor num_ds = 0;
19829e39c5baSBill Taylor
19839e39c5baSBill Taylor /* Check for valid number of SGL entries */
19849e39c5baSBill Taylor if (wr->wr_nds > srq->srq_wq_sgl) {
19859e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID);
19869e39c5baSBill Taylor }
19879e39c5baSBill Taylor
19889e39c5baSBill Taylor /*
19899e39c5baSBill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
19909e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon
19919e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
19929e39c5baSBill Taylor * the encoding for zero means a 2GB transfer. Because of this special
19939e39c5baSBill Taylor * encoding in the hardware, we mask the requested length with
19949e39c5baSBill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
19959e39c5baSBill Taylor * zero.)
19969e39c5baSBill Taylor */
19979e39c5baSBill Taylor for (i = 0; i < wr->wr_nds; i++) {
19989e39c5baSBill Taylor if (wr->wr_sgl[i].ds_len == 0) {
19999e39c5baSBill Taylor continue;
20009e39c5baSBill Taylor }
20019e39c5baSBill Taylor
20029e39c5baSBill Taylor /*
20039e39c5baSBill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
20049e39c5baSBill Taylor * information contained in the scatter-gather list of the
20059e39c5baSBill Taylor * work request.
20069e39c5baSBill Taylor */
20079e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &wr->wr_sgl[i]);
20089e39c5baSBill Taylor num_ds++;
20099e39c5baSBill Taylor }
20109e39c5baSBill Taylor
20119e39c5baSBill Taylor /*
20129e39c5baSBill Taylor * put in the null sgl pointer as well, if needed
20139e39c5baSBill Taylor */
20149e39c5baSBill Taylor if (num_ds < srq->srq_wq_sgl) {
20159e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &null_sgl);
20169e39c5baSBill Taylor }
20179e39c5baSBill Taylor
20189e39c5baSBill Taylor return (DDI_SUCCESS);
20199e39c5baSBill Taylor }
20209e39c5baSBill Taylor
20219e39c5baSBill Taylor
20229e39c5baSBill Taylor /*
20239e39c5baSBill Taylor * hermon_wr_get_immediate()
20249e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
20259e39c5baSBill Taylor */
20269e39c5baSBill Taylor static uint32_t
hermon_wr_get_immediate(ibt_send_wr_t * wr)20279e39c5baSBill Taylor hermon_wr_get_immediate(ibt_send_wr_t *wr)
20289e39c5baSBill Taylor {
20299e39c5baSBill Taylor /*
20309e39c5baSBill Taylor * This routine extracts the "immediate data" from the appropriate
20319e39c5baSBill Taylor * location in the IBTF work request. Because of the way the
20329e39c5baSBill Taylor * work request structure is defined, the location for this data
20339e39c5baSBill Taylor * depends on the actual work request operation type.
20349e39c5baSBill Taylor */
20359e39c5baSBill Taylor
20369e39c5baSBill Taylor /* For RDMA Write, test if RC or UC */
20379e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_RDMAW) {
20389e39c5baSBill Taylor if (wr->wr_trans == IBT_RC_SRV) {
20399e39c5baSBill Taylor return (wr->wr.rc.rcwr.rdma.rdma_immed);
20409e39c5baSBill Taylor } else { /* IBT_UC_SRV */
20419e39c5baSBill Taylor return (wr->wr.uc.ucwr.rdma.rdma_immed);
20429e39c5baSBill Taylor }
20439e39c5baSBill Taylor }
20449e39c5baSBill Taylor
20459e39c5baSBill Taylor /* For Send, test if RC, UD, or UC */
20469e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) {
20479e39c5baSBill Taylor if (wr->wr_trans == IBT_RC_SRV) {
20489e39c5baSBill Taylor return (wr->wr.rc.rcwr.send_immed);
20499e39c5baSBill Taylor } else if (wr->wr_trans == IBT_UD_SRV) {
20509e39c5baSBill Taylor return (wr->wr.ud.udwr_immed);
20519e39c5baSBill Taylor } else { /* IBT_UC_SRV */
20529e39c5baSBill Taylor return (wr->wr.uc.ucwr.send_immed);
20539e39c5baSBill Taylor }
20549e39c5baSBill Taylor }
20559e39c5baSBill Taylor
20569e39c5baSBill Taylor /*
20579e39c5baSBill Taylor * If any other type of request, then immediate is undefined
20589e39c5baSBill Taylor */
20599e39c5baSBill Taylor return (0);
20609e39c5baSBill Taylor }
20619e39c5baSBill Taylor
20629e39c5baSBill Taylor /*
20639e39c5baSBill Taylor * hermon_wqe_headroom()
20649e39c5baSBill Taylor * Context: can be called from interrupt or base, currently only from
20659e39c5baSBill Taylor * base context.
20669e39c5baSBill Taylor * Routine that fills in the headroom for the Send Queue
20679e39c5baSBill Taylor */
20689e39c5baSBill Taylor
20699e39c5baSBill Taylor static void
hermon_wqe_headroom(uint_t from,hermon_qphdl_t qp)20709e39c5baSBill Taylor hermon_wqe_headroom(uint_t from, hermon_qphdl_t qp)
20719e39c5baSBill Taylor {
20729e39c5baSBill Taylor uint32_t *wqe_start, *wqe_top, *wqe_base, qsize;
20739e39c5baSBill Taylor int hdrmwqes, wqesizebytes, sectperwqe;
20749e39c5baSBill Taylor uint32_t invalue;
20759e39c5baSBill Taylor int i, j;
20769e39c5baSBill Taylor
20779e39c5baSBill Taylor qsize = qp->qp_sq_bufsz;
20789e39c5baSBill Taylor wqesizebytes = 1 << qp->qp_sq_log_wqesz;
20799e39c5baSBill Taylor sectperwqe = wqesizebytes >> 6; /* 64 bytes/section */
20809e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes;
20819e39c5baSBill Taylor wqe_base = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, 0);
20829e39c5baSBill Taylor wqe_top = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, qsize);
20839e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, from);
20849e39c5baSBill Taylor
20859e39c5baSBill Taylor for (i = 0; i < hdrmwqes; i++) {
20869e39c5baSBill Taylor for (j = 0; j < sectperwqe; j++) {
20879e39c5baSBill Taylor if (j == 0) { /* 1st section of wqe */
20889e39c5baSBill Taylor /* perserve ownership bit */
20899e39c5baSBill Taylor invalue = ddi_get32(qp->qp_wqinfo.qa_acchdl,
20909e39c5baSBill Taylor wqe_start) | 0x7FFFFFFF;
20919e39c5baSBill Taylor } else {
20929e39c5baSBill Taylor /* or just invalidate it */
20939e39c5baSBill Taylor invalue = 0xFFFFFFFF;
20949e39c5baSBill Taylor }
20959e39c5baSBill Taylor ddi_put32(qp->qp_wqinfo.qa_acchdl, wqe_start, invalue);
20969e39c5baSBill Taylor wqe_start += 16; /* move 64 bytes */
20979e39c5baSBill Taylor }
20989e39c5baSBill Taylor if (wqe_start == wqe_top) /* hit the end of the queue */
20999e39c5baSBill Taylor wqe_start = wqe_base; /* wrap to start */
21009e39c5baSBill Taylor }
21019e39c5baSBill Taylor }
21029e39c5baSBill Taylor
21039e39c5baSBill Taylor /*
21049e39c5baSBill Taylor * hermon_wr_bind_check()
21059e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
21069e39c5baSBill Taylor */
21079e39c5baSBill Taylor /* ARGSUSED */
21089e39c5baSBill Taylor static int
hermon_wr_bind_check(hermon_state_t * state,ibt_send_wr_t * wr)21099e39c5baSBill Taylor hermon_wr_bind_check(hermon_state_t *state, ibt_send_wr_t *wr)
21109e39c5baSBill Taylor {
21119e39c5baSBill Taylor ibt_bind_flags_t bind_flags;
21129e39c5baSBill Taylor uint64_t vaddr, len;
21139e39c5baSBill Taylor uint64_t reg_start_addr, reg_end_addr;
21149e39c5baSBill Taylor hermon_mwhdl_t mw;
21159e39c5baSBill Taylor hermon_mrhdl_t mr;
21169e39c5baSBill Taylor hermon_rsrc_t *mpt;
21179e39c5baSBill Taylor uint32_t new_rkey;
21189e39c5baSBill Taylor
21199e39c5baSBill Taylor /* Check for a valid Memory Window handle in the WR */
21209e39c5baSBill Taylor mw = (hermon_mwhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mw_hdl;
21219e39c5baSBill Taylor if (mw == NULL) {
21229e39c5baSBill Taylor return (IBT_MW_HDL_INVALID);
21239e39c5baSBill Taylor }
21249e39c5baSBill Taylor
21259e39c5baSBill Taylor /* Check for a valid Memory Region handle in the WR */
21269e39c5baSBill Taylor mr = (hermon_mrhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mr_hdl;
21279e39c5baSBill Taylor if (mr == NULL) {
21289e39c5baSBill Taylor return (IBT_MR_HDL_INVALID);
21299e39c5baSBill Taylor }
21309e39c5baSBill Taylor
21319e39c5baSBill Taylor mutex_enter(&mr->mr_lock);
21329e39c5baSBill Taylor mutex_enter(&mw->mr_lock);
21339e39c5baSBill Taylor
21349e39c5baSBill Taylor /*
21359e39c5baSBill Taylor * Check here to see if the memory region has already been partially
21369e39c5baSBill Taylor * deregistered as a result of a hermon_umap_umemlock_cb() callback.
21379e39c5baSBill Taylor * If so, this is an error, return failure.
21389e39c5baSBill Taylor */
21399e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
21409e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
21419e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
21429e39c5baSBill Taylor return (IBT_MR_HDL_INVALID);
21439e39c5baSBill Taylor }
21449e39c5baSBill Taylor
21459e39c5baSBill Taylor /* Check for a valid Memory Window RKey (i.e. a matching RKey) */
21469e39c5baSBill Taylor if (mw->mr_rkey != wr->wr.rc.rcwr.bind->bind_rkey) {
21479e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
21489e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
21499e39c5baSBill Taylor return (IBT_MR_RKEY_INVALID);
21509e39c5baSBill Taylor }
21519e39c5baSBill Taylor
21529e39c5baSBill Taylor /* Check for a valid Memory Region LKey (i.e. a matching LKey) */
21539e39c5baSBill Taylor if (mr->mr_lkey != wr->wr.rc.rcwr.bind->bind_lkey) {
21549e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
21559e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
21569e39c5baSBill Taylor return (IBT_MR_LKEY_INVALID);
21579e39c5baSBill Taylor }
21589e39c5baSBill Taylor
21599e39c5baSBill Taylor /*
21609e39c5baSBill Taylor * Now check for valid "vaddr" and "len". Note: We don't check the
21619e39c5baSBill Taylor * "vaddr" range when "len == 0" (i.e. on unbind operations)
21629e39c5baSBill Taylor */
21639e39c5baSBill Taylor len = wr->wr.rc.rcwr.bind->bind_len;
21649e39c5baSBill Taylor if (len != 0) {
21659e39c5baSBill Taylor vaddr = wr->wr.rc.rcwr.bind->bind_va;
21669e39c5baSBill Taylor reg_start_addr = mr->mr_bindinfo.bi_addr;
21679e39c5baSBill Taylor reg_end_addr = mr->mr_bindinfo.bi_addr +
21689e39c5baSBill Taylor (mr->mr_bindinfo.bi_len - 1);
21699e39c5baSBill Taylor if ((vaddr < reg_start_addr) || (vaddr > reg_end_addr)) {
21709e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
21719e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
21729e39c5baSBill Taylor return (IBT_MR_VA_INVALID);
21739e39c5baSBill Taylor }
21749e39c5baSBill Taylor vaddr = (vaddr + len) - 1;
21759e39c5baSBill Taylor if (vaddr > reg_end_addr) {
21769e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
21779e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
21789e39c5baSBill Taylor return (IBT_MR_LEN_INVALID);
21799e39c5baSBill Taylor }
21809e39c5baSBill Taylor }
21819e39c5baSBill Taylor
21829e39c5baSBill Taylor /*
21839e39c5baSBill Taylor * Validate the bind access flags. Remote Write and Atomic access for
21849e39c5baSBill Taylor * the Memory Window require that Local Write access be set in the
21859e39c5baSBill Taylor * corresponding Memory Region.
21869e39c5baSBill Taylor */
21879e39c5baSBill Taylor bind_flags = wr->wr.rc.rcwr.bind->bind_flags;
21889e39c5baSBill Taylor if (((bind_flags & IBT_WR_BIND_WRITE) ||
21899e39c5baSBill Taylor (bind_flags & IBT_WR_BIND_ATOMIC)) &&
21909e39c5baSBill Taylor !(mr->mr_accflag & IBT_MR_LOCAL_WRITE)) {
21919e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
21929e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
21939e39c5baSBill Taylor return (IBT_MR_ACCESS_REQ_INVALID);
21949e39c5baSBill Taylor }
21959e39c5baSBill Taylor
21969e39c5baSBill Taylor /* Calculate the new RKey for the Memory Window */
21979e39c5baSBill Taylor mpt = mw->mr_mptrsrcp;
21989e39c5baSBill Taylor new_rkey = hermon_mr_keycalc(mpt->hr_indx);
21999e39c5baSBill Taylor new_rkey = hermon_mr_key_swap(new_rkey);
22009e39c5baSBill Taylor
22019e39c5baSBill Taylor wr->wr.rc.rcwr.bind->bind_rkey_out = new_rkey;
22029e39c5baSBill Taylor mw->mr_rkey = new_rkey;
22039e39c5baSBill Taylor
22049e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
22059e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
22069e39c5baSBill Taylor return (DDI_SUCCESS);
22079e39c5baSBill Taylor }
22089e39c5baSBill Taylor
22099e39c5baSBill Taylor
22109e39c5baSBill Taylor /*
22119e39c5baSBill Taylor * hermon_wrid_from_reset_handling()
22129e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
22139e39c5baSBill Taylor */
22149e39c5baSBill Taylor /* ARGSUSED */
22159e39c5baSBill Taylor int
hermon_wrid_from_reset_handling(hermon_state_t * state,hermon_qphdl_t qp)22169e39c5baSBill Taylor hermon_wrid_from_reset_handling(hermon_state_t *state, hermon_qphdl_t qp)
22179e39c5baSBill Taylor {
22189e39c5baSBill Taylor hermon_workq_hdr_t *swq, *rwq;
22199e39c5baSBill Taylor
2220*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP)
22219e39c5baSBill Taylor return (DDI_SUCCESS);
22229e39c5baSBill Taylor
22239e39c5baSBill Taylor #ifdef __lock_lint
2224*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock);
22259e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock);
22269e39c5baSBill Taylor #else
2227*17a2b317SBill Taylor /* grab the cq lock(s) to modify the wqavl tree */
2228*17a2b317SBill Taylor if (qp->qp_rq_cqhdl)
2229*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock);
2230*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl &&
2231*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL)
22329e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock);
22339e39c5baSBill Taylor #endif
22349e39c5baSBill Taylor
22359e39c5baSBill Taylor /* Chain the newly allocated work queue header to the CQ's list */
2236*17a2b317SBill Taylor if (qp->qp_sq_cqhdl)
2237*17a2b317SBill Taylor hermon_cq_workq_add(qp->qp_sq_cqhdl, &qp->qp_sq_wqavl);
22389e39c5baSBill Taylor
22399e39c5baSBill Taylor swq = qp->qp_sq_wqhdr;
22409e39c5baSBill Taylor swq->wq_head = 0;
22419e39c5baSBill Taylor swq->wq_tail = 0;
22429e39c5baSBill Taylor swq->wq_full = 0;
22439e39c5baSBill Taylor
22449e39c5baSBill Taylor /*
22459e39c5baSBill Taylor * Now we repeat all the above operations for the receive work queue,
22469e39c5baSBill Taylor * or shared receive work queue.
22479e39c5baSBill Taylor *
22489e39c5baSBill Taylor * Note: We still use the 'qp_rq_cqhdl' even in the SRQ case.
22499e39c5baSBill Taylor */
22509e39c5baSBill Taylor
22519e39c5baSBill Taylor #ifdef __lock_lint
22529e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock);
22539e39c5baSBill Taylor #else
2254*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
22559e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock);
22569e39c5baSBill Taylor } else {
22579e39c5baSBill Taylor rwq = qp->qp_rq_wqhdr;
22589e39c5baSBill Taylor rwq->wq_head = 0;
22599e39c5baSBill Taylor rwq->wq_tail = 0;
22609e39c5baSBill Taylor rwq->wq_full = 0;
22619e39c5baSBill Taylor qp->qp_rq_wqecntr = 0;
22629e39c5baSBill Taylor }
22639e39c5baSBill Taylor #endif
22649e39c5baSBill Taylor hermon_cq_workq_add(qp->qp_rq_cqhdl, &qp->qp_rq_wqavl);
22659e39c5baSBill Taylor
22669e39c5baSBill Taylor #ifdef __lock_lint
22679e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock);
22689e39c5baSBill Taylor #else
2269*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
22709e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock);
22719e39c5baSBill Taylor }
22729e39c5baSBill Taylor #endif
22739e39c5baSBill Taylor
22749e39c5baSBill Taylor #ifdef __lock_lint
22759e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock);
2276*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock);
22779e39c5baSBill Taylor #else
2278*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl &&
2279*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL)
22809e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock);
2281*17a2b317SBill Taylor if (qp->qp_rq_cqhdl)
2282*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock);
22839e39c5baSBill Taylor #endif
22849e39c5baSBill Taylor return (DDI_SUCCESS);
22859e39c5baSBill Taylor }
22869e39c5baSBill Taylor
22879e39c5baSBill Taylor
22889e39c5baSBill Taylor /*
22899e39c5baSBill Taylor * hermon_wrid_to_reset_handling()
22909e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
22919e39c5baSBill Taylor */
22929e39c5baSBill Taylor int
hermon_wrid_to_reset_handling(hermon_state_t * state,hermon_qphdl_t qp)22939e39c5baSBill Taylor hermon_wrid_to_reset_handling(hermon_state_t *state, hermon_qphdl_t qp)
22949e39c5baSBill Taylor {
2295*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP)
22969e39c5baSBill Taylor return (DDI_SUCCESS);
22979e39c5baSBill Taylor
22989e39c5baSBill Taylor /*
22999e39c5baSBill Taylor * If there are unpolled entries in these CQs, they are
23009e39c5baSBill Taylor * polled/flushed.
23019e39c5baSBill Taylor * Grab the CQ lock(s) before manipulating the lists.
23029e39c5baSBill Taylor */
23039e39c5baSBill Taylor #ifdef __lock_lint
2304*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock);
23059e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock);
23069e39c5baSBill Taylor #else
2307*17a2b317SBill Taylor /* grab the cq lock(s) to modify the wqavl tree */
2308*17a2b317SBill Taylor if (qp->qp_rq_cqhdl)
2309*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock);
2310*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl &&
2311*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL)
23129e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock);
23139e39c5baSBill Taylor #endif
23149e39c5baSBill Taylor
23159e39c5baSBill Taylor #ifdef __lock_lint
23169e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock);
23179e39c5baSBill Taylor #else
2318*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
23199e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock);
23209e39c5baSBill Taylor }
23219e39c5baSBill Taylor #endif
23229e39c5baSBill Taylor /*
23239e39c5baSBill Taylor * Flush the entries on the CQ for this QP's QPN.
23249e39c5baSBill Taylor */
23259e39c5baSBill Taylor hermon_cq_entries_flush(state, qp);
23269e39c5baSBill Taylor
23279e39c5baSBill Taylor #ifdef __lock_lint
23289e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock);
23299e39c5baSBill Taylor #else
2330*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
23319e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock);
23329e39c5baSBill Taylor }
23339e39c5baSBill Taylor #endif
23349e39c5baSBill Taylor
23359e39c5baSBill Taylor hermon_cq_workq_remove(qp->qp_rq_cqhdl, &qp->qp_rq_wqavl);
2336*17a2b317SBill Taylor if (qp->qp_sq_cqhdl != NULL)
2337*17a2b317SBill Taylor hermon_cq_workq_remove(qp->qp_sq_cqhdl, &qp->qp_sq_wqavl);
23389e39c5baSBill Taylor
23399e39c5baSBill Taylor #ifdef __lock_lint
23409e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock);
2341*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock);
23429e39c5baSBill Taylor #else
2343*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl &&
2344*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL)
23459e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock);
2346*17a2b317SBill Taylor if (qp->qp_rq_cqhdl)
2347*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock);
23489e39c5baSBill Taylor #endif
23499e39c5baSBill Taylor
23509e39c5baSBill Taylor return (IBT_SUCCESS);
23519e39c5baSBill Taylor }
23529e39c5baSBill Taylor
23539e39c5baSBill Taylor
23549e39c5baSBill Taylor /*
23559e39c5baSBill Taylor * hermon_wrid_get_entry()
23569e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
23579e39c5baSBill Taylor */
23589e39c5baSBill Taylor uint64_t
hermon_wrid_get_entry(hermon_cqhdl_t cq,hermon_hw_cqe_t * cqe)23599e39c5baSBill Taylor hermon_wrid_get_entry(hermon_cqhdl_t cq, hermon_hw_cqe_t *cqe)
23609e39c5baSBill Taylor {
23619e39c5baSBill Taylor hermon_workq_avl_t *wqa;
23629e39c5baSBill Taylor hermon_workq_hdr_t *wq;
23639e39c5baSBill Taylor uint64_t wrid;
23649e39c5baSBill Taylor uint_t send_or_recv, qpnum;
23659e39c5baSBill Taylor uint32_t indx;
23669e39c5baSBill Taylor
23679e39c5baSBill Taylor /*
23689e39c5baSBill Taylor * Determine whether this CQE is a send or receive completion.
23699e39c5baSBill Taylor */
23709e39c5baSBill Taylor send_or_recv = HERMON_CQE_SENDRECV_GET(cq, cqe);
23719e39c5baSBill Taylor
23729e39c5baSBill Taylor /* Find the work queue for this QP number (send or receive side) */
23739e39c5baSBill Taylor qpnum = HERMON_CQE_QPNUM_GET(cq, cqe);
23749e39c5baSBill Taylor wqa = hermon_wrid_wqavl_find(cq, qpnum, send_or_recv);
23759e39c5baSBill Taylor wq = wqa->wqa_wq;
23769e39c5baSBill Taylor
23779e39c5baSBill Taylor /*
23789e39c5baSBill Taylor * Regardless of whether the completion is the result of a "success"
23799e39c5baSBill Taylor * or a "failure", we lock the list of "containers" and attempt to
23809e39c5baSBill Taylor * search for the the first matching completion (i.e. the first WR
23819e39c5baSBill Taylor * with a matching WQE addr and size). Once we find it, we pull out
23829e39c5baSBill Taylor * the "wrid" field and return it (see below). XXX Note: One possible
23839e39c5baSBill Taylor * future enhancement would be to enable this routine to skip over
23849e39c5baSBill Taylor * any "unsignaled" completions to go directly to the next "signaled"
23859e39c5baSBill Taylor * entry on success.
23869e39c5baSBill Taylor */
23879e39c5baSBill Taylor indx = HERMON_CQE_WQEADDRSZ_GET(cq, cqe) & wq->wq_mask;
23889e39c5baSBill Taylor wrid = wq->wq_wrid[indx];
23899e39c5baSBill Taylor if (wqa->wqa_srq_en) {
23909e39c5baSBill Taylor struct hermon_sw_srq_s *srq;
23919e39c5baSBill Taylor uint64_t *desc;
23929e39c5baSBill Taylor
23939e39c5baSBill Taylor /* put wqe back on the srq free list */
23949e39c5baSBill Taylor srq = wqa->wqa_srq;
23959e39c5baSBill Taylor mutex_enter(&srq->srq_lock);
23969e39c5baSBill Taylor desc = HERMON_SRQ_WQE_ADDR(srq, wq->wq_tail);
23979e39c5baSBill Taylor ((uint16_t *)desc)[1] = htons(indx);
23989e39c5baSBill Taylor wq->wq_tail = indx;
23999e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
24009e39c5baSBill Taylor } else {
24019e39c5baSBill Taylor wq->wq_head = (indx + 1) & wq->wq_mask;
24029e39c5baSBill Taylor wq->wq_full = 0;
24039e39c5baSBill Taylor }
24049e39c5baSBill Taylor
24059e39c5baSBill Taylor return (wrid);
24069e39c5baSBill Taylor }
24079e39c5baSBill Taylor
24089e39c5baSBill Taylor
24099e39c5baSBill Taylor int
hermon_wrid_workq_compare(const void * p1,const void * p2)24109e39c5baSBill Taylor hermon_wrid_workq_compare(const void *p1, const void *p2)
24119e39c5baSBill Taylor {
24129e39c5baSBill Taylor hermon_workq_compare_t *cmpp;
24139e39c5baSBill Taylor hermon_workq_avl_t *curr;
24149e39c5baSBill Taylor
24159e39c5baSBill Taylor cmpp = (hermon_workq_compare_t *)p1;
24169e39c5baSBill Taylor curr = (hermon_workq_avl_t *)p2;
24179e39c5baSBill Taylor
24189e39c5baSBill Taylor if (cmpp->cmp_qpn < curr->wqa_qpn)
24199e39c5baSBill Taylor return (-1);
24209e39c5baSBill Taylor else if (cmpp->cmp_qpn > curr->wqa_qpn)
24219e39c5baSBill Taylor return (+1);
24229e39c5baSBill Taylor else if (cmpp->cmp_type < curr->wqa_type)
24239e39c5baSBill Taylor return (-1);
24249e39c5baSBill Taylor else if (cmpp->cmp_type > curr->wqa_type)
24259e39c5baSBill Taylor return (+1);
24269e39c5baSBill Taylor else
24279e39c5baSBill Taylor return (0);
24289e39c5baSBill Taylor }
24299e39c5baSBill Taylor
24309e39c5baSBill Taylor
24319e39c5baSBill Taylor /*
24329e39c5baSBill Taylor * hermon_wrid_workq_find()
24339e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
24349e39c5baSBill Taylor */
24359e39c5baSBill Taylor static hermon_workq_avl_t *
hermon_wrid_wqavl_find(hermon_cqhdl_t cq,uint_t qpn,uint_t wq_type)24369e39c5baSBill Taylor hermon_wrid_wqavl_find(hermon_cqhdl_t cq, uint_t qpn, uint_t wq_type)
24379e39c5baSBill Taylor {
24389e39c5baSBill Taylor hermon_workq_avl_t *curr;
24399e39c5baSBill Taylor hermon_workq_compare_t cmp;
24409e39c5baSBill Taylor
24419e39c5baSBill Taylor /*
24429e39c5baSBill Taylor * Walk the CQ's work queue list, trying to find a send or recv queue
24439e39c5baSBill Taylor * with the same QP number. We do this even if we are going to later
24449e39c5baSBill Taylor * create a new entry because it helps us easily find the end of the
24459e39c5baSBill Taylor * list.
24469e39c5baSBill Taylor */
24479e39c5baSBill Taylor cmp.cmp_qpn = qpn;
24489e39c5baSBill Taylor cmp.cmp_type = wq_type;
24499e39c5baSBill Taylor #ifdef __lock_lint
24509e39c5baSBill Taylor hermon_wrid_workq_compare(NULL, NULL);
24519e39c5baSBill Taylor #endif
24529e39c5baSBill Taylor curr = avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, NULL);
24539e39c5baSBill Taylor
24549e39c5baSBill Taylor return (curr);
24559e39c5baSBill Taylor }
24569e39c5baSBill Taylor
24579e39c5baSBill Taylor
24589e39c5baSBill Taylor /*
24599e39c5baSBill Taylor * hermon_wrid_wqhdr_create()
24609e39c5baSBill Taylor * Context: Can be called from base context.
24619e39c5baSBill Taylor */
24629e39c5baSBill Taylor /* ARGSUSED */
24639e39c5baSBill Taylor hermon_workq_hdr_t *
hermon_wrid_wqhdr_create(int bufsz)24649e39c5baSBill Taylor hermon_wrid_wqhdr_create(int bufsz)
24659e39c5baSBill Taylor {
24669e39c5baSBill Taylor hermon_workq_hdr_t *wqhdr;
24679e39c5baSBill Taylor
24689e39c5baSBill Taylor /*
24699e39c5baSBill Taylor * Allocate space for the wqhdr, and an array to record all the wrids.
24709e39c5baSBill Taylor */
24719e39c5baSBill Taylor wqhdr = (hermon_workq_hdr_t *)kmem_zalloc(sizeof (*wqhdr), KM_NOSLEEP);
24729e39c5baSBill Taylor if (wqhdr == NULL) {
24739e39c5baSBill Taylor return (NULL);
24749e39c5baSBill Taylor }
24759e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqhdr))
24769e39c5baSBill Taylor wqhdr->wq_wrid = kmem_zalloc(bufsz * sizeof (uint64_t), KM_NOSLEEP);
24779e39c5baSBill Taylor if (wqhdr->wq_wrid == NULL) {
24789e39c5baSBill Taylor kmem_free(wqhdr, sizeof (*wqhdr));
24799e39c5baSBill Taylor return (NULL);
24809e39c5baSBill Taylor }
24819e39c5baSBill Taylor wqhdr->wq_size = bufsz;
24829e39c5baSBill Taylor wqhdr->wq_mask = bufsz - 1;
24839e39c5baSBill Taylor
24849e39c5baSBill Taylor return (wqhdr);
24859e39c5baSBill Taylor }
24869e39c5baSBill Taylor
24879e39c5baSBill Taylor void
hermon_wrid_wqhdr_destroy(hermon_workq_hdr_t * wqhdr)24889e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(hermon_workq_hdr_t *wqhdr)
24899e39c5baSBill Taylor {
24909e39c5baSBill Taylor kmem_free(wqhdr->wq_wrid, wqhdr->wq_size * sizeof (uint64_t));
24919e39c5baSBill Taylor kmem_free(wqhdr, sizeof (*wqhdr));
24929e39c5baSBill Taylor }
24939e39c5baSBill Taylor
24949e39c5baSBill Taylor
24959e39c5baSBill Taylor /*
24969e39c5baSBill Taylor * hermon_cq_workq_add()
24979e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
24989e39c5baSBill Taylor */
24999e39c5baSBill Taylor static void
hermon_cq_workq_add(hermon_cqhdl_t cq,hermon_workq_avl_t * wqavl)25009e39c5baSBill Taylor hermon_cq_workq_add(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl)
25019e39c5baSBill Taylor {
25029e39c5baSBill Taylor hermon_workq_compare_t cmp;
25039e39c5baSBill Taylor avl_index_t where;
25049e39c5baSBill Taylor
25059e39c5baSBill Taylor cmp.cmp_qpn = wqavl->wqa_qpn;
25069e39c5baSBill Taylor cmp.cmp_type = wqavl->wqa_type;
25079e39c5baSBill Taylor #ifdef __lock_lint
25089e39c5baSBill Taylor hermon_wrid_workq_compare(NULL, NULL);
25099e39c5baSBill Taylor #endif
25109e39c5baSBill Taylor (void) avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, &where);
25119e39c5baSBill Taylor avl_insert(&cq->cq_wrid_wqhdr_avl_tree, wqavl, where);
25129e39c5baSBill Taylor }
25139e39c5baSBill Taylor
25149e39c5baSBill Taylor
25159e39c5baSBill Taylor /*
25169e39c5baSBill Taylor * hermon_cq_workq_remove()
25179e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
25189e39c5baSBill Taylor */
25199e39c5baSBill Taylor static void
hermon_cq_workq_remove(hermon_cqhdl_t cq,hermon_workq_avl_t * wqavl)25209e39c5baSBill Taylor hermon_cq_workq_remove(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl)
25219e39c5baSBill Taylor {
25229e39c5baSBill Taylor #ifdef __lock_lint
25239e39c5baSBill Taylor hermon_wrid_workq_compare(NULL, NULL);
25249e39c5baSBill Taylor #endif
25259e39c5baSBill Taylor avl_remove(&cq->cq_wrid_wqhdr_avl_tree, wqavl);
25269e39c5baSBill Taylor }
2527