19e39c5baSBill Taylor /* 29e39c5baSBill Taylor * CDDL HEADER START 39e39c5baSBill Taylor * 49e39c5baSBill Taylor * The contents of this file are subject to the terms of the 59e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 69e39c5baSBill Taylor * You may not use this file except in compliance with the License. 79e39c5baSBill Taylor * 89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 109e39c5baSBill Taylor * See the License for the specific language governing permissions 119e39c5baSBill Taylor * and limitations under the License. 129e39c5baSBill Taylor * 139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 189e39c5baSBill Taylor * 199e39c5baSBill Taylor * CDDL HEADER END 209e39c5baSBill Taylor */ 219e39c5baSBill Taylor 229e39c5baSBill Taylor /* 23*17a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 249e39c5baSBill Taylor */ 259e39c5baSBill Taylor 269e39c5baSBill Taylor /* 279e39c5baSBill Taylor * hermon_wr.c 289e39c5baSBill Taylor * Hermon Work Request Processing Routines 299e39c5baSBill Taylor * 309e39c5baSBill Taylor * Implements all the routines necessary to provide the PostSend(), 319e39c5baSBill Taylor * PostRecv() and PostSRQ() verbs. Also contains all the code 329e39c5baSBill Taylor * necessary to implement the Hermon WRID tracking mechanism. 339e39c5baSBill Taylor */ 349e39c5baSBill Taylor 359e39c5baSBill Taylor #include <sys/types.h> 369e39c5baSBill Taylor #include <sys/conf.h> 379e39c5baSBill Taylor #include <sys/ddi.h> 389e39c5baSBill Taylor #include <sys/sunddi.h> 399e39c5baSBill Taylor #include <sys/modctl.h> 409e39c5baSBill Taylor #include <sys/avl.h> 419e39c5baSBill Taylor 429e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 439e39c5baSBill Taylor 449e39c5baSBill Taylor static uint32_t hermon_wr_get_immediate(ibt_send_wr_t *wr); 459e39c5baSBill Taylor static int hermon_wr_bind_check(hermon_state_t *state, ibt_send_wr_t *wr); 469e39c5baSBill Taylor static int hermon_wqe_send_build(hermon_state_t *state, hermon_qphdl_t qp, 479e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size); 489e39c5baSBill Taylor static int hermon_wqe_mlx_build(hermon_state_t *state, hermon_qphdl_t qp, 499e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size); 509e39c5baSBill Taylor static void hermon_wqe_headroom(uint_t from, hermon_qphdl_t qp); 519e39c5baSBill Taylor static int hermon_wqe_recv_build(hermon_state_t *state, hermon_qphdl_t qp, 529e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc); 539e39c5baSBill Taylor static int hermon_wqe_srq_build(hermon_state_t *state, hermon_srqhdl_t srq, 549e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc); 559e39c5baSBill Taylor static hermon_workq_avl_t *hermon_wrid_wqavl_find(hermon_cqhdl_t cq, uint_t qpn, 569e39c5baSBill Taylor uint_t send_or_recv); 579e39c5baSBill Taylor static void hermon_cq_workq_add(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl); 589e39c5baSBill Taylor static void hermon_cq_workq_remove(hermon_cqhdl_t cq, 599e39c5baSBill Taylor hermon_workq_avl_t *wqavl); 609e39c5baSBill Taylor 619e39c5baSBill Taylor static ibt_wr_ds_t null_sgl = { 0, 0x00000100, 0 }; 629e39c5baSBill Taylor 63c7facc54SBill Taylor /* 64c7facc54SBill Taylor * Add ability to try to debug RDMA_READ/RDMA_WRITE failures. 65c7facc54SBill Taylor * 66c7facc54SBill Taylor * 0x1 - print rkey used during post_send 67c7facc54SBill Taylor * 0x2 - print sgls used during post_send 68c7facc54SBill Taylor * 0x4 - print FMR comings and goings 69c7facc54SBill Taylor */ 70c7facc54SBill Taylor int hermon_rdma_debug = 0x0; 71c7facc54SBill Taylor 729e39c5baSBill Taylor static int 739e39c5baSBill Taylor hermon_post_send_ud(hermon_state_t *state, hermon_qphdl_t qp, 749e39c5baSBill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted) 759e39c5baSBill Taylor { 769e39c5baSBill Taylor hermon_hw_snd_wqe_ud_t *ud; 779e39c5baSBill Taylor hermon_workq_hdr_t *wq; 789e39c5baSBill Taylor hermon_ahhdl_t ah; 79*17a2b317SBill Taylor ibt_wr_rfci_send_t *rfci; 80*17a2b317SBill Taylor ibt_wr_init_send_t *is; 819e39c5baSBill Taylor ibt_ud_dest_t *dest; 829e39c5baSBill Taylor uint64_t *desc; 839e39c5baSBill Taylor uint32_t desc_sz; 849e39c5baSBill Taylor uint32_t signaled_dbd, solicited; 859e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk; 869e39c5baSBill Taylor uint32_t hdrmwqes; 879e39c5baSBill Taylor uint32_t nopcode, fence, immed_data = 0; 889c865d64SRajkumar Sivaprakasam hermon_hw_wqe_sgl_t *ds, *old_ds; 899e39c5baSBill Taylor ibt_wr_ds_t *sgl; 90*17a2b317SBill Taylor int nds; 919e39c5baSBill Taylor int i, j, last_ds, num_ds, status; 929e39c5baSBill Taylor uint32_t *wqe_start; 939e39c5baSBill Taylor int sectperwqe; 949e39c5baSBill Taylor uint_t posted_cnt = 0; 95*17a2b317SBill Taylor int total_len, strong_order, fc_bits, cksum; 96*17a2b317SBill Taylor 979e39c5baSBill Taylor 989e39c5baSBill Taylor /* initialize the FMA retry loop */ 999e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num); 1009e39c5baSBill Taylor 1019e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock)); 1029e39c5baSBill Taylor _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock)) 1039e39c5baSBill Taylor 1049e39c5baSBill Taylor /* Grab the lock for the WRID list */ 1059e39c5baSBill Taylor membar_consumer(); 1069e39c5baSBill Taylor 1079e39c5baSBill Taylor /* Save away some initial QP state */ 1089e39c5baSBill Taylor wq = qp->qp_sq_wqhdr; 1099e39c5baSBill Taylor qsize_msk = wq->wq_mask; 1109e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes; /* in WQEs */ 1119e39c5baSBill Taylor sectperwqe = 1 << (qp->qp_sq_log_wqesz - 2); 1129e39c5baSBill Taylor 1139e39c5baSBill Taylor tail = wq->wq_tail; 1149e39c5baSBill Taylor head = wq->wq_head; 1159e39c5baSBill Taylor status = DDI_SUCCESS; 1169e39c5baSBill Taylor 1179e39c5baSBill Taylor post_next: 1189e39c5baSBill Taylor /* 1199e39c5baSBill Taylor * Check for "queue full" condition. If the queue 1209e39c5baSBill Taylor * is already full, then no more WQEs can be posted. 1219e39c5baSBill Taylor * So break out, ring a doorbell (if necessary) and 1229e39c5baSBill Taylor * return an error 1239e39c5baSBill Taylor */ 1249e39c5baSBill Taylor if (wq->wq_full != 0) { 1259e39c5baSBill Taylor status = IBT_QP_FULL; 1269e39c5baSBill Taylor goto done; 1279e39c5baSBill Taylor } 1289e39c5baSBill Taylor 1299e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk; 1309e39c5baSBill Taylor if (((tail + hdrmwqes) & qsize_msk) == head) { 1319e39c5baSBill Taylor wq->wq_full = 1; 1329e39c5baSBill Taylor } 1339e39c5baSBill Taylor 1349e39c5baSBill Taylor desc = HERMON_QP_SQ_ENTRY(qp, tail); 1359e39c5baSBill Taylor 1369e39c5baSBill Taylor nds = wr->wr_nds; 1379e39c5baSBill Taylor sgl = wr->wr_sgl; 1389e39c5baSBill Taylor num_ds = 0; 139*17a2b317SBill Taylor strong_order = 0; 140*17a2b317SBill Taylor fc_bits = 0; 141*17a2b317SBill Taylor cksum = 0; 1429e39c5baSBill Taylor 1439e39c5baSBill Taylor /* 1449e39c5baSBill Taylor * Build a Send or Send_LSO WQE 1459e39c5baSBill Taylor */ 146*17a2b317SBill Taylor switch (wr->wr_opcode) { 147*17a2b317SBill Taylor case IBT_WRC_SEND_LSO: 148*17a2b317SBill Taylor if (wr->wr_trans != IBT_UD_SRV) { 149*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 150*17a2b317SBill Taylor goto done; 151*17a2b317SBill Taylor } 1529e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LSO; 153*17a2b317SBill Taylor if (wr->wr_flags & IBT_WR_SEND_CKSUM) 154*17a2b317SBill Taylor cksum = 0x30; 1559c865d64SRajkumar Sivaprakasam if (wr->wr.ud_lso.lso_hdr_sz > 60) { 1569c865d64SRajkumar Sivaprakasam nopcode |= (1 << 6); /* ReRead bit must be set */ 1579c865d64SRajkumar Sivaprakasam } 1589e39c5baSBill Taylor dest = wr->wr.ud_lso.lso_ud_dest; 1599e39c5baSBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah; 1609e39c5baSBill Taylor if (ah == NULL) { 1619e39c5baSBill Taylor status = IBT_AH_HDL_INVALID; 1629e39c5baSBill Taylor goto done; 1639e39c5baSBill Taylor } 164*17a2b317SBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc + 165*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 166*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud + 167*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ud_t)); 1689e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, dest); 1699e39c5baSBill Taylor 1709e39c5baSBill Taylor total_len = (4 + 0xf + wr->wr.ud_lso.lso_hdr_sz) & ~0xf; 1719e39c5baSBill Taylor if ((uintptr_t)ds + total_len + (nds * 16) > 1729e39c5baSBill Taylor (uintptr_t)desc + (1 << qp->qp_sq_log_wqesz)) { 1739e39c5baSBill Taylor status = IBT_QP_SGL_LEN_INVALID; 1749e39c5baSBill Taylor goto done; 1759e39c5baSBill Taylor } 1769e39c5baSBill Taylor old_ds = ds; 1779c865d64SRajkumar Sivaprakasam bcopy(wr->wr.ud_lso.lso_hdr, (uint32_t *)old_ds + 1, 1789e39c5baSBill Taylor wr->wr.ud_lso.lso_hdr_sz); 1799c865d64SRajkumar Sivaprakasam ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + total_len); 1809c865d64SRajkumar Sivaprakasam i = 0; 181*17a2b317SBill Taylor break; 182*17a2b317SBill Taylor 183*17a2b317SBill Taylor case IBT_WRC_SEND: 184*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SEND; 185*17a2b317SBill Taylor if (qp->qp_serv_type == HERMON_QP_UD) { 186*17a2b317SBill Taylor if (wr->wr_trans != IBT_UD_SRV) { 187*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 188*17a2b317SBill Taylor goto done; 189*17a2b317SBill Taylor } 190*17a2b317SBill Taylor if (wr->wr_flags & IBT_WR_SEND_CKSUM) 191*17a2b317SBill Taylor cksum = 0x30; 192*17a2b317SBill Taylor dest = wr->wr.ud.udwr_dest; 193*17a2b317SBill Taylor } else if (qp->qp_serv_type == HERMON_QP_RFCI) { 194*17a2b317SBill Taylor if (wr->wr_trans != IBT_RFCI_SRV) { 195*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 196*17a2b317SBill Taylor goto done; 197*17a2b317SBill Taylor } 198*17a2b317SBill Taylor rfci = &wr->wr.fc.rfci_send; 199*17a2b317SBill Taylor if ((wr->wr_flags & IBT_WR_SEND_FC_CRC) != 0) { 200*17a2b317SBill Taylor nopcode |= (rfci->rfci_eof << 16); 201*17a2b317SBill Taylor fc_bits = 0x40; /* set FCRC */ 202*17a2b317SBill Taylor } 203*17a2b317SBill Taylor dest = rfci->rfci_dest; 204*17a2b317SBill Taylor } else { 205*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID; 206*17a2b317SBill Taylor goto done; 207*17a2b317SBill Taylor } 2089e39c5baSBill Taylor if (wr->wr_flags & IBT_WR_SEND_IMMED) { 209*17a2b317SBill Taylor /* "|=" changes 0xa to 0xb without touching FCEOF */ 210*17a2b317SBill Taylor nopcode |= HERMON_WQE_SEND_NOPCODE_SENDI; 2119e39c5baSBill Taylor immed_data = wr->wr.ud.udwr_immed; 2129e39c5baSBill Taylor } 2139e39c5baSBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah; 2149e39c5baSBill Taylor if (ah == NULL) { 2159e39c5baSBill Taylor status = IBT_AH_HDL_INVALID; 2169e39c5baSBill Taylor goto done; 2179e39c5baSBill Taylor } 218*17a2b317SBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc + 219*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 220*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud + 221*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ud_t)); 2229e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, dest); 2239e39c5baSBill Taylor i = 0; 224*17a2b317SBill Taylor break; 225*17a2b317SBill Taylor 226*17a2b317SBill Taylor case IBT_WRC_INIT_SEND_FCMD: 227*17a2b317SBill Taylor if (qp->qp_serv_type != HERMON_QP_FCMND) { 228*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID; 229*17a2b317SBill Taylor goto done; 230*17a2b317SBill Taylor } 231*17a2b317SBill Taylor if (wr->wr_trans != IBT_FCMD_SRV) { 232*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 233*17a2b317SBill Taylor goto done; 234*17a2b317SBill Taylor } 235*17a2b317SBill Taylor nopcode = HERMON_WQE_FCP_OPCODE_INIT_AND_SEND; 236*17a2b317SBill Taylor is = wr->wr.fc.fc_is; 237*17a2b317SBill Taylor dest = is->is_ctl.fc_dest; 238*17a2b317SBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah; 239*17a2b317SBill Taylor if (ah == NULL) { 240*17a2b317SBill Taylor status = IBT_AH_HDL_INVALID; 241*17a2b317SBill Taylor goto done; 242*17a2b317SBill Taylor } 243*17a2b317SBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc + 244*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 245*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud + 246*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ud_t)); 247*17a2b317SBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, dest); 248*17a2b317SBill Taylor old_ds = ds; 249*17a2b317SBill Taylor /* move ds beyond the FCP-3 Init Segment */ 250*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + 0x10); 251*17a2b317SBill Taylor i = 0; 252*17a2b317SBill Taylor break; 253*17a2b317SBill Taylor 254*17a2b317SBill Taylor case IBT_WRC_FAST_REG_PMR: 255*17a2b317SBill Taylor { 256*17a2b317SBill Taylor hermon_hw_snd_wqe_frwr_t *frwr; 257*17a2b317SBill Taylor 258*17a2b317SBill Taylor if (qp->qp_serv_type != HERMON_QP_FCMND) { 259*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID; 260*17a2b317SBill Taylor goto done; 261*17a2b317SBill Taylor } 262*17a2b317SBill Taylor if (wr->wr_trans != IBT_FCMD_SRV) { 263*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 264*17a2b317SBill Taylor goto done; 265*17a2b317SBill Taylor } 266*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_FRWR; 267*17a2b317SBill Taylor frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc + 268*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 269*17a2b317SBill Taylor HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.fc.reg_pmr); 270*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)frwr + 271*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_frwr_t)); 272*17a2b317SBill Taylor nds = 0; 273*17a2b317SBill Taylor strong_order = 0x80; 274*17a2b317SBill Taylor break; 275*17a2b317SBill Taylor } 276*17a2b317SBill Taylor 277*17a2b317SBill Taylor #if 0 278*17a2b317SBill Taylor /* firmware does not support this */ 279*17a2b317SBill Taylor case IBT_WRC_LOCAL_INVALIDATE: 280*17a2b317SBill Taylor { 281*17a2b317SBill Taylor hermon_hw_snd_wqe_local_inv_t *li; 282*17a2b317SBill Taylor 283*17a2b317SBill Taylor if (qp->qp_serv_type != HERMON_QP_FCMND) { 284*17a2b317SBill Taylor status = IBT_QP_OP_TYPE_INVALID; 285*17a2b317SBill Taylor goto done; 286*17a2b317SBill Taylor } 287*17a2b317SBill Taylor if (wr->wr_trans != IBT_FCMD_SRV) { 288*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 289*17a2b317SBill Taylor goto done; 290*17a2b317SBill Taylor } 291*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LCL_INV; 292*17a2b317SBill Taylor li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc + 293*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 294*17a2b317SBill Taylor HERMON_WQE_BUILD_LI(qp, li, wr->wr.fc.li); 295*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)li + 296*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_local_inv_t)); 297*17a2b317SBill Taylor nds = 0; 298*17a2b317SBill Taylor strong_order = 0x80; 299*17a2b317SBill Taylor break; 300*17a2b317SBill Taylor } 301*17a2b317SBill Taylor #endif 302*17a2b317SBill Taylor default: 3039e39c5baSBill Taylor status = IBT_QP_OP_TYPE_INVALID; 3049e39c5baSBill Taylor goto done; 3059e39c5baSBill Taylor } 3069e39c5baSBill Taylor 3079e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) { 3089e39c5baSBill Taylor status = IBT_QP_SGL_LEN_INVALID; 3099e39c5baSBill Taylor goto done; 3109e39c5baSBill Taylor } 3119e39c5baSBill Taylor for (last_ds = num_ds, j = i; j < nds; j++) { 3129e39c5baSBill Taylor if (sgl[j].ds_len != 0) 3139e39c5baSBill Taylor last_ds++; /* real last ds of wqe to fill */ 3149e39c5baSBill Taylor } 3159e39c5baSBill Taylor desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4; 3169e39c5baSBill Taylor for (j = nds; --j >= i; ) { 3179e39c5baSBill Taylor if (sgl[j].ds_len == 0) { 3189e39c5baSBill Taylor continue; 3199e39c5baSBill Taylor } 3209e39c5baSBill Taylor 3219e39c5baSBill Taylor /* 3229e39c5baSBill Taylor * Fill in the Data Segment(s) for the current WQE, using the 3239e39c5baSBill Taylor * information contained in the scatter-gather list of the 3249e39c5baSBill Taylor * work request. 3259e39c5baSBill Taylor */ 3269e39c5baSBill Taylor last_ds--; 3279e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[j]); 3289e39c5baSBill Taylor } 3299e39c5baSBill Taylor 3309c865d64SRajkumar Sivaprakasam membar_producer(); 3319c865d64SRajkumar Sivaprakasam 3329c865d64SRajkumar Sivaprakasam if (wr->wr_opcode == IBT_WRC_SEND_LSO) { 3339c865d64SRajkumar Sivaprakasam HERMON_WQE_BUILD_LSO(qp, old_ds, wr->wr.ud_lso.lso_mss, 3349c865d64SRajkumar Sivaprakasam wr->wr.ud_lso.lso_hdr_sz); 335*17a2b317SBill Taylor } else if (wr->wr_opcode == IBT_WRC_INIT_SEND_FCMD) { 336*17a2b317SBill Taylor /* This sits in the STAMP, so must be set after setting SGL */ 337*17a2b317SBill Taylor HERMON_WQE_BUILD_FCP3_INIT(old_ds, is->is_ctl.fc_frame_ctrl, 338*17a2b317SBill Taylor is->is_cs_priority, is->is_tx_seq_id, is->is_fc_mtu, 339*17a2b317SBill Taylor is->is_dest_id, is->is_op, is->is_rem_exch, 340*17a2b317SBill Taylor is->is_exch_qp_idx); 341*17a2b317SBill Taylor 342*17a2b317SBill Taylor /* The following will be used in HERMON_WQE_SET_CTRL_SEGMENT */ 343*17a2b317SBill Taylor /* SIT bit in FCP-3 ctrl segment */ 344*17a2b317SBill Taylor desc_sz |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_SIT) ? 0x80 : 0; 345*17a2b317SBill Taylor /* LS bit in FCP-3 ctrl segment */ 346*17a2b317SBill Taylor fc_bits |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_LAST_SEQ) ? 347*17a2b317SBill Taylor 0x10000 : 0; 348*17a2b317SBill Taylor fc_bits |= ((is->is_ctl.fc_routing_ctrl & 0xF) << 20) | 349*17a2b317SBill Taylor (is->is_ctl.fc_seq_id << 24); 350*17a2b317SBill Taylor immed_data = is->is_ctl.fc_parameter; 3519c865d64SRajkumar Sivaprakasam } 3529c865d64SRajkumar Sivaprakasam 3539e39c5baSBill Taylor fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0; 3549e39c5baSBill Taylor 3559e39c5baSBill Taylor signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) || 356*17a2b317SBill Taylor (wr->wr_flags & IBT_WR_SEND_SIGNAL)) ? 0xC : 0; 3579e39c5baSBill Taylor 358*17a2b317SBill Taylor solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0; 3599e39c5baSBill Taylor 3609e39c5baSBill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data, 361*17a2b317SBill Taylor solicited, signaled_dbd, cksum, qp, strong_order, fc_bits); 3629e39c5baSBill Taylor 3639e39c5baSBill Taylor wq->wq_wrid[tail] = wr->wr_id; 3649e39c5baSBill Taylor 3659e39c5baSBill Taylor tail = next_tail; 3669e39c5baSBill Taylor 3679e39c5baSBill Taylor /* Update some of the state in the QP */ 3689e39c5baSBill Taylor wq->wq_tail = tail; 3699e39c5baSBill Taylor 3709e39c5baSBill Taylor membar_producer(); 3719e39c5baSBill Taylor 3729e39c5baSBill Taylor /* Now set the ownership bit and opcode (first dword). */ 3739e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode); 3749e39c5baSBill Taylor 3759e39c5baSBill Taylor posted_cnt++; 3769e39c5baSBill Taylor if (--num_wr > 0) { 3779e39c5baSBill Taylor /* do the invalidate of the headroom */ 3789e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, 3799e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk); 3809e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) { 3819e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF; 3829e39c5baSBill Taylor } 3839e39c5baSBill Taylor 3849e39c5baSBill Taylor wr++; 3859e39c5baSBill Taylor goto post_next; 3869e39c5baSBill Taylor } 3879e39c5baSBill Taylor done: 3889e39c5baSBill Taylor if (posted_cnt != 0) { 3899e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state); 3909e39c5baSBill Taylor 3919e39c5baSBill Taylor membar_producer(); 3929e39c5baSBill Taylor 3939e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell register. */ 3949e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt, 3959e39c5baSBill Taylor fm_status, fm_test_num); 3969e39c5baSBill Taylor 3979e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl, 3989e39c5baSBill Taylor (uint64_t *)(void *)&state->hs_uar->send, 3999e39c5baSBill Taylor (uint64_t)qp->qp_ring); 4009e39c5baSBill Taylor 4019e39c5baSBill Taylor /* the FMA retry loop ends. */ 4029e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt, 4039e39c5baSBill Taylor fm_status, fm_test_num); 4049e39c5baSBill Taylor 4059e39c5baSBill Taylor /* do the invalidate of the headroom */ 4069e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, 4079e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk); 4089e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) { 4099e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF; 4109e39c5baSBill Taylor } 4119e39c5baSBill Taylor } 4129e39c5baSBill Taylor if (num_posted != NULL) 4139e39c5baSBill Taylor *num_posted = posted_cnt; 4149e39c5baSBill Taylor 4159e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock); 4169e39c5baSBill Taylor 4179e39c5baSBill Taylor return (status); 4189e39c5baSBill Taylor 4199e39c5baSBill Taylor pio_error: 4209e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock); 4219e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 4229e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 4239e39c5baSBill Taylor } 4249e39c5baSBill Taylor 4259e39c5baSBill Taylor static int 4269e39c5baSBill Taylor hermon_post_send_rc(hermon_state_t *state, hermon_qphdl_t qp, 4279e39c5baSBill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted) 4289e39c5baSBill Taylor { 4299e39c5baSBill Taylor uint64_t *desc; 4309e39c5baSBill Taylor hermon_workq_hdr_t *wq; 4319e39c5baSBill Taylor uint32_t desc_sz; 4329e39c5baSBill Taylor uint32_t signaled_dbd, solicited; 4339e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk; 4349e39c5baSBill Taylor uint32_t hdrmwqes; 4359e39c5baSBill Taylor int status; 4369e39c5baSBill Taylor uint32_t nopcode, fence, immed_data = 0; 4379e39c5baSBill Taylor hermon_hw_snd_wqe_remaddr_t *rc; 4389e39c5baSBill Taylor hermon_hw_snd_wqe_atomic_t *at; 4399e39c5baSBill Taylor hermon_hw_snd_wqe_bind_t *bn; 440*17a2b317SBill Taylor hermon_hw_snd_wqe_frwr_t *frwr; 441*17a2b317SBill Taylor hermon_hw_snd_wqe_local_inv_t *li; 4429e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds; 4439e39c5baSBill Taylor ibt_wr_ds_t *sgl; 444*17a2b317SBill Taylor int nds; 4459e39c5baSBill Taylor int i, last_ds, num_ds; 4469e39c5baSBill Taylor uint32_t *wqe_start; 4479e39c5baSBill Taylor int sectperwqe; 4489e39c5baSBill Taylor uint_t posted_cnt = 0; 449*17a2b317SBill Taylor int strong_order; 450c7facc54SBill Taylor int print_rdma; 451c7facc54SBill Taylor int rlen; 452c7facc54SBill Taylor uint32_t rkey; 453c7facc54SBill Taylor uint64_t raddr; 4549e39c5baSBill Taylor 4559e39c5baSBill Taylor /* initialize the FMA retry loop */ 4569e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num); 4579e39c5baSBill Taylor 4589e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock)); 4599e39c5baSBill Taylor _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock)) 4609e39c5baSBill Taylor 4619e39c5baSBill Taylor /* Save away some initial QP state */ 4629e39c5baSBill Taylor wq = qp->qp_sq_wqhdr; 4639e39c5baSBill Taylor qsize_msk = wq->wq_mask; 4649e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes; /* in WQEs */ 4659e39c5baSBill Taylor sectperwqe = 1 << (qp->qp_sq_log_wqesz - 2); 4669e39c5baSBill Taylor 4679e39c5baSBill Taylor tail = wq->wq_tail; 4689e39c5baSBill Taylor head = wq->wq_head; 4699e39c5baSBill Taylor status = DDI_SUCCESS; 4709e39c5baSBill Taylor 4719e39c5baSBill Taylor post_next: 472c7facc54SBill Taylor print_rdma = 0; 473c7facc54SBill Taylor rlen = 0; 474*17a2b317SBill Taylor strong_order = 0; 475c7facc54SBill Taylor 4769e39c5baSBill Taylor /* 4779e39c5baSBill Taylor * Check for "queue full" condition. If the queue 4789e39c5baSBill Taylor * is already full, then no more WQEs can be posted. 4799e39c5baSBill Taylor * So break out, ring a doorbell (if necessary) and 4809e39c5baSBill Taylor * return an error 4819e39c5baSBill Taylor */ 4829e39c5baSBill Taylor if (wq->wq_full != 0) { 4839e39c5baSBill Taylor status = IBT_QP_FULL; 4849e39c5baSBill Taylor goto done; 4859e39c5baSBill Taylor } 4869e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk; 4879e39c5baSBill Taylor if (((tail + hdrmwqes) & qsize_msk) == head) { 4889e39c5baSBill Taylor wq->wq_full = 1; 4899e39c5baSBill Taylor } 4909e39c5baSBill Taylor 4919e39c5baSBill Taylor desc = HERMON_QP_SQ_ENTRY(qp, tail); 4929e39c5baSBill Taylor 4939e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc + 4949e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 4959e39c5baSBill Taylor nds = wr->wr_nds; 4969e39c5baSBill Taylor sgl = wr->wr_sgl; 4979e39c5baSBill Taylor num_ds = 0; 498*17a2b317SBill Taylor if (wr->wr_trans != IBT_RC_SRV) { 499*17a2b317SBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 500*17a2b317SBill Taylor goto done; 501*17a2b317SBill Taylor } 5029e39c5baSBill Taylor 5039e39c5baSBill Taylor /* 5049e39c5baSBill Taylor * Validate the operation type. For RC requests, we allow 5059e39c5baSBill Taylor * "Send", "RDMA Read", "RDMA Write", various "Atomic" 5069e39c5baSBill Taylor * operations, and memory window "Bind" 5079e39c5baSBill Taylor */ 5089e39c5baSBill Taylor switch (wr->wr_opcode) { 5099e39c5baSBill Taylor default: 5109e39c5baSBill Taylor status = IBT_QP_OP_TYPE_INVALID; 5119e39c5baSBill Taylor goto done; 5129e39c5baSBill Taylor 5139e39c5baSBill Taylor case IBT_WRC_SEND: 514*17a2b317SBill Taylor if (wr->wr_flags & IBT_WR_SEND_REMOTE_INVAL) { 515*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SND_INV; 516*17a2b317SBill Taylor immed_data = wr->wr.rc.rcwr.send_inval; 517*17a2b317SBill Taylor } else if (wr->wr_flags & IBT_WR_SEND_IMMED) { 5189e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SENDI; 5199e39c5baSBill Taylor immed_data = wr->wr.rc.rcwr.send_immed; 5209e39c5baSBill Taylor } else { 5219e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SEND; 5229e39c5baSBill Taylor } 5239e39c5baSBill Taylor break; 5249e39c5baSBill Taylor 5259e39c5baSBill Taylor /* 5269e39c5baSBill Taylor * If this is an RDMA Read or RDMA Write request, then fill 5279e39c5baSBill Taylor * in the "Remote Address" header fields. 5289e39c5baSBill Taylor */ 5299e39c5baSBill Taylor case IBT_WRC_RDMAW: 5309e39c5baSBill Taylor if (wr->wr_flags & IBT_WR_SEND_IMMED) { 5319e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAWI; 5329e39c5baSBill Taylor immed_data = wr->wr.rc.rcwr.rdma.rdma_immed; 5339e39c5baSBill Taylor } else { 5349e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAW; 5359e39c5baSBill Taylor } 5369e39c5baSBill Taylor /* FALLTHROUGH */ 5379e39c5baSBill Taylor case IBT_WRC_RDMAR: 5389e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_RDMAR) 5399e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAR; 5409e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc + 5419e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 5429e39c5baSBill Taylor 5439e39c5baSBill Taylor /* 5449e39c5baSBill Taylor * Build the Remote Address Segment for the WQE, using 5459e39c5baSBill Taylor * the information from the RC work request. 5469e39c5baSBill Taylor */ 5479e39c5baSBill Taylor HERMON_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma); 5489e39c5baSBill Taylor 549c7facc54SBill Taylor if (hermon_rdma_debug) { 550c7facc54SBill Taylor print_rdma = hermon_rdma_debug; 551c7facc54SBill Taylor rkey = wr->wr.rc.rcwr.rdma.rdma_rkey; 552c7facc54SBill Taylor raddr = wr->wr.rc.rcwr.rdma.rdma_raddr; 553c7facc54SBill Taylor } 554c7facc54SBill Taylor 5559e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */ 5569e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)rc + 5579e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t)); 5589e39c5baSBill Taylor break; 5599e39c5baSBill Taylor 5609e39c5baSBill Taylor /* 5619e39c5baSBill Taylor * If this is one of the Atomic type operations (i.e 5629e39c5baSBill Taylor * Compare-Swap or Fetch-Add), then fill in both the "Remote 5639e39c5baSBill Taylor * Address" header fields and the "Atomic" header fields. 5649e39c5baSBill Taylor */ 5659e39c5baSBill Taylor case IBT_WRC_CSWAP: 5669e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMCS; 5679e39c5baSBill Taylor /* FALLTHROUGH */ 5689e39c5baSBill Taylor case IBT_WRC_FADD: 5699e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_FADD) 5709e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMFA; 5719e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc + 5729e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 5739e39c5baSBill Taylor at = (hermon_hw_snd_wqe_atomic_t *)((uintptr_t)rc + 5749e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t)); 5759e39c5baSBill Taylor 5769e39c5baSBill Taylor /* 5779e39c5baSBill Taylor * Build the Remote Address and Atomic Segments for 5789e39c5baSBill Taylor * the WQE, using the information from the RC Atomic 5799e39c5baSBill Taylor * work request. 5809e39c5baSBill Taylor */ 5819e39c5baSBill Taylor HERMON_WQE_BUILD_RC_ATOMIC_REMADDR(qp, rc, wr); 5829e39c5baSBill Taylor HERMON_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic); 5839e39c5baSBill Taylor 5849e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */ 5859e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)at + 5869e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_atomic_t)); 5879e39c5baSBill Taylor 5889e39c5baSBill Taylor /* 5899e39c5baSBill Taylor * Update "nds" and "sgl" because Atomic requests have 5909e39c5baSBill Taylor * only a single Data Segment. 5919e39c5baSBill Taylor */ 5929e39c5baSBill Taylor nds = 1; 5939e39c5baSBill Taylor sgl = wr->wr_sgl; 5949e39c5baSBill Taylor break; 5959e39c5baSBill Taylor 5969e39c5baSBill Taylor /* 5979e39c5baSBill Taylor * If this is memory window Bind operation, then we call the 5989e39c5baSBill Taylor * hermon_wr_bind_check() routine to validate the request and 5999e39c5baSBill Taylor * to generate the updated RKey. If this is successful, then 6009e39c5baSBill Taylor * we fill in the WQE's "Bind" header fields. 6019e39c5baSBill Taylor */ 6029e39c5baSBill Taylor case IBT_WRC_BIND: 6039e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_BIND; 6049e39c5baSBill Taylor status = hermon_wr_bind_check(state, wr); 6059e39c5baSBill Taylor if (status != DDI_SUCCESS) 6069e39c5baSBill Taylor goto done; 6079e39c5baSBill Taylor 6089e39c5baSBill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc + 6099e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 6109e39c5baSBill Taylor 6119e39c5baSBill Taylor /* 6129e39c5baSBill Taylor * Build the Bind Memory Window Segments for the WQE, 6139e39c5baSBill Taylor * using the information from the RC Bind memory 6149e39c5baSBill Taylor * window work request. 6159e39c5baSBill Taylor */ 6169e39c5baSBill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind); 6179e39c5baSBill Taylor 6189e39c5baSBill Taylor /* 6199e39c5baSBill Taylor * Update the "ds" pointer. Even though the "bind" 6209e39c5baSBill Taylor * operation requires no SGLs, this is necessary to 6219e39c5baSBill Taylor * facilitate the correct descriptor size calculations 6229e39c5baSBill Taylor * (below). 6239e39c5baSBill Taylor */ 6249e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)bn + 6259e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_bind_t)); 6269e39c5baSBill Taylor nds = 0; 627*17a2b317SBill Taylor break; 628*17a2b317SBill Taylor 629*17a2b317SBill Taylor case IBT_WRC_FAST_REG_PMR: 630*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_FRWR; 631*17a2b317SBill Taylor frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc + 632*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 633*17a2b317SBill Taylor HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.rc.rcwr.reg_pmr); 634*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)frwr + 635*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_frwr_t)); 636*17a2b317SBill Taylor nds = 0; 637*17a2b317SBill Taylor strong_order = 0x80; 638*17a2b317SBill Taylor break; 639*17a2b317SBill Taylor 640*17a2b317SBill Taylor case IBT_WRC_LOCAL_INVALIDATE: 641*17a2b317SBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LCL_INV; 642*17a2b317SBill Taylor li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc + 643*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 644*17a2b317SBill Taylor HERMON_WQE_BUILD_LI(qp, li, wr->wr.rc.rcwr.li); 645*17a2b317SBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)li + 646*17a2b317SBill Taylor sizeof (hermon_hw_snd_wqe_local_inv_t)); 647*17a2b317SBill Taylor nds = 0; 648*17a2b317SBill Taylor strong_order = 0x80; 649*17a2b317SBill Taylor break; 6509e39c5baSBill Taylor } 6519e39c5baSBill Taylor 6529e39c5baSBill Taylor /* 6539e39c5baSBill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based 6549e39c5baSBill Taylor * on the values setup above (i.e. "sgl", "nds", and the "ds" 6559e39c5baSBill Taylor * pointer. Start by checking for a valid number of SGL entries 6569e39c5baSBill Taylor */ 6579e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) { 6589e39c5baSBill Taylor status = IBT_QP_SGL_LEN_INVALID; 6599e39c5baSBill Taylor goto done; 6609e39c5baSBill Taylor } 6619e39c5baSBill Taylor 6629e39c5baSBill Taylor for (last_ds = num_ds, i = 0; i < nds; i++) { 6639e39c5baSBill Taylor if (sgl[i].ds_len != 0) 6649e39c5baSBill Taylor last_ds++; /* real last ds of wqe to fill */ 6659e39c5baSBill Taylor } 6669e39c5baSBill Taylor desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4; 6679e39c5baSBill Taylor for (i = nds; --i >= 0; ) { 6689e39c5baSBill Taylor if (sgl[i].ds_len == 0) { 6699e39c5baSBill Taylor continue; 6709e39c5baSBill Taylor } 671c7facc54SBill Taylor rlen += sgl[i].ds_len; 672c7facc54SBill Taylor if (print_rdma & 0x2) 673c7facc54SBill Taylor IBTF_DPRINTF_L2("rdma", "post: [%d]: laddr %llx " 674c7facc54SBill Taylor "llen %x", i, sgl[i].ds_va, sgl[i].ds_len); 6759e39c5baSBill Taylor 6769e39c5baSBill Taylor /* 6779e39c5baSBill Taylor * Fill in the Data Segment(s) for the current WQE, using the 6789e39c5baSBill Taylor * information contained in the scatter-gather list of the 6799e39c5baSBill Taylor * work request. 6809e39c5baSBill Taylor */ 6819e39c5baSBill Taylor last_ds--; 6829e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[i]); 6839e39c5baSBill Taylor } 684*17a2b317SBill Taylor /* ensure RDMA READ does not exceed HCA limit */ 685*17a2b317SBill Taylor if ((wr->wr_opcode == IBT_WRC_RDMAR) && (desc_sz > 686*17a2b317SBill Taylor state->hs_ibtfinfo.hca_attr->hca_conn_rdma_read_sgl_sz + 2)) { 687*17a2b317SBill Taylor status = IBT_QP_SGL_LEN_INVALID; 688*17a2b317SBill Taylor goto done; 689*17a2b317SBill Taylor } 6909e39c5baSBill Taylor 691c7facc54SBill Taylor if (print_rdma & 0x1) { 692c7facc54SBill Taylor IBTF_DPRINTF_L2("rdma", "post: indx %x rkey %x raddr %llx " 693c7facc54SBill Taylor "total len %x", tail, rkey, raddr, rlen); 694c7facc54SBill Taylor } 695c7facc54SBill Taylor 6969e39c5baSBill Taylor fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0; 6979e39c5baSBill Taylor 6989e39c5baSBill Taylor signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) || 699*17a2b317SBill Taylor (wr->wr_flags & IBT_WR_SEND_SIGNAL)) ? 0xC : 0; 7009e39c5baSBill Taylor 701*17a2b317SBill Taylor solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0; 7029e39c5baSBill Taylor 7039e39c5baSBill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data, solicited, 704*17a2b317SBill Taylor signaled_dbd, 0, qp, strong_order, 0); 7059e39c5baSBill Taylor 7069e39c5baSBill Taylor wq->wq_wrid[tail] = wr->wr_id; 7079e39c5baSBill Taylor 7089e39c5baSBill Taylor tail = next_tail; 7099e39c5baSBill Taylor 7109e39c5baSBill Taylor /* Update some of the state in the QP */ 7119e39c5baSBill Taylor wq->wq_tail = tail; 7129e39c5baSBill Taylor 7139e39c5baSBill Taylor membar_producer(); 7149e39c5baSBill Taylor 7159e39c5baSBill Taylor /* Now set the ownership bit of the first one in the chain. */ 7169e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode); 7179e39c5baSBill Taylor 7189e39c5baSBill Taylor posted_cnt++; 7199e39c5baSBill Taylor if (--num_wr > 0) { 7209e39c5baSBill Taylor /* do the invalidate of the headroom */ 7219e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, 7229e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk); 7239e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) { 7249e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF; 7259e39c5baSBill Taylor } 7269e39c5baSBill Taylor 7279e39c5baSBill Taylor wr++; 7289e39c5baSBill Taylor goto post_next; 7299e39c5baSBill Taylor } 7309e39c5baSBill Taylor done: 7319e39c5baSBill Taylor 7329e39c5baSBill Taylor if (posted_cnt != 0) { 7339e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state); 7349e39c5baSBill Taylor 7359e39c5baSBill Taylor membar_producer(); 7369e39c5baSBill Taylor 7379e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell register. */ 7389e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt, 7399e39c5baSBill Taylor fm_status, fm_test_num); 7409e39c5baSBill Taylor 7419e39c5baSBill Taylor /* Ring the doorbell */ 7429e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl, 7439e39c5baSBill Taylor (uint64_t *)(void *)&state->hs_uar->send, 7449e39c5baSBill Taylor (uint64_t)qp->qp_ring); 7459e39c5baSBill Taylor 7469e39c5baSBill Taylor /* the FMA retry loop ends. */ 7479e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt, 7489e39c5baSBill Taylor fm_status, fm_test_num); 7499e39c5baSBill Taylor 7509e39c5baSBill Taylor /* do the invalidate of the headroom */ 7519e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, 7529e39c5baSBill Taylor (tail + hdrmwqes) & qsize_msk); 7539e39c5baSBill Taylor for (i = 16; i < sectperwqe; i += 16) { 7549e39c5baSBill Taylor wqe_start[i] = 0xFFFFFFFF; 7559e39c5baSBill Taylor } 7569e39c5baSBill Taylor } 7579e39c5baSBill Taylor /* 7589e39c5baSBill Taylor * Update the "num_posted" return value (if necessary). 7599e39c5baSBill Taylor * Then drop the locks and return success. 7609e39c5baSBill Taylor */ 7619e39c5baSBill Taylor if (num_posted != NULL) { 7629e39c5baSBill Taylor *num_posted = posted_cnt; 7639e39c5baSBill Taylor } 7649e39c5baSBill Taylor 7659e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock); 7669e39c5baSBill Taylor return (status); 7679e39c5baSBill Taylor 7689e39c5baSBill Taylor pio_error: 7699e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock); 7709e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 7719e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 7729e39c5baSBill Taylor } 7739e39c5baSBill Taylor 7749e39c5baSBill Taylor /* 7759e39c5baSBill Taylor * hermon_post_send() 7769e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 7779e39c5baSBill Taylor */ 7789e39c5baSBill Taylor int 7799e39c5baSBill Taylor hermon_post_send(hermon_state_t *state, hermon_qphdl_t qp, 7809e39c5baSBill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted) 7819e39c5baSBill Taylor { 7829e39c5baSBill Taylor ibt_send_wr_t *curr_wr; 7839e39c5baSBill Taylor hermon_workq_hdr_t *wq; 7849e39c5baSBill Taylor hermon_ahhdl_t ah; 7859e39c5baSBill Taylor uint64_t *desc, *prev; 7869e39c5baSBill Taylor uint32_t desc_sz; 7879e39c5baSBill Taylor uint32_t signaled_dbd, solicited; 7889e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk; 7899e39c5baSBill Taylor uint32_t hdrmwqes; 7909e39c5baSBill Taylor uint_t currindx, wrindx, numremain; 7919e39c5baSBill Taylor uint_t chainlen; 7929e39c5baSBill Taylor uint_t posted_cnt, maxstat; 7939e39c5baSBill Taylor uint_t total_posted; 7949e39c5baSBill Taylor int status; 7959e39c5baSBill Taylor uint32_t nopcode, fence, immed_data = 0; 7969e39c5baSBill Taylor uint32_t prev_nopcode; 797*17a2b317SBill Taylor uint_t qp_state; 7989e39c5baSBill Taylor 7999e39c5baSBill Taylor /* initialize the FMA retry loop */ 8009e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test); 8019e39c5baSBill Taylor 8029e39c5baSBill Taylor /* 8039e39c5baSBill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel 8049e39c5baSBill Taylor * clients to post to QP memory that is accessible directly by the 8059e39c5baSBill Taylor * user. If the QP memory is user accessible, then return an error. 8069e39c5baSBill Taylor */ 807*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP) { 8089e39c5baSBill Taylor return (IBT_QP_HDL_INVALID); 8099e39c5baSBill Taylor } 8109e39c5baSBill Taylor 811*17a2b317SBill Taylor mutex_enter(&qp->qp_sq_lock); 8129e39c5baSBill Taylor 8139e39c5baSBill Taylor /* 8149e39c5baSBill Taylor * Check QP state. Can not post Send requests from the "Reset", 8159e39c5baSBill Taylor * "Init", or "RTR" states 8169e39c5baSBill Taylor */ 817*17a2b317SBill Taylor qp_state = qp->qp_state_for_post_send; 818*17a2b317SBill Taylor if ((qp_state == HERMON_QP_RESET) || 819*17a2b317SBill Taylor (qp_state == HERMON_QP_INIT) || 820*17a2b317SBill Taylor (qp_state == HERMON_QP_RTR)) { 821*17a2b317SBill Taylor mutex_exit(&qp->qp_sq_lock); 8229e39c5baSBill Taylor return (IBT_QP_STATE_INVALID); 8239e39c5baSBill Taylor } 8249e39c5baSBill Taylor 8259e39c5baSBill Taylor if (qp->qp_is_special) 8269e39c5baSBill Taylor goto post_many; 8279e39c5baSBill Taylor 8289e39c5baSBill Taylor /* Use these optimized functions most of the time */ 829*17a2b317SBill Taylor if (qp->qp_type == IBT_UD_RQP) { 8309e39c5baSBill Taylor return (hermon_post_send_ud(state, qp, wr, num_wr, num_posted)); 831c7facc54SBill Taylor } 8329e39c5baSBill Taylor 833c7facc54SBill Taylor if (qp->qp_serv_type == HERMON_QP_RC) { 8349e39c5baSBill Taylor return (hermon_post_send_rc(state, qp, wr, num_wr, num_posted)); 835c7facc54SBill Taylor } 8369e39c5baSBill Taylor 8379e39c5baSBill Taylor if (qp->qp_serv_type == HERMON_QP_UC) 8389e39c5baSBill Taylor goto post_many; 8399e39c5baSBill Taylor 8409e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock); 8419e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID); 8429e39c5baSBill Taylor 8439e39c5baSBill Taylor post_many: 8449e39c5baSBill Taylor /* general loop for non-optimized posting */ 8459e39c5baSBill Taylor 8469e39c5baSBill Taylor /* Save away some initial QP state */ 8479e39c5baSBill Taylor wq = qp->qp_sq_wqhdr; 8489e39c5baSBill Taylor qsize_msk = wq->wq_mask; 8499e39c5baSBill Taylor tail = wq->wq_tail; 8509e39c5baSBill Taylor head = wq->wq_head; 8519e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes; /* in WQEs */ 8529e39c5baSBill Taylor 8539e39c5baSBill Taylor /* Initialize posted_cnt */ 8549e39c5baSBill Taylor posted_cnt = 0; 8559e39c5baSBill Taylor total_posted = 0; 8569e39c5baSBill Taylor 8579e39c5baSBill Taylor /* 8589e39c5baSBill Taylor * For each ibt_send_wr_t in the wr[] list passed in, parse the 8599e39c5baSBill Taylor * request and build a Send WQE. NOTE: Because we are potentially 8609e39c5baSBill Taylor * building a chain of WQEs to post, we want to build them all first, 8619e39c5baSBill Taylor * and set the valid (HW Ownership) bit on all but the first. 8629e39c5baSBill Taylor * However, we do not want to validate the first one until the 8639e39c5baSBill Taylor * entire chain of WQEs has been built. Then in the final 8649e39c5baSBill Taylor * we set the valid bit in the first, flush if needed, and as a last 8659e39c5baSBill Taylor * step ring the appropriate doorbell. NOTE: the doorbell ring may 8669e39c5baSBill Taylor * NOT be needed if the HCA is already processing, but the doorbell 8679e39c5baSBill Taylor * ring will be done regardless. NOTE ALSO: It is possible for 8689e39c5baSBill Taylor * more Work Requests to be posted than the HW will support at one 8699e39c5baSBill Taylor * shot. If this happens, we need to be able to post and ring 8709e39c5baSBill Taylor * several chains here until the the entire request is complete. 8719e39c5baSBill Taylor * NOTE ALSO: the term "chain" is used to differentiate it from 8729e39c5baSBill Taylor * Work Request List passed in; and because that's the terminology 8739e39c5baSBill Taylor * from the previous generations of HCA - but the WQEs are not, in fact 8749e39c5baSBill Taylor * chained together for Hermon 8759e39c5baSBill Taylor */ 8769e39c5baSBill Taylor 8779e39c5baSBill Taylor wrindx = 0; 8789e39c5baSBill Taylor numremain = num_wr; 8799e39c5baSBill Taylor status = DDI_SUCCESS; 8809e39c5baSBill Taylor while ((wrindx < num_wr) && (status == DDI_SUCCESS)) { 8819e39c5baSBill Taylor /* 8829e39c5baSBill Taylor * For the first WQE on a new chain we need "prev" to point 8839e39c5baSBill Taylor * to the current descriptor. 8849e39c5baSBill Taylor */ 8859e39c5baSBill Taylor prev = HERMON_QP_SQ_ENTRY(qp, tail); 8869e39c5baSBill Taylor 8879e39c5baSBill Taylor /* 8889e39c5baSBill Taylor * Break the request up into lists that are less than or 8899e39c5baSBill Taylor * equal to the maximum number of WQEs that can be posted 8909e39c5baSBill Taylor * per doorbell ring - 256 currently 8919e39c5baSBill Taylor */ 8929e39c5baSBill Taylor chainlen = (numremain > HERMON_QP_MAXDESC_PER_DB) ? 8939e39c5baSBill Taylor HERMON_QP_MAXDESC_PER_DB : numremain; 8949e39c5baSBill Taylor numremain -= chainlen; 8959e39c5baSBill Taylor 8969e39c5baSBill Taylor for (currindx = 0; currindx < chainlen; currindx++, wrindx++) { 8979e39c5baSBill Taylor /* 8989e39c5baSBill Taylor * Check for "queue full" condition. If the queue 8999e39c5baSBill Taylor * is already full, then no more WQEs can be posted. 9009e39c5baSBill Taylor * So break out, ring a doorbell (if necessary) and 9019e39c5baSBill Taylor * return an error 9029e39c5baSBill Taylor */ 9039e39c5baSBill Taylor if (wq->wq_full != 0) { 9049e39c5baSBill Taylor status = IBT_QP_FULL; 9059e39c5baSBill Taylor break; 9069e39c5baSBill Taylor } 9079e39c5baSBill Taylor 9089e39c5baSBill Taylor /* 9099e39c5baSBill Taylor * Increment the "tail index". Check for "queue 9109e39c5baSBill Taylor * full" condition incl. headroom. If we detect that 9119e39c5baSBill Taylor * the current work request is going to fill the work 9129e39c5baSBill Taylor * queue, then we mark this condition and continue. 9139e39c5baSBill Taylor * Don't need >=, because going one-by-one we have to 9149e39c5baSBill Taylor * hit it exactly sooner or later 9159e39c5baSBill Taylor */ 9169e39c5baSBill Taylor 9179e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk; 9189e39c5baSBill Taylor if (((tail + hdrmwqes) & qsize_msk) == head) { 9199e39c5baSBill Taylor wq->wq_full = 1; 9209e39c5baSBill Taylor } 9219e39c5baSBill Taylor 9229e39c5baSBill Taylor /* 9239e39c5baSBill Taylor * Get the address of the location where the next 9249e39c5baSBill Taylor * Send WQE should be built 9259e39c5baSBill Taylor */ 9269e39c5baSBill Taylor desc = HERMON_QP_SQ_ENTRY(qp, tail); 9279e39c5baSBill Taylor /* 9289e39c5baSBill Taylor * Call hermon_wqe_send_build() to build the WQE 9299e39c5baSBill Taylor * at the given address. This routine uses the 9309e39c5baSBill Taylor * information in the ibt_send_wr_t list (wr[]) and 9319e39c5baSBill Taylor * returns the size of the WQE when it returns. 9329e39c5baSBill Taylor */ 9339e39c5baSBill Taylor status = hermon_wqe_send_build(state, qp, 9349e39c5baSBill Taylor &wr[wrindx], desc, &desc_sz); 9359e39c5baSBill Taylor if (status != DDI_SUCCESS) { 9369e39c5baSBill Taylor break; 9379e39c5baSBill Taylor } 9389e39c5baSBill Taylor 9399e39c5baSBill Taylor /* 9409e39c5baSBill Taylor * Now, build the Ctrl Segment based on 9419e39c5baSBill Taylor * what was just done 9429e39c5baSBill Taylor */ 9439e39c5baSBill Taylor curr_wr = &wr[wrindx]; 9449e39c5baSBill Taylor 9459e39c5baSBill Taylor switch (curr_wr->wr_opcode) { 9469e39c5baSBill Taylor case IBT_WRC_RDMAW: 9479e39c5baSBill Taylor if (curr_wr->wr_flags & IBT_WR_SEND_IMMED) { 9489e39c5baSBill Taylor nopcode = 9499e39c5baSBill Taylor HERMON_WQE_SEND_NOPCODE_RDMAWI; 9509e39c5baSBill Taylor immed_data = 9519e39c5baSBill Taylor hermon_wr_get_immediate(curr_wr); 9529e39c5baSBill Taylor } else { 9539e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAW; 9549e39c5baSBill Taylor } 9559e39c5baSBill Taylor break; 9569e39c5baSBill Taylor 9579e39c5baSBill Taylor case IBT_WRC_SEND: 9589e39c5baSBill Taylor if (curr_wr->wr_flags & IBT_WR_SEND_IMMED) { 9599e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SENDI; 9609e39c5baSBill Taylor immed_data = 9619e39c5baSBill Taylor hermon_wr_get_immediate(curr_wr); 9629e39c5baSBill Taylor } else { 9639e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_SEND; 9649e39c5baSBill Taylor } 9659e39c5baSBill Taylor break; 9669e39c5baSBill Taylor 9679e39c5baSBill Taylor case IBT_WRC_SEND_LSO: 9689e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_LSO; 9699e39c5baSBill Taylor break; 9709e39c5baSBill Taylor 9719e39c5baSBill Taylor case IBT_WRC_RDMAR: 9729e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_RDMAR; 9739e39c5baSBill Taylor break; 9749e39c5baSBill Taylor 9759e39c5baSBill Taylor case IBT_WRC_CSWAP: 9769e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMCS; 9779e39c5baSBill Taylor break; 9789e39c5baSBill Taylor 9799e39c5baSBill Taylor case IBT_WRC_FADD: 9809e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_ATMFA; 9819e39c5baSBill Taylor break; 9829e39c5baSBill Taylor 9839e39c5baSBill Taylor case IBT_WRC_BIND: 9849e39c5baSBill Taylor nopcode = HERMON_WQE_SEND_NOPCODE_BIND; 9859e39c5baSBill Taylor break; 9869e39c5baSBill Taylor } 9879e39c5baSBill Taylor 9889e39c5baSBill Taylor fence = (curr_wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0; 9899e39c5baSBill Taylor 9909e39c5baSBill Taylor /* 9919e39c5baSBill Taylor * now, build up the control segment, leaving the 9929e39c5baSBill Taylor * owner bit as it is 9939e39c5baSBill Taylor */ 9949e39c5baSBill Taylor 9959e39c5baSBill Taylor if ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) || 9969e39c5baSBill Taylor (curr_wr->wr_flags & IBT_WR_SEND_SIGNAL)) { 997*17a2b317SBill Taylor signaled_dbd = 0xC; 9989e39c5baSBill Taylor } else { 9999e39c5baSBill Taylor signaled_dbd = 0; 10009e39c5baSBill Taylor } 10019e39c5baSBill Taylor if (curr_wr->wr_flags & IBT_WR_SEND_SOLICIT) 1002*17a2b317SBill Taylor solicited = 0x2; 10039e39c5baSBill Taylor else 10049e39c5baSBill Taylor solicited = 0; 10059e39c5baSBill Taylor 10069e39c5baSBill Taylor if (qp->qp_is_special) { 1007949b58c7SBill Taylor /* Ensure correctness, set the ReRead bit */ 1008949b58c7SBill Taylor nopcode |= (1 << 6); 10099e39c5baSBill Taylor ah = (hermon_ahhdl_t) 10109e39c5baSBill Taylor curr_wr->wr.ud.udwr_dest->ud_ah; 10119e39c5baSBill Taylor mutex_enter(&ah->ah_lock); 10129e39c5baSBill Taylor maxstat = ah->ah_udav->max_stat_rate; 10139e39c5baSBill Taylor HERMON_WQE_SET_MLX_CTRL_SEGMENT(desc, desc_sz, 10149e39c5baSBill Taylor signaled_dbd, maxstat, ah->ah_udav->rlid, 10159e39c5baSBill Taylor qp, ah->ah_udav->sl); 10169e39c5baSBill Taylor mutex_exit(&ah->ah_lock); 10179e39c5baSBill Taylor } else { 10189e39c5baSBill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, 10199e39c5baSBill Taylor fence, immed_data, solicited, 1020*17a2b317SBill Taylor signaled_dbd, 0, qp, 0, 0); 10219e39c5baSBill Taylor } 10229e39c5baSBill Taylor wq->wq_wrid[tail] = curr_wr->wr_id; 10239e39c5baSBill Taylor 10249e39c5baSBill Taylor /* 10259e39c5baSBill Taylor * If this is not the first descriptor on the current 10269e39c5baSBill Taylor * chain, then set the ownership bit. 10279e39c5baSBill Taylor */ 10289e39c5baSBill Taylor if (currindx != 0) { /* not the first */ 10299e39c5baSBill Taylor membar_producer(); 10309e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp, 10319e39c5baSBill Taylor (uint32_t *)desc, nopcode); 10329e39c5baSBill Taylor } else 10339e39c5baSBill Taylor prev_nopcode = nopcode; 10349e39c5baSBill Taylor 10359e39c5baSBill Taylor /* 10369e39c5baSBill Taylor * Update the current "tail index" and increment 10379e39c5baSBill Taylor * "posted_cnt" 10389e39c5baSBill Taylor */ 10399e39c5baSBill Taylor tail = next_tail; 10409e39c5baSBill Taylor posted_cnt++; 10419e39c5baSBill Taylor } 10429e39c5baSBill Taylor 10439e39c5baSBill Taylor /* 10449e39c5baSBill Taylor * If we reach here and there are one or more WQEs which have 10459e39c5baSBill Taylor * been successfully built as a chain, we have to finish up 10469e39c5baSBill Taylor * and prepare them for writing to the HW 10479e39c5baSBill Taylor * The steps are: 10489e39c5baSBill Taylor * 1. do the headroom fixup 10499e39c5baSBill Taylor * 2. add in the size of the headroom for the sync 10509e39c5baSBill Taylor * 3. write the owner bit for the first WQE 10519e39c5baSBill Taylor * 4. sync them 10529e39c5baSBill Taylor * 5. fix up the structures 10539e39c5baSBill Taylor * 6. hit the doorbell in UAR 10549e39c5baSBill Taylor */ 10559e39c5baSBill Taylor if (posted_cnt != 0) { 10569e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state); 10579e39c5baSBill Taylor 10589e39c5baSBill Taylor /* do the invalidate of the headroom */ 10599e39c5baSBill Taylor 10609e39c5baSBill Taylor hermon_wqe_headroom(tail, qp); 10619e39c5baSBill Taylor 10629e39c5baSBill Taylor /* Update some of the state in the QP */ 10639e39c5baSBill Taylor wq->wq_tail = tail; 10649e39c5baSBill Taylor total_posted += posted_cnt; 10659e39c5baSBill Taylor posted_cnt = 0; 10669e39c5baSBill Taylor 10679e39c5baSBill Taylor membar_producer(); 10689e39c5baSBill Taylor 10699e39c5baSBill Taylor /* 10709e39c5baSBill Taylor * Now set the ownership bit of the first 10719e39c5baSBill Taylor * one in the chain 10729e39c5baSBill Taylor */ 10739e39c5baSBill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)prev, 10749e39c5baSBill Taylor prev_nopcode); 10759e39c5baSBill Taylor 10769e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell. */ 10779e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt, 10789e39c5baSBill Taylor fm_status, fm_test); 10799e39c5baSBill Taylor 10809e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl, 10819e39c5baSBill Taylor (uint64_t *)(void *)&state->hs_uar->send, 10829e39c5baSBill Taylor (uint64_t)qp->qp_ring); 10839e39c5baSBill Taylor 10849e39c5baSBill Taylor /* the FMA retry loop ends. */ 10859e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt, 10869e39c5baSBill Taylor fm_status, fm_test); 10879e39c5baSBill Taylor } 10889e39c5baSBill Taylor } 10899e39c5baSBill Taylor 10909e39c5baSBill Taylor /* 10919e39c5baSBill Taylor * Update the "num_posted" return value (if necessary). 10929e39c5baSBill Taylor * Then drop the locks and return success. 10939e39c5baSBill Taylor */ 10949e39c5baSBill Taylor if (num_posted != NULL) { 10959e39c5baSBill Taylor *num_posted = total_posted; 10969e39c5baSBill Taylor } 10979e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock); 10989e39c5baSBill Taylor return (status); 10999e39c5baSBill Taylor 11009e39c5baSBill Taylor pio_error: 11019e39c5baSBill Taylor mutex_exit(&qp->qp_sq_lock); 11029e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 11039e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 11049e39c5baSBill Taylor } 11059e39c5baSBill Taylor 11069e39c5baSBill Taylor 11079e39c5baSBill Taylor /* 11089e39c5baSBill Taylor * hermon_post_recv() 11099e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 11109e39c5baSBill Taylor */ 11119e39c5baSBill Taylor int 11129e39c5baSBill Taylor hermon_post_recv(hermon_state_t *state, hermon_qphdl_t qp, 11139e39c5baSBill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted) 11149e39c5baSBill Taylor { 11159e39c5baSBill Taylor uint64_t *desc; 11169e39c5baSBill Taylor hermon_workq_hdr_t *wq; 11179e39c5baSBill Taylor uint32_t head, tail, next_tail, qsize_msk; 11189e39c5baSBill Taylor uint_t wrindx; 11199e39c5baSBill Taylor uint_t posted_cnt; 11209e39c5baSBill Taylor int status; 11219e39c5baSBill Taylor 11229e39c5baSBill Taylor /* 11239e39c5baSBill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel 11249e39c5baSBill Taylor * clients to post to QP memory that is accessible directly by the 11259e39c5baSBill Taylor * user. If the QP memory is user accessible, then return an error. 11269e39c5baSBill Taylor */ 1127*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP) { 11289e39c5baSBill Taylor return (IBT_QP_HDL_INVALID); 11299e39c5baSBill Taylor } 11309e39c5baSBill Taylor 11319e39c5baSBill Taylor /* Initialize posted_cnt */ 11329e39c5baSBill Taylor posted_cnt = 0; 11339e39c5baSBill Taylor 11349e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 11359e39c5baSBill Taylor 11369e39c5baSBill Taylor /* 11379e39c5baSBill Taylor * Check if QP is associated with an SRQ 11389e39c5baSBill Taylor */ 1139*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) { 11409e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 11419e39c5baSBill Taylor return (IBT_SRQ_IN_USE); 11429e39c5baSBill Taylor } 11439e39c5baSBill Taylor 11449e39c5baSBill Taylor /* 11459e39c5baSBill Taylor * Check QP state. Can not post Recv requests from the "Reset" state 11469e39c5baSBill Taylor */ 11479e39c5baSBill Taylor if (qp->qp_state == HERMON_QP_RESET) { 11489e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 11499e39c5baSBill Taylor return (IBT_QP_STATE_INVALID); 11509e39c5baSBill Taylor } 11519e39c5baSBill Taylor 11529e39c5baSBill Taylor /* Check that work request transport type is valid */ 1153*17a2b317SBill Taylor if ((qp->qp_type != IBT_UD_RQP) && 11549e39c5baSBill Taylor (qp->qp_serv_type != HERMON_QP_RC) && 11559e39c5baSBill Taylor (qp->qp_serv_type != HERMON_QP_UC)) { 11569e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 11579e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID); 11589e39c5baSBill Taylor } 11599e39c5baSBill Taylor 11609e39c5baSBill Taylor /* 11619e39c5baSBill Taylor * Grab the lock for the WRID list, i.e., membar_consumer(). 11629e39c5baSBill Taylor * This is not needed because the mutex_enter() above has 11639e39c5baSBill Taylor * the same effect. 11649e39c5baSBill Taylor */ 11659e39c5baSBill Taylor 11669e39c5baSBill Taylor /* Save away some initial QP state */ 11679e39c5baSBill Taylor wq = qp->qp_rq_wqhdr; 11689e39c5baSBill Taylor qsize_msk = wq->wq_mask; 11699e39c5baSBill Taylor tail = wq->wq_tail; 11709e39c5baSBill Taylor head = wq->wq_head; 11719e39c5baSBill Taylor 11729e39c5baSBill Taylor wrindx = 0; 11739e39c5baSBill Taylor status = DDI_SUCCESS; 11749e39c5baSBill Taylor 11759e39c5baSBill Taylor for (wrindx = 0; wrindx < num_wr; wrindx++) { 11769e39c5baSBill Taylor if (wq->wq_full != 0) { 11779e39c5baSBill Taylor status = IBT_QP_FULL; 11789e39c5baSBill Taylor break; 11799e39c5baSBill Taylor } 11809e39c5baSBill Taylor next_tail = (tail + 1) & qsize_msk; 11819e39c5baSBill Taylor if (next_tail == head) { 11829e39c5baSBill Taylor wq->wq_full = 1; 11839e39c5baSBill Taylor } 11849e39c5baSBill Taylor desc = HERMON_QP_RQ_ENTRY(qp, tail); 11859e39c5baSBill Taylor status = hermon_wqe_recv_build(state, qp, &wr[wrindx], desc); 11869e39c5baSBill Taylor if (status != DDI_SUCCESS) { 11879e39c5baSBill Taylor break; 11889e39c5baSBill Taylor } 11899e39c5baSBill Taylor 11909e39c5baSBill Taylor wq->wq_wrid[tail] = wr[wrindx].wr_id; 11919e39c5baSBill Taylor qp->qp_rq_wqecntr++; 11929e39c5baSBill Taylor 11939e39c5baSBill Taylor tail = next_tail; 11949e39c5baSBill Taylor posted_cnt++; 11959e39c5baSBill Taylor } 11969e39c5baSBill Taylor 11979e39c5baSBill Taylor if (posted_cnt != 0) { 11989e39c5baSBill Taylor 11999e39c5baSBill Taylor wq->wq_tail = tail; 12009e39c5baSBill Taylor 12019e39c5baSBill Taylor membar_producer(); /* ensure wrids are visible */ 12029e39c5baSBill Taylor 12039e39c5baSBill Taylor /* Update the doorbell record w/ wqecntr */ 12049e39c5baSBill Taylor HERMON_UAR_DB_RECORD_WRITE(qp->qp_rq_vdbr, 12059e39c5baSBill Taylor qp->qp_rq_wqecntr & 0xFFFF); 12069e39c5baSBill Taylor } 12079e39c5baSBill Taylor 12089e39c5baSBill Taylor if (num_posted != NULL) { 12099e39c5baSBill Taylor *num_posted = posted_cnt; 12109e39c5baSBill Taylor } 12119e39c5baSBill Taylor 12129e39c5baSBill Taylor 1213*17a2b317SBill Taylor mutex_exit(&qp->qp_lock); 12149e39c5baSBill Taylor return (status); 12159e39c5baSBill Taylor } 12169e39c5baSBill Taylor 12179e39c5baSBill Taylor /* 12189e39c5baSBill Taylor * hermon_post_srq() 12199e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 12209e39c5baSBill Taylor */ 12219e39c5baSBill Taylor int 12229e39c5baSBill Taylor hermon_post_srq(hermon_state_t *state, hermon_srqhdl_t srq, 12239e39c5baSBill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted) 12249e39c5baSBill Taylor { 12259e39c5baSBill Taylor uint64_t *desc; 12269e39c5baSBill Taylor hermon_workq_hdr_t *wq; 12279e39c5baSBill Taylor uint_t indx, wrindx; 12289e39c5baSBill Taylor uint_t posted_cnt; 12299e39c5baSBill Taylor int status; 12309e39c5baSBill Taylor 12319e39c5baSBill Taylor mutex_enter(&srq->srq_lock); 12329e39c5baSBill Taylor 12339e39c5baSBill Taylor /* 12349e39c5baSBill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel 12359e39c5baSBill Taylor * clients to post to QP memory that is accessible directly by the 12369e39c5baSBill Taylor * user. If the QP memory is user accessible, then return an error. 12379e39c5baSBill Taylor */ 12389e39c5baSBill Taylor if (srq->srq_is_umap) { 12399e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 12409e39c5baSBill Taylor return (IBT_SRQ_HDL_INVALID); 12419e39c5baSBill Taylor } 12429e39c5baSBill Taylor 12439e39c5baSBill Taylor /* 12449e39c5baSBill Taylor * Check SRQ state. Can not post Recv requests when SRQ is in error 12459e39c5baSBill Taylor */ 12469e39c5baSBill Taylor if (srq->srq_state == HERMON_SRQ_STATE_ERROR) { 12479e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 12489e39c5baSBill Taylor return (IBT_QP_STATE_INVALID); 12499e39c5baSBill Taylor } 12509e39c5baSBill Taylor 12519e39c5baSBill Taylor status = DDI_SUCCESS; 12529e39c5baSBill Taylor posted_cnt = 0; 12539e39c5baSBill Taylor wq = srq->srq_wq_wqhdr; 12549e39c5baSBill Taylor indx = wq->wq_head; 12559e39c5baSBill Taylor 12569e39c5baSBill Taylor for (wrindx = 0; wrindx < num_wr; wrindx++) { 12579e39c5baSBill Taylor 12589e39c5baSBill Taylor if (indx == wq->wq_tail) { 12599e39c5baSBill Taylor status = IBT_QP_FULL; 12609e39c5baSBill Taylor break; 12619e39c5baSBill Taylor } 12629e39c5baSBill Taylor desc = HERMON_SRQ_WQE_ADDR(srq, indx); 12639e39c5baSBill Taylor 12649e39c5baSBill Taylor wq->wq_wrid[indx] = wr[wrindx].wr_id; 12659e39c5baSBill Taylor 12669e39c5baSBill Taylor status = hermon_wqe_srq_build(state, srq, &wr[wrindx], desc); 12679e39c5baSBill Taylor if (status != DDI_SUCCESS) { 12689e39c5baSBill Taylor break; 12699e39c5baSBill Taylor } 12709e39c5baSBill Taylor 12719e39c5baSBill Taylor posted_cnt++; 12729e39c5baSBill Taylor indx = htons(((uint16_t *)desc)[1]); 12739e39c5baSBill Taylor wq->wq_head = indx; 12749e39c5baSBill Taylor } 12759e39c5baSBill Taylor 12769e39c5baSBill Taylor if (posted_cnt != 0) { 12779e39c5baSBill Taylor 12789e39c5baSBill Taylor srq->srq_wq_wqecntr += posted_cnt; 12799e39c5baSBill Taylor 12809e39c5baSBill Taylor membar_producer(); /* ensure wrids are visible */ 12819e39c5baSBill Taylor 12829e39c5baSBill Taylor /* Ring the doorbell w/ wqecntr */ 12839e39c5baSBill Taylor HERMON_UAR_DB_RECORD_WRITE(srq->srq_wq_vdbr, 12849e39c5baSBill Taylor srq->srq_wq_wqecntr & 0xFFFF); 12859e39c5baSBill Taylor } 12869e39c5baSBill Taylor 12879e39c5baSBill Taylor if (num_posted != NULL) { 12889e39c5baSBill Taylor *num_posted = posted_cnt; 12899e39c5baSBill Taylor } 12909e39c5baSBill Taylor 12919e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 12929e39c5baSBill Taylor return (status); 12939e39c5baSBill Taylor } 12949e39c5baSBill Taylor 12959e39c5baSBill Taylor 12969e39c5baSBill Taylor /* 12979e39c5baSBill Taylor * hermon_wqe_send_build() 12989e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 12999e39c5baSBill Taylor */ 13009e39c5baSBill Taylor static int 13019e39c5baSBill Taylor hermon_wqe_send_build(hermon_state_t *state, hermon_qphdl_t qp, 13029e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size) 13039e39c5baSBill Taylor { 13049e39c5baSBill Taylor hermon_hw_snd_wqe_ud_t *ud; 13059e39c5baSBill Taylor hermon_hw_snd_wqe_remaddr_t *rc; 13069e39c5baSBill Taylor hermon_hw_snd_wqe_atomic_t *at; 13079e39c5baSBill Taylor hermon_hw_snd_wqe_remaddr_t *uc; 13089e39c5baSBill Taylor hermon_hw_snd_wqe_bind_t *bn; 13099e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds, *old_ds; 13109e39c5baSBill Taylor ibt_ud_dest_t *dest; 13119e39c5baSBill Taylor ibt_wr_ds_t *sgl; 13129e39c5baSBill Taylor hermon_ahhdl_t ah; 13139e39c5baSBill Taylor uint32_t nds; 13149e39c5baSBill Taylor int i, j, last_ds, num_ds, status; 13159e39c5baSBill Taylor int tmpsize; 13169e39c5baSBill Taylor 13179e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock)); 13189e39c5baSBill Taylor 13199e39c5baSBill Taylor /* Initialize the information for the Data Segments */ 13209e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc + 13219e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 13229e39c5baSBill Taylor nds = wr->wr_nds; 13239e39c5baSBill Taylor sgl = wr->wr_sgl; 13249e39c5baSBill Taylor num_ds = 0; 13259e39c5baSBill Taylor i = 0; 13269e39c5baSBill Taylor 13279e39c5baSBill Taylor /* 13289e39c5baSBill Taylor * Build a Send WQE depends first and foremost on the transport 13299e39c5baSBill Taylor * type of Work Request (i.e. UD, RC, or UC) 13309e39c5baSBill Taylor */ 13319e39c5baSBill Taylor switch (wr->wr_trans) { 13329e39c5baSBill Taylor case IBT_UD_SRV: 13339e39c5baSBill Taylor /* Ensure that work request transport type matches QP type */ 13349e39c5baSBill Taylor if (qp->qp_serv_type != HERMON_QP_UD) { 13359e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID); 13369e39c5baSBill Taylor } 13379e39c5baSBill Taylor 13389e39c5baSBill Taylor /* 13399e39c5baSBill Taylor * Validate the operation type. For UD requests, only the 13409e39c5baSBill Taylor * "Send" and "Send LSO" operations are valid. 13419e39c5baSBill Taylor */ 13429e39c5baSBill Taylor if (wr->wr_opcode != IBT_WRC_SEND && 13439e39c5baSBill Taylor wr->wr_opcode != IBT_WRC_SEND_LSO) { 13449e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID); 13459e39c5baSBill Taylor } 13469e39c5baSBill Taylor 13479e39c5baSBill Taylor /* 13489e39c5baSBill Taylor * If this is a Special QP (QP0 or QP1), then we need to 13499e39c5baSBill Taylor * build MLX WQEs instead. So jump to hermon_wqe_mlx_build() 13509e39c5baSBill Taylor * and return whatever status it returns 13519e39c5baSBill Taylor */ 13529e39c5baSBill Taylor if (qp->qp_is_special) { 13539e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND_LSO) { 13549e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID); 13559e39c5baSBill Taylor } 13569e39c5baSBill Taylor status = hermon_wqe_mlx_build(state, qp, 13579e39c5baSBill Taylor wr, desc, size); 13589e39c5baSBill Taylor return (status); 13599e39c5baSBill Taylor } 13609e39c5baSBill Taylor 13619e39c5baSBill Taylor /* 13629e39c5baSBill Taylor * Otherwise, if this is a normal UD Send request, then fill 13639e39c5baSBill Taylor * all the fields in the Hermon UD header for the WQE. Note: 13649e39c5baSBill Taylor * to do this we'll need to extract some information from the 13659e39c5baSBill Taylor * Address Handle passed with the work request. 13669e39c5baSBill Taylor */ 13679e39c5baSBill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc + 13689e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 13699e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) { 13709e39c5baSBill Taylor dest = wr->wr.ud.udwr_dest; 13719e39c5baSBill Taylor } else { 13729e39c5baSBill Taylor dest = wr->wr.ud_lso.lso_ud_dest; 13739e39c5baSBill Taylor } 13749e39c5baSBill Taylor ah = (hermon_ahhdl_t)dest->ud_ah; 13759e39c5baSBill Taylor if (ah == NULL) { 13769e39c5baSBill Taylor return (IBT_AH_HDL_INVALID); 13779e39c5baSBill Taylor } 13789e39c5baSBill Taylor 13799e39c5baSBill Taylor /* 13809e39c5baSBill Taylor * Build the Unreliable Datagram Segment for the WQE, using 13819e39c5baSBill Taylor * the information from the address handle and the work 13829e39c5baSBill Taylor * request. 13839e39c5baSBill Taylor */ 13849e39c5baSBill Taylor /* mutex_enter(&ah->ah_lock); */ 13859e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) { 13869e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, wr->wr.ud.udwr_dest); 13879e39c5baSBill Taylor } else { /* IBT_WRC_SEND_LSO */ 13889e39c5baSBill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, 13899e39c5baSBill Taylor wr->wr.ud_lso.lso_ud_dest); 13909e39c5baSBill Taylor } 13919e39c5baSBill Taylor /* mutex_exit(&ah->ah_lock); */ 13929e39c5baSBill Taylor 13939e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */ 13949e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ud + 13959e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ud_t)); 13969e39c5baSBill Taylor 13979e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND_LSO) { 13989e39c5baSBill Taylor int total_len; 13999e39c5baSBill Taylor 14009e39c5baSBill Taylor total_len = (4 + 0xf + wr->wr.ud_lso.lso_hdr_sz) & ~0xf; 14019e39c5baSBill Taylor if ((uintptr_t)ds + total_len + (nds * 16) > 14029e39c5baSBill Taylor (uintptr_t)desc + (1 << qp->qp_sq_log_wqesz)) 14039e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID); 14049e39c5baSBill Taylor 14059e39c5baSBill Taylor bcopy(wr->wr.ud_lso.lso_hdr, (uint32_t *)ds + 1, 14069e39c5baSBill Taylor wr->wr.ud_lso.lso_hdr_sz); 14079e39c5baSBill Taylor old_ds = ds; 14089e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + total_len); 14099e39c5baSBill Taylor for (; i < nds; i++) { 14109e39c5baSBill Taylor if (sgl[i].ds_len == 0) 14119e39c5baSBill Taylor continue; 14129e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[num_ds], 14139e39c5baSBill Taylor &sgl[i]); 14149e39c5baSBill Taylor num_ds++; 14159e39c5baSBill Taylor i++; 14169e39c5baSBill Taylor break; 14179e39c5baSBill Taylor } 14189e39c5baSBill Taylor membar_producer(); 14199e39c5baSBill Taylor HERMON_WQE_BUILD_LSO(qp, old_ds, wr->wr.ud_lso.lso_mss, 14209e39c5baSBill Taylor wr->wr.ud_lso.lso_hdr_sz); 14219e39c5baSBill Taylor } 14229e39c5baSBill Taylor 14239e39c5baSBill Taylor break; 14249e39c5baSBill Taylor 14259e39c5baSBill Taylor case IBT_RC_SRV: 14269e39c5baSBill Taylor /* Ensure that work request transport type matches QP type */ 14279e39c5baSBill Taylor if (qp->qp_serv_type != HERMON_QP_RC) { 14289e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID); 14299e39c5baSBill Taylor } 14309e39c5baSBill Taylor 14319e39c5baSBill Taylor /* 14329e39c5baSBill Taylor * Validate the operation type. For RC requests, we allow 14339e39c5baSBill Taylor * "Send", "RDMA Read", "RDMA Write", various "Atomic" 14349e39c5baSBill Taylor * operations, and memory window "Bind" 14359e39c5baSBill Taylor */ 14369e39c5baSBill Taylor if ((wr->wr_opcode != IBT_WRC_SEND) && 14379e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_RDMAR) && 14389e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_RDMAW) && 14399e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_CSWAP) && 14409e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_FADD) && 14419e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_BIND)) { 14429e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID); 14439e39c5baSBill Taylor } 14449e39c5baSBill Taylor 14459e39c5baSBill Taylor /* 14469e39c5baSBill Taylor * If this is a Send request, then all we need to do is break 14479e39c5baSBill Taylor * out and here and begin the Data Segment processing below 14489e39c5baSBill Taylor */ 14499e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) { 14509e39c5baSBill Taylor break; 14519e39c5baSBill Taylor } 14529e39c5baSBill Taylor 14539e39c5baSBill Taylor /* 14549e39c5baSBill Taylor * If this is an RDMA Read or RDMA Write request, then fill 14559e39c5baSBill Taylor * in the "Remote Address" header fields. 14569e39c5baSBill Taylor */ 14579e39c5baSBill Taylor if ((wr->wr_opcode == IBT_WRC_RDMAR) || 14589e39c5baSBill Taylor (wr->wr_opcode == IBT_WRC_RDMAW)) { 14599e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc + 14609e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 14619e39c5baSBill Taylor 14629e39c5baSBill Taylor /* 14639e39c5baSBill Taylor * Build the Remote Address Segment for the WQE, using 14649e39c5baSBill Taylor * the information from the RC work request. 14659e39c5baSBill Taylor */ 14669e39c5baSBill Taylor HERMON_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma); 14679e39c5baSBill Taylor 14689e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */ 14699e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)rc + 14709e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t)); 14719e39c5baSBill Taylor break; 14729e39c5baSBill Taylor } 14739e39c5baSBill Taylor 14749e39c5baSBill Taylor /* 14759e39c5baSBill Taylor * If this is one of the Atomic type operations (i.e 14769e39c5baSBill Taylor * Compare-Swap or Fetch-Add), then fill in both the "Remote 14779e39c5baSBill Taylor * Address" header fields and the "Atomic" header fields. 14789e39c5baSBill Taylor */ 14799e39c5baSBill Taylor if ((wr->wr_opcode == IBT_WRC_CSWAP) || 14809e39c5baSBill Taylor (wr->wr_opcode == IBT_WRC_FADD)) { 14819e39c5baSBill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc + 14829e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 14839e39c5baSBill Taylor at = (hermon_hw_snd_wqe_atomic_t *)((uintptr_t)rc + 14849e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t)); 14859e39c5baSBill Taylor 14869e39c5baSBill Taylor /* 14879e39c5baSBill Taylor * Build the Remote Address and Atomic Segments for 14889e39c5baSBill Taylor * the WQE, using the information from the RC Atomic 14899e39c5baSBill Taylor * work request. 14909e39c5baSBill Taylor */ 14919e39c5baSBill Taylor HERMON_WQE_BUILD_RC_ATOMIC_REMADDR(qp, rc, wr); 14929e39c5baSBill Taylor HERMON_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic); 14939e39c5baSBill Taylor 14949e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */ 14959e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)at + 14969e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_atomic_t)); 14979e39c5baSBill Taylor 14989e39c5baSBill Taylor /* 14999e39c5baSBill Taylor * Update "nds" and "sgl" because Atomic requests have 15009e39c5baSBill Taylor * only a single Data Segment (and they are encoded 15019e39c5baSBill Taylor * somewhat differently in the work request. 15029e39c5baSBill Taylor */ 15039e39c5baSBill Taylor nds = 1; 15049e39c5baSBill Taylor sgl = wr->wr_sgl; 15059e39c5baSBill Taylor break; 15069e39c5baSBill Taylor } 15079e39c5baSBill Taylor 15089e39c5baSBill Taylor /* 15099e39c5baSBill Taylor * If this is memory window Bind operation, then we call the 15109e39c5baSBill Taylor * hermon_wr_bind_check() routine to validate the request and 15119e39c5baSBill Taylor * to generate the updated RKey. If this is successful, then 15129e39c5baSBill Taylor * we fill in the WQE's "Bind" header fields. 15139e39c5baSBill Taylor */ 15149e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_BIND) { 15159e39c5baSBill Taylor status = hermon_wr_bind_check(state, wr); 15169e39c5baSBill Taylor if (status != DDI_SUCCESS) { 15179e39c5baSBill Taylor return (status); 15189e39c5baSBill Taylor } 15199e39c5baSBill Taylor 15209e39c5baSBill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc + 15219e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 15229e39c5baSBill Taylor 15239e39c5baSBill Taylor /* 15249e39c5baSBill Taylor * Build the Bind Memory Window Segments for the WQE, 15259e39c5baSBill Taylor * using the information from the RC Bind memory 15269e39c5baSBill Taylor * window work request. 15279e39c5baSBill Taylor */ 15289e39c5baSBill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind); 15299e39c5baSBill Taylor 15309e39c5baSBill Taylor /* 15319e39c5baSBill Taylor * Update the "ds" pointer. Even though the "bind" 15329e39c5baSBill Taylor * operation requires no SGLs, this is necessary to 15339e39c5baSBill Taylor * facilitate the correct descriptor size calculations 15349e39c5baSBill Taylor * (below). 15359e39c5baSBill Taylor */ 15369e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)bn + 15379e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_bind_t)); 15389e39c5baSBill Taylor nds = 0; 15399e39c5baSBill Taylor } 15409e39c5baSBill Taylor break; 15419e39c5baSBill Taylor 15429e39c5baSBill Taylor case IBT_UC_SRV: 15439e39c5baSBill Taylor /* Ensure that work request transport type matches QP type */ 15449e39c5baSBill Taylor if (qp->qp_serv_type != HERMON_QP_UC) { 15459e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID); 15469e39c5baSBill Taylor } 15479e39c5baSBill Taylor 15489e39c5baSBill Taylor /* 15499e39c5baSBill Taylor * Validate the operation type. For UC requests, we only 15509e39c5baSBill Taylor * allow "Send", "RDMA Write", and memory window "Bind". 15519e39c5baSBill Taylor * Note: Unlike RC, UC does not allow "RDMA Read" or "Atomic" 15529e39c5baSBill Taylor * operations 15539e39c5baSBill Taylor */ 15549e39c5baSBill Taylor if ((wr->wr_opcode != IBT_WRC_SEND) && 15559e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_RDMAW) && 15569e39c5baSBill Taylor (wr->wr_opcode != IBT_WRC_BIND)) { 15579e39c5baSBill Taylor return (IBT_QP_OP_TYPE_INVALID); 15589e39c5baSBill Taylor } 15599e39c5baSBill Taylor 15609e39c5baSBill Taylor /* 15619e39c5baSBill Taylor * If this is a Send request, then all we need to do is break 15629e39c5baSBill Taylor * out and here and begin the Data Segment processing below 15639e39c5baSBill Taylor */ 15649e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) { 15659e39c5baSBill Taylor break; 15669e39c5baSBill Taylor } 15679e39c5baSBill Taylor 15689e39c5baSBill Taylor /* 15699e39c5baSBill Taylor * If this is an RDMA Write request, then fill in the "Remote 15709e39c5baSBill Taylor * Address" header fields. 15719e39c5baSBill Taylor */ 15729e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_RDMAW) { 15739e39c5baSBill Taylor uc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc + 15749e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 15759e39c5baSBill Taylor 15769e39c5baSBill Taylor /* 15779e39c5baSBill Taylor * Build the Remote Address Segment for the WQE, using 15789e39c5baSBill Taylor * the information from the UC work request. 15799e39c5baSBill Taylor */ 15809e39c5baSBill Taylor HERMON_WQE_BUILD_REMADDR(qp, uc, &wr->wr.uc.ucwr.rdma); 15819e39c5baSBill Taylor 15829e39c5baSBill Taylor /* Update "ds" for filling in Data Segments (below) */ 15839e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)uc + 15849e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_remaddr_t)); 15859e39c5baSBill Taylor break; 15869e39c5baSBill Taylor } 15879e39c5baSBill Taylor 15889e39c5baSBill Taylor /* 15899e39c5baSBill Taylor * If this is memory window Bind operation, then we call the 15909e39c5baSBill Taylor * hermon_wr_bind_check() routine to validate the request and 15919e39c5baSBill Taylor * to generate the updated RKey. If this is successful, then 15929e39c5baSBill Taylor * we fill in the WQE's "Bind" header fields. 15939e39c5baSBill Taylor */ 15949e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_BIND) { 15959e39c5baSBill Taylor status = hermon_wr_bind_check(state, wr); 15969e39c5baSBill Taylor if (status != DDI_SUCCESS) { 15979e39c5baSBill Taylor return (status); 15989e39c5baSBill Taylor } 15999e39c5baSBill Taylor 16009e39c5baSBill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc + 16019e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)); 16029e39c5baSBill Taylor 16039e39c5baSBill Taylor /* 16049e39c5baSBill Taylor * Build the Bind Memory Window Segments for the WQE, 16059e39c5baSBill Taylor * using the information from the UC Bind memory 16069e39c5baSBill Taylor * window work request. 16079e39c5baSBill Taylor */ 16089e39c5baSBill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.uc.ucwr.bind); 16099e39c5baSBill Taylor 16109e39c5baSBill Taylor /* 16119e39c5baSBill Taylor * Update the "ds" pointer. Even though the "bind" 16129e39c5baSBill Taylor * operation requires no SGLs, this is necessary to 16139e39c5baSBill Taylor * facilitate the correct descriptor size calculations 16149e39c5baSBill Taylor * (below). 16159e39c5baSBill Taylor */ 16169e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)bn + 16179e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_bind_t)); 16189e39c5baSBill Taylor nds = 0; 16199e39c5baSBill Taylor } 16209e39c5baSBill Taylor break; 16219e39c5baSBill Taylor 16229e39c5baSBill Taylor default: 16239e39c5baSBill Taylor return (IBT_QP_SRV_TYPE_INVALID); 16249e39c5baSBill Taylor } 16259e39c5baSBill Taylor 16269e39c5baSBill Taylor /* 16279e39c5baSBill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based on 16289e39c5baSBill Taylor * the values setup above (i.e. "sgl", "nds", and the "ds" pointer 16299e39c5baSBill Taylor * Start by checking for a valid number of SGL entries 16309e39c5baSBill Taylor */ 16319e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) { 16329e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID); 16339e39c5baSBill Taylor } 16349e39c5baSBill Taylor 16359e39c5baSBill Taylor /* 16369e39c5baSBill Taylor * For each SGL in the Send Work Request, fill in the Send WQE's data 16379e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon 16389e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually 16399e39c5baSBill Taylor * the encoding for zero means a 2GB transfer. 16409e39c5baSBill Taylor */ 16419e39c5baSBill Taylor for (last_ds = num_ds, j = i; j < nds; j++) { 16429e39c5baSBill Taylor if (sgl[j].ds_len != 0) 16439e39c5baSBill Taylor last_ds++; /* real last ds of wqe to fill */ 16449e39c5baSBill Taylor } 16459e39c5baSBill Taylor 16469e39c5baSBill Taylor /* 16479e39c5baSBill Taylor * Return the size of descriptor (in 16-byte chunks) 16489e39c5baSBill Taylor * For Hermon, we want them (for now) to be on stride size 16499e39c5baSBill Taylor * boundaries, which was implicit in Tavor/Arbel 16509e39c5baSBill Taylor * 16519e39c5baSBill Taylor */ 16529e39c5baSBill Taylor tmpsize = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc); 16539e39c5baSBill Taylor 16549e39c5baSBill Taylor *size = tmpsize >> 0x4; 16559e39c5baSBill Taylor 16569e39c5baSBill Taylor for (j = nds; --j >= i; ) { 16579e39c5baSBill Taylor if (sgl[j].ds_len == 0) { 16589e39c5baSBill Taylor continue; 16599e39c5baSBill Taylor } 16609e39c5baSBill Taylor 16619e39c5baSBill Taylor /* 16629e39c5baSBill Taylor * Fill in the Data Segment(s) for the current WQE, using the 16639e39c5baSBill Taylor * information contained in the scatter-gather list of the 16649e39c5baSBill Taylor * work request. 16659e39c5baSBill Taylor */ 16669e39c5baSBill Taylor last_ds--; 16679e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[j]); 16689e39c5baSBill Taylor } 16699e39c5baSBill Taylor 16709e39c5baSBill Taylor return (DDI_SUCCESS); 16719e39c5baSBill Taylor } 16729e39c5baSBill Taylor 16739e39c5baSBill Taylor 16749e39c5baSBill Taylor 16759e39c5baSBill Taylor /* 16769e39c5baSBill Taylor * hermon_wqe_mlx_build() 16779e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 16789e39c5baSBill Taylor */ 16799e39c5baSBill Taylor static int 16809e39c5baSBill Taylor hermon_wqe_mlx_build(hermon_state_t *state, hermon_qphdl_t qp, 16819e39c5baSBill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size) 16829e39c5baSBill Taylor { 16839e39c5baSBill Taylor hermon_ahhdl_t ah; 16849e39c5baSBill Taylor hermon_hw_udav_t *udav; 16859e39c5baSBill Taylor ib_lrh_hdr_t *lrh; 16869e39c5baSBill Taylor ib_grh_t *grh; 16879e39c5baSBill Taylor ib_bth_hdr_t *bth; 16889e39c5baSBill Taylor ib_deth_hdr_t *deth; 16899e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds; 16909e39c5baSBill Taylor ibt_wr_ds_t *sgl; 16919e39c5baSBill Taylor uint8_t *mgmtclass, *hpoint, *hcount; 16929e39c5baSBill Taylor uint32_t nds, offset, pktlen; 16939e39c5baSBill Taylor uint32_t desc_sz; 16949e39c5baSBill Taylor int i, num_ds; 16959e39c5baSBill Taylor int tmpsize; 16969e39c5baSBill Taylor 16979e39c5baSBill Taylor ASSERT(MUTEX_HELD(&qp->qp_sq_lock)); 16989e39c5baSBill Taylor 16999e39c5baSBill Taylor /* Initialize the information for the Data Segments */ 17009e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc + 17019e39c5baSBill Taylor sizeof (hermon_hw_mlx_wqe_nextctrl_t)); 17029e39c5baSBill Taylor 17039e39c5baSBill Taylor /* 17049e39c5baSBill Taylor * Pull the address handle from the work request. The UDAV will 17059e39c5baSBill Taylor * be used to answer some questions about the request. 17069e39c5baSBill Taylor */ 17079e39c5baSBill Taylor ah = (hermon_ahhdl_t)wr->wr.ud.udwr_dest->ud_ah; 17089e39c5baSBill Taylor if (ah == NULL) { 17099e39c5baSBill Taylor return (IBT_AH_HDL_INVALID); 17109e39c5baSBill Taylor } 17119e39c5baSBill Taylor mutex_enter(&ah->ah_lock); 17129e39c5baSBill Taylor udav = ah->ah_udav; 17139e39c5baSBill Taylor 17149e39c5baSBill Taylor /* 17159e39c5baSBill Taylor * If the request is for QP1 and the destination LID is equal to 17169e39c5baSBill Taylor * the Permissive LID, then return an error. This combination is 17179e39c5baSBill Taylor * not allowed 17189e39c5baSBill Taylor */ 17199e39c5baSBill Taylor if ((udav->rlid == IB_LID_PERMISSIVE) && 17209e39c5baSBill Taylor (qp->qp_is_special == HERMON_QP_GSI)) { 17219e39c5baSBill Taylor mutex_exit(&ah->ah_lock); 17229e39c5baSBill Taylor return (IBT_AH_HDL_INVALID); 17239e39c5baSBill Taylor } 17249e39c5baSBill Taylor 17259e39c5baSBill Taylor /* 17269e39c5baSBill Taylor * Calculate the size of the packet headers, including the GRH 17279e39c5baSBill Taylor * (if necessary) 17289e39c5baSBill Taylor */ 17299e39c5baSBill Taylor desc_sz = sizeof (ib_lrh_hdr_t) + sizeof (ib_bth_hdr_t) + 17309e39c5baSBill Taylor sizeof (ib_deth_hdr_t); 17319e39c5baSBill Taylor if (udav->grh) { 17329e39c5baSBill Taylor desc_sz += sizeof (ib_grh_t); 17339e39c5baSBill Taylor } 17349e39c5baSBill Taylor 17359e39c5baSBill Taylor /* 17369e39c5baSBill Taylor * Begin to build the first "inline" data segment for the packet 17379e39c5baSBill Taylor * headers. Note: By specifying "inline" we can build the contents 17389e39c5baSBill Taylor * of the MAD packet headers directly into the work queue (as part 17399e39c5baSBill Taylor * descriptor). This has the advantage of both speeding things up 17409e39c5baSBill Taylor * and of not requiring the driver to allocate/register any additional 17419e39c5baSBill Taylor * memory for the packet headers. 17429e39c5baSBill Taylor */ 17439e39c5baSBill Taylor HERMON_WQE_BUILD_INLINE(qp, &ds[0], desc_sz); 17449e39c5baSBill Taylor desc_sz += 4; 17459e39c5baSBill Taylor 17469e39c5baSBill Taylor /* 17479e39c5baSBill Taylor * Build Local Route Header (LRH) 17489e39c5baSBill Taylor * We start here by building the LRH into a temporary location. 17499e39c5baSBill Taylor * When we have finished we copy the LRH data into the descriptor. 17509e39c5baSBill Taylor * 17519e39c5baSBill Taylor * Notice that the VL values are hardcoded. This is not a problem 17529e39c5baSBill Taylor * because VL15 is decided later based on the value in the MLX 17539e39c5baSBill Taylor * transport "next/ctrl" header (see the "vl15" bit below), and it 17549e39c5baSBill Taylor * is otherwise (meaning for QP1) chosen from the SL-to-VL table 17559e39c5baSBill Taylor * values. This rule does not hold for loopback packets however 17569e39c5baSBill Taylor * (all of which bypass the SL-to-VL tables) and it is the reason 17579e39c5baSBill Taylor * that non-QP0 MADs are setup with VL hardcoded to zero below. 17589e39c5baSBill Taylor * 17599e39c5baSBill Taylor * Notice also that Source LID is hardcoded to the Permissive LID 17609e39c5baSBill Taylor * (0xFFFF). This is also not a problem because if the Destination 17619e39c5baSBill Taylor * LID is not the Permissive LID, then the "slr" value in the MLX 17629e39c5baSBill Taylor * transport "next/ctrl" header will be set to zero and the hardware 17639e39c5baSBill Taylor * will pull the LID from value in the port. 17649e39c5baSBill Taylor */ 17659e39c5baSBill Taylor lrh = (ib_lrh_hdr_t *)((uintptr_t)&ds[0] + 4); 17669e39c5baSBill Taylor pktlen = (desc_sz + 0x100) >> 2; 17679e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_LRH(lrh, qp, udav, pktlen); 17689e39c5baSBill Taylor 17699e39c5baSBill Taylor /* 17709e39c5baSBill Taylor * Build Global Route Header (GRH) 17719e39c5baSBill Taylor * This is only built if necessary as defined by the "grh" bit in 17729e39c5baSBill Taylor * the address vector. Note: We also calculate the offset to the 17739e39c5baSBill Taylor * next header (BTH) based on whether or not the "grh" bit is set. 17749e39c5baSBill Taylor */ 17759e39c5baSBill Taylor if (udav->grh) { 17769e39c5baSBill Taylor /* 17779e39c5baSBill Taylor * If the request is for QP0, then return an error. The 17789e39c5baSBill Taylor * combination of global routine (GRH) and QP0 is not allowed. 17799e39c5baSBill Taylor */ 17809e39c5baSBill Taylor if (qp->qp_is_special == HERMON_QP_SMI) { 17819e39c5baSBill Taylor mutex_exit(&ah->ah_lock); 17829e39c5baSBill Taylor return (IBT_AH_HDL_INVALID); 17839e39c5baSBill Taylor } 17849e39c5baSBill Taylor grh = (ib_grh_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t)); 17859e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_GRH(state, grh, qp, udav, pktlen); 17869e39c5baSBill Taylor 17879e39c5baSBill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)grh + sizeof (ib_grh_t)); 17889e39c5baSBill Taylor } else { 17899e39c5baSBill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t)); 17909e39c5baSBill Taylor } 17919e39c5baSBill Taylor mutex_exit(&ah->ah_lock); 17929e39c5baSBill Taylor 17939e39c5baSBill Taylor 17949e39c5baSBill Taylor /* 17959e39c5baSBill Taylor * Build Base Transport Header (BTH) 17969e39c5baSBill Taylor * Notice that the M, PadCnt, and TVer fields are all set 17979e39c5baSBill Taylor * to zero implicitly. This is true for all Management Datagrams 17989e39c5baSBill Taylor * MADs whether GSI are SMI. 17999e39c5baSBill Taylor */ 18009e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_BTH(state, bth, qp, wr); 18019e39c5baSBill Taylor 18029e39c5baSBill Taylor /* 18039e39c5baSBill Taylor * Build Datagram Extended Transport Header (DETH) 18049e39c5baSBill Taylor */ 18059e39c5baSBill Taylor deth = (ib_deth_hdr_t *)((uintptr_t)bth + sizeof (ib_bth_hdr_t)); 18069e39c5baSBill Taylor HERMON_WQE_BUILD_MLX_DETH(deth, qp); 18079e39c5baSBill Taylor 18089e39c5baSBill Taylor /* Ensure that the Data Segment is aligned on a 16-byte boundary */ 18099e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)deth + sizeof (ib_deth_hdr_t)); 18109e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)(((uintptr_t)ds + 0xF) & ~0xF); 18119e39c5baSBill Taylor nds = wr->wr_nds; 18129e39c5baSBill Taylor sgl = wr->wr_sgl; 18139e39c5baSBill Taylor num_ds = 0; 18149e39c5baSBill Taylor 18159e39c5baSBill Taylor /* 18169e39c5baSBill Taylor * Now fill in the Data Segments (SGL) for the MLX WQE based on the 18179e39c5baSBill Taylor * values set up above (i.e. "sgl", "nds", and the "ds" pointer 18189e39c5baSBill Taylor * Start by checking for a valid number of SGL entries 18199e39c5baSBill Taylor */ 18209e39c5baSBill Taylor if (nds > qp->qp_sq_sgl) { 18219e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID); 18229e39c5baSBill Taylor } 18239e39c5baSBill Taylor 18249e39c5baSBill Taylor /* 18259e39c5baSBill Taylor * For each SGL in the Send Work Request, fill in the MLX WQE's data 18269e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon 18279e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually 18289e39c5baSBill Taylor * the encoding for zero means a 2GB transfer. Because of this special 18299e39c5baSBill Taylor * encoding in the hardware, we mask the requested length with 18309e39c5baSBill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as 18319e39c5baSBill Taylor * zero.) 18329e39c5baSBill Taylor */ 18339e39c5baSBill Taylor mgmtclass = hpoint = hcount = NULL; 18349e39c5baSBill Taylor offset = 0; 18359e39c5baSBill Taylor for (i = 0; i < nds; i++) { 18369e39c5baSBill Taylor if (sgl[i].ds_len == 0) { 18379e39c5baSBill Taylor continue; 18389e39c5baSBill Taylor } 18399e39c5baSBill Taylor 18409e39c5baSBill Taylor /* 18419e39c5baSBill Taylor * Fill in the Data Segment(s) for the MLX send WQE, using 18429e39c5baSBill Taylor * the information contained in the scatter-gather list of 18439e39c5baSBill Taylor * the work request. 18449e39c5baSBill Taylor */ 18459e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[num_ds], &sgl[i]); 18469e39c5baSBill Taylor 18479e39c5baSBill Taylor /* 18489e39c5baSBill Taylor * Search through the contents of all MADs posted to QP0 to 18499e39c5baSBill Taylor * initialize pointers to the places where Directed Route "hop 18509e39c5baSBill Taylor * pointer", "hop count", and "mgmtclass" would be. Hermon 18519e39c5baSBill Taylor * needs these updated (i.e. incremented or decremented, as 18529e39c5baSBill Taylor * necessary) by software. 18539e39c5baSBill Taylor */ 18549e39c5baSBill Taylor if (qp->qp_is_special == HERMON_QP_SMI) { 18559e39c5baSBill Taylor 18569e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_GET_MGMTCLASS(mgmtclass, 18579e39c5baSBill Taylor offset, sgl[i].ds_va, sgl[i].ds_len); 18589e39c5baSBill Taylor 18599e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_GET_HOPPOINTER(hpoint, 18609e39c5baSBill Taylor offset, sgl[i].ds_va, sgl[i].ds_len); 18619e39c5baSBill Taylor 18629e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_GET_HOPCOUNT(hcount, 18639e39c5baSBill Taylor offset, sgl[i].ds_va, sgl[i].ds_len); 18649e39c5baSBill Taylor 18659e39c5baSBill Taylor offset += sgl[i].ds_len; 18669e39c5baSBill Taylor } 18679e39c5baSBill Taylor num_ds++; 18689e39c5baSBill Taylor } 18699e39c5baSBill Taylor 18709e39c5baSBill Taylor /* 18719e39c5baSBill Taylor * Hermon's Directed Route MADs need to have the "hop pointer" 18729e39c5baSBill Taylor * incremented/decremented (as necessary) depending on whether it is 18739e39c5baSBill Taylor * currently less than or greater than the "hop count" (i.e. whether 18749e39c5baSBill Taylor * the MAD is a request or a response.) 18759e39c5baSBill Taylor */ 18769e39c5baSBill Taylor if (qp->qp_is_special == HERMON_QP_SMI) { 18779e39c5baSBill Taylor HERMON_SPECIAL_QP_DRMAD_DO_HOPPOINTER_MODIFY(*mgmtclass, 18789e39c5baSBill Taylor *hpoint, *hcount); 18799e39c5baSBill Taylor } 18809e39c5baSBill Taylor 18819e39c5baSBill Taylor /* 18829e39c5baSBill Taylor * Now fill in the ICRC Data Segment. This data segment is inlined 18839e39c5baSBill Taylor * just like the packets headers above, but it is only four bytes and 18849e39c5baSBill Taylor * set to zero (to indicate that we wish the hardware to generate ICRC. 18859e39c5baSBill Taylor */ 18869e39c5baSBill Taylor HERMON_WQE_BUILD_INLINE_ICRC(qp, &ds[num_ds], 4, 0); 18879e39c5baSBill Taylor num_ds++; 18889e39c5baSBill Taylor 18899e39c5baSBill Taylor /* 18909e39c5baSBill Taylor * Return the size of descriptor (in 16-byte chunks) 18919e39c5baSBill Taylor * For Hermon, we want them (for now) to be on stride size 18929e39c5baSBill Taylor * boundaries, which was implicit in Tavor/Arbel 18939e39c5baSBill Taylor */ 18949e39c5baSBill Taylor tmpsize = ((uintptr_t)&ds[num_ds] - (uintptr_t)desc); 18959e39c5baSBill Taylor 18969e39c5baSBill Taylor *size = tmpsize >> 0x04; 18979e39c5baSBill Taylor 18989e39c5baSBill Taylor return (DDI_SUCCESS); 18999e39c5baSBill Taylor } 19009e39c5baSBill Taylor 19019e39c5baSBill Taylor 19029e39c5baSBill Taylor 19039e39c5baSBill Taylor /* 19049e39c5baSBill Taylor * hermon_wqe_recv_build() 19059e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 19069e39c5baSBill Taylor */ 19079e39c5baSBill Taylor /* ARGSUSED */ 19089e39c5baSBill Taylor static int 19099e39c5baSBill Taylor hermon_wqe_recv_build(hermon_state_t *state, hermon_qphdl_t qp, 19109e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc) 19119e39c5baSBill Taylor { 19129e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds; 19139e39c5baSBill Taylor int i, num_ds; 19149e39c5baSBill Taylor 1915*17a2b317SBill Taylor ASSERT(MUTEX_HELD(&qp->qp_lock)); 19169e39c5baSBill Taylor 19179e39c5baSBill Taylor /* 19189e39c5baSBill Taylor * Fill in the Data Segments (SGL) for the Recv WQE - don't 19199e39c5baSBill Taylor * need to have a reserved for the ctrl, there is none on the 19209e39c5baSBill Taylor * recv queue for hermon, but will need to put an invalid 19219e39c5baSBill Taylor * (null) scatter pointer per PRM 19229e39c5baSBill Taylor */ 19239e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)(uintptr_t)desc; 19249e39c5baSBill Taylor num_ds = 0; 19259e39c5baSBill Taylor 19269e39c5baSBill Taylor /* Check for valid number of SGL entries */ 19279e39c5baSBill Taylor if (wr->wr_nds > qp->qp_rq_sgl) { 19289e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID); 19299e39c5baSBill Taylor } 19309e39c5baSBill Taylor 19319e39c5baSBill Taylor /* 19329e39c5baSBill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data 19339e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon 19349e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually 19359e39c5baSBill Taylor * the encoding for zero means a 2GB transfer. Because of this special 19369e39c5baSBill Taylor * encoding in the hardware, we mask the requested length with 19379e39c5baSBill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as 19389e39c5baSBill Taylor * zero.) 19399e39c5baSBill Taylor */ 19409e39c5baSBill Taylor for (i = 0; i < wr->wr_nds; i++) { 19419e39c5baSBill Taylor if (wr->wr_sgl[i].ds_len == 0) { 19429e39c5baSBill Taylor continue; 19439e39c5baSBill Taylor } 19449e39c5baSBill Taylor 19459e39c5baSBill Taylor /* 19469e39c5baSBill Taylor * Fill in the Data Segment(s) for the receive WQE, using the 19479e39c5baSBill Taylor * information contained in the scatter-gather list of the 19489e39c5baSBill Taylor * work request. 19499e39c5baSBill Taylor */ 19509e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &wr->wr_sgl[i]); 19519e39c5baSBill Taylor num_ds++; 19529e39c5baSBill Taylor } 19539e39c5baSBill Taylor 19549e39c5baSBill Taylor /* put the null sgl pointer as well if needed */ 19559e39c5baSBill Taylor if (num_ds < qp->qp_rq_sgl) { 19569e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &null_sgl); 19579e39c5baSBill Taylor } 19589e39c5baSBill Taylor 19599e39c5baSBill Taylor return (DDI_SUCCESS); 19609e39c5baSBill Taylor } 19619e39c5baSBill Taylor 19629e39c5baSBill Taylor 19639e39c5baSBill Taylor 19649e39c5baSBill Taylor /* 19659e39c5baSBill Taylor * hermon_wqe_srq_build() 19669e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 19679e39c5baSBill Taylor */ 19689e39c5baSBill Taylor /* ARGSUSED */ 19699e39c5baSBill Taylor static int 19709e39c5baSBill Taylor hermon_wqe_srq_build(hermon_state_t *state, hermon_srqhdl_t srq, 19719e39c5baSBill Taylor ibt_recv_wr_t *wr, uint64_t *desc) 19729e39c5baSBill Taylor { 19739e39c5baSBill Taylor hermon_hw_wqe_sgl_t *ds; 19749e39c5baSBill Taylor int i, num_ds; 19759e39c5baSBill Taylor 19769e39c5baSBill Taylor ASSERT(MUTEX_HELD(&srq->srq_lock)); 19779e39c5baSBill Taylor 19789e39c5baSBill Taylor /* Fill in the Data Segments (SGL) for the Recv WQE */ 19799e39c5baSBill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)desc + 19809e39c5baSBill Taylor sizeof (hermon_hw_srq_wqe_next_t)); 19819e39c5baSBill Taylor num_ds = 0; 19829e39c5baSBill Taylor 19839e39c5baSBill Taylor /* Check for valid number of SGL entries */ 19849e39c5baSBill Taylor if (wr->wr_nds > srq->srq_wq_sgl) { 19859e39c5baSBill Taylor return (IBT_QP_SGL_LEN_INVALID); 19869e39c5baSBill Taylor } 19879e39c5baSBill Taylor 19889e39c5baSBill Taylor /* 19899e39c5baSBill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data 19909e39c5baSBill Taylor * segments. Note: We skip any SGL with zero size because Hermon 19919e39c5baSBill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually 19929e39c5baSBill Taylor * the encoding for zero means a 2GB transfer. Because of this special 19939e39c5baSBill Taylor * encoding in the hardware, we mask the requested length with 19949e39c5baSBill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as 19959e39c5baSBill Taylor * zero.) 19969e39c5baSBill Taylor */ 19979e39c5baSBill Taylor for (i = 0; i < wr->wr_nds; i++) { 19989e39c5baSBill Taylor if (wr->wr_sgl[i].ds_len == 0) { 19999e39c5baSBill Taylor continue; 20009e39c5baSBill Taylor } 20019e39c5baSBill Taylor 20029e39c5baSBill Taylor /* 20039e39c5baSBill Taylor * Fill in the Data Segment(s) for the receive WQE, using the 20049e39c5baSBill Taylor * information contained in the scatter-gather list of the 20059e39c5baSBill Taylor * work request. 20069e39c5baSBill Taylor */ 20079e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &wr->wr_sgl[i]); 20089e39c5baSBill Taylor num_ds++; 20099e39c5baSBill Taylor } 20109e39c5baSBill Taylor 20119e39c5baSBill Taylor /* 20129e39c5baSBill Taylor * put in the null sgl pointer as well, if needed 20139e39c5baSBill Taylor */ 20149e39c5baSBill Taylor if (num_ds < srq->srq_wq_sgl) { 20159e39c5baSBill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &null_sgl); 20169e39c5baSBill Taylor } 20179e39c5baSBill Taylor 20189e39c5baSBill Taylor return (DDI_SUCCESS); 20199e39c5baSBill Taylor } 20209e39c5baSBill Taylor 20219e39c5baSBill Taylor 20229e39c5baSBill Taylor /* 20239e39c5baSBill Taylor * hermon_wr_get_immediate() 20249e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 20259e39c5baSBill Taylor */ 20269e39c5baSBill Taylor static uint32_t 20279e39c5baSBill Taylor hermon_wr_get_immediate(ibt_send_wr_t *wr) 20289e39c5baSBill Taylor { 20299e39c5baSBill Taylor /* 20309e39c5baSBill Taylor * This routine extracts the "immediate data" from the appropriate 20319e39c5baSBill Taylor * location in the IBTF work request. Because of the way the 20329e39c5baSBill Taylor * work request structure is defined, the location for this data 20339e39c5baSBill Taylor * depends on the actual work request operation type. 20349e39c5baSBill Taylor */ 20359e39c5baSBill Taylor 20369e39c5baSBill Taylor /* For RDMA Write, test if RC or UC */ 20379e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_RDMAW) { 20389e39c5baSBill Taylor if (wr->wr_trans == IBT_RC_SRV) { 20399e39c5baSBill Taylor return (wr->wr.rc.rcwr.rdma.rdma_immed); 20409e39c5baSBill Taylor } else { /* IBT_UC_SRV */ 20419e39c5baSBill Taylor return (wr->wr.uc.ucwr.rdma.rdma_immed); 20429e39c5baSBill Taylor } 20439e39c5baSBill Taylor } 20449e39c5baSBill Taylor 20459e39c5baSBill Taylor /* For Send, test if RC, UD, or UC */ 20469e39c5baSBill Taylor if (wr->wr_opcode == IBT_WRC_SEND) { 20479e39c5baSBill Taylor if (wr->wr_trans == IBT_RC_SRV) { 20489e39c5baSBill Taylor return (wr->wr.rc.rcwr.send_immed); 20499e39c5baSBill Taylor } else if (wr->wr_trans == IBT_UD_SRV) { 20509e39c5baSBill Taylor return (wr->wr.ud.udwr_immed); 20519e39c5baSBill Taylor } else { /* IBT_UC_SRV */ 20529e39c5baSBill Taylor return (wr->wr.uc.ucwr.send_immed); 20539e39c5baSBill Taylor } 20549e39c5baSBill Taylor } 20559e39c5baSBill Taylor 20569e39c5baSBill Taylor /* 20579e39c5baSBill Taylor * If any other type of request, then immediate is undefined 20589e39c5baSBill Taylor */ 20599e39c5baSBill Taylor return (0); 20609e39c5baSBill Taylor } 20619e39c5baSBill Taylor 20629e39c5baSBill Taylor /* 20639e39c5baSBill Taylor * hermon_wqe_headroom() 20649e39c5baSBill Taylor * Context: can be called from interrupt or base, currently only from 20659e39c5baSBill Taylor * base context. 20669e39c5baSBill Taylor * Routine that fills in the headroom for the Send Queue 20679e39c5baSBill Taylor */ 20689e39c5baSBill Taylor 20699e39c5baSBill Taylor static void 20709e39c5baSBill Taylor hermon_wqe_headroom(uint_t from, hermon_qphdl_t qp) 20719e39c5baSBill Taylor { 20729e39c5baSBill Taylor uint32_t *wqe_start, *wqe_top, *wqe_base, qsize; 20739e39c5baSBill Taylor int hdrmwqes, wqesizebytes, sectperwqe; 20749e39c5baSBill Taylor uint32_t invalue; 20759e39c5baSBill Taylor int i, j; 20769e39c5baSBill Taylor 20779e39c5baSBill Taylor qsize = qp->qp_sq_bufsz; 20789e39c5baSBill Taylor wqesizebytes = 1 << qp->qp_sq_log_wqesz; 20799e39c5baSBill Taylor sectperwqe = wqesizebytes >> 6; /* 64 bytes/section */ 20809e39c5baSBill Taylor hdrmwqes = qp->qp_sq_hdrmwqes; 20819e39c5baSBill Taylor wqe_base = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, 0); 20829e39c5baSBill Taylor wqe_top = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, qsize); 20839e39c5baSBill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, from); 20849e39c5baSBill Taylor 20859e39c5baSBill Taylor for (i = 0; i < hdrmwqes; i++) { 20869e39c5baSBill Taylor for (j = 0; j < sectperwqe; j++) { 20879e39c5baSBill Taylor if (j == 0) { /* 1st section of wqe */ 20889e39c5baSBill Taylor /* perserve ownership bit */ 20899e39c5baSBill Taylor invalue = ddi_get32(qp->qp_wqinfo.qa_acchdl, 20909e39c5baSBill Taylor wqe_start) | 0x7FFFFFFF; 20919e39c5baSBill Taylor } else { 20929e39c5baSBill Taylor /* or just invalidate it */ 20939e39c5baSBill Taylor invalue = 0xFFFFFFFF; 20949e39c5baSBill Taylor } 20959e39c5baSBill Taylor ddi_put32(qp->qp_wqinfo.qa_acchdl, wqe_start, invalue); 20969e39c5baSBill Taylor wqe_start += 16; /* move 64 bytes */ 20979e39c5baSBill Taylor } 20989e39c5baSBill Taylor if (wqe_start == wqe_top) /* hit the end of the queue */ 20999e39c5baSBill Taylor wqe_start = wqe_base; /* wrap to start */ 21009e39c5baSBill Taylor } 21019e39c5baSBill Taylor } 21029e39c5baSBill Taylor 21039e39c5baSBill Taylor /* 21049e39c5baSBill Taylor * hermon_wr_bind_check() 21059e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 21069e39c5baSBill Taylor */ 21079e39c5baSBill Taylor /* ARGSUSED */ 21089e39c5baSBill Taylor static int 21099e39c5baSBill Taylor hermon_wr_bind_check(hermon_state_t *state, ibt_send_wr_t *wr) 21109e39c5baSBill Taylor { 21119e39c5baSBill Taylor ibt_bind_flags_t bind_flags; 21129e39c5baSBill Taylor uint64_t vaddr, len; 21139e39c5baSBill Taylor uint64_t reg_start_addr, reg_end_addr; 21149e39c5baSBill Taylor hermon_mwhdl_t mw; 21159e39c5baSBill Taylor hermon_mrhdl_t mr; 21169e39c5baSBill Taylor hermon_rsrc_t *mpt; 21179e39c5baSBill Taylor uint32_t new_rkey; 21189e39c5baSBill Taylor 21199e39c5baSBill Taylor /* Check for a valid Memory Window handle in the WR */ 21209e39c5baSBill Taylor mw = (hermon_mwhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mw_hdl; 21219e39c5baSBill Taylor if (mw == NULL) { 21229e39c5baSBill Taylor return (IBT_MW_HDL_INVALID); 21239e39c5baSBill Taylor } 21249e39c5baSBill Taylor 21259e39c5baSBill Taylor /* Check for a valid Memory Region handle in the WR */ 21269e39c5baSBill Taylor mr = (hermon_mrhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mr_hdl; 21279e39c5baSBill Taylor if (mr == NULL) { 21289e39c5baSBill Taylor return (IBT_MR_HDL_INVALID); 21299e39c5baSBill Taylor } 21309e39c5baSBill Taylor 21319e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 21329e39c5baSBill Taylor mutex_enter(&mw->mr_lock); 21339e39c5baSBill Taylor 21349e39c5baSBill Taylor /* 21359e39c5baSBill Taylor * Check here to see if the memory region has already been partially 21369e39c5baSBill Taylor * deregistered as a result of a hermon_umap_umemlock_cb() callback. 21379e39c5baSBill Taylor * If so, this is an error, return failure. 21389e39c5baSBill Taylor */ 21399e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) { 21409e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21419e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 21429e39c5baSBill Taylor return (IBT_MR_HDL_INVALID); 21439e39c5baSBill Taylor } 21449e39c5baSBill Taylor 21459e39c5baSBill Taylor /* Check for a valid Memory Window RKey (i.e. a matching RKey) */ 21469e39c5baSBill Taylor if (mw->mr_rkey != wr->wr.rc.rcwr.bind->bind_rkey) { 21479e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21489e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 21499e39c5baSBill Taylor return (IBT_MR_RKEY_INVALID); 21509e39c5baSBill Taylor } 21519e39c5baSBill Taylor 21529e39c5baSBill Taylor /* Check for a valid Memory Region LKey (i.e. a matching LKey) */ 21539e39c5baSBill Taylor if (mr->mr_lkey != wr->wr.rc.rcwr.bind->bind_lkey) { 21549e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21559e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 21569e39c5baSBill Taylor return (IBT_MR_LKEY_INVALID); 21579e39c5baSBill Taylor } 21589e39c5baSBill Taylor 21599e39c5baSBill Taylor /* 21609e39c5baSBill Taylor * Now check for valid "vaddr" and "len". Note: We don't check the 21619e39c5baSBill Taylor * "vaddr" range when "len == 0" (i.e. on unbind operations) 21629e39c5baSBill Taylor */ 21639e39c5baSBill Taylor len = wr->wr.rc.rcwr.bind->bind_len; 21649e39c5baSBill Taylor if (len != 0) { 21659e39c5baSBill Taylor vaddr = wr->wr.rc.rcwr.bind->bind_va; 21669e39c5baSBill Taylor reg_start_addr = mr->mr_bindinfo.bi_addr; 21679e39c5baSBill Taylor reg_end_addr = mr->mr_bindinfo.bi_addr + 21689e39c5baSBill Taylor (mr->mr_bindinfo.bi_len - 1); 21699e39c5baSBill Taylor if ((vaddr < reg_start_addr) || (vaddr > reg_end_addr)) { 21709e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21719e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 21729e39c5baSBill Taylor return (IBT_MR_VA_INVALID); 21739e39c5baSBill Taylor } 21749e39c5baSBill Taylor vaddr = (vaddr + len) - 1; 21759e39c5baSBill Taylor if (vaddr > reg_end_addr) { 21769e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21779e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 21789e39c5baSBill Taylor return (IBT_MR_LEN_INVALID); 21799e39c5baSBill Taylor } 21809e39c5baSBill Taylor } 21819e39c5baSBill Taylor 21829e39c5baSBill Taylor /* 21839e39c5baSBill Taylor * Validate the bind access flags. Remote Write and Atomic access for 21849e39c5baSBill Taylor * the Memory Window require that Local Write access be set in the 21859e39c5baSBill Taylor * corresponding Memory Region. 21869e39c5baSBill Taylor */ 21879e39c5baSBill Taylor bind_flags = wr->wr.rc.rcwr.bind->bind_flags; 21889e39c5baSBill Taylor if (((bind_flags & IBT_WR_BIND_WRITE) || 21899e39c5baSBill Taylor (bind_flags & IBT_WR_BIND_ATOMIC)) && 21909e39c5baSBill Taylor !(mr->mr_accflag & IBT_MR_LOCAL_WRITE)) { 21919e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21929e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 21939e39c5baSBill Taylor return (IBT_MR_ACCESS_REQ_INVALID); 21949e39c5baSBill Taylor } 21959e39c5baSBill Taylor 21969e39c5baSBill Taylor /* Calculate the new RKey for the Memory Window */ 21979e39c5baSBill Taylor mpt = mw->mr_mptrsrcp; 21989e39c5baSBill Taylor new_rkey = hermon_mr_keycalc(mpt->hr_indx); 21999e39c5baSBill Taylor new_rkey = hermon_mr_key_swap(new_rkey); 22009e39c5baSBill Taylor 22019e39c5baSBill Taylor wr->wr.rc.rcwr.bind->bind_rkey_out = new_rkey; 22029e39c5baSBill Taylor mw->mr_rkey = new_rkey; 22039e39c5baSBill Taylor 22049e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 22059e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 22069e39c5baSBill Taylor return (DDI_SUCCESS); 22079e39c5baSBill Taylor } 22089e39c5baSBill Taylor 22099e39c5baSBill Taylor 22109e39c5baSBill Taylor /* 22119e39c5baSBill Taylor * hermon_wrid_from_reset_handling() 22129e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 22139e39c5baSBill Taylor */ 22149e39c5baSBill Taylor /* ARGSUSED */ 22159e39c5baSBill Taylor int 22169e39c5baSBill Taylor hermon_wrid_from_reset_handling(hermon_state_t *state, hermon_qphdl_t qp) 22179e39c5baSBill Taylor { 22189e39c5baSBill Taylor hermon_workq_hdr_t *swq, *rwq; 22199e39c5baSBill Taylor 2220*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP) 22219e39c5baSBill Taylor return (DDI_SUCCESS); 22229e39c5baSBill Taylor 22239e39c5baSBill Taylor #ifdef __lock_lint 2224*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock); 22259e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock); 22269e39c5baSBill Taylor #else 2227*17a2b317SBill Taylor /* grab the cq lock(s) to modify the wqavl tree */ 2228*17a2b317SBill Taylor if (qp->qp_rq_cqhdl) 2229*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock); 2230*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl && 2231*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL) 22329e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock); 22339e39c5baSBill Taylor #endif 22349e39c5baSBill Taylor 22359e39c5baSBill Taylor /* Chain the newly allocated work queue header to the CQ's list */ 2236*17a2b317SBill Taylor if (qp->qp_sq_cqhdl) 2237*17a2b317SBill Taylor hermon_cq_workq_add(qp->qp_sq_cqhdl, &qp->qp_sq_wqavl); 22389e39c5baSBill Taylor 22399e39c5baSBill Taylor swq = qp->qp_sq_wqhdr; 22409e39c5baSBill Taylor swq->wq_head = 0; 22419e39c5baSBill Taylor swq->wq_tail = 0; 22429e39c5baSBill Taylor swq->wq_full = 0; 22439e39c5baSBill Taylor 22449e39c5baSBill Taylor /* 22459e39c5baSBill Taylor * Now we repeat all the above operations for the receive work queue, 22469e39c5baSBill Taylor * or shared receive work queue. 22479e39c5baSBill Taylor * 22489e39c5baSBill Taylor * Note: We still use the 'qp_rq_cqhdl' even in the SRQ case. 22499e39c5baSBill Taylor */ 22509e39c5baSBill Taylor 22519e39c5baSBill Taylor #ifdef __lock_lint 22529e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock); 22539e39c5baSBill Taylor #else 2254*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) { 22559e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock); 22569e39c5baSBill Taylor } else { 22579e39c5baSBill Taylor rwq = qp->qp_rq_wqhdr; 22589e39c5baSBill Taylor rwq->wq_head = 0; 22599e39c5baSBill Taylor rwq->wq_tail = 0; 22609e39c5baSBill Taylor rwq->wq_full = 0; 22619e39c5baSBill Taylor qp->qp_rq_wqecntr = 0; 22629e39c5baSBill Taylor } 22639e39c5baSBill Taylor #endif 22649e39c5baSBill Taylor hermon_cq_workq_add(qp->qp_rq_cqhdl, &qp->qp_rq_wqavl); 22659e39c5baSBill Taylor 22669e39c5baSBill Taylor #ifdef __lock_lint 22679e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock); 22689e39c5baSBill Taylor #else 2269*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) { 22709e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock); 22719e39c5baSBill Taylor } 22729e39c5baSBill Taylor #endif 22739e39c5baSBill Taylor 22749e39c5baSBill Taylor #ifdef __lock_lint 22759e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock); 2276*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock); 22779e39c5baSBill Taylor #else 2278*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl && 2279*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL) 22809e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock); 2281*17a2b317SBill Taylor if (qp->qp_rq_cqhdl) 2282*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock); 22839e39c5baSBill Taylor #endif 22849e39c5baSBill Taylor return (DDI_SUCCESS); 22859e39c5baSBill Taylor } 22869e39c5baSBill Taylor 22879e39c5baSBill Taylor 22889e39c5baSBill Taylor /* 22899e39c5baSBill Taylor * hermon_wrid_to_reset_handling() 22909e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 22919e39c5baSBill Taylor */ 22929e39c5baSBill Taylor int 22939e39c5baSBill Taylor hermon_wrid_to_reset_handling(hermon_state_t *state, hermon_qphdl_t qp) 22949e39c5baSBill Taylor { 2295*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP) 22969e39c5baSBill Taylor return (DDI_SUCCESS); 22979e39c5baSBill Taylor 22989e39c5baSBill Taylor /* 22999e39c5baSBill Taylor * If there are unpolled entries in these CQs, they are 23009e39c5baSBill Taylor * polled/flushed. 23019e39c5baSBill Taylor * Grab the CQ lock(s) before manipulating the lists. 23029e39c5baSBill Taylor */ 23039e39c5baSBill Taylor #ifdef __lock_lint 2304*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock); 23059e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock); 23069e39c5baSBill Taylor #else 2307*17a2b317SBill Taylor /* grab the cq lock(s) to modify the wqavl tree */ 2308*17a2b317SBill Taylor if (qp->qp_rq_cqhdl) 2309*17a2b317SBill Taylor mutex_enter(&qp->qp_rq_cqhdl->cq_lock); 2310*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl && 2311*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL) 23129e39c5baSBill Taylor mutex_enter(&qp->qp_sq_cqhdl->cq_lock); 23139e39c5baSBill Taylor #endif 23149e39c5baSBill Taylor 23159e39c5baSBill Taylor #ifdef __lock_lint 23169e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock); 23179e39c5baSBill Taylor #else 2318*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) { 23199e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock); 23209e39c5baSBill Taylor } 23219e39c5baSBill Taylor #endif 23229e39c5baSBill Taylor /* 23239e39c5baSBill Taylor * Flush the entries on the CQ for this QP's QPN. 23249e39c5baSBill Taylor */ 23259e39c5baSBill Taylor hermon_cq_entries_flush(state, qp); 23269e39c5baSBill Taylor 23279e39c5baSBill Taylor #ifdef __lock_lint 23289e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock); 23299e39c5baSBill Taylor #else 2330*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) { 23319e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock); 23329e39c5baSBill Taylor } 23339e39c5baSBill Taylor #endif 23349e39c5baSBill Taylor 23359e39c5baSBill Taylor hermon_cq_workq_remove(qp->qp_rq_cqhdl, &qp->qp_rq_wqavl); 2336*17a2b317SBill Taylor if (qp->qp_sq_cqhdl != NULL) 2337*17a2b317SBill Taylor hermon_cq_workq_remove(qp->qp_sq_cqhdl, &qp->qp_sq_wqavl); 23389e39c5baSBill Taylor 23399e39c5baSBill Taylor #ifdef __lock_lint 23409e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock); 2341*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock); 23429e39c5baSBill Taylor #else 2343*17a2b317SBill Taylor if (qp->qp_rq_cqhdl != qp->qp_sq_cqhdl && 2344*17a2b317SBill Taylor qp->qp_sq_cqhdl != NULL) 23459e39c5baSBill Taylor mutex_exit(&qp->qp_sq_cqhdl->cq_lock); 2346*17a2b317SBill Taylor if (qp->qp_rq_cqhdl) 2347*17a2b317SBill Taylor mutex_exit(&qp->qp_rq_cqhdl->cq_lock); 23489e39c5baSBill Taylor #endif 23499e39c5baSBill Taylor 23509e39c5baSBill Taylor return (IBT_SUCCESS); 23519e39c5baSBill Taylor } 23529e39c5baSBill Taylor 23539e39c5baSBill Taylor 23549e39c5baSBill Taylor /* 23559e39c5baSBill Taylor * hermon_wrid_get_entry() 23569e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 23579e39c5baSBill Taylor */ 23589e39c5baSBill Taylor uint64_t 23599e39c5baSBill Taylor hermon_wrid_get_entry(hermon_cqhdl_t cq, hermon_hw_cqe_t *cqe) 23609e39c5baSBill Taylor { 23619e39c5baSBill Taylor hermon_workq_avl_t *wqa; 23629e39c5baSBill Taylor hermon_workq_hdr_t *wq; 23639e39c5baSBill Taylor uint64_t wrid; 23649e39c5baSBill Taylor uint_t send_or_recv, qpnum; 23659e39c5baSBill Taylor uint32_t indx; 23669e39c5baSBill Taylor 23679e39c5baSBill Taylor /* 23689e39c5baSBill Taylor * Determine whether this CQE is a send or receive completion. 23699e39c5baSBill Taylor */ 23709e39c5baSBill Taylor send_or_recv = HERMON_CQE_SENDRECV_GET(cq, cqe); 23719e39c5baSBill Taylor 23729e39c5baSBill Taylor /* Find the work queue for this QP number (send or receive side) */ 23739e39c5baSBill Taylor qpnum = HERMON_CQE_QPNUM_GET(cq, cqe); 23749e39c5baSBill Taylor wqa = hermon_wrid_wqavl_find(cq, qpnum, send_or_recv); 23759e39c5baSBill Taylor wq = wqa->wqa_wq; 23769e39c5baSBill Taylor 23779e39c5baSBill Taylor /* 23789e39c5baSBill Taylor * Regardless of whether the completion is the result of a "success" 23799e39c5baSBill Taylor * or a "failure", we lock the list of "containers" and attempt to 23809e39c5baSBill Taylor * search for the the first matching completion (i.e. the first WR 23819e39c5baSBill Taylor * with a matching WQE addr and size). Once we find it, we pull out 23829e39c5baSBill Taylor * the "wrid" field and return it (see below). XXX Note: One possible 23839e39c5baSBill Taylor * future enhancement would be to enable this routine to skip over 23849e39c5baSBill Taylor * any "unsignaled" completions to go directly to the next "signaled" 23859e39c5baSBill Taylor * entry on success. 23869e39c5baSBill Taylor */ 23879e39c5baSBill Taylor indx = HERMON_CQE_WQEADDRSZ_GET(cq, cqe) & wq->wq_mask; 23889e39c5baSBill Taylor wrid = wq->wq_wrid[indx]; 23899e39c5baSBill Taylor if (wqa->wqa_srq_en) { 23909e39c5baSBill Taylor struct hermon_sw_srq_s *srq; 23919e39c5baSBill Taylor uint64_t *desc; 23929e39c5baSBill Taylor 23939e39c5baSBill Taylor /* put wqe back on the srq free list */ 23949e39c5baSBill Taylor srq = wqa->wqa_srq; 23959e39c5baSBill Taylor mutex_enter(&srq->srq_lock); 23969e39c5baSBill Taylor desc = HERMON_SRQ_WQE_ADDR(srq, wq->wq_tail); 23979e39c5baSBill Taylor ((uint16_t *)desc)[1] = htons(indx); 23989e39c5baSBill Taylor wq->wq_tail = indx; 23999e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 24009e39c5baSBill Taylor } else { 24019e39c5baSBill Taylor wq->wq_head = (indx + 1) & wq->wq_mask; 24029e39c5baSBill Taylor wq->wq_full = 0; 24039e39c5baSBill Taylor } 24049e39c5baSBill Taylor 24059e39c5baSBill Taylor return (wrid); 24069e39c5baSBill Taylor } 24079e39c5baSBill Taylor 24089e39c5baSBill Taylor 24099e39c5baSBill Taylor int 24109e39c5baSBill Taylor hermon_wrid_workq_compare(const void *p1, const void *p2) 24119e39c5baSBill Taylor { 24129e39c5baSBill Taylor hermon_workq_compare_t *cmpp; 24139e39c5baSBill Taylor hermon_workq_avl_t *curr; 24149e39c5baSBill Taylor 24159e39c5baSBill Taylor cmpp = (hermon_workq_compare_t *)p1; 24169e39c5baSBill Taylor curr = (hermon_workq_avl_t *)p2; 24179e39c5baSBill Taylor 24189e39c5baSBill Taylor if (cmpp->cmp_qpn < curr->wqa_qpn) 24199e39c5baSBill Taylor return (-1); 24209e39c5baSBill Taylor else if (cmpp->cmp_qpn > curr->wqa_qpn) 24219e39c5baSBill Taylor return (+1); 24229e39c5baSBill Taylor else if (cmpp->cmp_type < curr->wqa_type) 24239e39c5baSBill Taylor return (-1); 24249e39c5baSBill Taylor else if (cmpp->cmp_type > curr->wqa_type) 24259e39c5baSBill Taylor return (+1); 24269e39c5baSBill Taylor else 24279e39c5baSBill Taylor return (0); 24289e39c5baSBill Taylor } 24299e39c5baSBill Taylor 24309e39c5baSBill Taylor 24319e39c5baSBill Taylor /* 24329e39c5baSBill Taylor * hermon_wrid_workq_find() 24339e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 24349e39c5baSBill Taylor */ 24359e39c5baSBill Taylor static hermon_workq_avl_t * 24369e39c5baSBill Taylor hermon_wrid_wqavl_find(hermon_cqhdl_t cq, uint_t qpn, uint_t wq_type) 24379e39c5baSBill Taylor { 24389e39c5baSBill Taylor hermon_workq_avl_t *curr; 24399e39c5baSBill Taylor hermon_workq_compare_t cmp; 24409e39c5baSBill Taylor 24419e39c5baSBill Taylor /* 24429e39c5baSBill Taylor * Walk the CQ's work queue list, trying to find a send or recv queue 24439e39c5baSBill Taylor * with the same QP number. We do this even if we are going to later 24449e39c5baSBill Taylor * create a new entry because it helps us easily find the end of the 24459e39c5baSBill Taylor * list. 24469e39c5baSBill Taylor */ 24479e39c5baSBill Taylor cmp.cmp_qpn = qpn; 24489e39c5baSBill Taylor cmp.cmp_type = wq_type; 24499e39c5baSBill Taylor #ifdef __lock_lint 24509e39c5baSBill Taylor hermon_wrid_workq_compare(NULL, NULL); 24519e39c5baSBill Taylor #endif 24529e39c5baSBill Taylor curr = avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, NULL); 24539e39c5baSBill Taylor 24549e39c5baSBill Taylor return (curr); 24559e39c5baSBill Taylor } 24569e39c5baSBill Taylor 24579e39c5baSBill Taylor 24589e39c5baSBill Taylor /* 24599e39c5baSBill Taylor * hermon_wrid_wqhdr_create() 24609e39c5baSBill Taylor * Context: Can be called from base context. 24619e39c5baSBill Taylor */ 24629e39c5baSBill Taylor /* ARGSUSED */ 24639e39c5baSBill Taylor hermon_workq_hdr_t * 24649e39c5baSBill Taylor hermon_wrid_wqhdr_create(int bufsz) 24659e39c5baSBill Taylor { 24669e39c5baSBill Taylor hermon_workq_hdr_t *wqhdr; 24679e39c5baSBill Taylor 24689e39c5baSBill Taylor /* 24699e39c5baSBill Taylor * Allocate space for the wqhdr, and an array to record all the wrids. 24709e39c5baSBill Taylor */ 24719e39c5baSBill Taylor wqhdr = (hermon_workq_hdr_t *)kmem_zalloc(sizeof (*wqhdr), KM_NOSLEEP); 24729e39c5baSBill Taylor if (wqhdr == NULL) { 24739e39c5baSBill Taylor return (NULL); 24749e39c5baSBill Taylor } 24759e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqhdr)) 24769e39c5baSBill Taylor wqhdr->wq_wrid = kmem_zalloc(bufsz * sizeof (uint64_t), KM_NOSLEEP); 24779e39c5baSBill Taylor if (wqhdr->wq_wrid == NULL) { 24789e39c5baSBill Taylor kmem_free(wqhdr, sizeof (*wqhdr)); 24799e39c5baSBill Taylor return (NULL); 24809e39c5baSBill Taylor } 24819e39c5baSBill Taylor wqhdr->wq_size = bufsz; 24829e39c5baSBill Taylor wqhdr->wq_mask = bufsz - 1; 24839e39c5baSBill Taylor 24849e39c5baSBill Taylor return (wqhdr); 24859e39c5baSBill Taylor } 24869e39c5baSBill Taylor 24879e39c5baSBill Taylor void 24889e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(hermon_workq_hdr_t *wqhdr) 24899e39c5baSBill Taylor { 24909e39c5baSBill Taylor kmem_free(wqhdr->wq_wrid, wqhdr->wq_size * sizeof (uint64_t)); 24919e39c5baSBill Taylor kmem_free(wqhdr, sizeof (*wqhdr)); 24929e39c5baSBill Taylor } 24939e39c5baSBill Taylor 24949e39c5baSBill Taylor 24959e39c5baSBill Taylor /* 24969e39c5baSBill Taylor * hermon_cq_workq_add() 24979e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 24989e39c5baSBill Taylor */ 24999e39c5baSBill Taylor static void 25009e39c5baSBill Taylor hermon_cq_workq_add(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl) 25019e39c5baSBill Taylor { 25029e39c5baSBill Taylor hermon_workq_compare_t cmp; 25039e39c5baSBill Taylor avl_index_t where; 25049e39c5baSBill Taylor 25059e39c5baSBill Taylor cmp.cmp_qpn = wqavl->wqa_qpn; 25069e39c5baSBill Taylor cmp.cmp_type = wqavl->wqa_type; 25079e39c5baSBill Taylor #ifdef __lock_lint 25089e39c5baSBill Taylor hermon_wrid_workq_compare(NULL, NULL); 25099e39c5baSBill Taylor #endif 25109e39c5baSBill Taylor (void) avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, &where); 25119e39c5baSBill Taylor avl_insert(&cq->cq_wrid_wqhdr_avl_tree, wqavl, where); 25129e39c5baSBill Taylor } 25139e39c5baSBill Taylor 25149e39c5baSBill Taylor 25159e39c5baSBill Taylor /* 25169e39c5baSBill Taylor * hermon_cq_workq_remove() 25179e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 25189e39c5baSBill Taylor */ 25199e39c5baSBill Taylor static void 25209e39c5baSBill Taylor hermon_cq_workq_remove(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl) 25219e39c5baSBill Taylor { 25229e39c5baSBill Taylor #ifdef __lock_lint 25239e39c5baSBill Taylor hermon_wrid_workq_compare(NULL, NULL); 25249e39c5baSBill Taylor #endif 25259e39c5baSBill Taylor avl_remove(&cq->cq_wrid_wqhdr_avl_tree, wqavl); 25269e39c5baSBill Taylor } 2527