1*9e39c5baSBill Taylor /* 2*9e39c5baSBill Taylor * CDDL HEADER START 3*9e39c5baSBill Taylor * 4*9e39c5baSBill Taylor * The contents of this file are subject to the terms of the 5*9e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 6*9e39c5baSBill Taylor * You may not use this file except in compliance with the License. 7*9e39c5baSBill Taylor * 8*9e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*9e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 10*9e39c5baSBill Taylor * See the License for the specific language governing permissions 11*9e39c5baSBill Taylor * and limitations under the License. 12*9e39c5baSBill Taylor * 13*9e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 14*9e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*9e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 16*9e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 17*9e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 18*9e39c5baSBill Taylor * 19*9e39c5baSBill Taylor * CDDL HEADER END 20*9e39c5baSBill Taylor */ 21*9e39c5baSBill Taylor 22*9e39c5baSBill Taylor /* 23*9e39c5baSBill Taylor * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24*9e39c5baSBill Taylor * Use is subject to license terms. 25*9e39c5baSBill Taylor */ 26*9e39c5baSBill Taylor 27*9e39c5baSBill Taylor /* 28*9e39c5baSBill Taylor * hermon_qp.c 29*9e39c5baSBill Taylor * Hermon Queue Pair Processing Routines 30*9e39c5baSBill Taylor * 31*9e39c5baSBill Taylor * Implements all the routines necessary for allocating, freeing, and 32*9e39c5baSBill Taylor * querying the Hermon queue pairs. 33*9e39c5baSBill Taylor */ 34*9e39c5baSBill Taylor 35*9e39c5baSBill Taylor #include <sys/types.h> 36*9e39c5baSBill Taylor #include <sys/conf.h> 37*9e39c5baSBill Taylor #include <sys/ddi.h> 38*9e39c5baSBill Taylor #include <sys/sunddi.h> 39*9e39c5baSBill Taylor #include <sys/modctl.h> 40*9e39c5baSBill Taylor #include <sys/bitmap.h> 41*9e39c5baSBill Taylor #include <sys/sysmacros.h> 42*9e39c5baSBill Taylor 43*9e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 44*9e39c5baSBill Taylor #include <sys/ib/ib_pkt_hdrs.h> 45*9e39c5baSBill Taylor 46*9e39c5baSBill Taylor static int hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp, 47*9e39c5baSBill Taylor hermon_rsrc_t *qpc); 48*9e39c5baSBill Taylor static int hermon_qpn_avl_compare(const void *q, const void *e); 49*9e39c5baSBill Taylor static int hermon_special_qp_rsrc_alloc(hermon_state_t *state, 50*9e39c5baSBill Taylor ibt_sqp_type_t type, uint_t port, hermon_rsrc_t **qp_rsrc); 51*9e39c5baSBill Taylor static int hermon_special_qp_rsrc_free(hermon_state_t *state, 52*9e39c5baSBill Taylor ibt_sqp_type_t type, uint_t port); 53*9e39c5baSBill Taylor static void hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl, 54*9e39c5baSBill Taylor uint_t real_max_sgl, hermon_qp_wq_type_t wq_type, 55*9e39c5baSBill Taylor uint_t *logwqesz, uint_t *max_sgl); 56*9e39c5baSBill Taylor 57*9e39c5baSBill Taylor /* 58*9e39c5baSBill Taylor * hermon_qp_alloc() 59*9e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 60*9e39c5baSBill Taylor */ 61*9e39c5baSBill Taylor int 62*9e39c5baSBill Taylor hermon_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo, 63*9e39c5baSBill Taylor uint_t sleepflag) 64*9e39c5baSBill Taylor { 65*9e39c5baSBill Taylor hermon_rsrc_t *qpc, *rsrc; 66*9e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 67*9e39c5baSBill Taylor hermon_qphdl_t qp; 68*9e39c5baSBill Taylor ibt_qp_alloc_attr_t *attr_p; 69*9e39c5baSBill Taylor ibt_qp_type_t type; 70*9e39c5baSBill Taylor hermon_qp_wq_type_t swq_type; 71*9e39c5baSBill Taylor ibtl_qp_hdl_t ibt_qphdl; 72*9e39c5baSBill Taylor ibt_chan_sizes_t *queuesz_p; 73*9e39c5baSBill Taylor ib_qpn_t *qpn; 74*9e39c5baSBill Taylor hermon_qphdl_t *qphdl; 75*9e39c5baSBill Taylor ibt_mr_attr_t mr_attr; 76*9e39c5baSBill Taylor hermon_mr_options_t mr_op; 77*9e39c5baSBill Taylor hermon_srqhdl_t srq; 78*9e39c5baSBill Taylor hermon_pdhdl_t pd; 79*9e39c5baSBill Taylor hermon_cqhdl_t sq_cq, rq_cq; 80*9e39c5baSBill Taylor hermon_mrhdl_t mr; 81*9e39c5baSBill Taylor uint64_t value, qp_desc_off; 82*9e39c5baSBill Taylor uint64_t *thewqe, thewqesz; 83*9e39c5baSBill Taylor uint32_t *sq_buf, *rq_buf; 84*9e39c5baSBill Taylor uint32_t log_qp_sq_size, log_qp_rq_size; 85*9e39c5baSBill Taylor uint32_t sq_size, rq_size; 86*9e39c5baSBill Taylor uint32_t sq_depth, rq_depth; 87*9e39c5baSBill Taylor uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift; 88*9e39c5baSBill Taylor uint32_t max_sgl, max_recv_sgl, uarpg; 89*9e39c5baSBill Taylor uint_t qp_is_umap; 90*9e39c5baSBill Taylor uint_t qp_srq_en, i, j; 91*9e39c5baSBill Taylor int status, flag; 92*9e39c5baSBill Taylor 93*9e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p)) 94*9e39c5baSBill Taylor 95*9e39c5baSBill Taylor /* 96*9e39c5baSBill Taylor * Extract the necessary info from the hermon_qp_info_t structure 97*9e39c5baSBill Taylor */ 98*9e39c5baSBill Taylor attr_p = qpinfo->qpi_attrp; 99*9e39c5baSBill Taylor type = qpinfo->qpi_type; 100*9e39c5baSBill Taylor ibt_qphdl = qpinfo->qpi_ibt_qphdl; 101*9e39c5baSBill Taylor queuesz_p = qpinfo->qpi_queueszp; 102*9e39c5baSBill Taylor qpn = qpinfo->qpi_qpn; 103*9e39c5baSBill Taylor qphdl = &qpinfo->qpi_qphdl; 104*9e39c5baSBill Taylor 105*9e39c5baSBill Taylor /* 106*9e39c5baSBill Taylor * Determine whether QP is being allocated for userland access or 107*9e39c5baSBill Taylor * whether it is being allocated for kernel access. If the QP is 108*9e39c5baSBill Taylor * being allocated for userland access, then lookup the UAR 109*9e39c5baSBill Taylor * page number for the current process. Note: If this is not found 110*9e39c5baSBill Taylor * (e.g. if the process has not previously open()'d the Hermon driver), 111*9e39c5baSBill Taylor * then an error is returned. 112*9e39c5baSBill Taylor */ 113*9e39c5baSBill Taylor 114*9e39c5baSBill Taylor 115*9e39c5baSBill Taylor qp_is_umap = (attr_p->qp_alloc_flags & IBT_QP_USER_MAP) ? 1 : 0; 116*9e39c5baSBill Taylor if (qp_is_umap) { 117*9e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(), 118*9e39c5baSBill Taylor MLNX_UMAP_UARPG_RSRC, &value, 0, NULL); 119*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 120*9e39c5baSBill Taylor status = IBT_INVALID_PARAM; 121*9e39c5baSBill Taylor goto qpalloc_fail; 122*9e39c5baSBill Taylor } 123*9e39c5baSBill Taylor uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx; 124*9e39c5baSBill Taylor } else { 125*9e39c5baSBill Taylor uarpg = state->hs_kernel_uar_index; 126*9e39c5baSBill Taylor } 127*9e39c5baSBill Taylor 128*9e39c5baSBill Taylor /* 129*9e39c5baSBill Taylor * Determine whether QP is being associated with an SRQ 130*9e39c5baSBill Taylor */ 131*9e39c5baSBill Taylor qp_srq_en = (attr_p->qp_alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0; 132*9e39c5baSBill Taylor if (qp_srq_en) { 133*9e39c5baSBill Taylor /* 134*9e39c5baSBill Taylor * Check for valid SRQ handle pointers 135*9e39c5baSBill Taylor */ 136*9e39c5baSBill Taylor if (attr_p->qp_ibc_srq_hdl == NULL) { 137*9e39c5baSBill Taylor status = IBT_SRQ_HDL_INVALID; 138*9e39c5baSBill Taylor goto qpalloc_fail; 139*9e39c5baSBill Taylor } 140*9e39c5baSBill Taylor srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl; 141*9e39c5baSBill Taylor } 142*9e39c5baSBill Taylor 143*9e39c5baSBill Taylor /* 144*9e39c5baSBill Taylor * Check for valid QP service type (only UD/RC/UC supported) 145*9e39c5baSBill Taylor */ 146*9e39c5baSBill Taylor if (((type != IBT_UD_RQP) && (type != IBT_RC_RQP) && 147*9e39c5baSBill Taylor (type != IBT_UC_RQP))) { 148*9e39c5baSBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 149*9e39c5baSBill Taylor goto qpalloc_fail; 150*9e39c5baSBill Taylor } 151*9e39c5baSBill Taylor 152*9e39c5baSBill Taylor 153*9e39c5baSBill Taylor /* 154*9e39c5baSBill Taylor * Check for valid PD handle pointer 155*9e39c5baSBill Taylor */ 156*9e39c5baSBill Taylor if (attr_p->qp_pd_hdl == NULL) { 157*9e39c5baSBill Taylor status = IBT_PD_HDL_INVALID; 158*9e39c5baSBill Taylor goto qpalloc_fail; 159*9e39c5baSBill Taylor } 160*9e39c5baSBill Taylor pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl; 161*9e39c5baSBill Taylor 162*9e39c5baSBill Taylor /* 163*9e39c5baSBill Taylor * If on an SRQ, check to make sure the PD is the same 164*9e39c5baSBill Taylor */ 165*9e39c5baSBill Taylor if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) { 166*9e39c5baSBill Taylor status = IBT_PD_HDL_INVALID; 167*9e39c5baSBill Taylor goto qpalloc_fail; 168*9e39c5baSBill Taylor } 169*9e39c5baSBill Taylor 170*9e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */ 171*9e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 172*9e39c5baSBill Taylor 173*9e39c5baSBill Taylor /* 174*9e39c5baSBill Taylor * Check for valid CQ handle pointers 175*9e39c5baSBill Taylor */ 176*9e39c5baSBill Taylor if ((attr_p->qp_ibc_scq_hdl == NULL) || 177*9e39c5baSBill Taylor (attr_p->qp_ibc_rcq_hdl == NULL)) { 178*9e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 179*9e39c5baSBill Taylor goto qpalloc_fail1; 180*9e39c5baSBill Taylor } 181*9e39c5baSBill Taylor sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl; 182*9e39c5baSBill Taylor rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl; 183*9e39c5baSBill Taylor 184*9e39c5baSBill Taylor /* 185*9e39c5baSBill Taylor * Increment the reference count on the CQs. One or both of these 186*9e39c5baSBill Taylor * could return error if we determine that the given CQ is already 187*9e39c5baSBill Taylor * being used with a special (SMI/GSI) QP. 188*9e39c5baSBill Taylor */ 189*9e39c5baSBill Taylor status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL); 190*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 191*9e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 192*9e39c5baSBill Taylor goto qpalloc_fail1; 193*9e39c5baSBill Taylor } 194*9e39c5baSBill Taylor status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL); 195*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 196*9e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 197*9e39c5baSBill Taylor goto qpalloc_fail2; 198*9e39c5baSBill Taylor } 199*9e39c5baSBill Taylor 200*9e39c5baSBill Taylor /* 201*9e39c5baSBill Taylor * Allocate an QP context entry. This will be filled in with all 202*9e39c5baSBill Taylor * the necessary parameters to define the Queue Pair. Unlike 203*9e39c5baSBill Taylor * other Hermon hardware resources, ownership is not immediately 204*9e39c5baSBill Taylor * given to hardware in the final step here. Instead, we must 205*9e39c5baSBill Taylor * wait until the QP is later transitioned to the "Init" state before 206*9e39c5baSBill Taylor * passing the QP to hardware. If we fail here, we must undo all 207*9e39c5baSBill Taylor * the reference count (CQ and PD). 208*9e39c5baSBill Taylor */ 209*9e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_QPC, 1, sleepflag, &qpc); 210*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 211*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 212*9e39c5baSBill Taylor goto qpalloc_fail3; 213*9e39c5baSBill Taylor } 214*9e39c5baSBill Taylor 215*9e39c5baSBill Taylor /* 216*9e39c5baSBill Taylor * Allocate the software structure for tracking the queue pair 217*9e39c5baSBill Taylor * (i.e. the Hermon Queue Pair handle). If we fail here, we must 218*9e39c5baSBill Taylor * undo the reference counts and the previous resource allocation. 219*9e39c5baSBill Taylor */ 220*9e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc); 221*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 222*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 223*9e39c5baSBill Taylor goto qpalloc_fail4; 224*9e39c5baSBill Taylor } 225*9e39c5baSBill Taylor qp = (hermon_qphdl_t)rsrc->hr_addr; 226*9e39c5baSBill Taylor bzero(qp, sizeof (struct hermon_sw_qp_s)); 227*9e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp)) 228*9e39c5baSBill Taylor 229*9e39c5baSBill Taylor /* 230*9e39c5baSBill Taylor * Calculate the QP number from QPC index. This routine handles 231*9e39c5baSBill Taylor * all of the operations necessary to keep track of used, unused, 232*9e39c5baSBill Taylor * and released QP numbers. 233*9e39c5baSBill Taylor */ 234*9e39c5baSBill Taylor status = hermon_qp_create_qpn(state, qp, qpc); 235*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 236*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 237*9e39c5baSBill Taylor goto qpalloc_fail5; 238*9e39c5baSBill Taylor } 239*9e39c5baSBill Taylor 240*9e39c5baSBill Taylor /* 241*9e39c5baSBill Taylor * If this will be a user-mappable QP, then allocate an entry for 242*9e39c5baSBill Taylor * the "userland resources database". This will later be added to 243*9e39c5baSBill Taylor * the database (after all further QP operations are successful). 244*9e39c5baSBill Taylor * If we fail here, we must undo the reference counts and the 245*9e39c5baSBill Taylor * previous resource allocation. 246*9e39c5baSBill Taylor */ 247*9e39c5baSBill Taylor if (qp_is_umap) { 248*9e39c5baSBill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance, qp->qp_qpnum, 249*9e39c5baSBill Taylor MLNX_UMAP_QPMEM_RSRC, (uint64_t)(uintptr_t)rsrc); 250*9e39c5baSBill Taylor if (umapdb == NULL) { 251*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 252*9e39c5baSBill Taylor goto qpalloc_fail6; 253*9e39c5baSBill Taylor } 254*9e39c5baSBill Taylor } 255*9e39c5baSBill Taylor 256*9e39c5baSBill Taylor /* 257*9e39c5baSBill Taylor * Allocate the doorbell record. Hermon just needs one for the RQ, 258*9e39c5baSBill Taylor * if the QP is not associated with an SRQ, and use uarpg (above) as 259*9e39c5baSBill Taylor * the uar index 260*9e39c5baSBill Taylor */ 261*9e39c5baSBill Taylor 262*9e39c5baSBill Taylor if (!qp_srq_en) { 263*9e39c5baSBill Taylor status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl, 264*9e39c5baSBill Taylor &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset); 265*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 266*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 267*9e39c5baSBill Taylor goto qpalloc_fail6; 268*9e39c5baSBill Taylor } 269*9e39c5baSBill Taylor } 270*9e39c5baSBill Taylor 271*9e39c5baSBill Taylor qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO); 272*9e39c5baSBill Taylor 273*9e39c5baSBill Taylor /* 274*9e39c5baSBill Taylor * We verify that the requested number of SGL is valid (i.e. 275*9e39c5baSBill Taylor * consistent with the device limits and/or software-configured 276*9e39c5baSBill Taylor * limits). If not, then obviously the same cleanup needs to be done. 277*9e39c5baSBill Taylor */ 278*9e39c5baSBill Taylor if (type == IBT_UD_RQP) { 279*9e39c5baSBill Taylor max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz; 280*9e39c5baSBill Taylor swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD; 281*9e39c5baSBill Taylor } else { 282*9e39c5baSBill Taylor max_sgl = state->hs_ibtfinfo.hca_attr->hca_conn_send_sgl_sz; 283*9e39c5baSBill Taylor swq_type = HERMON_QP_WQ_TYPE_SENDQ_CONN; 284*9e39c5baSBill Taylor } 285*9e39c5baSBill Taylor max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz; 286*9e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) || 287*9e39c5baSBill Taylor (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) { 288*9e39c5baSBill Taylor status = IBT_HCA_SGL_EXCEEDED; 289*9e39c5baSBill Taylor goto qpalloc_fail7; 290*9e39c5baSBill Taylor } 291*9e39c5baSBill Taylor 292*9e39c5baSBill Taylor /* 293*9e39c5baSBill Taylor * Determine this QP's WQE stride (for both the Send and Recv WQEs). 294*9e39c5baSBill Taylor * This will depend on the requested number of SGLs. Note: this 295*9e39c5baSBill Taylor * has the side-effect of also calculating the real number of SGLs 296*9e39c5baSBill Taylor * (for the calculated WQE size). 297*9e39c5baSBill Taylor * 298*9e39c5baSBill Taylor * For QP's on an SRQ, we set these to 0. 299*9e39c5baSBill Taylor */ 300*9e39c5baSBill Taylor if (qp_srq_en) { 301*9e39c5baSBill Taylor qp->qp_rq_log_wqesz = 0; 302*9e39c5baSBill Taylor qp->qp_rq_sgl = 0; 303*9e39c5baSBill Taylor } else { 304*9e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl, 305*9e39c5baSBill Taylor max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ, 306*9e39c5baSBill Taylor &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl); 307*9e39c5baSBill Taylor } 308*9e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl, 309*9e39c5baSBill Taylor max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl); 310*9e39c5baSBill Taylor 311*9e39c5baSBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 312*9e39c5baSBill Taylor 313*9e39c5baSBill Taylor /* NOTE: currently policy in driver, later maybe IBTF interface */ 314*9e39c5baSBill Taylor qp->qp_no_prefetch = 0; 315*9e39c5baSBill Taylor 316*9e39c5baSBill Taylor /* 317*9e39c5baSBill Taylor * for prefetching, we need to add the number of wqes in 318*9e39c5baSBill Taylor * the 2k area plus one to the number requested, but 319*9e39c5baSBill Taylor * ONLY for send queue. If no_prefetch == 1 (prefetch off) 320*9e39c5baSBill Taylor * it's exactly TWO wqes for the headroom 321*9e39c5baSBill Taylor */ 322*9e39c5baSBill Taylor if (qp->qp_no_prefetch) 323*9e39c5baSBill Taylor qp->qp_sq_headroom = 2 * sq_wqe_size; 324*9e39c5baSBill Taylor else 325*9e39c5baSBill Taylor qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE; 326*9e39c5baSBill Taylor /* 327*9e39c5baSBill Taylor * hdrm wqes must be integral since both sq_wqe_size & 328*9e39c5baSBill Taylor * HERMON_QP_OH_SIZE are power of 2 329*9e39c5baSBill Taylor */ 330*9e39c5baSBill Taylor qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size); 331*9e39c5baSBill Taylor 332*9e39c5baSBill Taylor 333*9e39c5baSBill Taylor /* 334*9e39c5baSBill Taylor * Calculate the appropriate size for the work queues. 335*9e39c5baSBill Taylor * For send queue, add in the headroom wqes to the calculation. 336*9e39c5baSBill Taylor * Note: All Hermon QP work queues must be a power-of-2 in size. Also 337*9e39c5baSBill Taylor * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is 338*9e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2 339*9e39c5baSBill Taylor */ 340*9e39c5baSBill Taylor /* first, adjust to a minimum and tell the caller the change */ 341*9e39c5baSBill Taylor attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq, 342*9e39c5baSBill Taylor HERMON_QP_MIN_SIZE); 343*9e39c5baSBill Taylor attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq, 344*9e39c5baSBill Taylor HERMON_QP_MIN_SIZE); 345*9e39c5baSBill Taylor /* 346*9e39c5baSBill Taylor * now, calculate the alloc size, taking into account 347*9e39c5baSBill Taylor * the headroom for the sq 348*9e39c5baSBill Taylor */ 349*9e39c5baSBill Taylor log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes); 350*9e39c5baSBill Taylor /* if the total is a power of two, reduce it */ 351*9e39c5baSBill Taylor if (((attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes) & 352*9e39c5baSBill Taylor (attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes - 1)) == 0) { 353*9e39c5baSBill Taylor log_qp_sq_size = log_qp_sq_size - 1; 354*9e39c5baSBill Taylor } 355*9e39c5baSBill Taylor 356*9e39c5baSBill Taylor log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq); 357*9e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) { 358*9e39c5baSBill Taylor log_qp_rq_size = log_qp_rq_size - 1; 359*9e39c5baSBill Taylor } 360*9e39c5baSBill Taylor 361*9e39c5baSBill Taylor /* 362*9e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent 363*9e39c5baSBill Taylor * with the device limits and/or software-configured limits). If not, 364*9e39c5baSBill Taylor * then obviously we have a lot of cleanup to do before returning. 365*9e39c5baSBill Taylor * 366*9e39c5baSBill Taylor * NOTE: the first condition deals with the (test) case of cs_sq 367*9e39c5baSBill Taylor * being just less than 2^32. In this case, the headroom addition 368*9e39c5baSBill Taylor * to the requested cs_sq will pass the test when it should not. 369*9e39c5baSBill Taylor * This test no longer lets that case slip through the check. 370*9e39c5baSBill Taylor */ 371*9e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_sq > 372*9e39c5baSBill Taylor (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) || 373*9e39c5baSBill Taylor (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) || 374*9e39c5baSBill Taylor (!qp_srq_en && (log_qp_rq_size > 375*9e39c5baSBill Taylor state->hs_cfg_profile->cp_log_max_qp_sz))) { 376*9e39c5baSBill Taylor status = IBT_HCA_WR_EXCEEDED; 377*9e39c5baSBill Taylor goto qpalloc_fail7; 378*9e39c5baSBill Taylor } 379*9e39c5baSBill Taylor 380*9e39c5baSBill Taylor /* 381*9e39c5baSBill Taylor * Allocate the memory for QP work queues. Since Hermon work queues 382*9e39c5baSBill Taylor * are not allowed to cross a 32-bit (4GB) boundary, the alignment of 383*9e39c5baSBill Taylor * the work queue memory is very important. We used to allocate 384*9e39c5baSBill Taylor * work queues (the combined receive and send queues) so that they 385*9e39c5baSBill Taylor * would be aligned on their combined size. That alignment guaranteed 386*9e39c5baSBill Taylor * that they would never cross the 4GB boundary (Hermon work queues 387*9e39c5baSBill Taylor * are on the order of MBs at maximum). Now we are able to relax 388*9e39c5baSBill Taylor * this alignment constraint by ensuring that the IB address assigned 389*9e39c5baSBill Taylor * to the queue memory (as a result of the hermon_mr_register() call) 390*9e39c5baSBill Taylor * is offset from zero. 391*9e39c5baSBill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to 392*9e39c5baSBill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass 393*9e39c5baSBill Taylor * mode we found that we were not allowed to specify any alignment 394*9e39c5baSBill Taylor * that was more restrictive than the system page size. 395*9e39c5baSBill Taylor * So we avoided this constraint by passing two alignment values, 396*9e39c5baSBill Taylor * one for the memory allocation itself and the other for the DMA 397*9e39c5baSBill Taylor * handle (for later bind). This used to cause more memory than 398*9e39c5baSBill Taylor * necessary to be allocated (in order to guarantee the more 399*9e39c5baSBill Taylor * restrictive alignment contraint). But by guaranteeing the 400*9e39c5baSBill Taylor * zero-based IB virtual address for the queue, we are able to 401*9e39c5baSBill Taylor * conserve this memory. 402*9e39c5baSBill Taylor */ 403*9e39c5baSBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 404*9e39c5baSBill Taylor sq_depth = 1 << log_qp_sq_size; 405*9e39c5baSBill Taylor sq_size = sq_depth * sq_wqe_size; 406*9e39c5baSBill Taylor 407*9e39c5baSBill Taylor /* QP on SRQ sets these to 0 */ 408*9e39c5baSBill Taylor if (qp_srq_en) { 409*9e39c5baSBill Taylor rq_wqe_size = 0; 410*9e39c5baSBill Taylor rq_size = 0; 411*9e39c5baSBill Taylor } else { 412*9e39c5baSBill Taylor rq_wqe_size = 1 << qp->qp_rq_log_wqesz; 413*9e39c5baSBill Taylor rq_depth = 1 << log_qp_rq_size; 414*9e39c5baSBill Taylor rq_size = rq_depth * rq_wqe_size; 415*9e39c5baSBill Taylor } 416*9e39c5baSBill Taylor 417*9e39c5baSBill Taylor qp->qp_wqinfo.qa_size = sq_size + rq_size; 418*9e39c5baSBill Taylor 419*9e39c5baSBill Taylor qp->qp_wqinfo.qa_alloc_align = PAGESIZE; 420*9e39c5baSBill Taylor qp->qp_wqinfo.qa_bind_align = PAGESIZE; 421*9e39c5baSBill Taylor 422*9e39c5baSBill Taylor if (qp_is_umap) { 423*9e39c5baSBill Taylor qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND; 424*9e39c5baSBill Taylor } else { 425*9e39c5baSBill Taylor qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL; 426*9e39c5baSBill Taylor } 427*9e39c5baSBill Taylor status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag); 428*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 429*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 430*9e39c5baSBill Taylor goto qpalloc_fail7; 431*9e39c5baSBill Taylor } 432*9e39c5baSBill Taylor 433*9e39c5baSBill Taylor /* 434*9e39c5baSBill Taylor * Sort WQs in memory according to stride (*q_wqe_size), largest first 435*9e39c5baSBill Taylor * If they are equal, still put the SQ first 436*9e39c5baSBill Taylor */ 437*9e39c5baSBill Taylor qp->qp_sq_baseaddr = 0; 438*9e39c5baSBill Taylor qp->qp_rq_baseaddr = 0; 439*9e39c5baSBill Taylor if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) { 440*9e39c5baSBill Taylor sq_buf = qp->qp_wqinfo.qa_buf_aligned; 441*9e39c5baSBill Taylor 442*9e39c5baSBill Taylor /* if this QP is on an SRQ, set the rq_buf to NULL */ 443*9e39c5baSBill Taylor if (qp_srq_en) { 444*9e39c5baSBill Taylor rq_buf = NULL; 445*9e39c5baSBill Taylor } else { 446*9e39c5baSBill Taylor rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size); 447*9e39c5baSBill Taylor qp->qp_rq_baseaddr = sq_size; 448*9e39c5baSBill Taylor } 449*9e39c5baSBill Taylor } else { 450*9e39c5baSBill Taylor rq_buf = qp->qp_wqinfo.qa_buf_aligned; 451*9e39c5baSBill Taylor sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size); 452*9e39c5baSBill Taylor qp->qp_sq_baseaddr = rq_size; 453*9e39c5baSBill Taylor } 454*9e39c5baSBill Taylor 455*9e39c5baSBill Taylor if (qp_is_umap == 0) { 456*9e39c5baSBill Taylor qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth); 457*9e39c5baSBill Taylor if (qp->qp_sq_wqhdr == NULL) { 458*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 459*9e39c5baSBill Taylor goto qpalloc_fail8; 460*9e39c5baSBill Taylor } 461*9e39c5baSBill Taylor if (qp_srq_en) { 462*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr; 463*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_srq_en = 1; 464*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_srq = srq; 465*9e39c5baSBill Taylor } else { 466*9e39c5baSBill Taylor qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth); 467*9e39c5baSBill Taylor if (qp->qp_rq_wqhdr == NULL) { 468*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 469*9e39c5baSBill Taylor goto qpalloc_fail8; 470*9e39c5baSBill Taylor } 471*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr; 472*9e39c5baSBill Taylor } 473*9e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum; 474*9e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND; 475*9e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr; 476*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum; 477*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV; 478*9e39c5baSBill Taylor } 479*9e39c5baSBill Taylor 480*9e39c5baSBill Taylor /* 481*9e39c5baSBill Taylor * Register the memory for the QP work queues. The memory for the 482*9e39c5baSBill Taylor * QP must be registered in the Hermon cMPT tables. This gives us the 483*9e39c5baSBill Taylor * LKey to specify in the QP context later. Note: The memory for 484*9e39c5baSBill Taylor * Hermon work queues (both Send and Recv) must be contiguous and 485*9e39c5baSBill Taylor * registered as a single memory region. Note: If the QP memory is 486*9e39c5baSBill Taylor * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to 487*9e39c5baSBill Taylor * meet the alignment restriction, we pass the "mro_bind_override_addr" 488*9e39c5baSBill Taylor * flag in the call to hermon_mr_register(). This guarantees that the 489*9e39c5baSBill Taylor * resulting IB vaddr will be zero-based (modulo the offset into the 490*9e39c5baSBill Taylor * first page). If we fail here, we still have the bunch of resource 491*9e39c5baSBill Taylor * and reference count cleanup to do. 492*9e39c5baSBill Taylor */ 493*9e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : 494*9e39c5baSBill Taylor IBT_MR_NOSLEEP; 495*9e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned; 496*9e39c5baSBill Taylor mr_attr.mr_len = qp->qp_wqinfo.qa_size; 497*9e39c5baSBill Taylor mr_attr.mr_as = NULL; 498*9e39c5baSBill Taylor mr_attr.mr_flags = flag; 499*9e39c5baSBill Taylor if (qp_is_umap) { 500*9e39c5baSBill Taylor mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass; 501*9e39c5baSBill Taylor } else { 502*9e39c5baSBill Taylor /* HERMON_QUEUE_LOCATION_NORMAL */ 503*9e39c5baSBill Taylor mr_op.mro_bind_type = 504*9e39c5baSBill Taylor state->hs_cfg_profile->cp_iommu_bypass; 505*9e39c5baSBill Taylor } 506*9e39c5baSBill Taylor mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl; 507*9e39c5baSBill Taylor mr_op.mro_bind_override_addr = 1; 508*9e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, 509*9e39c5baSBill Taylor &mr_op, HERMON_QP_CMPT); 510*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 511*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 512*9e39c5baSBill Taylor goto qpalloc_fail9; 513*9e39c5baSBill Taylor } 514*9e39c5baSBill Taylor 515*9e39c5baSBill Taylor /* 516*9e39c5baSBill Taylor * Calculate the offset between the kernel virtual address space 517*9e39c5baSBill Taylor * and the IB virtual address space. This will be used when 518*9e39c5baSBill Taylor * posting work requests to properly initialize each WQE. 519*9e39c5baSBill Taylor */ 520*9e39c5baSBill Taylor qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned - 521*9e39c5baSBill Taylor (uint64_t)mr->mr_bindinfo.bi_addr; 522*9e39c5baSBill Taylor 523*9e39c5baSBill Taylor /* 524*9e39c5baSBill Taylor * Fill in all the return arguments (if necessary). This includes 525*9e39c5baSBill Taylor * real work queue sizes (in wqes), real SGLs, and QP number 526*9e39c5baSBill Taylor */ 527*9e39c5baSBill Taylor if (queuesz_p != NULL) { 528*9e39c5baSBill Taylor queuesz_p->cs_sq = 529*9e39c5baSBill Taylor (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes; 530*9e39c5baSBill Taylor queuesz_p->cs_sq_sgl = qp->qp_sq_sgl; 531*9e39c5baSBill Taylor 532*9e39c5baSBill Taylor /* if this QP is on an SRQ, set these to 0 */ 533*9e39c5baSBill Taylor if (qp_srq_en) { 534*9e39c5baSBill Taylor queuesz_p->cs_rq = 0; 535*9e39c5baSBill Taylor queuesz_p->cs_rq_sgl = 0; 536*9e39c5baSBill Taylor } else { 537*9e39c5baSBill Taylor queuesz_p->cs_rq = (1 << log_qp_rq_size); 538*9e39c5baSBill Taylor queuesz_p->cs_rq_sgl = qp->qp_rq_sgl; 539*9e39c5baSBill Taylor } 540*9e39c5baSBill Taylor } 541*9e39c5baSBill Taylor if (qpn != NULL) { 542*9e39c5baSBill Taylor *qpn = (ib_qpn_t)qp->qp_qpnum; 543*9e39c5baSBill Taylor } 544*9e39c5baSBill Taylor 545*9e39c5baSBill Taylor /* 546*9e39c5baSBill Taylor * Fill in the rest of the Hermon Queue Pair handle. 547*9e39c5baSBill Taylor */ 548*9e39c5baSBill Taylor qp->qp_qpcrsrcp = qpc; 549*9e39c5baSBill Taylor qp->qp_rsrcp = rsrc; 550*9e39c5baSBill Taylor qp->qp_state = HERMON_QP_RESET; 551*9e39c5baSBill Taylor qp->qp_pdhdl = pd; 552*9e39c5baSBill Taylor qp->qp_mrhdl = mr; 553*9e39c5baSBill Taylor qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ? 554*9e39c5baSBill Taylor HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED; 555*9e39c5baSBill Taylor qp->qp_is_special = 0; 556*9e39c5baSBill Taylor qp->qp_is_umap = qp_is_umap; 557*9e39c5baSBill Taylor qp->qp_uarpg = uarpg; 558*9e39c5baSBill Taylor qp->qp_umap_dhp = (devmap_cookie_t)NULL; 559*9e39c5baSBill Taylor qp->qp_sq_cqhdl = sq_cq; 560*9e39c5baSBill Taylor qp->qp_sq_bufsz = (1 << log_qp_sq_size); 561*9e39c5baSBill Taylor qp->qp_sq_logqsz = log_qp_sq_size; 562*9e39c5baSBill Taylor qp->qp_sq_buf = sq_buf; 563*9e39c5baSBill Taylor qp->qp_desc_off = qp_desc_off; 564*9e39c5baSBill Taylor qp->qp_rq_cqhdl = rq_cq; 565*9e39c5baSBill Taylor qp->qp_rq_buf = rq_buf; 566*9e39c5baSBill Taylor qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) != 567*9e39c5baSBill Taylor 0; 568*9e39c5baSBill Taylor 569*9e39c5baSBill Taylor /* if this QP is on an SRQ, set rq_bufsz to 0 */ 570*9e39c5baSBill Taylor if (qp_srq_en) { 571*9e39c5baSBill Taylor qp->qp_rq_bufsz = 0; 572*9e39c5baSBill Taylor qp->qp_rq_logqsz = 0; 573*9e39c5baSBill Taylor } else { 574*9e39c5baSBill Taylor qp->qp_rq_bufsz = (1 << log_qp_rq_size); 575*9e39c5baSBill Taylor qp->qp_rq_logqsz = log_qp_rq_size; 576*9e39c5baSBill Taylor } 577*9e39c5baSBill Taylor 578*9e39c5baSBill Taylor qp->qp_forward_sqd_event = 0; 579*9e39c5baSBill Taylor qp->qp_sqd_still_draining = 0; 580*9e39c5baSBill Taylor qp->qp_hdlrarg = (void *)ibt_qphdl; 581*9e39c5baSBill Taylor qp->qp_mcg_refcnt = 0; 582*9e39c5baSBill Taylor 583*9e39c5baSBill Taylor /* 584*9e39c5baSBill Taylor * If this QP is to be associated with an SRQ, set the SRQ handle 585*9e39c5baSBill Taylor */ 586*9e39c5baSBill Taylor if (qp_srq_en) { 587*9e39c5baSBill Taylor qp->qp_srqhdl = srq; 588*9e39c5baSBill Taylor qp->qp_srq_en = HERMON_QP_SRQ_ENABLED; 589*9e39c5baSBill Taylor hermon_srq_refcnt_inc(qp->qp_srqhdl); 590*9e39c5baSBill Taylor } else { 591*9e39c5baSBill Taylor qp->qp_srqhdl = NULL; 592*9e39c5baSBill Taylor qp->qp_srq_en = HERMON_QP_SRQ_DISABLED; 593*9e39c5baSBill Taylor } 594*9e39c5baSBill Taylor 595*9e39c5baSBill Taylor /* Determine the QP service type */ 596*9e39c5baSBill Taylor if (type == IBT_RC_RQP) { 597*9e39c5baSBill Taylor qp->qp_serv_type = HERMON_QP_RC; 598*9e39c5baSBill Taylor } else if (type == IBT_UD_RQP) { 599*9e39c5baSBill Taylor qp->qp_serv_type = HERMON_QP_UD; 600*9e39c5baSBill Taylor } else { 601*9e39c5baSBill Taylor qp->qp_serv_type = HERMON_QP_UC; 602*9e39c5baSBill Taylor } 603*9e39c5baSBill Taylor 604*9e39c5baSBill Taylor /* 605*9e39c5baSBill Taylor * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed 606*9e39c5baSBill Taylor */ 607*9e39c5baSBill Taylor 608*9e39c5baSBill Taylor /* 609*9e39c5baSBill Taylor * Initialize the SQ WQEs - all that needs to be done is every 64 bytes 610*9e39c5baSBill Taylor * set the quadword to all F's - high-order bit is owner (init to one) 611*9e39c5baSBill Taylor * and the rest for the headroom definition of prefetching 612*9e39c5baSBill Taylor * 613*9e39c5baSBill Taylor */ 614*9e39c5baSBill Taylor wqesz_shift = qp->qp_sq_log_wqesz; 615*9e39c5baSBill Taylor thewqesz = 1 << wqesz_shift; 616*9e39c5baSBill Taylor thewqe = (uint64_t *)(void *)(qp->qp_sq_buf); 617*9e39c5baSBill Taylor if (qp_is_umap == 0) { 618*9e39c5baSBill Taylor for (i = 0; i < sq_depth; i++) { 619*9e39c5baSBill Taylor /* 620*9e39c5baSBill Taylor * for each stride, go through and every 64 bytes 621*9e39c5baSBill Taylor * write the init value - having set the address 622*9e39c5baSBill Taylor * once, just keep incrementing it 623*9e39c5baSBill Taylor */ 624*9e39c5baSBill Taylor for (j = 0; j < thewqesz; j += 64, thewqe += 8) { 625*9e39c5baSBill Taylor *(uint32_t *)thewqe = 0xFFFFFFFF; 626*9e39c5baSBill Taylor } 627*9e39c5baSBill Taylor } 628*9e39c5baSBill Taylor } 629*9e39c5baSBill Taylor 630*9e39c5baSBill Taylor /* Zero out the QP context */ 631*9e39c5baSBill Taylor bzero(&qp->qpc, sizeof (hermon_hw_qpc_t)); 632*9e39c5baSBill Taylor 633*9e39c5baSBill Taylor /* 634*9e39c5baSBill Taylor * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the 635*9e39c5baSBill Taylor * "qphdl" and return success 636*9e39c5baSBill Taylor */ 637*9e39c5baSBill Taylor ASSERT(state->hs_qphdl[qpc->hr_indx] == NULL); 638*9e39c5baSBill Taylor state->hs_qphdl[qpc->hr_indx] = qp; 639*9e39c5baSBill Taylor 640*9e39c5baSBill Taylor /* 641*9e39c5baSBill Taylor * If this is a user-mappable QP, then we need to insert the previously 642*9e39c5baSBill Taylor * allocated entry into the "userland resources database". This will 643*9e39c5baSBill Taylor * allow for later lookup during devmap() (i.e. mmap()) calls. 644*9e39c5baSBill Taylor */ 645*9e39c5baSBill Taylor if (qp_is_umap) { 646*9e39c5baSBill Taylor hermon_umap_db_add(umapdb); 647*9e39c5baSBill Taylor } 648*9e39c5baSBill Taylor mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER, 649*9e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 650*9e39c5baSBill Taylor mutex_init(&qp->qp_rq_lock, NULL, MUTEX_DRIVER, 651*9e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 652*9e39c5baSBill Taylor 653*9e39c5baSBill Taylor *qphdl = qp; 654*9e39c5baSBill Taylor 655*9e39c5baSBill Taylor return (DDI_SUCCESS); 656*9e39c5baSBill Taylor 657*9e39c5baSBill Taylor /* 658*9e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 659*9e39c5baSBill Taylor */ 660*9e39c5baSBill Taylor qpalloc_fail9: 661*9e39c5baSBill Taylor hermon_queue_free(&qp->qp_wqinfo); 662*9e39c5baSBill Taylor qpalloc_fail8: 663*9e39c5baSBill Taylor if (qp->qp_sq_wqhdr) 664*9e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr); 665*9e39c5baSBill Taylor if (qp->qp_rq_wqhdr) 666*9e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr); 667*9e39c5baSBill Taylor qpalloc_fail7: 668*9e39c5baSBill Taylor if (qp_is_umap) { 669*9e39c5baSBill Taylor hermon_umap_db_free(umapdb); 670*9e39c5baSBill Taylor } 671*9e39c5baSBill Taylor if (!qp_srq_en) { 672*9e39c5baSBill Taylor hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr); 673*9e39c5baSBill Taylor } 674*9e39c5baSBill Taylor 675*9e39c5baSBill Taylor qpalloc_fail6: 676*9e39c5baSBill Taylor /* 677*9e39c5baSBill Taylor * Releasing the QPN will also free up the QPC context. Update 678*9e39c5baSBill Taylor * the QPC context pointer to indicate this. 679*9e39c5baSBill Taylor */ 680*9e39c5baSBill Taylor hermon_qp_release_qpn(state, qp->qp_qpn_hdl, HERMON_QPN_RELEASE); 681*9e39c5baSBill Taylor qpc = NULL; 682*9e39c5baSBill Taylor qpalloc_fail5: 683*9e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 684*9e39c5baSBill Taylor qpalloc_fail4: 685*9e39c5baSBill Taylor if (qpc) { 686*9e39c5baSBill Taylor hermon_rsrc_free(state, &qpc); 687*9e39c5baSBill Taylor } 688*9e39c5baSBill Taylor qpalloc_fail3: 689*9e39c5baSBill Taylor hermon_cq_refcnt_dec(rq_cq); 690*9e39c5baSBill Taylor qpalloc_fail2: 691*9e39c5baSBill Taylor hermon_cq_refcnt_dec(sq_cq); 692*9e39c5baSBill Taylor qpalloc_fail1: 693*9e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 694*9e39c5baSBill Taylor qpalloc_fail: 695*9e39c5baSBill Taylor return (status); 696*9e39c5baSBill Taylor } 697*9e39c5baSBill Taylor 698*9e39c5baSBill Taylor 699*9e39c5baSBill Taylor 700*9e39c5baSBill Taylor /* 701*9e39c5baSBill Taylor * hermon_special_qp_alloc() 702*9e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 703*9e39c5baSBill Taylor */ 704*9e39c5baSBill Taylor int 705*9e39c5baSBill Taylor hermon_special_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo, 706*9e39c5baSBill Taylor uint_t sleepflag) 707*9e39c5baSBill Taylor { 708*9e39c5baSBill Taylor hermon_rsrc_t *qpc, *rsrc; 709*9e39c5baSBill Taylor hermon_qphdl_t qp; 710*9e39c5baSBill Taylor ibt_qp_alloc_attr_t *attr_p; 711*9e39c5baSBill Taylor ibt_sqp_type_t type; 712*9e39c5baSBill Taylor uint8_t port; 713*9e39c5baSBill Taylor ibtl_qp_hdl_t ibt_qphdl; 714*9e39c5baSBill Taylor ibt_chan_sizes_t *queuesz_p; 715*9e39c5baSBill Taylor hermon_qphdl_t *qphdl; 716*9e39c5baSBill Taylor ibt_mr_attr_t mr_attr; 717*9e39c5baSBill Taylor hermon_mr_options_t mr_op; 718*9e39c5baSBill Taylor hermon_pdhdl_t pd; 719*9e39c5baSBill Taylor hermon_cqhdl_t sq_cq, rq_cq; 720*9e39c5baSBill Taylor hermon_mrhdl_t mr; 721*9e39c5baSBill Taylor uint64_t qp_desc_off; 722*9e39c5baSBill Taylor uint64_t *thewqe, thewqesz; 723*9e39c5baSBill Taylor uint32_t *sq_buf, *rq_buf; 724*9e39c5baSBill Taylor uint32_t log_qp_sq_size, log_qp_rq_size; 725*9e39c5baSBill Taylor uint32_t sq_size, rq_size, max_sgl; 726*9e39c5baSBill Taylor uint32_t uarpg; 727*9e39c5baSBill Taylor uint32_t sq_depth; 728*9e39c5baSBill Taylor uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift; 729*9e39c5baSBill Taylor int status, flag, i, j; 730*9e39c5baSBill Taylor 731*9e39c5baSBill Taylor /* 732*9e39c5baSBill Taylor * Extract the necessary info from the hermon_qp_info_t structure 733*9e39c5baSBill Taylor */ 734*9e39c5baSBill Taylor attr_p = qpinfo->qpi_attrp; 735*9e39c5baSBill Taylor type = qpinfo->qpi_type; 736*9e39c5baSBill Taylor port = qpinfo->qpi_port; 737*9e39c5baSBill Taylor ibt_qphdl = qpinfo->qpi_ibt_qphdl; 738*9e39c5baSBill Taylor queuesz_p = qpinfo->qpi_queueszp; 739*9e39c5baSBill Taylor qphdl = &qpinfo->qpi_qphdl; 740*9e39c5baSBill Taylor 741*9e39c5baSBill Taylor /* 742*9e39c5baSBill Taylor * Check for valid special QP type (only SMI & GSI supported) 743*9e39c5baSBill Taylor */ 744*9e39c5baSBill Taylor if ((type != IBT_SMI_SQP) && (type != IBT_GSI_SQP)) { 745*9e39c5baSBill Taylor status = IBT_QP_SPECIAL_TYPE_INVALID; 746*9e39c5baSBill Taylor goto spec_qpalloc_fail; 747*9e39c5baSBill Taylor } 748*9e39c5baSBill Taylor 749*9e39c5baSBill Taylor /* 750*9e39c5baSBill Taylor * Check for valid port number 751*9e39c5baSBill Taylor */ 752*9e39c5baSBill Taylor if (!hermon_portnum_is_valid(state, port)) { 753*9e39c5baSBill Taylor status = IBT_HCA_PORT_INVALID; 754*9e39c5baSBill Taylor goto spec_qpalloc_fail; 755*9e39c5baSBill Taylor } 756*9e39c5baSBill Taylor port = port - 1; 757*9e39c5baSBill Taylor 758*9e39c5baSBill Taylor /* 759*9e39c5baSBill Taylor * Check for valid PD handle pointer 760*9e39c5baSBill Taylor */ 761*9e39c5baSBill Taylor if (attr_p->qp_pd_hdl == NULL) { 762*9e39c5baSBill Taylor status = IBT_PD_HDL_INVALID; 763*9e39c5baSBill Taylor goto spec_qpalloc_fail; 764*9e39c5baSBill Taylor } 765*9e39c5baSBill Taylor pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl; 766*9e39c5baSBill Taylor 767*9e39c5baSBill Taylor /* Increment the reference count on the PD */ 768*9e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 769*9e39c5baSBill Taylor 770*9e39c5baSBill Taylor /* 771*9e39c5baSBill Taylor * Check for valid CQ handle pointers 772*9e39c5baSBill Taylor */ 773*9e39c5baSBill Taylor if ((attr_p->qp_ibc_scq_hdl == NULL) || 774*9e39c5baSBill Taylor (attr_p->qp_ibc_rcq_hdl == NULL)) { 775*9e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 776*9e39c5baSBill Taylor goto spec_qpalloc_fail1; 777*9e39c5baSBill Taylor } 778*9e39c5baSBill Taylor sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl; 779*9e39c5baSBill Taylor rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl; 780*9e39c5baSBill Taylor 781*9e39c5baSBill Taylor /* 782*9e39c5baSBill Taylor * Increment the reference count on the CQs. One or both of these 783*9e39c5baSBill Taylor * could return error if we determine that the given CQ is already 784*9e39c5baSBill Taylor * being used with a non-special QP (i.e. a normal QP). 785*9e39c5baSBill Taylor */ 786*9e39c5baSBill Taylor status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_SPECIAL); 787*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 788*9e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 789*9e39c5baSBill Taylor goto spec_qpalloc_fail1; 790*9e39c5baSBill Taylor } 791*9e39c5baSBill Taylor status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_SPECIAL); 792*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 793*9e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 794*9e39c5baSBill Taylor goto spec_qpalloc_fail2; 795*9e39c5baSBill Taylor } 796*9e39c5baSBill Taylor 797*9e39c5baSBill Taylor /* 798*9e39c5baSBill Taylor * Allocate the special QP resources. Essentially, this allocation 799*9e39c5baSBill Taylor * amounts to checking if the request special QP has already been 800*9e39c5baSBill Taylor * allocated. If successful, the QP context return is an actual 801*9e39c5baSBill Taylor * QP context that has been "aliased" to act as a special QP of the 802*9e39c5baSBill Taylor * appropriate type (and for the appropriate port). Just as in 803*9e39c5baSBill Taylor * hermon_qp_alloc() above, ownership for this QP context is not 804*9e39c5baSBill Taylor * immediately given to hardware in the final step here. Instead, we 805*9e39c5baSBill Taylor * wait until the QP is later transitioned to the "Init" state before 806*9e39c5baSBill Taylor * passing the QP to hardware. If we fail here, we must undo all 807*9e39c5baSBill Taylor * the reference count (CQ and PD). 808*9e39c5baSBill Taylor */ 809*9e39c5baSBill Taylor status = hermon_special_qp_rsrc_alloc(state, type, port, &qpc); 810*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 811*9e39c5baSBill Taylor goto spec_qpalloc_fail3; 812*9e39c5baSBill Taylor } 813*9e39c5baSBill Taylor 814*9e39c5baSBill Taylor /* 815*9e39c5baSBill Taylor * Allocate the software structure for tracking the special queue 816*9e39c5baSBill Taylor * pair (i.e. the Hermon Queue Pair handle). If we fail here, we 817*9e39c5baSBill Taylor * must undo the reference counts and the previous resource allocation. 818*9e39c5baSBill Taylor */ 819*9e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc); 820*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 821*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 822*9e39c5baSBill Taylor goto spec_qpalloc_fail4; 823*9e39c5baSBill Taylor } 824*9e39c5baSBill Taylor qp = (hermon_qphdl_t)rsrc->hr_addr; 825*9e39c5baSBill Taylor 826*9e39c5baSBill Taylor bzero(qp, sizeof (struct hermon_sw_qp_s)); 827*9e39c5baSBill Taylor 828*9e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp)) 829*9e39c5baSBill Taylor 830*9e39c5baSBill Taylor /* 831*9e39c5baSBill Taylor * Actual QP number is a combination of the index of the QPC and 832*9e39c5baSBill Taylor * the port number. This is because the special QP contexts must 833*9e39c5baSBill Taylor * be allocated two-at-a-time. 834*9e39c5baSBill Taylor */ 835*9e39c5baSBill Taylor qp->qp_qpnum = qpc->hr_indx + port; 836*9e39c5baSBill Taylor qp->qp_ring = qp->qp_qpnum << 8; 837*9e39c5baSBill Taylor 838*9e39c5baSBill Taylor uarpg = state->hs_kernel_uar_index; /* must be for spec qp */ 839*9e39c5baSBill Taylor /* 840*9e39c5baSBill Taylor * Allocate the doorbell record. Hermon uses only one for the RQ so 841*9e39c5baSBill Taylor * alloc a qp doorbell, using uarpg (above) as the uar index 842*9e39c5baSBill Taylor */ 843*9e39c5baSBill Taylor 844*9e39c5baSBill Taylor status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl, 845*9e39c5baSBill Taylor &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset); 846*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 847*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 848*9e39c5baSBill Taylor goto spec_qpalloc_fail5; 849*9e39c5baSBill Taylor } 850*9e39c5baSBill Taylor /* 851*9e39c5baSBill Taylor * Calculate the appropriate size for the work queues. 852*9e39c5baSBill Taylor * Note: All Hermon QP work queues must be a power-of-2 in size. Also 853*9e39c5baSBill Taylor * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is 854*9e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2 855*9e39c5baSBill Taylor */ 856*9e39c5baSBill Taylor attr_p->qp_sizes.cs_sq = 857*9e39c5baSBill Taylor max(attr_p->qp_sizes.cs_sq, HERMON_QP_MIN_SIZE); 858*9e39c5baSBill Taylor attr_p->qp_sizes.cs_rq = 859*9e39c5baSBill Taylor max(attr_p->qp_sizes.cs_rq, HERMON_QP_MIN_SIZE); 860*9e39c5baSBill Taylor log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq); 861*9e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_sq & (attr_p->qp_sizes.cs_sq - 1)) == 0) { 862*9e39c5baSBill Taylor log_qp_sq_size = log_qp_sq_size - 1; 863*9e39c5baSBill Taylor } 864*9e39c5baSBill Taylor log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq); 865*9e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_rq & (attr_p->qp_sizes.cs_rq - 1)) == 0) { 866*9e39c5baSBill Taylor log_qp_rq_size = log_qp_rq_size - 1; 867*9e39c5baSBill Taylor } 868*9e39c5baSBill Taylor 869*9e39c5baSBill Taylor /* 870*9e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent 871*9e39c5baSBill Taylor * with the device limits and/or software-configured limits). If not, 872*9e39c5baSBill Taylor * then obviously we have a bit of cleanup to do before returning. 873*9e39c5baSBill Taylor */ 874*9e39c5baSBill Taylor if ((log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) || 875*9e39c5baSBill Taylor (log_qp_rq_size > state->hs_cfg_profile->cp_log_max_qp_sz)) { 876*9e39c5baSBill Taylor status = IBT_HCA_WR_EXCEEDED; 877*9e39c5baSBill Taylor goto spec_qpalloc_fail5a; 878*9e39c5baSBill Taylor } 879*9e39c5baSBill Taylor 880*9e39c5baSBill Taylor /* 881*9e39c5baSBill Taylor * Next we verify that the requested number of SGL is valid (i.e. 882*9e39c5baSBill Taylor * consistent with the device limits and/or software-configured 883*9e39c5baSBill Taylor * limits). If not, then obviously the same cleanup needs to be done. 884*9e39c5baSBill Taylor */ 885*9e39c5baSBill Taylor max_sgl = state->hs_cfg_profile->cp_wqe_real_max_sgl; 886*9e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) || 887*9e39c5baSBill Taylor (attr_p->qp_sizes.cs_rq_sgl > max_sgl)) { 888*9e39c5baSBill Taylor status = IBT_HCA_SGL_EXCEEDED; 889*9e39c5baSBill Taylor goto spec_qpalloc_fail5a; 890*9e39c5baSBill Taylor } 891*9e39c5baSBill Taylor 892*9e39c5baSBill Taylor /* 893*9e39c5baSBill Taylor * Determine this QP's WQE stride (for both the Send and Recv WQEs). 894*9e39c5baSBill Taylor * This will depend on the requested number of SGLs. Note: this 895*9e39c5baSBill Taylor * has the side-effect of also calculating the real number of SGLs 896*9e39c5baSBill Taylor * (for the calculated WQE size). 897*9e39c5baSBill Taylor */ 898*9e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl, 899*9e39c5baSBill Taylor max_sgl, HERMON_QP_WQ_TYPE_RECVQ, 900*9e39c5baSBill Taylor &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl); 901*9e39c5baSBill Taylor if (type == IBT_SMI_SQP) { 902*9e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl, 903*9e39c5baSBill Taylor max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP0, 904*9e39c5baSBill Taylor &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl); 905*9e39c5baSBill Taylor } else { 906*9e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl, 907*9e39c5baSBill Taylor max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP1, 908*9e39c5baSBill Taylor &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl); 909*9e39c5baSBill Taylor } 910*9e39c5baSBill Taylor 911*9e39c5baSBill Taylor /* 912*9e39c5baSBill Taylor * Allocate the memory for QP work queues. Since Hermon work queues 913*9e39c5baSBill Taylor * are not allowed to cross a 32-bit (4GB) boundary, the alignment of 914*9e39c5baSBill Taylor * the work queue memory is very important. We used to allocate 915*9e39c5baSBill Taylor * work queues (the combined receive and send queues) so that they 916*9e39c5baSBill Taylor * would be aligned on their combined size. That alignment guaranteed 917*9e39c5baSBill Taylor * that they would never cross the 4GB boundary (Hermon work queues 918*9e39c5baSBill Taylor * are on the order of MBs at maximum). Now we are able to relax 919*9e39c5baSBill Taylor * this alignment constraint by ensuring that the IB address assigned 920*9e39c5baSBill Taylor * to the queue memory (as a result of the hermon_mr_register() call) 921*9e39c5baSBill Taylor * is offset from zero. 922*9e39c5baSBill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to 923*9e39c5baSBill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass 924*9e39c5baSBill Taylor * mode we found that we were not allowed to specify any alignment 925*9e39c5baSBill Taylor * that was more restrictive than the system page size. 926*9e39c5baSBill Taylor * So we avoided this constraint by passing two alignment values, 927*9e39c5baSBill Taylor * one for the memory allocation itself and the other for the DMA 928*9e39c5baSBill Taylor * handle (for later bind). This used to cause more memory than 929*9e39c5baSBill Taylor * necessary to be allocated (in order to guarantee the more 930*9e39c5baSBill Taylor * restrictive alignment contraint). But by guaranteeing the 931*9e39c5baSBill Taylor * zero-based IB virtual address for the queue, we are able to 932*9e39c5baSBill Taylor * conserve this memory. 933*9e39c5baSBill Taylor */ 934*9e39c5baSBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 935*9e39c5baSBill Taylor sq_depth = 1 << log_qp_sq_size; 936*9e39c5baSBill Taylor sq_size = (1 << log_qp_sq_size) * sq_wqe_size; 937*9e39c5baSBill Taylor 938*9e39c5baSBill Taylor rq_wqe_size = 1 << qp->qp_rq_log_wqesz; 939*9e39c5baSBill Taylor rq_size = (1 << log_qp_rq_size) * rq_wqe_size; 940*9e39c5baSBill Taylor 941*9e39c5baSBill Taylor qp->qp_wqinfo.qa_size = sq_size + rq_size; 942*9e39c5baSBill Taylor 943*9e39c5baSBill Taylor qp->qp_wqinfo.qa_alloc_align = PAGESIZE; 944*9e39c5baSBill Taylor qp->qp_wqinfo.qa_bind_align = PAGESIZE; 945*9e39c5baSBill Taylor qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL; 946*9e39c5baSBill Taylor 947*9e39c5baSBill Taylor status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag); 948*9e39c5baSBill Taylor if (status != NULL) { 949*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 950*9e39c5baSBill Taylor goto spec_qpalloc_fail5a; 951*9e39c5baSBill Taylor } 952*9e39c5baSBill Taylor 953*9e39c5baSBill Taylor /* 954*9e39c5baSBill Taylor * Sort WQs in memory according to depth, stride (*q_wqe_size), 955*9e39c5baSBill Taylor * biggest first. If equal, the Send Queue still goes first 956*9e39c5baSBill Taylor */ 957*9e39c5baSBill Taylor qp->qp_sq_baseaddr = 0; 958*9e39c5baSBill Taylor qp->qp_rq_baseaddr = 0; 959*9e39c5baSBill Taylor if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) { 960*9e39c5baSBill Taylor sq_buf = qp->qp_wqinfo.qa_buf_aligned; 961*9e39c5baSBill Taylor rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size); 962*9e39c5baSBill Taylor qp->qp_rq_baseaddr = sq_size; 963*9e39c5baSBill Taylor } else { 964*9e39c5baSBill Taylor rq_buf = qp->qp_wqinfo.qa_buf_aligned; 965*9e39c5baSBill Taylor sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size); 966*9e39c5baSBill Taylor qp->qp_sq_baseaddr = rq_size; 967*9e39c5baSBill Taylor } 968*9e39c5baSBill Taylor 969*9e39c5baSBill Taylor qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth); 970*9e39c5baSBill Taylor if (qp->qp_sq_wqhdr == NULL) { 971*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 972*9e39c5baSBill Taylor goto spec_qpalloc_fail6; 973*9e39c5baSBill Taylor } 974*9e39c5baSBill Taylor qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(1 << log_qp_rq_size); 975*9e39c5baSBill Taylor if (qp->qp_rq_wqhdr == NULL) { 976*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 977*9e39c5baSBill Taylor goto spec_qpalloc_fail6; 978*9e39c5baSBill Taylor } 979*9e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum; 980*9e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND; 981*9e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr; 982*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum; 983*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV; 984*9e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr; 985*9e39c5baSBill Taylor 986*9e39c5baSBill Taylor /* 987*9e39c5baSBill Taylor * Register the memory for the special QP work queues. The memory for 988*9e39c5baSBill Taylor * the special QP must be registered in the Hermon cMPT tables. This 989*9e39c5baSBill Taylor * gives us the LKey to specify in the QP context later. Note: The 990*9e39c5baSBill Taylor * memory for Hermon work queues (both Send and Recv) must be contiguous 991*9e39c5baSBill Taylor * and registered as a single memory region. Also, in order to meet the 992*9e39c5baSBill Taylor * alignment restriction, we pass the "mro_bind_override_addr" flag in 993*9e39c5baSBill Taylor * the call to hermon_mr_register(). This guarantees that the resulting 994*9e39c5baSBill Taylor * IB vaddr will be zero-based (modulo the offset into the first page). 995*9e39c5baSBill Taylor * If we fail here, we have a bunch of resource and reference count 996*9e39c5baSBill Taylor * cleanup to do. 997*9e39c5baSBill Taylor */ 998*9e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : 999*9e39c5baSBill Taylor IBT_MR_NOSLEEP; 1000*9e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned; 1001*9e39c5baSBill Taylor mr_attr.mr_len = qp->qp_wqinfo.qa_size; 1002*9e39c5baSBill Taylor mr_attr.mr_as = NULL; 1003*9e39c5baSBill Taylor mr_attr.mr_flags = flag; 1004*9e39c5baSBill Taylor 1005*9e39c5baSBill Taylor mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass; 1006*9e39c5baSBill Taylor mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl; 1007*9e39c5baSBill Taylor mr_op.mro_bind_override_addr = 1; 1008*9e39c5baSBill Taylor 1009*9e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &mr_op, 1010*9e39c5baSBill Taylor HERMON_QP_CMPT); 1011*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1012*9e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 1013*9e39c5baSBill Taylor goto spec_qpalloc_fail6; 1014*9e39c5baSBill Taylor } 1015*9e39c5baSBill Taylor 1016*9e39c5baSBill Taylor /* 1017*9e39c5baSBill Taylor * Calculate the offset between the kernel virtual address space 1018*9e39c5baSBill Taylor * and the IB virtual address space. This will be used when 1019*9e39c5baSBill Taylor * posting work requests to properly initialize each WQE. 1020*9e39c5baSBill Taylor */ 1021*9e39c5baSBill Taylor qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned - 1022*9e39c5baSBill Taylor (uint64_t)mr->mr_bindinfo.bi_addr; 1023*9e39c5baSBill Taylor 1024*9e39c5baSBill Taylor /* set the prefetch - initially, not prefetching */ 1025*9e39c5baSBill Taylor qp->qp_no_prefetch = 1; 1026*9e39c5baSBill Taylor 1027*9e39c5baSBill Taylor if (qp->qp_no_prefetch) 1028*9e39c5baSBill Taylor qp->qp_sq_headroom = 2 * sq_wqe_size; 1029*9e39c5baSBill Taylor else 1030*9e39c5baSBill Taylor qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE; 1031*9e39c5baSBill Taylor /* 1032*9e39c5baSBill Taylor * hdrm wqes must be integral since both sq_wqe_size & 1033*9e39c5baSBill Taylor * HERMON_QP_OH_SIZE are power of 2 1034*9e39c5baSBill Taylor */ 1035*9e39c5baSBill Taylor qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size); 1036*9e39c5baSBill Taylor /* 1037*9e39c5baSBill Taylor * Fill in all the return arguments (if necessary). This includes 1038*9e39c5baSBill Taylor * real work queue sizes, real SGLs, and QP number (which will be 1039*9e39c5baSBill Taylor * either zero or one, depending on the special QP type) 1040*9e39c5baSBill Taylor */ 1041*9e39c5baSBill Taylor if (queuesz_p != NULL) { 1042*9e39c5baSBill Taylor queuesz_p->cs_sq = 1043*9e39c5baSBill Taylor (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes; 1044*9e39c5baSBill Taylor queuesz_p->cs_sq_sgl = qp->qp_sq_sgl; 1045*9e39c5baSBill Taylor queuesz_p->cs_rq = (1 << log_qp_rq_size); 1046*9e39c5baSBill Taylor queuesz_p->cs_rq_sgl = qp->qp_rq_sgl; 1047*9e39c5baSBill Taylor } 1048*9e39c5baSBill Taylor 1049*9e39c5baSBill Taylor /* 1050*9e39c5baSBill Taylor * Fill in the rest of the Hermon Queue Pair handle. We can update 1051*9e39c5baSBill Taylor * the following fields for use in further operations on the QP. 1052*9e39c5baSBill Taylor */ 1053*9e39c5baSBill Taylor qp->qp_qpcrsrcp = qpc; 1054*9e39c5baSBill Taylor qp->qp_rsrcp = rsrc; 1055*9e39c5baSBill Taylor qp->qp_state = HERMON_QP_RESET; 1056*9e39c5baSBill Taylor qp->qp_pdhdl = pd; 1057*9e39c5baSBill Taylor qp->qp_mrhdl = mr; 1058*9e39c5baSBill Taylor qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ? 1059*9e39c5baSBill Taylor HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED; 1060*9e39c5baSBill Taylor qp->qp_is_special = (type == IBT_SMI_SQP) ? 1061*9e39c5baSBill Taylor HERMON_QP_SMI : HERMON_QP_GSI; 1062*9e39c5baSBill Taylor qp->qp_is_umap = 0; 1063*9e39c5baSBill Taylor qp->qp_uarpg = uarpg; 1064*9e39c5baSBill Taylor qp->qp_umap_dhp = (devmap_cookie_t)NULL; 1065*9e39c5baSBill Taylor qp->qp_sq_cqhdl = sq_cq; 1066*9e39c5baSBill Taylor qp->qp_sq_bufsz = (1 << log_qp_sq_size); 1067*9e39c5baSBill Taylor qp->qp_sq_buf = sq_buf; 1068*9e39c5baSBill Taylor qp->qp_sq_logqsz = log_qp_sq_size; 1069*9e39c5baSBill Taylor qp->qp_desc_off = qp_desc_off; 1070*9e39c5baSBill Taylor qp->qp_rq_cqhdl = rq_cq; 1071*9e39c5baSBill Taylor qp->qp_rq_bufsz = (1 << log_qp_rq_size); 1072*9e39c5baSBill Taylor qp->qp_rq_buf = rq_buf; 1073*9e39c5baSBill Taylor qp->qp_rq_logqsz = log_qp_rq_size; 1074*9e39c5baSBill Taylor qp->qp_portnum = port; 1075*9e39c5baSBill Taylor qp->qp_pkeyindx = 0; 1076*9e39c5baSBill Taylor qp->qp_forward_sqd_event = 0; 1077*9e39c5baSBill Taylor qp->qp_sqd_still_draining = 0; 1078*9e39c5baSBill Taylor qp->qp_hdlrarg = (void *)ibt_qphdl; 1079*9e39c5baSBill Taylor qp->qp_mcg_refcnt = 0; 1080*9e39c5baSBill Taylor qp->qp_srq_en = 0; 1081*9e39c5baSBill Taylor qp->qp_srqhdl = NULL; 1082*9e39c5baSBill Taylor 1083*9e39c5baSBill Taylor /* All special QPs are UD QP service type */ 1084*9e39c5baSBill Taylor qp->qp_serv_type = HERMON_QP_UD; 1085*9e39c5baSBill Taylor 1086*9e39c5baSBill Taylor /* 1087*9e39c5baSBill Taylor * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed 1088*9e39c5baSBill Taylor */ 1089*9e39c5baSBill Taylor 1090*9e39c5baSBill Taylor /* 1091*9e39c5baSBill Taylor * Initialize the SQ WQEs - all that needs to be done is every 64 bytes 1092*9e39c5baSBill Taylor * set the quadword to all F's - high-order bit is owner (init to one) 1093*9e39c5baSBill Taylor * and the rest for the headroom definition of prefetching 1094*9e39c5baSBill Taylor * 1095*9e39c5baSBill Taylor */ 1096*9e39c5baSBill Taylor 1097*9e39c5baSBill Taylor wqesz_shift = qp->qp_sq_log_wqesz; 1098*9e39c5baSBill Taylor thewqesz = 1 << wqesz_shift; 1099*9e39c5baSBill Taylor thewqe = (uint64_t *)(void *)(qp->qp_sq_buf); 1100*9e39c5baSBill Taylor for (i = 0; i < sq_depth; i++) { 1101*9e39c5baSBill Taylor /* 1102*9e39c5baSBill Taylor * for each stride, go through and every 64 bytes write the 1103*9e39c5baSBill Taylor * init value - having set the address once, just keep 1104*9e39c5baSBill Taylor * incrementing it 1105*9e39c5baSBill Taylor */ 1106*9e39c5baSBill Taylor for (j = 0; j < thewqesz; j += 64, thewqe += 8) { 1107*9e39c5baSBill Taylor *(uint32_t *)thewqe = 0xFFFFFFFF; 1108*9e39c5baSBill Taylor } 1109*9e39c5baSBill Taylor } 1110*9e39c5baSBill Taylor 1111*9e39c5baSBill Taylor 1112*9e39c5baSBill Taylor /* Zero out the QP context */ 1113*9e39c5baSBill Taylor bzero(&qp->qpc, sizeof (hermon_hw_qpc_t)); 1114*9e39c5baSBill Taylor 1115*9e39c5baSBill Taylor /* 1116*9e39c5baSBill Taylor * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the 1117*9e39c5baSBill Taylor * "qphdl" and return success 1118*9e39c5baSBill Taylor */ 1119*9e39c5baSBill Taylor ASSERT(state->hs_qphdl[qpc->hr_indx + port] == NULL); 1120*9e39c5baSBill Taylor state->hs_qphdl[qpc->hr_indx + port] = qp; 1121*9e39c5baSBill Taylor 1122*9e39c5baSBill Taylor *qphdl = qp; 1123*9e39c5baSBill Taylor 1124*9e39c5baSBill Taylor return (DDI_SUCCESS); 1125*9e39c5baSBill Taylor 1126*9e39c5baSBill Taylor /* 1127*9e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 1128*9e39c5baSBill Taylor */ 1129*9e39c5baSBill Taylor spec_qpalloc_fail6: 1130*9e39c5baSBill Taylor hermon_queue_free(&qp->qp_wqinfo); 1131*9e39c5baSBill Taylor if (qp->qp_sq_wqhdr) 1132*9e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr); 1133*9e39c5baSBill Taylor if (qp->qp_rq_wqhdr) 1134*9e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr); 1135*9e39c5baSBill Taylor spec_qpalloc_fail5a: 1136*9e39c5baSBill Taylor hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr); 1137*9e39c5baSBill Taylor spec_qpalloc_fail5: 1138*9e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 1139*9e39c5baSBill Taylor spec_qpalloc_fail4: 1140*9e39c5baSBill Taylor if (hermon_special_qp_rsrc_free(state, type, port) != DDI_SUCCESS) { 1141*9e39c5baSBill Taylor HERMON_WARNING(state, "failed to free special QP rsrc"); 1142*9e39c5baSBill Taylor } 1143*9e39c5baSBill Taylor spec_qpalloc_fail3: 1144*9e39c5baSBill Taylor hermon_cq_refcnt_dec(rq_cq); 1145*9e39c5baSBill Taylor spec_qpalloc_fail2: 1146*9e39c5baSBill Taylor hermon_cq_refcnt_dec(sq_cq); 1147*9e39c5baSBill Taylor spec_qpalloc_fail1: 1148*9e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 1149*9e39c5baSBill Taylor spec_qpalloc_fail: 1150*9e39c5baSBill Taylor return (status); 1151*9e39c5baSBill Taylor } 1152*9e39c5baSBill Taylor 1153*9e39c5baSBill Taylor 1154*9e39c5baSBill Taylor /* 1155*9e39c5baSBill Taylor * hermon_qp_free() 1156*9e39c5baSBill Taylor * This function frees up the QP resources. Depending on the value 1157*9e39c5baSBill Taylor * of the "free_qp_flags", the QP number may not be released until 1158*9e39c5baSBill Taylor * a subsequent call to hermon_qp_release_qpn(). 1159*9e39c5baSBill Taylor * 1160*9e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 1161*9e39c5baSBill Taylor */ 1162*9e39c5baSBill Taylor /* ARGSUSED */ 1163*9e39c5baSBill Taylor int 1164*9e39c5baSBill Taylor hermon_qp_free(hermon_state_t *state, hermon_qphdl_t *qphdl, 1165*9e39c5baSBill Taylor ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh, 1166*9e39c5baSBill Taylor uint_t sleepflag) 1167*9e39c5baSBill Taylor { 1168*9e39c5baSBill Taylor hermon_rsrc_t *qpc, *rsrc; 1169*9e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 1170*9e39c5baSBill Taylor hermon_qpn_entry_t *entry; 1171*9e39c5baSBill Taylor hermon_pdhdl_t pd; 1172*9e39c5baSBill Taylor hermon_mrhdl_t mr; 1173*9e39c5baSBill Taylor hermon_cqhdl_t sq_cq, rq_cq; 1174*9e39c5baSBill Taylor hermon_srqhdl_t srq; 1175*9e39c5baSBill Taylor hermon_qphdl_t qp; 1176*9e39c5baSBill Taylor uint64_t value; 1177*9e39c5baSBill Taylor uint_t type, port; 1178*9e39c5baSBill Taylor uint_t maxprot; 1179*9e39c5baSBill Taylor uint_t qp_srq_en; 1180*9e39c5baSBill Taylor int status; 1181*9e39c5baSBill Taylor 1182*9e39c5baSBill Taylor /* 1183*9e39c5baSBill Taylor * Pull all the necessary information from the Hermon Queue Pair 1184*9e39c5baSBill Taylor * handle. This is necessary here because the resource for the 1185*9e39c5baSBill Taylor * QP handle is going to be freed up as part of this operation. 1186*9e39c5baSBill Taylor */ 1187*9e39c5baSBill Taylor qp = *qphdl; 1188*9e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 1189*9e39c5baSBill Taylor qpc = qp->qp_qpcrsrcp; 1190*9e39c5baSBill Taylor rsrc = qp->qp_rsrcp; 1191*9e39c5baSBill Taylor pd = qp->qp_pdhdl; 1192*9e39c5baSBill Taylor srq = qp->qp_srqhdl; 1193*9e39c5baSBill Taylor mr = qp->qp_mrhdl; 1194*9e39c5baSBill Taylor rq_cq = qp->qp_rq_cqhdl; 1195*9e39c5baSBill Taylor sq_cq = qp->qp_sq_cqhdl; 1196*9e39c5baSBill Taylor port = qp->qp_portnum; 1197*9e39c5baSBill Taylor qp_srq_en = qp->qp_srq_en; 1198*9e39c5baSBill Taylor 1199*9e39c5baSBill Taylor /* 1200*9e39c5baSBill Taylor * If the QP is part of an MCG, then we fail the qp_free 1201*9e39c5baSBill Taylor */ 1202*9e39c5baSBill Taylor if (qp->qp_mcg_refcnt != 0) { 1203*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1204*9e39c5baSBill Taylor status = ibc_get_ci_failure(0); 1205*9e39c5baSBill Taylor goto qpfree_fail; 1206*9e39c5baSBill Taylor } 1207*9e39c5baSBill Taylor 1208*9e39c5baSBill Taylor /* 1209*9e39c5baSBill Taylor * If the QP is not already in "Reset" state, then transition to 1210*9e39c5baSBill Taylor * "Reset". This is necessary because software does not reclaim 1211*9e39c5baSBill Taylor * ownership of the QP context until the QP is in the "Reset" state. 1212*9e39c5baSBill Taylor * If the ownership transfer fails for any reason, then it is an 1213*9e39c5baSBill Taylor * indication that something (either in HW or SW) has gone seriously 1214*9e39c5baSBill Taylor * wrong. So we print a warning message and return. 1215*9e39c5baSBill Taylor */ 1216*9e39c5baSBill Taylor if (qp->qp_state != HERMON_QP_RESET) { 1217*9e39c5baSBill Taylor if (hermon_qp_to_reset(state, qp) != DDI_SUCCESS) { 1218*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1219*9e39c5baSBill Taylor HERMON_WARNING(state, "failed to reset QP context"); 1220*9e39c5baSBill Taylor status = ibc_get_ci_failure(0); 1221*9e39c5baSBill Taylor goto qpfree_fail; 1222*9e39c5baSBill Taylor } 1223*9e39c5baSBill Taylor qp->qp_state = HERMON_QP_RESET; 1224*9e39c5baSBill Taylor 1225*9e39c5baSBill Taylor /* 1226*9e39c5baSBill Taylor * Do any additional handling necessary for the transition 1227*9e39c5baSBill Taylor * to the "Reset" state (e.g. update the WRID lists) 1228*9e39c5baSBill Taylor */ 1229*9e39c5baSBill Taylor if (hermon_wrid_to_reset_handling(state, qp) != DDI_SUCCESS) { 1230*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1231*9e39c5baSBill Taylor HERMON_WARNING(state, "failed to reset QP WRID list"); 1232*9e39c5baSBill Taylor status = ibc_get_ci_failure(0); 1233*9e39c5baSBill Taylor goto qpfree_fail; 1234*9e39c5baSBill Taylor } 1235*9e39c5baSBill Taylor } 1236*9e39c5baSBill Taylor 1237*9e39c5baSBill Taylor /* 1238*9e39c5baSBill Taylor * If this was a user-mappable QP, then we need to remove its entry 1239*9e39c5baSBill Taylor * from the "userland resources database". If it is also currently 1240*9e39c5baSBill Taylor * mmap()'d out to a user process, then we need to call 1241*9e39c5baSBill Taylor * devmap_devmem_remap() to remap the QP memory to an invalid mapping. 1242*9e39c5baSBill Taylor * We also need to invalidate the QP tracking information for the 1243*9e39c5baSBill Taylor * user mapping. 1244*9e39c5baSBill Taylor */ 1245*9e39c5baSBill Taylor if (qp->qp_is_umap) { 1246*9e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, qp->qp_qpnum, 1247*9e39c5baSBill Taylor MLNX_UMAP_QPMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE, 1248*9e39c5baSBill Taylor &umapdb); 1249*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1250*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1251*9e39c5baSBill Taylor HERMON_WARNING(state, "failed to find in database"); 1252*9e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 1253*9e39c5baSBill Taylor } 1254*9e39c5baSBill Taylor hermon_umap_db_free(umapdb); 1255*9e39c5baSBill Taylor if (qp->qp_umap_dhp != NULL) { 1256*9e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 1257*9e39c5baSBill Taylor status = devmap_devmem_remap(qp->qp_umap_dhp, 1258*9e39c5baSBill Taylor state->hs_dip, 0, 0, qp->qp_wqinfo.qa_size, 1259*9e39c5baSBill Taylor maxprot, DEVMAP_MAPPING_INVALID, NULL); 1260*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1261*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1262*9e39c5baSBill Taylor HERMON_WARNING(state, "failed in QP memory " 1263*9e39c5baSBill Taylor "devmap_devmem_remap()"); 1264*9e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 1265*9e39c5baSBill Taylor } 1266*9e39c5baSBill Taylor qp->qp_umap_dhp = (devmap_cookie_t)NULL; 1267*9e39c5baSBill Taylor } 1268*9e39c5baSBill Taylor } 1269*9e39c5baSBill Taylor 1270*9e39c5baSBill Taylor 1271*9e39c5baSBill Taylor /* 1272*9e39c5baSBill Taylor * Put NULL into the Hermon QPNum-to-QPHdl list. This will allow any 1273*9e39c5baSBill Taylor * in-progress events to detect that the QP corresponding to this 1274*9e39c5baSBill Taylor * number has been freed. Note: it does depend in whether we are 1275*9e39c5baSBill Taylor * freeing a special QP or not. 1276*9e39c5baSBill Taylor */ 1277*9e39c5baSBill Taylor if (qp->qp_is_special) { 1278*9e39c5baSBill Taylor state->hs_qphdl[qpc->hr_indx + port] = NULL; 1279*9e39c5baSBill Taylor } else { 1280*9e39c5baSBill Taylor state->hs_qphdl[qpc->hr_indx] = NULL; 1281*9e39c5baSBill Taylor } 1282*9e39c5baSBill Taylor 1283*9e39c5baSBill Taylor /* 1284*9e39c5baSBill Taylor * Drop the QP lock 1285*9e39c5baSBill Taylor * At this point the lock is no longer necessary. We cannot 1286*9e39c5baSBill Taylor * protect from multiple simultaneous calls to free the same QP. 1287*9e39c5baSBill Taylor * In addition, since the QP lock is contained in the QP "software 1288*9e39c5baSBill Taylor * handle" resource, which we will free (see below), it is 1289*9e39c5baSBill Taylor * important that we have no further references to that memory. 1290*9e39c5baSBill Taylor */ 1291*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1292*9e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp)) 1293*9e39c5baSBill Taylor 1294*9e39c5baSBill Taylor /* 1295*9e39c5baSBill Taylor * Free the QP resources 1296*9e39c5baSBill Taylor * Start by deregistering and freeing the memory for work queues. 1297*9e39c5baSBill Taylor * Next free any previously allocated context information 1298*9e39c5baSBill Taylor * (depending on QP type) 1299*9e39c5baSBill Taylor * Finally, decrement the necessary reference counts. 1300*9e39c5baSBill Taylor * If this fails for any reason, then it is an indication that 1301*9e39c5baSBill Taylor * something (either in HW or SW) has gone seriously wrong. So we 1302*9e39c5baSBill Taylor * print a warning message and return. 1303*9e39c5baSBill Taylor */ 1304*9e39c5baSBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL, 1305*9e39c5baSBill Taylor sleepflag); 1306*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1307*9e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister QP memory"); 1308*9e39c5baSBill Taylor status = ibc_get_ci_failure(0); 1309*9e39c5baSBill Taylor goto qpfree_fail; 1310*9e39c5baSBill Taylor } 1311*9e39c5baSBill Taylor 1312*9e39c5baSBill Taylor /* Free the memory for the QP */ 1313*9e39c5baSBill Taylor hermon_queue_free(&qp->qp_wqinfo); 1314*9e39c5baSBill Taylor 1315*9e39c5baSBill Taylor if (qp->qp_sq_wqhdr) 1316*9e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr); 1317*9e39c5baSBill Taylor if (qp->qp_rq_wqhdr) 1318*9e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr); 1319*9e39c5baSBill Taylor 1320*9e39c5baSBill Taylor /* Free the dbr */ 1321*9e39c5baSBill Taylor if (!qp_srq_en) { 1322*9e39c5baSBill Taylor hermon_dbr_free(state, qp->qp_uarpg, qp->qp_rq_vdbr); 1323*9e39c5baSBill Taylor } 1324*9e39c5baSBill Taylor 1325*9e39c5baSBill Taylor /* 1326*9e39c5baSBill Taylor * Free up the remainder of the QP resources. Note: we have a few 1327*9e39c5baSBill Taylor * different resources to free up depending on whether the QP is a 1328*9e39c5baSBill Taylor * special QP or not. As described above, if any of these fail for 1329*9e39c5baSBill Taylor * any reason it is an indication that something (either in HW or SW) 1330*9e39c5baSBill Taylor * has gone seriously wrong. So we print a warning message and 1331*9e39c5baSBill Taylor * return. 1332*9e39c5baSBill Taylor */ 1333*9e39c5baSBill Taylor if (qp->qp_is_special) { 1334*9e39c5baSBill Taylor type = (qp->qp_is_special == HERMON_QP_SMI) ? 1335*9e39c5baSBill Taylor IBT_SMI_SQP : IBT_GSI_SQP; 1336*9e39c5baSBill Taylor 1337*9e39c5baSBill Taylor /* Free up resources for the special QP */ 1338*9e39c5baSBill Taylor status = hermon_special_qp_rsrc_free(state, type, port); 1339*9e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1340*9e39c5baSBill Taylor HERMON_WARNING(state, "failed to free special QP rsrc"); 1341*9e39c5baSBill Taylor status = ibc_get_ci_failure(0); 1342*9e39c5baSBill Taylor goto qpfree_fail; 1343*9e39c5baSBill Taylor } 1344*9e39c5baSBill Taylor 1345*9e39c5baSBill Taylor } else { 1346*9e39c5baSBill Taylor type = qp->qp_serv_type; 1347*9e39c5baSBill Taylor 1348*9e39c5baSBill Taylor /* 1349*9e39c5baSBill Taylor * Check the flags and determine whether to release the 1350*9e39c5baSBill Taylor * QPN or not, based on their value. 1351*9e39c5baSBill Taylor */ 1352*9e39c5baSBill Taylor if (free_qp_flags == IBC_FREE_QP_ONLY) { 1353*9e39c5baSBill Taylor entry = qp->qp_qpn_hdl; 1354*9e39c5baSBill Taylor hermon_qp_release_qpn(state, qp->qp_qpn_hdl, 1355*9e39c5baSBill Taylor HERMON_QPN_FREE_ONLY); 1356*9e39c5baSBill Taylor *qpnh = (ibc_qpn_hdl_t)entry; 1357*9e39c5baSBill Taylor } else { 1358*9e39c5baSBill Taylor hermon_qp_release_qpn(state, qp->qp_qpn_hdl, 1359*9e39c5baSBill Taylor HERMON_QPN_RELEASE); 1360*9e39c5baSBill Taylor } 1361*9e39c5baSBill Taylor } 1362*9e39c5baSBill Taylor mutex_destroy(&qp->qp_sq_lock); 1363*9e39c5baSBill Taylor mutex_destroy(&qp->qp_rq_lock); 1364*9e39c5baSBill Taylor 1365*9e39c5baSBill Taylor /* Free the Hermon Queue Pair handle */ 1366*9e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 1367*9e39c5baSBill Taylor 1368*9e39c5baSBill Taylor /* Decrement the reference counts on CQs, PD and SRQ (if needed) */ 1369*9e39c5baSBill Taylor hermon_cq_refcnt_dec(rq_cq); 1370*9e39c5baSBill Taylor hermon_cq_refcnt_dec(sq_cq); 1371*9e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 1372*9e39c5baSBill Taylor if (qp_srq_en == HERMON_QP_SRQ_ENABLED) { 1373*9e39c5baSBill Taylor hermon_srq_refcnt_dec(srq); 1374*9e39c5baSBill Taylor } 1375*9e39c5baSBill Taylor 1376*9e39c5baSBill Taylor /* Set the qphdl pointer to NULL and return success */ 1377*9e39c5baSBill Taylor *qphdl = NULL; 1378*9e39c5baSBill Taylor 1379*9e39c5baSBill Taylor return (DDI_SUCCESS); 1380*9e39c5baSBill Taylor 1381*9e39c5baSBill Taylor qpfree_fail: 1382*9e39c5baSBill Taylor return (status); 1383*9e39c5baSBill Taylor } 1384*9e39c5baSBill Taylor 1385*9e39c5baSBill Taylor 1386*9e39c5baSBill Taylor /* 1387*9e39c5baSBill Taylor * hermon_qp_query() 1388*9e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1389*9e39c5baSBill Taylor */ 1390*9e39c5baSBill Taylor int 1391*9e39c5baSBill Taylor hermon_qp_query(hermon_state_t *state, hermon_qphdl_t qp, 1392*9e39c5baSBill Taylor ibt_qp_query_attr_t *attr_p) 1393*9e39c5baSBill Taylor { 1394*9e39c5baSBill Taylor ibt_cep_state_t qp_state; 1395*9e39c5baSBill Taylor ibt_qp_ud_attr_t *ud; 1396*9e39c5baSBill Taylor ibt_qp_rc_attr_t *rc; 1397*9e39c5baSBill Taylor ibt_qp_uc_attr_t *uc; 1398*9e39c5baSBill Taylor ibt_cep_flags_t enable_flags; 1399*9e39c5baSBill Taylor hermon_hw_addr_path_t *qpc_path, *qpc_alt_path; 1400*9e39c5baSBill Taylor ibt_cep_path_t *path_ptr, *alt_path_ptr; 1401*9e39c5baSBill Taylor hermon_hw_qpc_t *qpc; 1402*9e39c5baSBill Taylor int status; 1403*9e39c5baSBill Taylor uint_t tmp_sched_q, tmp_alt_sched_q; 1404*9e39c5baSBill Taylor 1405*9e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 1406*9e39c5baSBill Taylor 1407*9e39c5baSBill Taylor /* 1408*9e39c5baSBill Taylor * Grab the temporary QPC entry from QP software state 1409*9e39c5baSBill Taylor */ 1410*9e39c5baSBill Taylor qpc = &qp->qpc; 1411*9e39c5baSBill Taylor 1412*9e39c5baSBill Taylor /* Convert the current Hermon QP state to IBTF QP state */ 1413*9e39c5baSBill Taylor switch (qp->qp_state) { 1414*9e39c5baSBill Taylor case HERMON_QP_RESET: 1415*9e39c5baSBill Taylor qp_state = IBT_STATE_RESET; /* "Reset" */ 1416*9e39c5baSBill Taylor break; 1417*9e39c5baSBill Taylor case HERMON_QP_INIT: 1418*9e39c5baSBill Taylor qp_state = IBT_STATE_INIT; /* Initialized */ 1419*9e39c5baSBill Taylor break; 1420*9e39c5baSBill Taylor case HERMON_QP_RTR: 1421*9e39c5baSBill Taylor qp_state = IBT_STATE_RTR; /* Ready to Receive */ 1422*9e39c5baSBill Taylor break; 1423*9e39c5baSBill Taylor case HERMON_QP_RTS: 1424*9e39c5baSBill Taylor qp_state = IBT_STATE_RTS; /* Ready to Send */ 1425*9e39c5baSBill Taylor break; 1426*9e39c5baSBill Taylor case HERMON_QP_SQERR: 1427*9e39c5baSBill Taylor qp_state = IBT_STATE_SQE; /* Send Queue Error */ 1428*9e39c5baSBill Taylor break; 1429*9e39c5baSBill Taylor case HERMON_QP_SQD: 1430*9e39c5baSBill Taylor if (qp->qp_sqd_still_draining) { 1431*9e39c5baSBill Taylor qp_state = IBT_STATE_SQDRAIN; /* SQ Draining */ 1432*9e39c5baSBill Taylor } else { 1433*9e39c5baSBill Taylor qp_state = IBT_STATE_SQD; /* SQ Drained */ 1434*9e39c5baSBill Taylor } 1435*9e39c5baSBill Taylor break; 1436*9e39c5baSBill Taylor case HERMON_QP_ERR: 1437*9e39c5baSBill Taylor qp_state = IBT_STATE_ERROR; /* Error */ 1438*9e39c5baSBill Taylor break; 1439*9e39c5baSBill Taylor default: 1440*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1441*9e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 1442*9e39c5baSBill Taylor } 1443*9e39c5baSBill Taylor attr_p->qp_info.qp_state = qp_state; 1444*9e39c5baSBill Taylor 1445*9e39c5baSBill Taylor /* SRQ Hook. */ 1446*9e39c5baSBill Taylor attr_p->qp_srq = NULL; 1447*9e39c5baSBill Taylor 1448*9e39c5baSBill Taylor /* 1449*9e39c5baSBill Taylor * The following QP information is always returned, regardless of 1450*9e39c5baSBill Taylor * the current QP state. Note: Some special handling is necessary 1451*9e39c5baSBill Taylor * for calculating the QP number on special QP (QP0 and QP1). 1452*9e39c5baSBill Taylor */ 1453*9e39c5baSBill Taylor attr_p->qp_sq_cq = qp->qp_sq_cqhdl->cq_hdlrarg; 1454*9e39c5baSBill Taylor attr_p->qp_rq_cq = qp->qp_rq_cqhdl->cq_hdlrarg; 1455*9e39c5baSBill Taylor if (qp->qp_is_special) { 1456*9e39c5baSBill Taylor attr_p->qp_qpn = (qp->qp_is_special == HERMON_QP_SMI) ? 0 : 1; 1457*9e39c5baSBill Taylor } else { 1458*9e39c5baSBill Taylor attr_p->qp_qpn = (ib_qpn_t)qp->qp_qpnum; 1459*9e39c5baSBill Taylor } 1460*9e39c5baSBill Taylor attr_p->qp_sq_sgl = qp->qp_sq_sgl; 1461*9e39c5baSBill Taylor attr_p->qp_rq_sgl = qp->qp_rq_sgl; 1462*9e39c5baSBill Taylor attr_p->qp_info.qp_sq_sz = qp->qp_sq_bufsz - qp->qp_sq_hdrmwqes; 1463*9e39c5baSBill Taylor attr_p->qp_info.qp_rq_sz = qp->qp_rq_bufsz; 1464*9e39c5baSBill Taylor 1465*9e39c5baSBill Taylor /* 1466*9e39c5baSBill Taylor * If QP is currently in the "Reset" state, then only the above are 1467*9e39c5baSBill Taylor * returned 1468*9e39c5baSBill Taylor */ 1469*9e39c5baSBill Taylor if (qp_state == IBT_STATE_RESET) { 1470*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1471*9e39c5baSBill Taylor return (DDI_SUCCESS); 1472*9e39c5baSBill Taylor } 1473*9e39c5baSBill Taylor 1474*9e39c5baSBill Taylor /* 1475*9e39c5baSBill Taylor * Post QUERY_QP command to firmware 1476*9e39c5baSBill Taylor * 1477*9e39c5baSBill Taylor * We do a HERMON_NOSLEEP here because we are holding the "qp_lock". 1478*9e39c5baSBill Taylor * Since we may be in the interrupt context (or subsequently raised 1479*9e39c5baSBill Taylor * to interrupt level by priority inversion), we do not want to block 1480*9e39c5baSBill Taylor * in this routine waiting for success. 1481*9e39c5baSBill Taylor */ 1482*9e39c5baSBill Taylor tmp_sched_q = qpc->pri_addr_path.sched_q; 1483*9e39c5baSBill Taylor tmp_alt_sched_q = qpc->alt_addr_path.sched_q; 1484*9e39c5baSBill Taylor status = hermon_cmn_query_cmd_post(state, QUERY_QP, 0, qp->qp_qpnum, 1485*9e39c5baSBill Taylor qpc, sizeof (hermon_hw_qpc_t), HERMON_CMD_NOSLEEP_SPIN); 1486*9e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 1487*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1488*9e39c5baSBill Taylor cmn_err(CE_WARN, "hermon%d: hermon_qp_query: QUERY_QP " 1489*9e39c5baSBill Taylor "command failed: %08x\n", state->hs_instance, status); 1490*9e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 1491*9e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 1492*9e39c5baSBill Taylor } 1493*9e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 1494*9e39c5baSBill Taylor } 1495*9e39c5baSBill Taylor qpc->pri_addr_path.sched_q = tmp_sched_q; 1496*9e39c5baSBill Taylor qpc->alt_addr_path.sched_q = tmp_alt_sched_q; 1497*9e39c5baSBill Taylor 1498*9e39c5baSBill Taylor /* 1499*9e39c5baSBill Taylor * Fill in the additional QP info based on the QP's transport type. 1500*9e39c5baSBill Taylor */ 1501*9e39c5baSBill Taylor if (qp->qp_serv_type == HERMON_QP_UD) { 1502*9e39c5baSBill Taylor 1503*9e39c5baSBill Taylor /* Fill in the UD-specific info */ 1504*9e39c5baSBill Taylor ud = &attr_p->qp_info.qp_transport.ud; 1505*9e39c5baSBill Taylor ud->ud_qkey = (ib_qkey_t)qpc->qkey; 1506*9e39c5baSBill Taylor ud->ud_sq_psn = qpc->next_snd_psn; 1507*9e39c5baSBill Taylor ud->ud_pkey_ix = qpc->pri_addr_path.pkey_indx; 1508*9e39c5baSBill Taylor /* port+1 for port 1/2 */ 1509*9e39c5baSBill Taylor ud->ud_port = 1510*9e39c5baSBill Taylor (uint8_t)(((qpc->pri_addr_path.sched_q >> 6) & 0x01) + 1); 1511*9e39c5baSBill Taylor 1512*9e39c5baSBill Taylor attr_p->qp_info.qp_trans = IBT_UD_SRV; 1513*9e39c5baSBill Taylor 1514*9e39c5baSBill Taylor } else if (qp->qp_serv_type == HERMON_QP_RC) { 1515*9e39c5baSBill Taylor 1516*9e39c5baSBill Taylor /* Fill in the RC-specific info */ 1517*9e39c5baSBill Taylor rc = &attr_p->qp_info.qp_transport.rc; 1518*9e39c5baSBill Taylor rc->rc_sq_psn = qpc->next_snd_psn; 1519*9e39c5baSBill Taylor rc->rc_rq_psn = qpc->next_rcv_psn; 1520*9e39c5baSBill Taylor rc->rc_dst_qpn = qpc->rem_qpn; 1521*9e39c5baSBill Taylor 1522*9e39c5baSBill Taylor /* Grab the path migration state information */ 1523*9e39c5baSBill Taylor if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) { 1524*9e39c5baSBill Taylor rc->rc_mig_state = IBT_STATE_MIGRATED; 1525*9e39c5baSBill Taylor } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) { 1526*9e39c5baSBill Taylor rc->rc_mig_state = IBT_STATE_REARMED; 1527*9e39c5baSBill Taylor } else { 1528*9e39c5baSBill Taylor rc->rc_mig_state = IBT_STATE_ARMED; 1529*9e39c5baSBill Taylor } 1530*9e39c5baSBill Taylor rc->rc_rdma_ra_out = (1 << qpc->sra_max); 1531*9e39c5baSBill Taylor rc->rc_rdma_ra_in = (1 << qpc->rra_max); 1532*9e39c5baSBill Taylor rc->rc_min_rnr_nak = qpc->min_rnr_nak; 1533*9e39c5baSBill Taylor rc->rc_path_mtu = qpc->mtu; 1534*9e39c5baSBill Taylor rc->rc_retry_cnt = qpc->retry_cnt; 1535*9e39c5baSBill Taylor 1536*9e39c5baSBill Taylor /* Get the common primary address path fields */ 1537*9e39c5baSBill Taylor qpc_path = &qpc->pri_addr_path; 1538*9e39c5baSBill Taylor path_ptr = &rc->rc_path; 1539*9e39c5baSBill Taylor hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect, 1540*9e39c5baSBill Taylor HERMON_ADDRPATH_QP); 1541*9e39c5baSBill Taylor 1542*9e39c5baSBill Taylor /* Fill in the additional primary address path fields */ 1543*9e39c5baSBill Taylor path_ptr->cep_pkey_ix = qpc_path->pkey_indx; 1544*9e39c5baSBill Taylor path_ptr->cep_hca_port_num = 1545*9e39c5baSBill Taylor path_ptr->cep_adds_vect.av_port_num = 1546*9e39c5baSBill Taylor (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1); 1547*9e39c5baSBill Taylor path_ptr->cep_timeout = qpc_path->ack_timeout; 1548*9e39c5baSBill Taylor 1549*9e39c5baSBill Taylor /* Get the common alternate address path fields */ 1550*9e39c5baSBill Taylor qpc_alt_path = &qpc->alt_addr_path; 1551*9e39c5baSBill Taylor alt_path_ptr = &rc->rc_alt_path; 1552*9e39c5baSBill Taylor hermon_get_addr_path(state, qpc_alt_path, 1553*9e39c5baSBill Taylor &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP); 1554*9e39c5baSBill Taylor 1555*9e39c5baSBill Taylor /* Fill in the additional alternate address path fields */ 1556*9e39c5baSBill Taylor alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx; 1557*9e39c5baSBill Taylor alt_path_ptr->cep_hca_port_num = 1558*9e39c5baSBill Taylor alt_path_ptr->cep_adds_vect.av_port_num = 1559*9e39c5baSBill Taylor (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1); 1560*9e39c5baSBill Taylor alt_path_ptr->cep_timeout = qpc_alt_path->ack_timeout; 1561*9e39c5baSBill Taylor 1562*9e39c5baSBill Taylor /* Get the RNR retry time from primary path */ 1563*9e39c5baSBill Taylor rc->rc_rnr_retry_cnt = qpc->rnr_retry; 1564*9e39c5baSBill Taylor 1565*9e39c5baSBill Taylor /* Set the enable flags based on RDMA/Atomic enable bits */ 1566*9e39c5baSBill Taylor enable_flags = IBT_CEP_NO_FLAGS; 1567*9e39c5baSBill Taylor enable_flags |= ((qpc->rre == 0) ? 0 : IBT_CEP_RDMA_RD); 1568*9e39c5baSBill Taylor enable_flags |= ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR); 1569*9e39c5baSBill Taylor enable_flags |= ((qpc->rae == 0) ? 0 : IBT_CEP_ATOMIC); 1570*9e39c5baSBill Taylor attr_p->qp_info.qp_flags = enable_flags; 1571*9e39c5baSBill Taylor 1572*9e39c5baSBill Taylor attr_p->qp_info.qp_trans = IBT_RC_SRV; 1573*9e39c5baSBill Taylor 1574*9e39c5baSBill Taylor } else if (qp->qp_serv_type == HERMON_QP_UC) { 1575*9e39c5baSBill Taylor 1576*9e39c5baSBill Taylor /* Fill in the UC-specific info */ 1577*9e39c5baSBill Taylor uc = &attr_p->qp_info.qp_transport.uc; 1578*9e39c5baSBill Taylor uc->uc_sq_psn = qpc->next_snd_psn; 1579*9e39c5baSBill Taylor uc->uc_rq_psn = qpc->next_rcv_psn; 1580*9e39c5baSBill Taylor uc->uc_dst_qpn = qpc->rem_qpn; 1581*9e39c5baSBill Taylor 1582*9e39c5baSBill Taylor /* Grab the path migration state information */ 1583*9e39c5baSBill Taylor if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) { 1584*9e39c5baSBill Taylor uc->uc_mig_state = IBT_STATE_MIGRATED; 1585*9e39c5baSBill Taylor } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) { 1586*9e39c5baSBill Taylor uc->uc_mig_state = IBT_STATE_REARMED; 1587*9e39c5baSBill Taylor } else { 1588*9e39c5baSBill Taylor uc->uc_mig_state = IBT_STATE_ARMED; 1589*9e39c5baSBill Taylor } 1590*9e39c5baSBill Taylor uc->uc_path_mtu = qpc->mtu; 1591*9e39c5baSBill Taylor 1592*9e39c5baSBill Taylor /* Get the common primary address path fields */ 1593*9e39c5baSBill Taylor qpc_path = &qpc->pri_addr_path; 1594*9e39c5baSBill Taylor path_ptr = &uc->uc_path; 1595*9e39c5baSBill Taylor hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect, 1596*9e39c5baSBill Taylor HERMON_ADDRPATH_QP); 1597*9e39c5baSBill Taylor 1598*9e39c5baSBill Taylor /* Fill in the additional primary address path fields */ 1599*9e39c5baSBill Taylor path_ptr->cep_pkey_ix = qpc_path->pkey_indx; 1600*9e39c5baSBill Taylor path_ptr->cep_hca_port_num = 1601*9e39c5baSBill Taylor path_ptr->cep_adds_vect.av_port_num = 1602*9e39c5baSBill Taylor (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1); 1603*9e39c5baSBill Taylor 1604*9e39c5baSBill Taylor /* Get the common alternate address path fields */ 1605*9e39c5baSBill Taylor qpc_alt_path = &qpc->alt_addr_path; 1606*9e39c5baSBill Taylor alt_path_ptr = &uc->uc_alt_path; 1607*9e39c5baSBill Taylor hermon_get_addr_path(state, qpc_alt_path, 1608*9e39c5baSBill Taylor &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP); 1609*9e39c5baSBill Taylor 1610*9e39c5baSBill Taylor /* Fill in the additional alternate address path fields */ 1611*9e39c5baSBill Taylor alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx; 1612*9e39c5baSBill Taylor alt_path_ptr->cep_hca_port_num = 1613*9e39c5baSBill Taylor alt_path_ptr->cep_adds_vect.av_port_num = 1614*9e39c5baSBill Taylor (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1); 1615*9e39c5baSBill Taylor 1616*9e39c5baSBill Taylor /* 1617*9e39c5baSBill Taylor * Set the enable flags based on RDMA enable bits (by 1618*9e39c5baSBill Taylor * definition UC doesn't support Atomic or RDMA Read) 1619*9e39c5baSBill Taylor */ 1620*9e39c5baSBill Taylor enable_flags = ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR); 1621*9e39c5baSBill Taylor attr_p->qp_info.qp_flags = enable_flags; 1622*9e39c5baSBill Taylor 1623*9e39c5baSBill Taylor attr_p->qp_info.qp_trans = IBT_UC_SRV; 1624*9e39c5baSBill Taylor 1625*9e39c5baSBill Taylor } else { 1626*9e39c5baSBill Taylor HERMON_WARNING(state, "unexpected QP transport type"); 1627*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1628*9e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 1629*9e39c5baSBill Taylor } 1630*9e39c5baSBill Taylor 1631*9e39c5baSBill Taylor /* 1632*9e39c5baSBill Taylor * Under certain circumstances it is possible for the Hermon hardware 1633*9e39c5baSBill Taylor * to transition to one of the error states without software directly 1634*9e39c5baSBill Taylor * knowing about it. The QueryQP() call is the one place where we 1635*9e39c5baSBill Taylor * have an opportunity to sample and update our view of the QP state. 1636*9e39c5baSBill Taylor */ 1637*9e39c5baSBill Taylor if (qpc->state == HERMON_QP_SQERR) { 1638*9e39c5baSBill Taylor attr_p->qp_info.qp_state = IBT_STATE_SQE; 1639*9e39c5baSBill Taylor qp->qp_state = HERMON_QP_SQERR; 1640*9e39c5baSBill Taylor } 1641*9e39c5baSBill Taylor if (qpc->state == HERMON_QP_ERR) { 1642*9e39c5baSBill Taylor attr_p->qp_info.qp_state = IBT_STATE_ERROR; 1643*9e39c5baSBill Taylor qp->qp_state = HERMON_QP_ERR; 1644*9e39c5baSBill Taylor } 1645*9e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 1646*9e39c5baSBill Taylor 1647*9e39c5baSBill Taylor return (DDI_SUCCESS); 1648*9e39c5baSBill Taylor } 1649*9e39c5baSBill Taylor 1650*9e39c5baSBill Taylor 1651*9e39c5baSBill Taylor /* 1652*9e39c5baSBill Taylor * hermon_qp_create_qpn() 1653*9e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1654*9e39c5baSBill Taylor */ 1655*9e39c5baSBill Taylor static int 1656*9e39c5baSBill Taylor hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp, 1657*9e39c5baSBill Taylor hermon_rsrc_t *qpc) 1658*9e39c5baSBill Taylor { 1659*9e39c5baSBill Taylor hermon_qpn_entry_t query; 1660*9e39c5baSBill Taylor hermon_qpn_entry_t *entry; 1661*9e39c5baSBill Taylor avl_index_t where; 1662*9e39c5baSBill Taylor 1663*9e39c5baSBill Taylor /* 1664*9e39c5baSBill Taylor * Build a query (for the AVL tree lookup) and attempt to find 1665*9e39c5baSBill Taylor * a previously added entry that has a matching QPC index. If 1666*9e39c5baSBill Taylor * no matching entry is found, then allocate, initialize, and 1667*9e39c5baSBill Taylor * add an entry to the AVL tree. 1668*9e39c5baSBill Taylor * If a matching entry is found, then increment its QPN counter 1669*9e39c5baSBill Taylor * and reference counter. 1670*9e39c5baSBill Taylor */ 1671*9e39c5baSBill Taylor query.qpn_indx = qpc->hr_indx; 1672*9e39c5baSBill Taylor mutex_enter(&state->hs_qpn_avl_lock); 1673*9e39c5baSBill Taylor entry = (hermon_qpn_entry_t *)avl_find(&state->hs_qpn_avl, 1674*9e39c5baSBill Taylor &query, &where); 1675*9e39c5baSBill Taylor if (entry == NULL) { 1676*9e39c5baSBill Taylor /* 1677*9e39c5baSBill Taylor * Allocate and initialize a QPN entry, then insert 1678*9e39c5baSBill Taylor * it into the AVL tree. 1679*9e39c5baSBill Taylor */ 1680*9e39c5baSBill Taylor entry = (hermon_qpn_entry_t *)kmem_zalloc( 1681*9e39c5baSBill Taylor sizeof (hermon_qpn_entry_t), KM_NOSLEEP); 1682*9e39c5baSBill Taylor if (entry == NULL) { 1683*9e39c5baSBill Taylor mutex_exit(&state->hs_qpn_avl_lock); 1684*9e39c5baSBill Taylor return (DDI_FAILURE); 1685*9e39c5baSBill Taylor } 1686*9e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry)) 1687*9e39c5baSBill Taylor 1688*9e39c5baSBill Taylor entry->qpn_indx = qpc->hr_indx; 1689*9e39c5baSBill Taylor entry->qpn_refcnt = 0; 1690*9e39c5baSBill Taylor entry->qpn_counter = 0; 1691*9e39c5baSBill Taylor 1692*9e39c5baSBill Taylor avl_insert(&state->hs_qpn_avl, entry, where); 1693*9e39c5baSBill Taylor } 1694*9e39c5baSBill Taylor 1695*9e39c5baSBill Taylor /* 1696*9e39c5baSBill Taylor * Make the AVL tree entry point to the QP context resource that 1697*9e39c5baSBill Taylor * it will be responsible for tracking 1698*9e39c5baSBill Taylor */ 1699*9e39c5baSBill Taylor entry->qpn_qpc = qpc; 1700*9e39c5baSBill Taylor 1701*9e39c5baSBill Taylor /* 1702*9e39c5baSBill Taylor * Setup the QP handle to point to the AVL tree entry. Then 1703*9e39c5baSBill Taylor * generate the new QP number from the entry's QPN counter value 1704*9e39c5baSBill Taylor * and the hardware's QP context table index. 1705*9e39c5baSBill Taylor */ 1706*9e39c5baSBill Taylor qp->qp_qpn_hdl = entry; 1707*9e39c5baSBill Taylor qp->qp_qpnum = ((entry->qpn_counter << 1708*9e39c5baSBill Taylor state->hs_cfg_profile->cp_log_num_qp) | qpc->hr_indx) & 1709*9e39c5baSBill Taylor HERMON_QP_MAXNUMBER_MSK; 1710*9e39c5baSBill Taylor qp->qp_ring = qp->qp_qpnum << 8; 1711*9e39c5baSBill Taylor 1712*9e39c5baSBill Taylor /* 1713*9e39c5baSBill Taylor * Increment the reference counter and QPN counter. The QPN 1714*9e39c5baSBill Taylor * counter always indicates the next available number for use. 1715*9e39c5baSBill Taylor */ 1716*9e39c5baSBill Taylor entry->qpn_counter++; 1717*9e39c5baSBill Taylor entry->qpn_refcnt++; 1718*9e39c5baSBill Taylor 1719*9e39c5baSBill Taylor mutex_exit(&state->hs_qpn_avl_lock); 1720*9e39c5baSBill Taylor 1721*9e39c5baSBill Taylor return (DDI_SUCCESS); 1722*9e39c5baSBill Taylor } 1723*9e39c5baSBill Taylor 1724*9e39c5baSBill Taylor 1725*9e39c5baSBill Taylor /* 1726*9e39c5baSBill Taylor * hermon_qp_release_qpn() 1727*9e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 1728*9e39c5baSBill Taylor */ 1729*9e39c5baSBill Taylor void 1730*9e39c5baSBill Taylor hermon_qp_release_qpn(hermon_state_t *state, hermon_qpn_entry_t *entry, 1731*9e39c5baSBill Taylor int flags) 1732*9e39c5baSBill Taylor { 1733*9e39c5baSBill Taylor ASSERT(entry != NULL); 1734*9e39c5baSBill Taylor 1735*9e39c5baSBill Taylor mutex_enter(&state->hs_qpn_avl_lock); 1736*9e39c5baSBill Taylor 1737*9e39c5baSBill Taylor /* 1738*9e39c5baSBill Taylor * If we are releasing the QP number here, then we decrement the 1739*9e39c5baSBill Taylor * reference count and check for zero references. If there are 1740*9e39c5baSBill Taylor * zero references, then we free the QPC context (if it hadn't 1741*9e39c5baSBill Taylor * already been freed during a HERMON_QPN_FREE_ONLY free, i.e. for 1742*9e39c5baSBill Taylor * reuse with another similar QP number) and remove the tracking 1743*9e39c5baSBill Taylor * structure from the QP number AVL tree and free the structure. 1744*9e39c5baSBill Taylor * If we are not releasing the QP number here, then, as long as we 1745*9e39c5baSBill Taylor * have not exhausted the usefulness of the QPC context (that is, 1746*9e39c5baSBill Taylor * re-used it too many times without the reference count having 1747*9e39c5baSBill Taylor * gone to zero), we free up the QPC context for use by another 1748*9e39c5baSBill Taylor * thread (which will use it to construct a different QP number 1749*9e39c5baSBill Taylor * from the same QPC table index). 1750*9e39c5baSBill Taylor */ 1751*9e39c5baSBill Taylor if (flags == HERMON_QPN_RELEASE) { 1752*9e39c5baSBill Taylor entry->qpn_refcnt--; 1753*9e39c5baSBill Taylor 1754*9e39c5baSBill Taylor /* 1755*9e39c5baSBill Taylor * If the reference count is zero, then we free the QPC 1756*9e39c5baSBill Taylor * context (if it hadn't already been freed in an early 1757*9e39c5baSBill Taylor * step, e.g. HERMON_QPN_FREE_ONLY) and remove/free the 1758*9e39c5baSBill Taylor * tracking structure from the QP number AVL tree. 1759*9e39c5baSBill Taylor */ 1760*9e39c5baSBill Taylor if (entry->qpn_refcnt == 0) { 1761*9e39c5baSBill Taylor if (entry->qpn_qpc != NULL) { 1762*9e39c5baSBill Taylor hermon_rsrc_free(state, &entry->qpn_qpc); 1763*9e39c5baSBill Taylor } 1764*9e39c5baSBill Taylor 1765*9e39c5baSBill Taylor /* 1766*9e39c5baSBill Taylor * If the current entry has served it's useful 1767*9e39c5baSBill Taylor * purpose (i.e. been reused the maximum allowable 1768*9e39c5baSBill Taylor * number of times), then remove it from QP number 1769*9e39c5baSBill Taylor * AVL tree and free it up. 1770*9e39c5baSBill Taylor */ 1771*9e39c5baSBill Taylor if (entry->qpn_counter >= (1 << 1772*9e39c5baSBill Taylor (24 - state->hs_cfg_profile->cp_log_num_qp))) { 1773*9e39c5baSBill Taylor avl_remove(&state->hs_qpn_avl, entry); 1774*9e39c5baSBill Taylor kmem_free(entry, sizeof (hermon_qpn_entry_t)); 1775*9e39c5baSBill Taylor } 1776*9e39c5baSBill Taylor } 1777*9e39c5baSBill Taylor 1778*9e39c5baSBill Taylor } else if (flags == HERMON_QPN_FREE_ONLY) { 1779*9e39c5baSBill Taylor /* 1780*9e39c5baSBill Taylor * Even if we are not freeing the QP number, that will not 1781*9e39c5baSBill Taylor * always prevent us from releasing the QPC context. In fact, 1782*9e39c5baSBill Taylor * since the QPC context only forms part of the whole QPN, 1783*9e39c5baSBill Taylor * we want to free it up for use by other consumers. But 1784*9e39c5baSBill Taylor * if the reference count is non-zero (which it will always 1785*9e39c5baSBill Taylor * be when we are doing HERMON_QPN_FREE_ONLY) and the counter 1786*9e39c5baSBill Taylor * has reached its maximum value, then we cannot reuse the 1787*9e39c5baSBill Taylor * QPC context until the reference count eventually reaches 1788*9e39c5baSBill Taylor * zero (in HERMON_QPN_RELEASE, above). 1789*9e39c5baSBill Taylor */ 1790*9e39c5baSBill Taylor if (entry->qpn_counter < (1 << 1791*9e39c5baSBill Taylor (24 - state->hs_cfg_profile->cp_log_num_qp))) { 1792*9e39c5baSBill Taylor hermon_rsrc_free(state, &entry->qpn_qpc); 1793*9e39c5baSBill Taylor } 1794*9e39c5baSBill Taylor } 1795*9e39c5baSBill Taylor mutex_exit(&state->hs_qpn_avl_lock); 1796*9e39c5baSBill Taylor } 1797*9e39c5baSBill Taylor 1798*9e39c5baSBill Taylor 1799*9e39c5baSBill Taylor /* 1800*9e39c5baSBill Taylor * hermon_qpn_avl_compare() 1801*9e39c5baSBill Taylor * Context: Can be called from user or kernel context. 1802*9e39c5baSBill Taylor */ 1803*9e39c5baSBill Taylor static int 1804*9e39c5baSBill Taylor hermon_qpn_avl_compare(const void *q, const void *e) 1805*9e39c5baSBill Taylor { 1806*9e39c5baSBill Taylor hermon_qpn_entry_t *entry, *query; 1807*9e39c5baSBill Taylor 1808*9e39c5baSBill Taylor entry = (hermon_qpn_entry_t *)e; 1809*9e39c5baSBill Taylor query = (hermon_qpn_entry_t *)q; 1810*9e39c5baSBill Taylor 1811*9e39c5baSBill Taylor if (query->qpn_indx < entry->qpn_indx) { 1812*9e39c5baSBill Taylor return (-1); 1813*9e39c5baSBill Taylor } else if (query->qpn_indx > entry->qpn_indx) { 1814*9e39c5baSBill Taylor return (+1); 1815*9e39c5baSBill Taylor } else { 1816*9e39c5baSBill Taylor return (0); 1817*9e39c5baSBill Taylor } 1818*9e39c5baSBill Taylor } 1819*9e39c5baSBill Taylor 1820*9e39c5baSBill Taylor 1821*9e39c5baSBill Taylor /* 1822*9e39c5baSBill Taylor * hermon_qpn_avl_init() 1823*9e39c5baSBill Taylor * Context: Only called from attach() path context 1824*9e39c5baSBill Taylor */ 1825*9e39c5baSBill Taylor void 1826*9e39c5baSBill Taylor hermon_qpn_avl_init(hermon_state_t *state) 1827*9e39c5baSBill Taylor { 1828*9e39c5baSBill Taylor /* Initialize the lock used for QP number (QPN) AVL tree access */ 1829*9e39c5baSBill Taylor mutex_init(&state->hs_qpn_avl_lock, NULL, MUTEX_DRIVER, 1830*9e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 1831*9e39c5baSBill Taylor 1832*9e39c5baSBill Taylor /* Initialize the AVL tree for the QP number (QPN) storage */ 1833*9e39c5baSBill Taylor avl_create(&state->hs_qpn_avl, hermon_qpn_avl_compare, 1834*9e39c5baSBill Taylor sizeof (hermon_qpn_entry_t), 1835*9e39c5baSBill Taylor offsetof(hermon_qpn_entry_t, qpn_avlnode)); 1836*9e39c5baSBill Taylor } 1837*9e39c5baSBill Taylor 1838*9e39c5baSBill Taylor 1839*9e39c5baSBill Taylor /* 1840*9e39c5baSBill Taylor * hermon_qpn_avl_fini() 1841*9e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts 1842*9e39c5baSBill Taylor */ 1843*9e39c5baSBill Taylor void 1844*9e39c5baSBill Taylor hermon_qpn_avl_fini(hermon_state_t *state) 1845*9e39c5baSBill Taylor { 1846*9e39c5baSBill Taylor hermon_qpn_entry_t *entry; 1847*9e39c5baSBill Taylor void *cookie; 1848*9e39c5baSBill Taylor 1849*9e39c5baSBill Taylor /* 1850*9e39c5baSBill Taylor * Empty all entries (if necessary) and destroy the AVL tree 1851*9e39c5baSBill Taylor * that was used for QP number (QPN) tracking. 1852*9e39c5baSBill Taylor */ 1853*9e39c5baSBill Taylor cookie = NULL; 1854*9e39c5baSBill Taylor while ((entry = (hermon_qpn_entry_t *)avl_destroy_nodes( 1855*9e39c5baSBill Taylor &state->hs_qpn_avl, &cookie)) != NULL) { 1856*9e39c5baSBill Taylor kmem_free(entry, sizeof (hermon_qpn_entry_t)); 1857*9e39c5baSBill Taylor } 1858*9e39c5baSBill Taylor avl_destroy(&state->hs_qpn_avl); 1859*9e39c5baSBill Taylor 1860*9e39c5baSBill Taylor /* Destroy the lock used for QP number (QPN) AVL tree access */ 1861*9e39c5baSBill Taylor mutex_destroy(&state->hs_qpn_avl_lock); 1862*9e39c5baSBill Taylor } 1863*9e39c5baSBill Taylor 1864*9e39c5baSBill Taylor 1865*9e39c5baSBill Taylor /* 1866*9e39c5baSBill Taylor * hermon_qphdl_from_qpnum() 1867*9e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1868*9e39c5baSBill Taylor * 1869*9e39c5baSBill Taylor * This routine is important because changing the unconstrained 1870*9e39c5baSBill Taylor * portion of the QP number is critical to the detection of a 1871*9e39c5baSBill Taylor * potential race condition in the QP event handler code (i.e. the case 1872*9e39c5baSBill Taylor * where a QP is freed and alloc'd again before an event for the 1873*9e39c5baSBill Taylor * "old" QP can be handled). 1874*9e39c5baSBill Taylor * 1875*9e39c5baSBill Taylor * While this is not a perfect solution (not sure that one exists) 1876*9e39c5baSBill Taylor * it does help to mitigate the chance that this race condition will 1877*9e39c5baSBill Taylor * cause us to deliver a "stale" event to the new QP owner. Note: 1878*9e39c5baSBill Taylor * this solution does not scale well because the number of constrained 1879*9e39c5baSBill Taylor * bits increases (and, hence, the number of unconstrained bits 1880*9e39c5baSBill Taylor * decreases) as the number of supported QPs grows. For small and 1881*9e39c5baSBill Taylor * intermediate values, it should hopefully provide sufficient 1882*9e39c5baSBill Taylor * protection. 1883*9e39c5baSBill Taylor */ 1884*9e39c5baSBill Taylor hermon_qphdl_t 1885*9e39c5baSBill Taylor hermon_qphdl_from_qpnum(hermon_state_t *state, uint_t qpnum) 1886*9e39c5baSBill Taylor { 1887*9e39c5baSBill Taylor uint_t qpindx, qpmask; 1888*9e39c5baSBill Taylor 1889*9e39c5baSBill Taylor /* Calculate the QP table index from the qpnum */ 1890*9e39c5baSBill Taylor qpmask = (1 << state->hs_cfg_profile->cp_log_num_qp) - 1; 1891*9e39c5baSBill Taylor qpindx = qpnum & qpmask; 1892*9e39c5baSBill Taylor return (state->hs_qphdl[qpindx]); 1893*9e39c5baSBill Taylor } 1894*9e39c5baSBill Taylor 1895*9e39c5baSBill Taylor 1896*9e39c5baSBill Taylor /* 1897*9e39c5baSBill Taylor * hermon_special_qp_rsrc_alloc 1898*9e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1899*9e39c5baSBill Taylor */ 1900*9e39c5baSBill Taylor static int 1901*9e39c5baSBill Taylor hermon_special_qp_rsrc_alloc(hermon_state_t *state, ibt_sqp_type_t type, 1902*9e39c5baSBill Taylor uint_t port, hermon_rsrc_t **qp_rsrc) 1903*9e39c5baSBill Taylor { 1904*9e39c5baSBill Taylor uint_t mask, flags; 1905*9e39c5baSBill Taylor int status; 1906*9e39c5baSBill Taylor 1907*9e39c5baSBill Taylor mutex_enter(&state->hs_spec_qplock); 1908*9e39c5baSBill Taylor flags = state->hs_spec_qpflags; 1909*9e39c5baSBill Taylor if (type == IBT_SMI_SQP) { 1910*9e39c5baSBill Taylor /* 1911*9e39c5baSBill Taylor * Check here to see if the driver has been configured 1912*9e39c5baSBill Taylor * to instruct the Hermon firmware to handle all incoming 1913*9e39c5baSBill Taylor * SMP messages (i.e. messages sent to SMA). If so, 1914*9e39c5baSBill Taylor * then we will treat QP0 as if it has already been 1915*9e39c5baSBill Taylor * allocated (for internal use). Otherwise, if we allow 1916*9e39c5baSBill Taylor * the allocation to happen, it will cause unexpected 1917*9e39c5baSBill Taylor * behaviors (e.g. Hermon SMA becomes unresponsive). 1918*9e39c5baSBill Taylor */ 1919*9e39c5baSBill Taylor if (state->hs_cfg_profile->cp_qp0_agents_in_fw != 0) { 1920*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 1921*9e39c5baSBill Taylor return (IBT_QP_IN_USE); 1922*9e39c5baSBill Taylor } 1923*9e39c5baSBill Taylor 1924*9e39c5baSBill Taylor /* 1925*9e39c5baSBill Taylor * If this is the first QP0 allocation, then post 1926*9e39c5baSBill Taylor * a CONF_SPECIAL_QP firmware command 1927*9e39c5baSBill Taylor */ 1928*9e39c5baSBill Taylor if ((flags & HERMON_SPECIAL_QP0_RSRC_MASK) == 0) { 1929*9e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 1930*9e39c5baSBill Taylor state->hs_spec_qp0->hr_indx, HERMON_CMD_QP_SMI, 1931*9e39c5baSBill Taylor HERMON_CMD_NOSLEEP_SPIN, 1932*9e39c5baSBill Taylor HERMON_CMD_SPEC_QP_OPMOD( 1933*9e39c5baSBill Taylor state->hs_cfg_profile->cp_qp0_agents_in_fw, 1934*9e39c5baSBill Taylor state->hs_cfg_profile->cp_qp1_agents_in_fw)); 1935*9e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 1936*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 1937*9e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 1938*9e39c5baSBill Taylor "command failed: %08x\n", 1939*9e39c5baSBill Taylor state->hs_instance, status); 1940*9e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 1941*9e39c5baSBill Taylor } 1942*9e39c5baSBill Taylor } 1943*9e39c5baSBill Taylor 1944*9e39c5baSBill Taylor /* 1945*9e39c5baSBill Taylor * Now check (and, if necessary, modify) the flags to indicate 1946*9e39c5baSBill Taylor * whether the allocation was successful 1947*9e39c5baSBill Taylor */ 1948*9e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port)); 1949*9e39c5baSBill Taylor if (flags & mask) { 1950*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 1951*9e39c5baSBill Taylor return (IBT_QP_IN_USE); 1952*9e39c5baSBill Taylor } 1953*9e39c5baSBill Taylor state->hs_spec_qpflags |= mask; 1954*9e39c5baSBill Taylor *qp_rsrc = state->hs_spec_qp0; 1955*9e39c5baSBill Taylor 1956*9e39c5baSBill Taylor } else { 1957*9e39c5baSBill Taylor /* 1958*9e39c5baSBill Taylor * If this is the first QP1 allocation, then post 1959*9e39c5baSBill Taylor * a CONF_SPECIAL_QP firmware command 1960*9e39c5baSBill Taylor */ 1961*9e39c5baSBill Taylor if ((flags & HERMON_SPECIAL_QP1_RSRC_MASK) == 0) { 1962*9e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 1963*9e39c5baSBill Taylor state->hs_spec_qp1->hr_indx, HERMON_CMD_QP_GSI, 1964*9e39c5baSBill Taylor HERMON_CMD_NOSLEEP_SPIN, 1965*9e39c5baSBill Taylor HERMON_CMD_SPEC_QP_OPMOD( 1966*9e39c5baSBill Taylor state->hs_cfg_profile->cp_qp0_agents_in_fw, 1967*9e39c5baSBill Taylor state->hs_cfg_profile->cp_qp1_agents_in_fw)); 1968*9e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 1969*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 1970*9e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 1971*9e39c5baSBill Taylor "command failed: %08x\n", 1972*9e39c5baSBill Taylor state->hs_instance, status); 1973*9e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 1974*9e39c5baSBill Taylor } 1975*9e39c5baSBill Taylor } 1976*9e39c5baSBill Taylor 1977*9e39c5baSBill Taylor /* 1978*9e39c5baSBill Taylor * Now check (and, if necessary, modify) the flags to indicate 1979*9e39c5baSBill Taylor * whether the allocation was successful 1980*9e39c5baSBill Taylor */ 1981*9e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port)); 1982*9e39c5baSBill Taylor if (flags & mask) { 1983*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 1984*9e39c5baSBill Taylor return (IBT_QP_IN_USE); 1985*9e39c5baSBill Taylor } 1986*9e39c5baSBill Taylor state->hs_spec_qpflags |= mask; 1987*9e39c5baSBill Taylor *qp_rsrc = state->hs_spec_qp1; 1988*9e39c5baSBill Taylor } 1989*9e39c5baSBill Taylor 1990*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 1991*9e39c5baSBill Taylor return (DDI_SUCCESS); 1992*9e39c5baSBill Taylor } 1993*9e39c5baSBill Taylor 1994*9e39c5baSBill Taylor 1995*9e39c5baSBill Taylor /* 1996*9e39c5baSBill Taylor * hermon_special_qp_rsrc_free 1997*9e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1998*9e39c5baSBill Taylor */ 1999*9e39c5baSBill Taylor static int 2000*9e39c5baSBill Taylor hermon_special_qp_rsrc_free(hermon_state_t *state, ibt_sqp_type_t type, 2001*9e39c5baSBill Taylor uint_t port) 2002*9e39c5baSBill Taylor { 2003*9e39c5baSBill Taylor uint_t mask, flags; 2004*9e39c5baSBill Taylor int status; 2005*9e39c5baSBill Taylor 2006*9e39c5baSBill Taylor mutex_enter(&state->hs_spec_qplock); 2007*9e39c5baSBill Taylor if (type == IBT_SMI_SQP) { 2008*9e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port)); 2009*9e39c5baSBill Taylor state->hs_spec_qpflags &= ~mask; 2010*9e39c5baSBill Taylor flags = state->hs_spec_qpflags; 2011*9e39c5baSBill Taylor 2012*9e39c5baSBill Taylor /* 2013*9e39c5baSBill Taylor * If this is the last QP0 free, then post a CONF_SPECIAL_QP 2014*9e39c5baSBill Taylor * NOW, If this is the last Special QP free, then post a 2015*9e39c5baSBill Taylor * CONF_SPECIAL_QP firmware command - it'll stop them all 2016*9e39c5baSBill Taylor */ 2017*9e39c5baSBill Taylor if (flags) { 2018*9e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 0, 2019*9e39c5baSBill Taylor HERMON_CMD_QP_SMI, HERMON_CMD_NOSLEEP_SPIN, 0); 2020*9e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 2021*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 2022*9e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 2023*9e39c5baSBill Taylor "command failed: %08x\n", 2024*9e39c5baSBill Taylor state->hs_instance, status); 2025*9e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 2026*9e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 2027*9e39c5baSBill Taylor HCA_ERR_SRV_LOST); 2028*9e39c5baSBill Taylor } 2029*9e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 2030*9e39c5baSBill Taylor } 2031*9e39c5baSBill Taylor } 2032*9e39c5baSBill Taylor } else { 2033*9e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port)); 2034*9e39c5baSBill Taylor state->hs_spec_qpflags &= ~mask; 2035*9e39c5baSBill Taylor flags = state->hs_spec_qpflags; 2036*9e39c5baSBill Taylor 2037*9e39c5baSBill Taylor /* 2038*9e39c5baSBill Taylor * If this is the last QP1 free, then post a CONF_SPECIAL_QP 2039*9e39c5baSBill Taylor * NOW, if this is the last special QP free, then post a 2040*9e39c5baSBill Taylor * CONF_SPECIAL_QP firmware command - it'll stop them all 2041*9e39c5baSBill Taylor */ 2042*9e39c5baSBill Taylor if (flags) { 2043*9e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 0, 2044*9e39c5baSBill Taylor HERMON_CMD_QP_GSI, HERMON_CMD_NOSLEEP_SPIN, 0); 2045*9e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 2046*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 2047*9e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 2048*9e39c5baSBill Taylor "command failed: %08x\n", 2049*9e39c5baSBill Taylor state->hs_instance, status); 2050*9e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 2051*9e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 2052*9e39c5baSBill Taylor HCA_ERR_SRV_LOST); 2053*9e39c5baSBill Taylor } 2054*9e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 2055*9e39c5baSBill Taylor } 2056*9e39c5baSBill Taylor } 2057*9e39c5baSBill Taylor } 2058*9e39c5baSBill Taylor 2059*9e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 2060*9e39c5baSBill Taylor return (DDI_SUCCESS); 2061*9e39c5baSBill Taylor } 2062*9e39c5baSBill Taylor 2063*9e39c5baSBill Taylor 2064*9e39c5baSBill Taylor /* 2065*9e39c5baSBill Taylor * hermon_qp_sgl_to_logwqesz() 2066*9e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 2067*9e39c5baSBill Taylor */ 2068*9e39c5baSBill Taylor static void 2069*9e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl, 2070*9e39c5baSBill Taylor uint_t real_max_sgl, hermon_qp_wq_type_t wq_type, 2071*9e39c5baSBill Taylor uint_t *logwqesz, uint_t *max_sgl) 2072*9e39c5baSBill Taylor { 2073*9e39c5baSBill Taylor uint_t max_size, log2, actual_sgl; 2074*9e39c5baSBill Taylor 2075*9e39c5baSBill Taylor switch (wq_type) { 2076*9e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDQ_UD: 2077*9e39c5baSBill Taylor /* 2078*9e39c5baSBill Taylor * Use requested maximum SGL to calculate max descriptor size 2079*9e39c5baSBill Taylor * (while guaranteeing that the descriptor size is a 2080*9e39c5baSBill Taylor * power-of-2 cachelines). 2081*9e39c5baSBill Taylor */ 2082*9e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4)); 2083*9e39c5baSBill Taylor log2 = highbit(max_size); 2084*9e39c5baSBill Taylor if ((max_size & (max_size - 1)) == 0) { 2085*9e39c5baSBill Taylor log2 = log2 - 1; 2086*9e39c5baSBill Taylor } 2087*9e39c5baSBill Taylor 2088*9e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 2089*9e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 2090*9e39c5baSBill Taylor 2091*9e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 2092*9e39c5baSBill Taylor actual_sgl = ((1 << log2) - 2093*9e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)) >> 4; 2094*9e39c5baSBill Taylor break; 2095*9e39c5baSBill Taylor 2096*9e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDQ_CONN: 2097*9e39c5baSBill Taylor /* 2098*9e39c5baSBill Taylor * Use requested maximum SGL to calculate max descriptor size 2099*9e39c5baSBill Taylor * (while guaranteeing that the descriptor size is a 2100*9e39c5baSBill Taylor * power-of-2 cachelines). 2101*9e39c5baSBill Taylor */ 2102*9e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4)); 2103*9e39c5baSBill Taylor log2 = highbit(max_size); 2104*9e39c5baSBill Taylor if ((max_size & (max_size - 1)) == 0) { 2105*9e39c5baSBill Taylor log2 = log2 - 1; 2106*9e39c5baSBill Taylor } 2107*9e39c5baSBill Taylor 2108*9e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 2109*9e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 2110*9e39c5baSBill Taylor 2111*9e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 2112*9e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SND_HDRS) >> 4; 2113*9e39c5baSBill Taylor break; 2114*9e39c5baSBill Taylor 2115*9e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_RECVQ: 2116*9e39c5baSBill Taylor /* 2117*9e39c5baSBill Taylor * Same as above (except for Recv WQEs) 2118*9e39c5baSBill Taylor */ 2119*9e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4)); 2120*9e39c5baSBill Taylor log2 = highbit(max_size); 2121*9e39c5baSBill Taylor if ((max_size & (max_size - 1)) == 0) { 2122*9e39c5baSBill Taylor log2 = log2 - 1; 2123*9e39c5baSBill Taylor } 2124*9e39c5baSBill Taylor 2125*9e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 2126*9e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 2127*9e39c5baSBill Taylor 2128*9e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 2129*9e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_RCV_HDRS) >> 4; 2130*9e39c5baSBill Taylor break; 2131*9e39c5baSBill Taylor 2132*9e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDMLX_QP0: 2133*9e39c5baSBill Taylor /* 2134*9e39c5baSBill Taylor * Same as above (except for MLX transport WQEs). For these 2135*9e39c5baSBill Taylor * WQEs we have to account for the space consumed by the 2136*9e39c5baSBill Taylor * "inline" packet headers. (This is smaller than for QP1 2137*9e39c5baSBill Taylor * below because QP0 is not allowed to send packets with a GRH. 2138*9e39c5baSBill Taylor */ 2139*9e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_QP0_HDRS + (num_sgl << 4)); 2140*9e39c5baSBill Taylor log2 = highbit(max_size); 2141*9e39c5baSBill Taylor if ((max_size & (max_size - 1)) == 0) { 2142*9e39c5baSBill Taylor log2 = log2 - 1; 2143*9e39c5baSBill Taylor } 2144*9e39c5baSBill Taylor 2145*9e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 2146*9e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 2147*9e39c5baSBill Taylor 2148*9e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 2149*9e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP0_HDRS) >> 4; 2150*9e39c5baSBill Taylor break; 2151*9e39c5baSBill Taylor 2152*9e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDMLX_QP1: 2153*9e39c5baSBill Taylor /* 2154*9e39c5baSBill Taylor * Same as above. For these WQEs we again have to account for 2155*9e39c5baSBill Taylor * the space consumed by the "inline" packet headers. (This 2156*9e39c5baSBill Taylor * is larger than for QP0 above because we have to account for 2157*9e39c5baSBill Taylor * the possibility of a GRH in each packet - and this 2158*9e39c5baSBill Taylor * introduces an alignment issue that causes us to consume 2159*9e39c5baSBill Taylor * an additional 8 bytes). 2160*9e39c5baSBill Taylor */ 2161*9e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (num_sgl << 4)); 2162*9e39c5baSBill Taylor log2 = highbit(max_size); 2163*9e39c5baSBill Taylor if ((max_size & (max_size - 1)) == 0) { 2164*9e39c5baSBill Taylor log2 = log2 - 1; 2165*9e39c5baSBill Taylor } 2166*9e39c5baSBill Taylor 2167*9e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 2168*9e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 2169*9e39c5baSBill Taylor 2170*9e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 2171*9e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4; 2172*9e39c5baSBill Taylor break; 2173*9e39c5baSBill Taylor 2174*9e39c5baSBill Taylor default: 2175*9e39c5baSBill Taylor HERMON_WARNING(state, "unexpected work queue type"); 2176*9e39c5baSBill Taylor break; 2177*9e39c5baSBill Taylor } 2178*9e39c5baSBill Taylor 2179*9e39c5baSBill Taylor /* Fill in the return values */ 2180*9e39c5baSBill Taylor *logwqesz = log2; 2181*9e39c5baSBill Taylor *max_sgl = min(real_max_sgl, actual_sgl); 2182*9e39c5baSBill Taylor } 2183