19e39c5baSBill Taylor /* 29e39c5baSBill Taylor * CDDL HEADER START 39e39c5baSBill Taylor * 49e39c5baSBill Taylor * The contents of this file are subject to the terms of the 59e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 69e39c5baSBill Taylor * You may not use this file except in compliance with the License. 79e39c5baSBill Taylor * 89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 109e39c5baSBill Taylor * See the License for the specific language governing permissions 119e39c5baSBill Taylor * and limitations under the License. 129e39c5baSBill Taylor * 139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 189e39c5baSBill Taylor * 199e39c5baSBill Taylor * CDDL HEADER END 209e39c5baSBill Taylor */ 219e39c5baSBill Taylor 229e39c5baSBill Taylor /* 2317a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 249e39c5baSBill Taylor */ 259e39c5baSBill Taylor 269e39c5baSBill Taylor /* 279e39c5baSBill Taylor * hermon_qp.c 289e39c5baSBill Taylor * Hermon Queue Pair Processing Routines 299e39c5baSBill Taylor * 309e39c5baSBill Taylor * Implements all the routines necessary for allocating, freeing, and 319e39c5baSBill Taylor * querying the Hermon queue pairs. 329e39c5baSBill Taylor */ 339e39c5baSBill Taylor 349e39c5baSBill Taylor #include <sys/types.h> 359e39c5baSBill Taylor #include <sys/conf.h> 369e39c5baSBill Taylor #include <sys/ddi.h> 379e39c5baSBill Taylor #include <sys/sunddi.h> 389e39c5baSBill Taylor #include <sys/modctl.h> 399e39c5baSBill Taylor #include <sys/bitmap.h> 409e39c5baSBill Taylor #include <sys/sysmacros.h> 419e39c5baSBill Taylor 429e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 439e39c5baSBill Taylor #include <sys/ib/ib_pkt_hdrs.h> 449e39c5baSBill Taylor 459e39c5baSBill Taylor static int hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp, 469e39c5baSBill Taylor hermon_rsrc_t *qpc); 479e39c5baSBill Taylor static int hermon_qpn_avl_compare(const void *q, const void *e); 489e39c5baSBill Taylor static int hermon_special_qp_rsrc_alloc(hermon_state_t *state, 499e39c5baSBill Taylor ibt_sqp_type_t type, uint_t port, hermon_rsrc_t **qp_rsrc); 509e39c5baSBill Taylor static int hermon_special_qp_rsrc_free(hermon_state_t *state, 519e39c5baSBill Taylor ibt_sqp_type_t type, uint_t port); 529e39c5baSBill Taylor static void hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl, 539e39c5baSBill Taylor uint_t real_max_sgl, hermon_qp_wq_type_t wq_type, 549e39c5baSBill Taylor uint_t *logwqesz, uint_t *max_sgl); 559e39c5baSBill Taylor 569e39c5baSBill Taylor /* 579e39c5baSBill Taylor * hermon_qp_alloc() 589e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 599e39c5baSBill Taylor */ 609e39c5baSBill Taylor int 619e39c5baSBill Taylor hermon_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo, 629e39c5baSBill Taylor uint_t sleepflag) 639e39c5baSBill Taylor { 649e39c5baSBill Taylor hermon_rsrc_t *qpc, *rsrc; 6517a2b317SBill Taylor hermon_rsrc_type_t rsrc_type; 669e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 679e39c5baSBill Taylor hermon_qphdl_t qp; 689e39c5baSBill Taylor ibt_qp_alloc_attr_t *attr_p; 6917a2b317SBill Taylor ibt_qp_alloc_flags_t alloc_flags; 709e39c5baSBill Taylor ibt_qp_type_t type; 719e39c5baSBill Taylor hermon_qp_wq_type_t swq_type; 729e39c5baSBill Taylor ibtl_qp_hdl_t ibt_qphdl; 739e39c5baSBill Taylor ibt_chan_sizes_t *queuesz_p; 749e39c5baSBill Taylor ib_qpn_t *qpn; 759e39c5baSBill Taylor hermon_qphdl_t *qphdl; 769e39c5baSBill Taylor ibt_mr_attr_t mr_attr; 779e39c5baSBill Taylor hermon_mr_options_t mr_op; 789e39c5baSBill Taylor hermon_srqhdl_t srq; 799e39c5baSBill Taylor hermon_pdhdl_t pd; 809e39c5baSBill Taylor hermon_cqhdl_t sq_cq, rq_cq; 819e39c5baSBill Taylor hermon_mrhdl_t mr; 829e39c5baSBill Taylor uint64_t value, qp_desc_off; 839e39c5baSBill Taylor uint64_t *thewqe, thewqesz; 849e39c5baSBill Taylor uint32_t *sq_buf, *rq_buf; 859e39c5baSBill Taylor uint32_t log_qp_sq_size, log_qp_rq_size; 869e39c5baSBill Taylor uint32_t sq_size, rq_size; 879e39c5baSBill Taylor uint32_t sq_depth, rq_depth; 889e39c5baSBill Taylor uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift; 899e39c5baSBill Taylor uint32_t max_sgl, max_recv_sgl, uarpg; 909e39c5baSBill Taylor uint_t qp_is_umap; 919e39c5baSBill Taylor uint_t qp_srq_en, i, j; 929e39c5baSBill Taylor int status, flag; 939e39c5baSBill Taylor 949e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p)) 959e39c5baSBill Taylor 969e39c5baSBill Taylor /* 979e39c5baSBill Taylor * Extract the necessary info from the hermon_qp_info_t structure 989e39c5baSBill Taylor */ 999e39c5baSBill Taylor attr_p = qpinfo->qpi_attrp; 1009e39c5baSBill Taylor type = qpinfo->qpi_type; 1019e39c5baSBill Taylor ibt_qphdl = qpinfo->qpi_ibt_qphdl; 1029e39c5baSBill Taylor queuesz_p = qpinfo->qpi_queueszp; 1039e39c5baSBill Taylor qpn = qpinfo->qpi_qpn; 1049e39c5baSBill Taylor qphdl = &qpinfo->qpi_qphdl; 10517a2b317SBill Taylor alloc_flags = attr_p->qp_alloc_flags; 10617a2b317SBill Taylor 10717a2b317SBill Taylor /* 10817a2b317SBill Taylor * Verify correctness of alloc_flags. 10917a2b317SBill Taylor * 11017a2b317SBill Taylor * 1. FEXCH and RSS are only allocated via qp_range. 11117a2b317SBill Taylor */ 11217a2b317SBill Taylor if (alloc_flags & (IBT_QP_USES_FEXCH | IBT_QP_USES_RSS)) { 11317a2b317SBill Taylor return (IBT_INVALID_PARAM); 11417a2b317SBill Taylor } 11517a2b317SBill Taylor rsrc_type = HERMON_QPC; 11617a2b317SBill Taylor qp_is_umap = 0; 11717a2b317SBill Taylor 11817a2b317SBill Taylor /* 2. Make sure only one of these flags is set. */ 11917a2b317SBill Taylor switch (alloc_flags & 12017a2b317SBill Taylor (IBT_QP_USER_MAP | IBT_QP_USES_RFCI | IBT_QP_USES_FCMD)) { 12117a2b317SBill Taylor case IBT_QP_USER_MAP: 12217a2b317SBill Taylor qp_is_umap = 1; 12317a2b317SBill Taylor break; 12417a2b317SBill Taylor case IBT_QP_USES_RFCI: 12517a2b317SBill Taylor if (type != IBT_UD_RQP) 12617a2b317SBill Taylor return (IBT_INVALID_PARAM); 12717a2b317SBill Taylor 12817a2b317SBill Taylor switch (attr_p->qp_fc.fc_hca_port) { 12917a2b317SBill Taylor case 1: 13017a2b317SBill Taylor rsrc_type = HERMON_QPC_RFCI_PORT1; 13117a2b317SBill Taylor break; 13217a2b317SBill Taylor case 2: 13317a2b317SBill Taylor rsrc_type = HERMON_QPC_RFCI_PORT2; 13417a2b317SBill Taylor break; 13517a2b317SBill Taylor default: 13617a2b317SBill Taylor return (IBT_INVALID_PARAM); 13717a2b317SBill Taylor } 13817a2b317SBill Taylor break; 13917a2b317SBill Taylor case IBT_QP_USES_FCMD: 14017a2b317SBill Taylor if (type != IBT_UD_RQP) 14117a2b317SBill Taylor return (IBT_INVALID_PARAM); 14217a2b317SBill Taylor break; 14317a2b317SBill Taylor case 0: 14417a2b317SBill Taylor break; 14517a2b317SBill Taylor default: 14617a2b317SBill Taylor return (IBT_INVALID_PARAM); /* conflicting flags set */ 14717a2b317SBill Taylor } 1489e39c5baSBill Taylor 1499e39c5baSBill Taylor /* 1509e39c5baSBill Taylor * Determine whether QP is being allocated for userland access or 1519e39c5baSBill Taylor * whether it is being allocated for kernel access. If the QP is 1529e39c5baSBill Taylor * being allocated for userland access, then lookup the UAR 1539e39c5baSBill Taylor * page number for the current process. Note: If this is not found 1549e39c5baSBill Taylor * (e.g. if the process has not previously open()'d the Hermon driver), 1559e39c5baSBill Taylor * then an error is returned. 1569e39c5baSBill Taylor */ 1579e39c5baSBill Taylor if (qp_is_umap) { 1589e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(), 1599e39c5baSBill Taylor MLNX_UMAP_UARPG_RSRC, &value, 0, NULL); 1609e39c5baSBill Taylor if (status != DDI_SUCCESS) { 16117a2b317SBill Taylor return (IBT_INVALID_PARAM); 1629e39c5baSBill Taylor } 1639e39c5baSBill Taylor uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx; 1649e39c5baSBill Taylor } else { 1659e39c5baSBill Taylor uarpg = state->hs_kernel_uar_index; 1669e39c5baSBill Taylor } 1679e39c5baSBill Taylor 1689e39c5baSBill Taylor /* 1699e39c5baSBill Taylor * Determine whether QP is being associated with an SRQ 1709e39c5baSBill Taylor */ 17117a2b317SBill Taylor qp_srq_en = (alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0; 1729e39c5baSBill Taylor if (qp_srq_en) { 1739e39c5baSBill Taylor /* 1749e39c5baSBill Taylor * Check for valid SRQ handle pointers 1759e39c5baSBill Taylor */ 1769e39c5baSBill Taylor if (attr_p->qp_ibc_srq_hdl == NULL) { 1779e39c5baSBill Taylor status = IBT_SRQ_HDL_INVALID; 1789e39c5baSBill Taylor goto qpalloc_fail; 1799e39c5baSBill Taylor } 1809e39c5baSBill Taylor srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl; 1819e39c5baSBill Taylor } 1829e39c5baSBill Taylor 1839e39c5baSBill Taylor /* 1849e39c5baSBill Taylor * Check for valid QP service type (only UD/RC/UC supported) 1859e39c5baSBill Taylor */ 1869e39c5baSBill Taylor if (((type != IBT_UD_RQP) && (type != IBT_RC_RQP) && 1879e39c5baSBill Taylor (type != IBT_UC_RQP))) { 1889e39c5baSBill Taylor status = IBT_QP_SRV_TYPE_INVALID; 1899e39c5baSBill Taylor goto qpalloc_fail; 1909e39c5baSBill Taylor } 1919e39c5baSBill Taylor 1929e39c5baSBill Taylor 1939e39c5baSBill Taylor /* 1949e39c5baSBill Taylor * Check for valid PD handle pointer 1959e39c5baSBill Taylor */ 1969e39c5baSBill Taylor if (attr_p->qp_pd_hdl == NULL) { 1979e39c5baSBill Taylor status = IBT_PD_HDL_INVALID; 1989e39c5baSBill Taylor goto qpalloc_fail; 1999e39c5baSBill Taylor } 2009e39c5baSBill Taylor pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl; 2019e39c5baSBill Taylor 2029e39c5baSBill Taylor /* 2039e39c5baSBill Taylor * If on an SRQ, check to make sure the PD is the same 2049e39c5baSBill Taylor */ 2059e39c5baSBill Taylor if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) { 2069e39c5baSBill Taylor status = IBT_PD_HDL_INVALID; 2079e39c5baSBill Taylor goto qpalloc_fail; 2089e39c5baSBill Taylor } 2099e39c5baSBill Taylor 2109e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */ 2119e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 2129e39c5baSBill Taylor 2139e39c5baSBill Taylor /* 2149e39c5baSBill Taylor * Check for valid CQ handle pointers 21517a2b317SBill Taylor * 21617a2b317SBill Taylor * FCMD QPs do not require a receive cq handle. 2179e39c5baSBill Taylor */ 21817a2b317SBill Taylor if (attr_p->qp_ibc_scq_hdl == NULL) { 2199e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 2209e39c5baSBill Taylor goto qpalloc_fail1; 2219e39c5baSBill Taylor } 2229e39c5baSBill Taylor sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl; 22317a2b317SBill Taylor if ((attr_p->qp_ibc_rcq_hdl == NULL)) { 22417a2b317SBill Taylor if ((alloc_flags & IBT_QP_USES_FCMD) == 0) { 22517a2b317SBill Taylor status = IBT_CQ_HDL_INVALID; 22617a2b317SBill Taylor goto qpalloc_fail1; 22717a2b317SBill Taylor } 22817a2b317SBill Taylor rq_cq = sq_cq; /* just use the send cq */ 22917a2b317SBill Taylor } else 23017a2b317SBill Taylor rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl; 2319e39c5baSBill Taylor 2329e39c5baSBill Taylor /* 2339e39c5baSBill Taylor * Increment the reference count on the CQs. One or both of these 2349e39c5baSBill Taylor * could return error if we determine that the given CQ is already 2359e39c5baSBill Taylor * being used with a special (SMI/GSI) QP. 2369e39c5baSBill Taylor */ 2379e39c5baSBill Taylor status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL); 2389e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2399e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 2409e39c5baSBill Taylor goto qpalloc_fail1; 2419e39c5baSBill Taylor } 2429e39c5baSBill Taylor status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL); 2439e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2449e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 2459e39c5baSBill Taylor goto qpalloc_fail2; 2469e39c5baSBill Taylor } 2479e39c5baSBill Taylor 2489e39c5baSBill Taylor /* 2499e39c5baSBill Taylor * Allocate an QP context entry. This will be filled in with all 2509e39c5baSBill Taylor * the necessary parameters to define the Queue Pair. Unlike 2519e39c5baSBill Taylor * other Hermon hardware resources, ownership is not immediately 2529e39c5baSBill Taylor * given to hardware in the final step here. Instead, we must 2539e39c5baSBill Taylor * wait until the QP is later transitioned to the "Init" state before 2549e39c5baSBill Taylor * passing the QP to hardware. If we fail here, we must undo all 2559e39c5baSBill Taylor * the reference count (CQ and PD). 2569e39c5baSBill Taylor */ 25717a2b317SBill Taylor status = hermon_rsrc_alloc(state, rsrc_type, 1, sleepflag, &qpc); 2589e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2599e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 2609e39c5baSBill Taylor goto qpalloc_fail3; 2619e39c5baSBill Taylor } 2629e39c5baSBill Taylor 2639e39c5baSBill Taylor /* 2649e39c5baSBill Taylor * Allocate the software structure for tracking the queue pair 2659e39c5baSBill Taylor * (i.e. the Hermon Queue Pair handle). If we fail here, we must 2669e39c5baSBill Taylor * undo the reference counts and the previous resource allocation. 2679e39c5baSBill Taylor */ 2689e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc); 2699e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2709e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 2719e39c5baSBill Taylor goto qpalloc_fail4; 2729e39c5baSBill Taylor } 2739e39c5baSBill Taylor qp = (hermon_qphdl_t)rsrc->hr_addr; 2749e39c5baSBill Taylor bzero(qp, sizeof (struct hermon_sw_qp_s)); 2759e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp)) 2769e39c5baSBill Taylor 27717a2b317SBill Taylor qp->qp_alloc_flags = alloc_flags; 27817a2b317SBill Taylor 2799e39c5baSBill Taylor /* 2809e39c5baSBill Taylor * Calculate the QP number from QPC index. This routine handles 2819e39c5baSBill Taylor * all of the operations necessary to keep track of used, unused, 2829e39c5baSBill Taylor * and released QP numbers. 2839e39c5baSBill Taylor */ 28417a2b317SBill Taylor if (type == IBT_UD_RQP) { 28517a2b317SBill Taylor qp->qp_qpnum = qpc->hr_indx; 28617a2b317SBill Taylor qp->qp_ring = qp->qp_qpnum << 8; 28717a2b317SBill Taylor qp->qp_qpn_hdl = NULL; 28817a2b317SBill Taylor } else { 28917a2b317SBill Taylor status = hermon_qp_create_qpn(state, qp, qpc); 29017a2b317SBill Taylor if (status != DDI_SUCCESS) { 29117a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 29217a2b317SBill Taylor goto qpalloc_fail5; 29317a2b317SBill Taylor } 2949e39c5baSBill Taylor } 2959e39c5baSBill Taylor 2969e39c5baSBill Taylor /* 2979e39c5baSBill Taylor * If this will be a user-mappable QP, then allocate an entry for 2989e39c5baSBill Taylor * the "userland resources database". This will later be added to 2999e39c5baSBill Taylor * the database (after all further QP operations are successful). 3009e39c5baSBill Taylor * If we fail here, we must undo the reference counts and the 3019e39c5baSBill Taylor * previous resource allocation. 3029e39c5baSBill Taylor */ 3039e39c5baSBill Taylor if (qp_is_umap) { 3049e39c5baSBill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance, qp->qp_qpnum, 3059e39c5baSBill Taylor MLNX_UMAP_QPMEM_RSRC, (uint64_t)(uintptr_t)rsrc); 3069e39c5baSBill Taylor if (umapdb == NULL) { 3079e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 3089e39c5baSBill Taylor goto qpalloc_fail6; 3099e39c5baSBill Taylor } 3109e39c5baSBill Taylor } 3119e39c5baSBill Taylor 3129e39c5baSBill Taylor /* 3139e39c5baSBill Taylor * Allocate the doorbell record. Hermon just needs one for the RQ, 3149e39c5baSBill Taylor * if the QP is not associated with an SRQ, and use uarpg (above) as 3159e39c5baSBill Taylor * the uar index 3169e39c5baSBill Taylor */ 3179e39c5baSBill Taylor 3189e39c5baSBill Taylor if (!qp_srq_en) { 3199e39c5baSBill Taylor status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl, 3209e39c5baSBill Taylor &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset); 3219e39c5baSBill Taylor if (status != DDI_SUCCESS) { 3229e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 3239e39c5baSBill Taylor goto qpalloc_fail6; 3249e39c5baSBill Taylor } 3259e39c5baSBill Taylor } 3269e39c5baSBill Taylor 3279e39c5baSBill Taylor qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO); 3289e39c5baSBill Taylor 3299e39c5baSBill Taylor /* 3309e39c5baSBill Taylor * We verify that the requested number of SGL is valid (i.e. 3319e39c5baSBill Taylor * consistent with the device limits and/or software-configured 3329e39c5baSBill Taylor * limits). If not, then obviously the same cleanup needs to be done. 3339e39c5baSBill Taylor */ 3349e39c5baSBill Taylor if (type == IBT_UD_RQP) { 3359e39c5baSBill Taylor max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz; 3369e39c5baSBill Taylor swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD; 3379e39c5baSBill Taylor } else { 3389e39c5baSBill Taylor max_sgl = state->hs_ibtfinfo.hca_attr->hca_conn_send_sgl_sz; 3399e39c5baSBill Taylor swq_type = HERMON_QP_WQ_TYPE_SENDQ_CONN; 3409e39c5baSBill Taylor } 3419e39c5baSBill Taylor max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz; 3429e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) || 3439e39c5baSBill Taylor (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) { 3449e39c5baSBill Taylor status = IBT_HCA_SGL_EXCEEDED; 3459e39c5baSBill Taylor goto qpalloc_fail7; 3469e39c5baSBill Taylor } 3479e39c5baSBill Taylor 3489e39c5baSBill Taylor /* 3499e39c5baSBill Taylor * Determine this QP's WQE stride (for both the Send and Recv WQEs). 3509e39c5baSBill Taylor * This will depend on the requested number of SGLs. Note: this 3519e39c5baSBill Taylor * has the side-effect of also calculating the real number of SGLs 3529e39c5baSBill Taylor * (for the calculated WQE size). 3539e39c5baSBill Taylor * 3549e39c5baSBill Taylor * For QP's on an SRQ, we set these to 0. 3559e39c5baSBill Taylor */ 3569e39c5baSBill Taylor if (qp_srq_en) { 3579e39c5baSBill Taylor qp->qp_rq_log_wqesz = 0; 3589e39c5baSBill Taylor qp->qp_rq_sgl = 0; 3599e39c5baSBill Taylor } else { 3609e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl, 3619e39c5baSBill Taylor max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ, 3629e39c5baSBill Taylor &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl); 3639e39c5baSBill Taylor } 3649e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl, 3659e39c5baSBill Taylor max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl); 3669e39c5baSBill Taylor 3679e39c5baSBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 3689e39c5baSBill Taylor 3699e39c5baSBill Taylor /* NOTE: currently policy in driver, later maybe IBTF interface */ 3709e39c5baSBill Taylor qp->qp_no_prefetch = 0; 3719e39c5baSBill Taylor 3729e39c5baSBill Taylor /* 3739e39c5baSBill Taylor * for prefetching, we need to add the number of wqes in 3749e39c5baSBill Taylor * the 2k area plus one to the number requested, but 3759e39c5baSBill Taylor * ONLY for send queue. If no_prefetch == 1 (prefetch off) 3769e39c5baSBill Taylor * it's exactly TWO wqes for the headroom 3779e39c5baSBill Taylor */ 3789e39c5baSBill Taylor if (qp->qp_no_prefetch) 3799e39c5baSBill Taylor qp->qp_sq_headroom = 2 * sq_wqe_size; 3809e39c5baSBill Taylor else 3819e39c5baSBill Taylor qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE; 3829e39c5baSBill Taylor /* 3839e39c5baSBill Taylor * hdrm wqes must be integral since both sq_wqe_size & 3849e39c5baSBill Taylor * HERMON_QP_OH_SIZE are power of 2 3859e39c5baSBill Taylor */ 3869e39c5baSBill Taylor qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size); 3879e39c5baSBill Taylor 3889e39c5baSBill Taylor 3899e39c5baSBill Taylor /* 3909e39c5baSBill Taylor * Calculate the appropriate size for the work queues. 3919e39c5baSBill Taylor * For send queue, add in the headroom wqes to the calculation. 3929e39c5baSBill Taylor * Note: All Hermon QP work queues must be a power-of-2 in size. Also 3939e39c5baSBill Taylor * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is 3949e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2 3959e39c5baSBill Taylor */ 3969e39c5baSBill Taylor /* first, adjust to a minimum and tell the caller the change */ 3979e39c5baSBill Taylor attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq, 3989e39c5baSBill Taylor HERMON_QP_MIN_SIZE); 3999e39c5baSBill Taylor attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq, 4009e39c5baSBill Taylor HERMON_QP_MIN_SIZE); 4019e39c5baSBill Taylor /* 4029e39c5baSBill Taylor * now, calculate the alloc size, taking into account 4039e39c5baSBill Taylor * the headroom for the sq 4049e39c5baSBill Taylor */ 4059e39c5baSBill Taylor log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes); 4069e39c5baSBill Taylor /* if the total is a power of two, reduce it */ 407*de710d24SJosef 'Jeff' Sipek if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes)) { 4089e39c5baSBill Taylor log_qp_sq_size = log_qp_sq_size - 1; 4099e39c5baSBill Taylor } 4109e39c5baSBill Taylor 4119e39c5baSBill Taylor log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq); 412*de710d24SJosef 'Jeff' Sipek if (ISP2(attr_p->qp_sizes.cs_rq)) { 4139e39c5baSBill Taylor log_qp_rq_size = log_qp_rq_size - 1; 4149e39c5baSBill Taylor } 4159e39c5baSBill Taylor 4169e39c5baSBill Taylor /* 4179e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent 4189e39c5baSBill Taylor * with the device limits and/or software-configured limits). If not, 4199e39c5baSBill Taylor * then obviously we have a lot of cleanup to do before returning. 4209e39c5baSBill Taylor * 4219e39c5baSBill Taylor * NOTE: the first condition deals with the (test) case of cs_sq 4229e39c5baSBill Taylor * being just less than 2^32. In this case, the headroom addition 4239e39c5baSBill Taylor * to the requested cs_sq will pass the test when it should not. 4249e39c5baSBill Taylor * This test no longer lets that case slip through the check. 4259e39c5baSBill Taylor */ 4269e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_sq > 4279e39c5baSBill Taylor (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) || 4289e39c5baSBill Taylor (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) || 4299e39c5baSBill Taylor (!qp_srq_en && (log_qp_rq_size > 4309e39c5baSBill Taylor state->hs_cfg_profile->cp_log_max_qp_sz))) { 4319e39c5baSBill Taylor status = IBT_HCA_WR_EXCEEDED; 4329e39c5baSBill Taylor goto qpalloc_fail7; 4339e39c5baSBill Taylor } 4349e39c5baSBill Taylor 4359e39c5baSBill Taylor /* 4369e39c5baSBill Taylor * Allocate the memory for QP work queues. Since Hermon work queues 4379e39c5baSBill Taylor * are not allowed to cross a 32-bit (4GB) boundary, the alignment of 4389e39c5baSBill Taylor * the work queue memory is very important. We used to allocate 4399e39c5baSBill Taylor * work queues (the combined receive and send queues) so that they 4409e39c5baSBill Taylor * would be aligned on their combined size. That alignment guaranteed 4419e39c5baSBill Taylor * that they would never cross the 4GB boundary (Hermon work queues 4429e39c5baSBill Taylor * are on the order of MBs at maximum). Now we are able to relax 4439e39c5baSBill Taylor * this alignment constraint by ensuring that the IB address assigned 4449e39c5baSBill Taylor * to the queue memory (as a result of the hermon_mr_register() call) 4459e39c5baSBill Taylor * is offset from zero. 4469e39c5baSBill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to 4479e39c5baSBill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass 4489e39c5baSBill Taylor * mode we found that we were not allowed to specify any alignment 4499e39c5baSBill Taylor * that was more restrictive than the system page size. 4509e39c5baSBill Taylor * So we avoided this constraint by passing two alignment values, 4519e39c5baSBill Taylor * one for the memory allocation itself and the other for the DMA 4529e39c5baSBill Taylor * handle (for later bind). This used to cause more memory than 4539e39c5baSBill Taylor * necessary to be allocated (in order to guarantee the more 4549e39c5baSBill Taylor * restrictive alignment contraint). But by guaranteeing the 4559e39c5baSBill Taylor * zero-based IB virtual address for the queue, we are able to 4569e39c5baSBill Taylor * conserve this memory. 4579e39c5baSBill Taylor */ 4589e39c5baSBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 4599e39c5baSBill Taylor sq_depth = 1 << log_qp_sq_size; 4609e39c5baSBill Taylor sq_size = sq_depth * sq_wqe_size; 4619e39c5baSBill Taylor 4629e39c5baSBill Taylor /* QP on SRQ sets these to 0 */ 4639e39c5baSBill Taylor if (qp_srq_en) { 4649e39c5baSBill Taylor rq_wqe_size = 0; 4659e39c5baSBill Taylor rq_size = 0; 4669e39c5baSBill Taylor } else { 4679e39c5baSBill Taylor rq_wqe_size = 1 << qp->qp_rq_log_wqesz; 4689e39c5baSBill Taylor rq_depth = 1 << log_qp_rq_size; 4699e39c5baSBill Taylor rq_size = rq_depth * rq_wqe_size; 4709e39c5baSBill Taylor } 4719e39c5baSBill Taylor 4729e39c5baSBill Taylor qp->qp_wqinfo.qa_size = sq_size + rq_size; 4739e39c5baSBill Taylor 4749e39c5baSBill Taylor qp->qp_wqinfo.qa_alloc_align = PAGESIZE; 4759e39c5baSBill Taylor qp->qp_wqinfo.qa_bind_align = PAGESIZE; 4769e39c5baSBill Taylor 4779e39c5baSBill Taylor if (qp_is_umap) { 4789e39c5baSBill Taylor qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND; 4799e39c5baSBill Taylor } else { 4809e39c5baSBill Taylor qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL; 4819e39c5baSBill Taylor } 4829e39c5baSBill Taylor status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag); 4839e39c5baSBill Taylor if (status != DDI_SUCCESS) { 4849e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 4859e39c5baSBill Taylor goto qpalloc_fail7; 4869e39c5baSBill Taylor } 4879e39c5baSBill Taylor 4889e39c5baSBill Taylor /* 4899e39c5baSBill Taylor * Sort WQs in memory according to stride (*q_wqe_size), largest first 4909e39c5baSBill Taylor * If they are equal, still put the SQ first 4919e39c5baSBill Taylor */ 4929e39c5baSBill Taylor qp->qp_sq_baseaddr = 0; 4939e39c5baSBill Taylor qp->qp_rq_baseaddr = 0; 4949e39c5baSBill Taylor if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) { 4959e39c5baSBill Taylor sq_buf = qp->qp_wqinfo.qa_buf_aligned; 4969e39c5baSBill Taylor 4979e39c5baSBill Taylor /* if this QP is on an SRQ, set the rq_buf to NULL */ 4989e39c5baSBill Taylor if (qp_srq_en) { 4999e39c5baSBill Taylor rq_buf = NULL; 5009e39c5baSBill Taylor } else { 5019e39c5baSBill Taylor rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size); 5029e39c5baSBill Taylor qp->qp_rq_baseaddr = sq_size; 5039e39c5baSBill Taylor } 5049e39c5baSBill Taylor } else { 5059e39c5baSBill Taylor rq_buf = qp->qp_wqinfo.qa_buf_aligned; 5069e39c5baSBill Taylor sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size); 5079e39c5baSBill Taylor qp->qp_sq_baseaddr = rq_size; 5089e39c5baSBill Taylor } 5099e39c5baSBill Taylor 5109e39c5baSBill Taylor if (qp_is_umap == 0) { 5119e39c5baSBill Taylor qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth); 5129e39c5baSBill Taylor if (qp->qp_sq_wqhdr == NULL) { 5139e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 5149e39c5baSBill Taylor goto qpalloc_fail8; 5159e39c5baSBill Taylor } 5169e39c5baSBill Taylor if (qp_srq_en) { 5179e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr; 5189e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_srq_en = 1; 5199e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_srq = srq; 5209e39c5baSBill Taylor } else { 5219e39c5baSBill Taylor qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth); 5229e39c5baSBill Taylor if (qp->qp_rq_wqhdr == NULL) { 5239e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 5249e39c5baSBill Taylor goto qpalloc_fail8; 5259e39c5baSBill Taylor } 5269e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr; 5279e39c5baSBill Taylor } 5289e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum; 5299e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND; 5309e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr; 5319e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum; 5329e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV; 5339e39c5baSBill Taylor } 5349e39c5baSBill Taylor 5359e39c5baSBill Taylor /* 5369e39c5baSBill Taylor * Register the memory for the QP work queues. The memory for the 5379e39c5baSBill Taylor * QP must be registered in the Hermon cMPT tables. This gives us the 5389e39c5baSBill Taylor * LKey to specify in the QP context later. Note: The memory for 5399e39c5baSBill Taylor * Hermon work queues (both Send and Recv) must be contiguous and 5409e39c5baSBill Taylor * registered as a single memory region. Note: If the QP memory is 5419e39c5baSBill Taylor * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to 5429e39c5baSBill Taylor * meet the alignment restriction, we pass the "mro_bind_override_addr" 5439e39c5baSBill Taylor * flag in the call to hermon_mr_register(). This guarantees that the 5449e39c5baSBill Taylor * resulting IB vaddr will be zero-based (modulo the offset into the 5459e39c5baSBill Taylor * first page). If we fail here, we still have the bunch of resource 5469e39c5baSBill Taylor * and reference count cleanup to do. 5479e39c5baSBill Taylor */ 5489e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : 5499e39c5baSBill Taylor IBT_MR_NOSLEEP; 5509e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned; 5519e39c5baSBill Taylor mr_attr.mr_len = qp->qp_wqinfo.qa_size; 5529e39c5baSBill Taylor mr_attr.mr_as = NULL; 5539e39c5baSBill Taylor mr_attr.mr_flags = flag; 5549e39c5baSBill Taylor if (qp_is_umap) { 5559e39c5baSBill Taylor mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass; 5569e39c5baSBill Taylor } else { 5579e39c5baSBill Taylor /* HERMON_QUEUE_LOCATION_NORMAL */ 5589e39c5baSBill Taylor mr_op.mro_bind_type = 5599e39c5baSBill Taylor state->hs_cfg_profile->cp_iommu_bypass; 5609e39c5baSBill Taylor } 5619e39c5baSBill Taylor mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl; 5629e39c5baSBill Taylor mr_op.mro_bind_override_addr = 1; 5639e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, 5649e39c5baSBill Taylor &mr_op, HERMON_QP_CMPT); 5659e39c5baSBill Taylor if (status != DDI_SUCCESS) { 5669e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 5679e39c5baSBill Taylor goto qpalloc_fail9; 5689e39c5baSBill Taylor } 5699e39c5baSBill Taylor 5709e39c5baSBill Taylor /* 5719e39c5baSBill Taylor * Calculate the offset between the kernel virtual address space 5729e39c5baSBill Taylor * and the IB virtual address space. This will be used when 5739e39c5baSBill Taylor * posting work requests to properly initialize each WQE. 5749e39c5baSBill Taylor */ 5759e39c5baSBill Taylor qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned - 5769e39c5baSBill Taylor (uint64_t)mr->mr_bindinfo.bi_addr; 5779e39c5baSBill Taylor 5789e39c5baSBill Taylor /* 5799e39c5baSBill Taylor * Fill in all the return arguments (if necessary). This includes 5809e39c5baSBill Taylor * real work queue sizes (in wqes), real SGLs, and QP number 5819e39c5baSBill Taylor */ 5829e39c5baSBill Taylor if (queuesz_p != NULL) { 5839e39c5baSBill Taylor queuesz_p->cs_sq = 5849e39c5baSBill Taylor (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes; 5859e39c5baSBill Taylor queuesz_p->cs_sq_sgl = qp->qp_sq_sgl; 5869e39c5baSBill Taylor 5879e39c5baSBill Taylor /* if this QP is on an SRQ, set these to 0 */ 5889e39c5baSBill Taylor if (qp_srq_en) { 5899e39c5baSBill Taylor queuesz_p->cs_rq = 0; 5909e39c5baSBill Taylor queuesz_p->cs_rq_sgl = 0; 5919e39c5baSBill Taylor } else { 5929e39c5baSBill Taylor queuesz_p->cs_rq = (1 << log_qp_rq_size); 5939e39c5baSBill Taylor queuesz_p->cs_rq_sgl = qp->qp_rq_sgl; 5949e39c5baSBill Taylor } 5959e39c5baSBill Taylor } 5969e39c5baSBill Taylor if (qpn != NULL) { 5979e39c5baSBill Taylor *qpn = (ib_qpn_t)qp->qp_qpnum; 5989e39c5baSBill Taylor } 5999e39c5baSBill Taylor 6009e39c5baSBill Taylor /* 6019e39c5baSBill Taylor * Fill in the rest of the Hermon Queue Pair handle. 6029e39c5baSBill Taylor */ 6039e39c5baSBill Taylor qp->qp_qpcrsrcp = qpc; 6049e39c5baSBill Taylor qp->qp_rsrcp = rsrc; 6059e39c5baSBill Taylor qp->qp_state = HERMON_QP_RESET; 60617a2b317SBill Taylor HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET); 6079e39c5baSBill Taylor qp->qp_pdhdl = pd; 6089e39c5baSBill Taylor qp->qp_mrhdl = mr; 6099e39c5baSBill Taylor qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ? 6109e39c5baSBill Taylor HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED; 6119e39c5baSBill Taylor qp->qp_is_special = 0; 6129e39c5baSBill Taylor qp->qp_uarpg = uarpg; 6139e39c5baSBill Taylor qp->qp_umap_dhp = (devmap_cookie_t)NULL; 6149e39c5baSBill Taylor qp->qp_sq_cqhdl = sq_cq; 6159e39c5baSBill Taylor qp->qp_sq_bufsz = (1 << log_qp_sq_size); 6169e39c5baSBill Taylor qp->qp_sq_logqsz = log_qp_sq_size; 6179e39c5baSBill Taylor qp->qp_sq_buf = sq_buf; 6189e39c5baSBill Taylor qp->qp_desc_off = qp_desc_off; 6199e39c5baSBill Taylor qp->qp_rq_cqhdl = rq_cq; 6209e39c5baSBill Taylor qp->qp_rq_buf = rq_buf; 6219e39c5baSBill Taylor qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) != 6229e39c5baSBill Taylor 0; 6239e39c5baSBill Taylor 6249e39c5baSBill Taylor /* if this QP is on an SRQ, set rq_bufsz to 0 */ 6259e39c5baSBill Taylor if (qp_srq_en) { 6269e39c5baSBill Taylor qp->qp_rq_bufsz = 0; 6279e39c5baSBill Taylor qp->qp_rq_logqsz = 0; 6289e39c5baSBill Taylor } else { 6299e39c5baSBill Taylor qp->qp_rq_bufsz = (1 << log_qp_rq_size); 6309e39c5baSBill Taylor qp->qp_rq_logqsz = log_qp_rq_size; 6319e39c5baSBill Taylor } 6329e39c5baSBill Taylor 6339e39c5baSBill Taylor qp->qp_forward_sqd_event = 0; 6349e39c5baSBill Taylor qp->qp_sqd_still_draining = 0; 6359e39c5baSBill Taylor qp->qp_hdlrarg = (void *)ibt_qphdl; 6369e39c5baSBill Taylor qp->qp_mcg_refcnt = 0; 6379e39c5baSBill Taylor 6389e39c5baSBill Taylor /* 6399e39c5baSBill Taylor * If this QP is to be associated with an SRQ, set the SRQ handle 6409e39c5baSBill Taylor */ 6419e39c5baSBill Taylor if (qp_srq_en) { 6429e39c5baSBill Taylor qp->qp_srqhdl = srq; 6439e39c5baSBill Taylor hermon_srq_refcnt_inc(qp->qp_srqhdl); 6449e39c5baSBill Taylor } else { 6459e39c5baSBill Taylor qp->qp_srqhdl = NULL; 6469e39c5baSBill Taylor } 6479e39c5baSBill Taylor 6489e39c5baSBill Taylor /* Determine the QP service type */ 64917a2b317SBill Taylor qp->qp_type = type; 6509e39c5baSBill Taylor if (type == IBT_RC_RQP) { 6519e39c5baSBill Taylor qp->qp_serv_type = HERMON_QP_RC; 6529e39c5baSBill Taylor } else if (type == IBT_UD_RQP) { 65317a2b317SBill Taylor if (alloc_flags & IBT_QP_USES_RFCI) 65417a2b317SBill Taylor qp->qp_serv_type = HERMON_QP_RFCI; 65517a2b317SBill Taylor else if (alloc_flags & IBT_QP_USES_FCMD) 65617a2b317SBill Taylor qp->qp_serv_type = HERMON_QP_FCMND; 65717a2b317SBill Taylor else 65817a2b317SBill Taylor qp->qp_serv_type = HERMON_QP_UD; 6599e39c5baSBill Taylor } else { 6609e39c5baSBill Taylor qp->qp_serv_type = HERMON_QP_UC; 6619e39c5baSBill Taylor } 6629e39c5baSBill Taylor 6639e39c5baSBill Taylor /* 6649e39c5baSBill Taylor * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed 6659e39c5baSBill Taylor */ 6669e39c5baSBill Taylor 6679e39c5baSBill Taylor /* 6689e39c5baSBill Taylor * Initialize the SQ WQEs - all that needs to be done is every 64 bytes 6699e39c5baSBill Taylor * set the quadword to all F's - high-order bit is owner (init to one) 6709e39c5baSBill Taylor * and the rest for the headroom definition of prefetching 6719e39c5baSBill Taylor * 6729e39c5baSBill Taylor */ 6739e39c5baSBill Taylor wqesz_shift = qp->qp_sq_log_wqesz; 6749e39c5baSBill Taylor thewqesz = 1 << wqesz_shift; 6759e39c5baSBill Taylor thewqe = (uint64_t *)(void *)(qp->qp_sq_buf); 6769e39c5baSBill Taylor if (qp_is_umap == 0) { 6779e39c5baSBill Taylor for (i = 0; i < sq_depth; i++) { 6789e39c5baSBill Taylor /* 6799e39c5baSBill Taylor * for each stride, go through and every 64 bytes 6809e39c5baSBill Taylor * write the init value - having set the address 6819e39c5baSBill Taylor * once, just keep incrementing it 6829e39c5baSBill Taylor */ 6839e39c5baSBill Taylor for (j = 0; j < thewqesz; j += 64, thewqe += 8) { 6849e39c5baSBill Taylor *(uint32_t *)thewqe = 0xFFFFFFFF; 6859e39c5baSBill Taylor } 6869e39c5baSBill Taylor } 6879e39c5baSBill Taylor } 6889e39c5baSBill Taylor 6899e39c5baSBill Taylor /* Zero out the QP context */ 6909e39c5baSBill Taylor bzero(&qp->qpc, sizeof (hermon_hw_qpc_t)); 6919e39c5baSBill Taylor 6929e39c5baSBill Taylor /* 6939e39c5baSBill Taylor * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the 6949e39c5baSBill Taylor * "qphdl" and return success 6959e39c5baSBill Taylor */ 69617a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx, qp); 6979e39c5baSBill Taylor 6989e39c5baSBill Taylor /* 6999e39c5baSBill Taylor * If this is a user-mappable QP, then we need to insert the previously 7009e39c5baSBill Taylor * allocated entry into the "userland resources database". This will 7019e39c5baSBill Taylor * allow for later lookup during devmap() (i.e. mmap()) calls. 7029e39c5baSBill Taylor */ 7039e39c5baSBill Taylor if (qp_is_umap) { 7049e39c5baSBill Taylor hermon_umap_db_add(umapdb); 7059e39c5baSBill Taylor } 7069e39c5baSBill Taylor mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER, 7079e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 7089e39c5baSBill Taylor 7099e39c5baSBill Taylor *qphdl = qp; 7109e39c5baSBill Taylor 7119e39c5baSBill Taylor return (DDI_SUCCESS); 7129e39c5baSBill Taylor 7139e39c5baSBill Taylor /* 7149e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 7159e39c5baSBill Taylor */ 7169e39c5baSBill Taylor qpalloc_fail9: 7179e39c5baSBill Taylor hermon_queue_free(&qp->qp_wqinfo); 7189e39c5baSBill Taylor qpalloc_fail8: 7199e39c5baSBill Taylor if (qp->qp_sq_wqhdr) 7209e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr); 7219e39c5baSBill Taylor if (qp->qp_rq_wqhdr) 7229e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr); 7239e39c5baSBill Taylor qpalloc_fail7: 7249e39c5baSBill Taylor if (qp_is_umap) { 7259e39c5baSBill Taylor hermon_umap_db_free(umapdb); 7269e39c5baSBill Taylor } 7279e39c5baSBill Taylor if (!qp_srq_en) { 7289e39c5baSBill Taylor hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr); 7299e39c5baSBill Taylor } 7309e39c5baSBill Taylor 7319e39c5baSBill Taylor qpalloc_fail6: 7329e39c5baSBill Taylor /* 7339e39c5baSBill Taylor * Releasing the QPN will also free up the QPC context. Update 7349e39c5baSBill Taylor * the QPC context pointer to indicate this. 7359e39c5baSBill Taylor */ 73617a2b317SBill Taylor if (qp->qp_qpn_hdl) { 73717a2b317SBill Taylor hermon_qp_release_qpn(state, qp->qp_qpn_hdl, 73817a2b317SBill Taylor HERMON_QPN_RELEASE); 73917a2b317SBill Taylor } else { 74017a2b317SBill Taylor hermon_rsrc_free(state, &qpc); 74117a2b317SBill Taylor } 7429e39c5baSBill Taylor qpc = NULL; 7439e39c5baSBill Taylor qpalloc_fail5: 7449e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 7459e39c5baSBill Taylor qpalloc_fail4: 7469e39c5baSBill Taylor if (qpc) { 7479e39c5baSBill Taylor hermon_rsrc_free(state, &qpc); 7489e39c5baSBill Taylor } 7499e39c5baSBill Taylor qpalloc_fail3: 7509e39c5baSBill Taylor hermon_cq_refcnt_dec(rq_cq); 7519e39c5baSBill Taylor qpalloc_fail2: 7529e39c5baSBill Taylor hermon_cq_refcnt_dec(sq_cq); 7539e39c5baSBill Taylor qpalloc_fail1: 7549e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 7559e39c5baSBill Taylor qpalloc_fail: 7569e39c5baSBill Taylor return (status); 7579e39c5baSBill Taylor } 7589e39c5baSBill Taylor 7599e39c5baSBill Taylor 7609e39c5baSBill Taylor 7619e39c5baSBill Taylor /* 7629e39c5baSBill Taylor * hermon_special_qp_alloc() 7639e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 7649e39c5baSBill Taylor */ 7659e39c5baSBill Taylor int 7669e39c5baSBill Taylor hermon_special_qp_alloc(hermon_state_t *state, hermon_qp_info_t *qpinfo, 7679e39c5baSBill Taylor uint_t sleepflag) 7689e39c5baSBill Taylor { 7699e39c5baSBill Taylor hermon_rsrc_t *qpc, *rsrc; 7709e39c5baSBill Taylor hermon_qphdl_t qp; 7719e39c5baSBill Taylor ibt_qp_alloc_attr_t *attr_p; 7729e39c5baSBill Taylor ibt_sqp_type_t type; 7739e39c5baSBill Taylor uint8_t port; 7749e39c5baSBill Taylor ibtl_qp_hdl_t ibt_qphdl; 7759e39c5baSBill Taylor ibt_chan_sizes_t *queuesz_p; 7769e39c5baSBill Taylor hermon_qphdl_t *qphdl; 7779e39c5baSBill Taylor ibt_mr_attr_t mr_attr; 7789e39c5baSBill Taylor hermon_mr_options_t mr_op; 7799e39c5baSBill Taylor hermon_pdhdl_t pd; 7809e39c5baSBill Taylor hermon_cqhdl_t sq_cq, rq_cq; 7819e39c5baSBill Taylor hermon_mrhdl_t mr; 7829e39c5baSBill Taylor uint64_t qp_desc_off; 7839e39c5baSBill Taylor uint64_t *thewqe, thewqesz; 7849e39c5baSBill Taylor uint32_t *sq_buf, *rq_buf; 7859e39c5baSBill Taylor uint32_t log_qp_sq_size, log_qp_rq_size; 7869e39c5baSBill Taylor uint32_t sq_size, rq_size, max_sgl; 7879e39c5baSBill Taylor uint32_t uarpg; 7889e39c5baSBill Taylor uint32_t sq_depth; 7899e39c5baSBill Taylor uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift; 7909e39c5baSBill Taylor int status, flag, i, j; 7919e39c5baSBill Taylor 7929e39c5baSBill Taylor /* 7939e39c5baSBill Taylor * Extract the necessary info from the hermon_qp_info_t structure 7949e39c5baSBill Taylor */ 7959e39c5baSBill Taylor attr_p = qpinfo->qpi_attrp; 7969e39c5baSBill Taylor type = qpinfo->qpi_type; 7979e39c5baSBill Taylor port = qpinfo->qpi_port; 7989e39c5baSBill Taylor ibt_qphdl = qpinfo->qpi_ibt_qphdl; 7999e39c5baSBill Taylor queuesz_p = qpinfo->qpi_queueszp; 8009e39c5baSBill Taylor qphdl = &qpinfo->qpi_qphdl; 8019e39c5baSBill Taylor 8029e39c5baSBill Taylor /* 8039e39c5baSBill Taylor * Check for valid special QP type (only SMI & GSI supported) 8049e39c5baSBill Taylor */ 8059e39c5baSBill Taylor if ((type != IBT_SMI_SQP) && (type != IBT_GSI_SQP)) { 8069e39c5baSBill Taylor status = IBT_QP_SPECIAL_TYPE_INVALID; 8079e39c5baSBill Taylor goto spec_qpalloc_fail; 8089e39c5baSBill Taylor } 8099e39c5baSBill Taylor 8109e39c5baSBill Taylor /* 8119e39c5baSBill Taylor * Check for valid port number 8129e39c5baSBill Taylor */ 8139e39c5baSBill Taylor if (!hermon_portnum_is_valid(state, port)) { 8149e39c5baSBill Taylor status = IBT_HCA_PORT_INVALID; 8159e39c5baSBill Taylor goto spec_qpalloc_fail; 8169e39c5baSBill Taylor } 8179e39c5baSBill Taylor port = port - 1; 8189e39c5baSBill Taylor 8199e39c5baSBill Taylor /* 8209e39c5baSBill Taylor * Check for valid PD handle pointer 8219e39c5baSBill Taylor */ 8229e39c5baSBill Taylor if (attr_p->qp_pd_hdl == NULL) { 8239e39c5baSBill Taylor status = IBT_PD_HDL_INVALID; 8249e39c5baSBill Taylor goto spec_qpalloc_fail; 8259e39c5baSBill Taylor } 8269e39c5baSBill Taylor pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl; 8279e39c5baSBill Taylor 8289e39c5baSBill Taylor /* Increment the reference count on the PD */ 8299e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 8309e39c5baSBill Taylor 8319e39c5baSBill Taylor /* 8329e39c5baSBill Taylor * Check for valid CQ handle pointers 8339e39c5baSBill Taylor */ 8349e39c5baSBill Taylor if ((attr_p->qp_ibc_scq_hdl == NULL) || 8359e39c5baSBill Taylor (attr_p->qp_ibc_rcq_hdl == NULL)) { 8369e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 8379e39c5baSBill Taylor goto spec_qpalloc_fail1; 8389e39c5baSBill Taylor } 8399e39c5baSBill Taylor sq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_scq_hdl; 8409e39c5baSBill Taylor rq_cq = (hermon_cqhdl_t)attr_p->qp_ibc_rcq_hdl; 8419e39c5baSBill Taylor 8429e39c5baSBill Taylor /* 8439e39c5baSBill Taylor * Increment the reference count on the CQs. One or both of these 8449e39c5baSBill Taylor * could return error if we determine that the given CQ is already 8459e39c5baSBill Taylor * being used with a non-special QP (i.e. a normal QP). 8469e39c5baSBill Taylor */ 8479e39c5baSBill Taylor status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_SPECIAL); 8489e39c5baSBill Taylor if (status != DDI_SUCCESS) { 8499e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 8509e39c5baSBill Taylor goto spec_qpalloc_fail1; 8519e39c5baSBill Taylor } 8529e39c5baSBill Taylor status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_SPECIAL); 8539e39c5baSBill Taylor if (status != DDI_SUCCESS) { 8549e39c5baSBill Taylor status = IBT_CQ_HDL_INVALID; 8559e39c5baSBill Taylor goto spec_qpalloc_fail2; 8569e39c5baSBill Taylor } 8579e39c5baSBill Taylor 8589e39c5baSBill Taylor /* 8599e39c5baSBill Taylor * Allocate the special QP resources. Essentially, this allocation 8609e39c5baSBill Taylor * amounts to checking if the request special QP has already been 8619e39c5baSBill Taylor * allocated. If successful, the QP context return is an actual 8629e39c5baSBill Taylor * QP context that has been "aliased" to act as a special QP of the 8639e39c5baSBill Taylor * appropriate type (and for the appropriate port). Just as in 8649e39c5baSBill Taylor * hermon_qp_alloc() above, ownership for this QP context is not 8659e39c5baSBill Taylor * immediately given to hardware in the final step here. Instead, we 8669e39c5baSBill Taylor * wait until the QP is later transitioned to the "Init" state before 8679e39c5baSBill Taylor * passing the QP to hardware. If we fail here, we must undo all 8689e39c5baSBill Taylor * the reference count (CQ and PD). 8699e39c5baSBill Taylor */ 8709e39c5baSBill Taylor status = hermon_special_qp_rsrc_alloc(state, type, port, &qpc); 8719e39c5baSBill Taylor if (status != DDI_SUCCESS) { 8729e39c5baSBill Taylor goto spec_qpalloc_fail3; 8739e39c5baSBill Taylor } 8749e39c5baSBill Taylor 8759e39c5baSBill Taylor /* 8769e39c5baSBill Taylor * Allocate the software structure for tracking the special queue 8779e39c5baSBill Taylor * pair (i.e. the Hermon Queue Pair handle). If we fail here, we 8789e39c5baSBill Taylor * must undo the reference counts and the previous resource allocation. 8799e39c5baSBill Taylor */ 8809e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc); 8819e39c5baSBill Taylor if (status != DDI_SUCCESS) { 8829e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 8839e39c5baSBill Taylor goto spec_qpalloc_fail4; 8849e39c5baSBill Taylor } 8859e39c5baSBill Taylor qp = (hermon_qphdl_t)rsrc->hr_addr; 8869e39c5baSBill Taylor 8879e39c5baSBill Taylor bzero(qp, sizeof (struct hermon_sw_qp_s)); 8889e39c5baSBill Taylor 8899e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp)) 89017a2b317SBill Taylor qp->qp_alloc_flags = attr_p->qp_alloc_flags; 8919e39c5baSBill Taylor 8929e39c5baSBill Taylor /* 8939e39c5baSBill Taylor * Actual QP number is a combination of the index of the QPC and 8949e39c5baSBill Taylor * the port number. This is because the special QP contexts must 8959e39c5baSBill Taylor * be allocated two-at-a-time. 8969e39c5baSBill Taylor */ 8979e39c5baSBill Taylor qp->qp_qpnum = qpc->hr_indx + port; 8989e39c5baSBill Taylor qp->qp_ring = qp->qp_qpnum << 8; 8999e39c5baSBill Taylor 9009e39c5baSBill Taylor uarpg = state->hs_kernel_uar_index; /* must be for spec qp */ 9019e39c5baSBill Taylor /* 9029e39c5baSBill Taylor * Allocate the doorbell record. Hermon uses only one for the RQ so 9039e39c5baSBill Taylor * alloc a qp doorbell, using uarpg (above) as the uar index 9049e39c5baSBill Taylor */ 9059e39c5baSBill Taylor 9069e39c5baSBill Taylor status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl, 9079e39c5baSBill Taylor &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset); 9089e39c5baSBill Taylor if (status != DDI_SUCCESS) { 9099e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 9109e39c5baSBill Taylor goto spec_qpalloc_fail5; 9119e39c5baSBill Taylor } 9129e39c5baSBill Taylor /* 9139e39c5baSBill Taylor * Calculate the appropriate size for the work queues. 9149e39c5baSBill Taylor * Note: All Hermon QP work queues must be a power-of-2 in size. Also 9159e39c5baSBill Taylor * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is 9169e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2 9179e39c5baSBill Taylor */ 9189e39c5baSBill Taylor attr_p->qp_sizes.cs_sq = 9199e39c5baSBill Taylor max(attr_p->qp_sizes.cs_sq, HERMON_QP_MIN_SIZE); 9209e39c5baSBill Taylor attr_p->qp_sizes.cs_rq = 9219e39c5baSBill Taylor max(attr_p->qp_sizes.cs_rq, HERMON_QP_MIN_SIZE); 9229e39c5baSBill Taylor log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq); 923*de710d24SJosef 'Jeff' Sipek if (ISP2(attr_p->qp_sizes.cs_sq)) { 9249e39c5baSBill Taylor log_qp_sq_size = log_qp_sq_size - 1; 9259e39c5baSBill Taylor } 9269e39c5baSBill Taylor log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq); 927*de710d24SJosef 'Jeff' Sipek if (ISP2(attr_p->qp_sizes.cs_rq)) { 9289e39c5baSBill Taylor log_qp_rq_size = log_qp_rq_size - 1; 9299e39c5baSBill Taylor } 9309e39c5baSBill Taylor 9319e39c5baSBill Taylor /* 9329e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent 9339e39c5baSBill Taylor * with the device limits and/or software-configured limits). If not, 9349e39c5baSBill Taylor * then obviously we have a bit of cleanup to do before returning. 9359e39c5baSBill Taylor */ 9369e39c5baSBill Taylor if ((log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) || 9379e39c5baSBill Taylor (log_qp_rq_size > state->hs_cfg_profile->cp_log_max_qp_sz)) { 9389e39c5baSBill Taylor status = IBT_HCA_WR_EXCEEDED; 9399e39c5baSBill Taylor goto spec_qpalloc_fail5a; 9409e39c5baSBill Taylor } 9419e39c5baSBill Taylor 9429e39c5baSBill Taylor /* 9439e39c5baSBill Taylor * Next we verify that the requested number of SGL is valid (i.e. 9449e39c5baSBill Taylor * consistent with the device limits and/or software-configured 9459e39c5baSBill Taylor * limits). If not, then obviously the same cleanup needs to be done. 9469e39c5baSBill Taylor */ 9479e39c5baSBill Taylor max_sgl = state->hs_cfg_profile->cp_wqe_real_max_sgl; 9489e39c5baSBill Taylor if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) || 9499e39c5baSBill Taylor (attr_p->qp_sizes.cs_rq_sgl > max_sgl)) { 9509e39c5baSBill Taylor status = IBT_HCA_SGL_EXCEEDED; 9519e39c5baSBill Taylor goto spec_qpalloc_fail5a; 9529e39c5baSBill Taylor } 9539e39c5baSBill Taylor 9549e39c5baSBill Taylor /* 9559e39c5baSBill Taylor * Determine this QP's WQE stride (for both the Send and Recv WQEs). 9569e39c5baSBill Taylor * This will depend on the requested number of SGLs. Note: this 9579e39c5baSBill Taylor * has the side-effect of also calculating the real number of SGLs 9589e39c5baSBill Taylor * (for the calculated WQE size). 9599e39c5baSBill Taylor */ 9609e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl, 9619e39c5baSBill Taylor max_sgl, HERMON_QP_WQ_TYPE_RECVQ, 9629e39c5baSBill Taylor &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl); 9639e39c5baSBill Taylor if (type == IBT_SMI_SQP) { 9649e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl, 9659e39c5baSBill Taylor max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP0, 9669e39c5baSBill Taylor &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl); 9679e39c5baSBill Taylor } else { 9689e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl, 9699e39c5baSBill Taylor max_sgl, HERMON_QP_WQ_TYPE_SENDMLX_QP1, 9709e39c5baSBill Taylor &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl); 9719e39c5baSBill Taylor } 9729e39c5baSBill Taylor 9739e39c5baSBill Taylor /* 9749e39c5baSBill Taylor * Allocate the memory for QP work queues. Since Hermon work queues 9759e39c5baSBill Taylor * are not allowed to cross a 32-bit (4GB) boundary, the alignment of 9769e39c5baSBill Taylor * the work queue memory is very important. We used to allocate 9779e39c5baSBill Taylor * work queues (the combined receive and send queues) so that they 9789e39c5baSBill Taylor * would be aligned on their combined size. That alignment guaranteed 9799e39c5baSBill Taylor * that they would never cross the 4GB boundary (Hermon work queues 9809e39c5baSBill Taylor * are on the order of MBs at maximum). Now we are able to relax 9819e39c5baSBill Taylor * this alignment constraint by ensuring that the IB address assigned 9829e39c5baSBill Taylor * to the queue memory (as a result of the hermon_mr_register() call) 9839e39c5baSBill Taylor * is offset from zero. 9849e39c5baSBill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to 9859e39c5baSBill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass 9869e39c5baSBill Taylor * mode we found that we were not allowed to specify any alignment 9879e39c5baSBill Taylor * that was more restrictive than the system page size. 9889e39c5baSBill Taylor * So we avoided this constraint by passing two alignment values, 9899e39c5baSBill Taylor * one for the memory allocation itself and the other for the DMA 9909e39c5baSBill Taylor * handle (for later bind). This used to cause more memory than 9919e39c5baSBill Taylor * necessary to be allocated (in order to guarantee the more 9929e39c5baSBill Taylor * restrictive alignment contraint). But by guaranteeing the 9939e39c5baSBill Taylor * zero-based IB virtual address for the queue, we are able to 9949e39c5baSBill Taylor * conserve this memory. 9959e39c5baSBill Taylor */ 9969e39c5baSBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 9979e39c5baSBill Taylor sq_depth = 1 << log_qp_sq_size; 9989e39c5baSBill Taylor sq_size = (1 << log_qp_sq_size) * sq_wqe_size; 9999e39c5baSBill Taylor 10009e39c5baSBill Taylor rq_wqe_size = 1 << qp->qp_rq_log_wqesz; 10019e39c5baSBill Taylor rq_size = (1 << log_qp_rq_size) * rq_wqe_size; 10029e39c5baSBill Taylor 10039e39c5baSBill Taylor qp->qp_wqinfo.qa_size = sq_size + rq_size; 10049e39c5baSBill Taylor 10059e39c5baSBill Taylor qp->qp_wqinfo.qa_alloc_align = PAGESIZE; 10069e39c5baSBill Taylor qp->qp_wqinfo.qa_bind_align = PAGESIZE; 10079e39c5baSBill Taylor qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL; 10089e39c5baSBill Taylor 10099e39c5baSBill Taylor status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag); 10109e39c5baSBill Taylor if (status != NULL) { 10119e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 10129e39c5baSBill Taylor goto spec_qpalloc_fail5a; 10139e39c5baSBill Taylor } 10149e39c5baSBill Taylor 10159e39c5baSBill Taylor /* 10169e39c5baSBill Taylor * Sort WQs in memory according to depth, stride (*q_wqe_size), 10179e39c5baSBill Taylor * biggest first. If equal, the Send Queue still goes first 10189e39c5baSBill Taylor */ 10199e39c5baSBill Taylor qp->qp_sq_baseaddr = 0; 10209e39c5baSBill Taylor qp->qp_rq_baseaddr = 0; 10219e39c5baSBill Taylor if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) { 10229e39c5baSBill Taylor sq_buf = qp->qp_wqinfo.qa_buf_aligned; 10239e39c5baSBill Taylor rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size); 10249e39c5baSBill Taylor qp->qp_rq_baseaddr = sq_size; 10259e39c5baSBill Taylor } else { 10269e39c5baSBill Taylor rq_buf = qp->qp_wqinfo.qa_buf_aligned; 10279e39c5baSBill Taylor sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size); 10289e39c5baSBill Taylor qp->qp_sq_baseaddr = rq_size; 10299e39c5baSBill Taylor } 10309e39c5baSBill Taylor 10319e39c5baSBill Taylor qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth); 10329e39c5baSBill Taylor if (qp->qp_sq_wqhdr == NULL) { 10339e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 10349e39c5baSBill Taylor goto spec_qpalloc_fail6; 10359e39c5baSBill Taylor } 10369e39c5baSBill Taylor qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(1 << log_qp_rq_size); 10379e39c5baSBill Taylor if (qp->qp_rq_wqhdr == NULL) { 10389e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 10399e39c5baSBill Taylor goto spec_qpalloc_fail6; 10409e39c5baSBill Taylor } 10419e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum; 10429e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND; 10439e39c5baSBill Taylor qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr; 10449e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum; 10459e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV; 10469e39c5baSBill Taylor qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr; 10479e39c5baSBill Taylor 10489e39c5baSBill Taylor /* 10499e39c5baSBill Taylor * Register the memory for the special QP work queues. The memory for 10509e39c5baSBill Taylor * the special QP must be registered in the Hermon cMPT tables. This 10519e39c5baSBill Taylor * gives us the LKey to specify in the QP context later. Note: The 10529e39c5baSBill Taylor * memory for Hermon work queues (both Send and Recv) must be contiguous 10539e39c5baSBill Taylor * and registered as a single memory region. Also, in order to meet the 10549e39c5baSBill Taylor * alignment restriction, we pass the "mro_bind_override_addr" flag in 10559e39c5baSBill Taylor * the call to hermon_mr_register(). This guarantees that the resulting 10569e39c5baSBill Taylor * IB vaddr will be zero-based (modulo the offset into the first page). 10579e39c5baSBill Taylor * If we fail here, we have a bunch of resource and reference count 10589e39c5baSBill Taylor * cleanup to do. 10599e39c5baSBill Taylor */ 10609e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : 10619e39c5baSBill Taylor IBT_MR_NOSLEEP; 10629e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned; 10639e39c5baSBill Taylor mr_attr.mr_len = qp->qp_wqinfo.qa_size; 10649e39c5baSBill Taylor mr_attr.mr_as = NULL; 10659e39c5baSBill Taylor mr_attr.mr_flags = flag; 10669e39c5baSBill Taylor 10679e39c5baSBill Taylor mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass; 10689e39c5baSBill Taylor mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl; 10699e39c5baSBill Taylor mr_op.mro_bind_override_addr = 1; 10709e39c5baSBill Taylor 10719e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &mr_op, 10729e39c5baSBill Taylor HERMON_QP_CMPT); 10739e39c5baSBill Taylor if (status != DDI_SUCCESS) { 10749e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 10759e39c5baSBill Taylor goto spec_qpalloc_fail6; 10769e39c5baSBill Taylor } 10779e39c5baSBill Taylor 10789e39c5baSBill Taylor /* 10799e39c5baSBill Taylor * Calculate the offset between the kernel virtual address space 10809e39c5baSBill Taylor * and the IB virtual address space. This will be used when 10819e39c5baSBill Taylor * posting work requests to properly initialize each WQE. 10829e39c5baSBill Taylor */ 10839e39c5baSBill Taylor qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned - 10849e39c5baSBill Taylor (uint64_t)mr->mr_bindinfo.bi_addr; 10859e39c5baSBill Taylor 10869e39c5baSBill Taylor /* set the prefetch - initially, not prefetching */ 10879e39c5baSBill Taylor qp->qp_no_prefetch = 1; 10889e39c5baSBill Taylor 10899e39c5baSBill Taylor if (qp->qp_no_prefetch) 10909e39c5baSBill Taylor qp->qp_sq_headroom = 2 * sq_wqe_size; 10919e39c5baSBill Taylor else 10929e39c5baSBill Taylor qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE; 10939e39c5baSBill Taylor /* 10949e39c5baSBill Taylor * hdrm wqes must be integral since both sq_wqe_size & 10959e39c5baSBill Taylor * HERMON_QP_OH_SIZE are power of 2 10969e39c5baSBill Taylor */ 10979e39c5baSBill Taylor qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size); 10989e39c5baSBill Taylor /* 10999e39c5baSBill Taylor * Fill in all the return arguments (if necessary). This includes 11009e39c5baSBill Taylor * real work queue sizes, real SGLs, and QP number (which will be 11019e39c5baSBill Taylor * either zero or one, depending on the special QP type) 11029e39c5baSBill Taylor */ 11039e39c5baSBill Taylor if (queuesz_p != NULL) { 11049e39c5baSBill Taylor queuesz_p->cs_sq = 11059e39c5baSBill Taylor (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes; 11069e39c5baSBill Taylor queuesz_p->cs_sq_sgl = qp->qp_sq_sgl; 11079e39c5baSBill Taylor queuesz_p->cs_rq = (1 << log_qp_rq_size); 11089e39c5baSBill Taylor queuesz_p->cs_rq_sgl = qp->qp_rq_sgl; 11099e39c5baSBill Taylor } 11109e39c5baSBill Taylor 11119e39c5baSBill Taylor /* 11129e39c5baSBill Taylor * Fill in the rest of the Hermon Queue Pair handle. We can update 11139e39c5baSBill Taylor * the following fields for use in further operations on the QP. 11149e39c5baSBill Taylor */ 11159e39c5baSBill Taylor qp->qp_qpcrsrcp = qpc; 11169e39c5baSBill Taylor qp->qp_rsrcp = rsrc; 11179e39c5baSBill Taylor qp->qp_state = HERMON_QP_RESET; 111817a2b317SBill Taylor HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET); 11199e39c5baSBill Taylor qp->qp_pdhdl = pd; 11209e39c5baSBill Taylor qp->qp_mrhdl = mr; 11219e39c5baSBill Taylor qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ? 11229e39c5baSBill Taylor HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED; 11239e39c5baSBill Taylor qp->qp_is_special = (type == IBT_SMI_SQP) ? 11249e39c5baSBill Taylor HERMON_QP_SMI : HERMON_QP_GSI; 11259e39c5baSBill Taylor qp->qp_uarpg = uarpg; 11269e39c5baSBill Taylor qp->qp_umap_dhp = (devmap_cookie_t)NULL; 11279e39c5baSBill Taylor qp->qp_sq_cqhdl = sq_cq; 11289e39c5baSBill Taylor qp->qp_sq_bufsz = (1 << log_qp_sq_size); 11299e39c5baSBill Taylor qp->qp_sq_buf = sq_buf; 11309e39c5baSBill Taylor qp->qp_sq_logqsz = log_qp_sq_size; 11319e39c5baSBill Taylor qp->qp_desc_off = qp_desc_off; 11329e39c5baSBill Taylor qp->qp_rq_cqhdl = rq_cq; 11339e39c5baSBill Taylor qp->qp_rq_bufsz = (1 << log_qp_rq_size); 11349e39c5baSBill Taylor qp->qp_rq_buf = rq_buf; 11359e39c5baSBill Taylor qp->qp_rq_logqsz = log_qp_rq_size; 11369e39c5baSBill Taylor qp->qp_portnum = port; 11379e39c5baSBill Taylor qp->qp_pkeyindx = 0; 11389e39c5baSBill Taylor qp->qp_forward_sqd_event = 0; 11399e39c5baSBill Taylor qp->qp_sqd_still_draining = 0; 11409e39c5baSBill Taylor qp->qp_hdlrarg = (void *)ibt_qphdl; 11419e39c5baSBill Taylor qp->qp_mcg_refcnt = 0; 11429e39c5baSBill Taylor qp->qp_srqhdl = NULL; 11439e39c5baSBill Taylor 11449e39c5baSBill Taylor /* All special QPs are UD QP service type */ 114517a2b317SBill Taylor qp->qp_type = IBT_UD_RQP; 11469e39c5baSBill Taylor qp->qp_serv_type = HERMON_QP_UD; 11479e39c5baSBill Taylor 11489e39c5baSBill Taylor /* 11499e39c5baSBill Taylor * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed 11509e39c5baSBill Taylor */ 11519e39c5baSBill Taylor 11529e39c5baSBill Taylor /* 11539e39c5baSBill Taylor * Initialize the SQ WQEs - all that needs to be done is every 64 bytes 11549e39c5baSBill Taylor * set the quadword to all F's - high-order bit is owner (init to one) 11559e39c5baSBill Taylor * and the rest for the headroom definition of prefetching 11569e39c5baSBill Taylor * 11579e39c5baSBill Taylor */ 11589e39c5baSBill Taylor 11599e39c5baSBill Taylor wqesz_shift = qp->qp_sq_log_wqesz; 11609e39c5baSBill Taylor thewqesz = 1 << wqesz_shift; 11619e39c5baSBill Taylor thewqe = (uint64_t *)(void *)(qp->qp_sq_buf); 11629e39c5baSBill Taylor for (i = 0; i < sq_depth; i++) { 11639e39c5baSBill Taylor /* 11649e39c5baSBill Taylor * for each stride, go through and every 64 bytes write the 11659e39c5baSBill Taylor * init value - having set the address once, just keep 11669e39c5baSBill Taylor * incrementing it 11679e39c5baSBill Taylor */ 11689e39c5baSBill Taylor for (j = 0; j < thewqesz; j += 64, thewqe += 8) { 11699e39c5baSBill Taylor *(uint32_t *)thewqe = 0xFFFFFFFF; 11709e39c5baSBill Taylor } 11719e39c5baSBill Taylor } 11729e39c5baSBill Taylor 11739e39c5baSBill Taylor 11749e39c5baSBill Taylor /* Zero out the QP context */ 11759e39c5baSBill Taylor bzero(&qp->qpc, sizeof (hermon_hw_qpc_t)); 11769e39c5baSBill Taylor 11779e39c5baSBill Taylor /* 11789e39c5baSBill Taylor * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the 11799e39c5baSBill Taylor * "qphdl" and return success 11809e39c5baSBill Taylor */ 118117a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + port, qp); 118217a2b317SBill Taylor 118317a2b317SBill Taylor mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER, 118417a2b317SBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 11859e39c5baSBill Taylor 11869e39c5baSBill Taylor *qphdl = qp; 11879e39c5baSBill Taylor 11889e39c5baSBill Taylor return (DDI_SUCCESS); 11899e39c5baSBill Taylor 11909e39c5baSBill Taylor /* 11919e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 11929e39c5baSBill Taylor */ 11939e39c5baSBill Taylor spec_qpalloc_fail6: 11949e39c5baSBill Taylor hermon_queue_free(&qp->qp_wqinfo); 11959e39c5baSBill Taylor if (qp->qp_sq_wqhdr) 11969e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr); 11979e39c5baSBill Taylor if (qp->qp_rq_wqhdr) 11989e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr); 11999e39c5baSBill Taylor spec_qpalloc_fail5a: 12009e39c5baSBill Taylor hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr); 12019e39c5baSBill Taylor spec_qpalloc_fail5: 12029e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 12039e39c5baSBill Taylor spec_qpalloc_fail4: 12049e39c5baSBill Taylor if (hermon_special_qp_rsrc_free(state, type, port) != DDI_SUCCESS) { 12059e39c5baSBill Taylor HERMON_WARNING(state, "failed to free special QP rsrc"); 12069e39c5baSBill Taylor } 12079e39c5baSBill Taylor spec_qpalloc_fail3: 12089e39c5baSBill Taylor hermon_cq_refcnt_dec(rq_cq); 12099e39c5baSBill Taylor spec_qpalloc_fail2: 12109e39c5baSBill Taylor hermon_cq_refcnt_dec(sq_cq); 12119e39c5baSBill Taylor spec_qpalloc_fail1: 12129e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 12139e39c5baSBill Taylor spec_qpalloc_fail: 12149e39c5baSBill Taylor return (status); 12159e39c5baSBill Taylor } 12169e39c5baSBill Taylor 12179e39c5baSBill Taylor 12189e39c5baSBill Taylor /* 121917a2b317SBill Taylor * hermon_qp_alloc_range() 12209e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 12219e39c5baSBill Taylor */ 12229e39c5baSBill Taylor int 122317a2b317SBill Taylor hermon_qp_alloc_range(hermon_state_t *state, uint_t log2, 122417a2b317SBill Taylor hermon_qp_info_t *qpinfo, ibtl_qp_hdl_t *ibt_qphdl, 122517a2b317SBill Taylor ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq, 122617a2b317SBill Taylor hermon_qphdl_t *qphdl, uint_t sleepflag) 12279e39c5baSBill Taylor { 122817a2b317SBill Taylor hermon_rsrc_t *qpc, *rsrc; 122917a2b317SBill Taylor hermon_rsrc_type_t rsrc_type; 123017a2b317SBill Taylor hermon_qphdl_t qp; 123117a2b317SBill Taylor hermon_qp_range_t *qp_range_p; 123217a2b317SBill Taylor ibt_qp_alloc_attr_t *attr_p; 123317a2b317SBill Taylor ibt_qp_type_t type; 123417a2b317SBill Taylor hermon_qp_wq_type_t swq_type; 123517a2b317SBill Taylor ibt_chan_sizes_t *queuesz_p; 123617a2b317SBill Taylor ibt_mr_attr_t mr_attr; 123717a2b317SBill Taylor hermon_mr_options_t mr_op; 123817a2b317SBill Taylor hermon_srqhdl_t srq; 123917a2b317SBill Taylor hermon_pdhdl_t pd; 124017a2b317SBill Taylor hermon_cqhdl_t sq_cq, rq_cq; 124117a2b317SBill Taylor hermon_mrhdl_t mr; 124217a2b317SBill Taylor uint64_t qp_desc_off; 124317a2b317SBill Taylor uint64_t *thewqe, thewqesz; 124417a2b317SBill Taylor uint32_t *sq_buf, *rq_buf; 124517a2b317SBill Taylor uint32_t log_qp_sq_size, log_qp_rq_size; 124617a2b317SBill Taylor uint32_t sq_size, rq_size; 124717a2b317SBill Taylor uint32_t sq_depth, rq_depth; 124817a2b317SBill Taylor uint32_t sq_wqe_size, rq_wqe_size, wqesz_shift; 124917a2b317SBill Taylor uint32_t max_sgl, max_recv_sgl, uarpg; 125017a2b317SBill Taylor uint_t qp_srq_en, i, j; 125117a2b317SBill Taylor int ii; /* loop counter for range */ 125217a2b317SBill Taylor int status, flag; 125317a2b317SBill Taylor uint_t serv_type; 125417a2b317SBill Taylor 125517a2b317SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p)) 12569e39c5baSBill Taylor 12579e39c5baSBill Taylor /* 125817a2b317SBill Taylor * Extract the necessary info from the hermon_qp_info_t structure 12599e39c5baSBill Taylor */ 126017a2b317SBill Taylor attr_p = qpinfo->qpi_attrp; 126117a2b317SBill Taylor type = qpinfo->qpi_type; 126217a2b317SBill Taylor queuesz_p = qpinfo->qpi_queueszp; 126317a2b317SBill Taylor 126417a2b317SBill Taylor if (attr_p->qp_alloc_flags & IBT_QP_USES_RSS) { 126517a2b317SBill Taylor if (log2 > state->hs_ibtfinfo.hca_attr->hca_rss_max_log2_table) 126617a2b317SBill Taylor return (IBT_INSUFF_RESOURCE); 126717a2b317SBill Taylor rsrc_type = HERMON_QPC; 126817a2b317SBill Taylor serv_type = HERMON_QP_UD; 126917a2b317SBill Taylor } else if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) { 127017a2b317SBill Taylor if (log2 > state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_qp) 127117a2b317SBill Taylor return (IBT_INSUFF_RESOURCE); 127217a2b317SBill Taylor switch (attr_p->qp_fc.fc_hca_port) { 127317a2b317SBill Taylor case 1: 127417a2b317SBill Taylor rsrc_type = HERMON_QPC_FEXCH_PORT1; 127517a2b317SBill Taylor break; 127617a2b317SBill Taylor case 2: 127717a2b317SBill Taylor rsrc_type = HERMON_QPC_FEXCH_PORT2; 127817a2b317SBill Taylor break; 127917a2b317SBill Taylor default: 128017a2b317SBill Taylor return (IBT_INVALID_PARAM); 128117a2b317SBill Taylor } 128217a2b317SBill Taylor serv_type = HERMON_QP_FEXCH; 128317a2b317SBill Taylor } else 128417a2b317SBill Taylor return (IBT_INVALID_PARAM); 12859e39c5baSBill Taylor 12869e39c5baSBill Taylor /* 128717a2b317SBill Taylor * Determine whether QP is being allocated for userland access or 128817a2b317SBill Taylor * whether it is being allocated for kernel access. If the QP is 128917a2b317SBill Taylor * being allocated for userland access, fail (too complex for now). 12909e39c5baSBill Taylor */ 129117a2b317SBill Taylor if (attr_p->qp_alloc_flags & IBT_QP_USER_MAP) { 129217a2b317SBill Taylor return (IBT_NOT_SUPPORTED); 129317a2b317SBill Taylor } else { 129417a2b317SBill Taylor uarpg = state->hs_kernel_uar_index; 12959e39c5baSBill Taylor } 12969e39c5baSBill Taylor 12979e39c5baSBill Taylor /* 129817a2b317SBill Taylor * Determine whether QP is being associated with an SRQ 12999e39c5baSBill Taylor */ 130017a2b317SBill Taylor qp_srq_en = (attr_p->qp_alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0; 130117a2b317SBill Taylor if (qp_srq_en) { 13029e39c5baSBill Taylor /* 130317a2b317SBill Taylor * Check for valid SRQ handle pointers 13049e39c5baSBill Taylor */ 130517a2b317SBill Taylor if (attr_p->qp_ibc_srq_hdl == NULL) { 130617a2b317SBill Taylor return (IBT_SRQ_HDL_INVALID); 13079e39c5baSBill Taylor } 130817a2b317SBill Taylor srq = (hermon_srqhdl_t)attr_p->qp_ibc_srq_hdl; 13099e39c5baSBill Taylor } 13109e39c5baSBill Taylor 13119e39c5baSBill Taylor /* 131217a2b317SBill Taylor * Check for valid QP service type (only UD supported) 13139e39c5baSBill Taylor */ 131417a2b317SBill Taylor if (type != IBT_UD_RQP) { 131517a2b317SBill Taylor return (IBT_QP_SRV_TYPE_INVALID); 13169e39c5baSBill Taylor } 13179e39c5baSBill Taylor 13189e39c5baSBill Taylor /* 131917a2b317SBill Taylor * Check for valid PD handle pointer 13209e39c5baSBill Taylor */ 132117a2b317SBill Taylor if (attr_p->qp_pd_hdl == NULL) { 132217a2b317SBill Taylor return (IBT_PD_HDL_INVALID); 13239e39c5baSBill Taylor } 132417a2b317SBill Taylor pd = (hermon_pdhdl_t)attr_p->qp_pd_hdl; 13259e39c5baSBill Taylor 13269e39c5baSBill Taylor /* 132717a2b317SBill Taylor * If on an SRQ, check to make sure the PD is the same 13289e39c5baSBill Taylor */ 132917a2b317SBill Taylor if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) { 133017a2b317SBill Taylor return (IBT_PD_HDL_INVALID); 133117a2b317SBill Taylor } 133217a2b317SBill Taylor 133317a2b317SBill Taylor /* set loop variable here, for freeing resources on error */ 133417a2b317SBill Taylor ii = 0; 13359e39c5baSBill Taylor 13369e39c5baSBill Taylor /* 133717a2b317SBill Taylor * Allocate 2^log2 contiguous/aligned QP context entries. This will 133817a2b317SBill Taylor * be filled in with all the necessary parameters to define the 133917a2b317SBill Taylor * Queue Pairs. Unlike other Hermon hardware resources, ownership 134017a2b317SBill Taylor * is not immediately given to hardware in the final step here. 134117a2b317SBill Taylor * Instead, we must wait until the QP is later transitioned to the 134217a2b317SBill Taylor * "Init" state before passing the QP to hardware. If we fail here, 134317a2b317SBill Taylor * we must undo all the reference count (CQ and PD). 13449e39c5baSBill Taylor */ 134517a2b317SBill Taylor status = hermon_rsrc_alloc(state, rsrc_type, 1 << log2, sleepflag, 134617a2b317SBill Taylor &qpc); 13479e39c5baSBill Taylor if (status != DDI_SUCCESS) { 134817a2b317SBill Taylor return (IBT_INSUFF_RESOURCE); 13499e39c5baSBill Taylor } 13509e39c5baSBill Taylor 135117a2b317SBill Taylor if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) 135217a2b317SBill Taylor /* 135317a2b317SBill Taylor * Need to init the MKEYs for the FEXCH QPs. 135417a2b317SBill Taylor * 135517a2b317SBill Taylor * For FEXCH QP subranges, we return the QPN base as 135617a2b317SBill Taylor * "relative" to the full FEXCH QP range for the port. 135717a2b317SBill Taylor */ 135817a2b317SBill Taylor *(qpinfo->qpi_qpn) = hermon_fcoib_fexch_relative_qpn(state, 135917a2b317SBill Taylor attr_p->qp_fc.fc_hca_port, qpc->hr_indx); 136017a2b317SBill Taylor else 136117a2b317SBill Taylor *(qpinfo->qpi_qpn) = (ib_qpn_t)qpc->hr_indx; 13629e39c5baSBill Taylor 136317a2b317SBill Taylor qp_range_p = kmem_alloc(sizeof (*qp_range_p), 136417a2b317SBill Taylor (sleepflag == HERMON_SLEEP) ? KM_SLEEP : KM_NOSLEEP); 136517a2b317SBill Taylor if (qp_range_p == NULL) { 136617a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 136717a2b317SBill Taylor goto qpalloc_fail0; 13689e39c5baSBill Taylor } 136917a2b317SBill Taylor mutex_init(&qp_range_p->hqpr_lock, NULL, MUTEX_DRIVER, 137017a2b317SBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 137117a2b317SBill Taylor mutex_enter(&qp_range_p->hqpr_lock); 137217a2b317SBill Taylor qp_range_p->hqpr_refcnt = 1 << log2; 137317a2b317SBill Taylor qp_range_p->hqpr_qpcrsrc = qpc; 137417a2b317SBill Taylor mutex_exit(&qp_range_p->hqpr_lock); 13759e39c5baSBill Taylor 137617a2b317SBill Taylor for_each_qp: 13779e39c5baSBill Taylor 137817a2b317SBill Taylor /* Increment the reference count on the protection domain (PD) */ 137917a2b317SBill Taylor hermon_pd_refcnt_inc(pd); 13809e39c5baSBill Taylor 138117a2b317SBill Taylor rq_cq = (hermon_cqhdl_t)recv_cq[ii]; 138217a2b317SBill Taylor sq_cq = (hermon_cqhdl_t)send_cq[ii]; 138317a2b317SBill Taylor if (sq_cq == NULL) { 138417a2b317SBill Taylor if (attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) { 138517a2b317SBill Taylor /* if no send completions, just use rq_cq */ 138617a2b317SBill Taylor sq_cq = rq_cq; 13879e39c5baSBill Taylor } else { 138817a2b317SBill Taylor status = IBT_CQ_HDL_INVALID; 138917a2b317SBill Taylor goto qpalloc_fail1; 13909e39c5baSBill Taylor } 13919e39c5baSBill Taylor } 13929e39c5baSBill Taylor 139317a2b317SBill Taylor /* 139417a2b317SBill Taylor * Increment the reference count on the CQs. One or both of these 139517a2b317SBill Taylor * could return error if we determine that the given CQ is already 139617a2b317SBill Taylor * being used with a special (SMI/GSI) QP. 139717a2b317SBill Taylor */ 139817a2b317SBill Taylor status = hermon_cq_refcnt_inc(sq_cq, HERMON_CQ_IS_NORMAL); 139917a2b317SBill Taylor if (status != DDI_SUCCESS) { 140017a2b317SBill Taylor status = IBT_CQ_HDL_INVALID; 140117a2b317SBill Taylor goto qpalloc_fail1; 140217a2b317SBill Taylor } 140317a2b317SBill Taylor status = hermon_cq_refcnt_inc(rq_cq, HERMON_CQ_IS_NORMAL); 140417a2b317SBill Taylor if (status != DDI_SUCCESS) { 140517a2b317SBill Taylor status = IBT_CQ_HDL_INVALID; 140617a2b317SBill Taylor goto qpalloc_fail2; 140717a2b317SBill Taylor } 140817a2b317SBill Taylor 140917a2b317SBill Taylor /* 141017a2b317SBill Taylor * Allocate the software structure for tracking the queue pair 141117a2b317SBill Taylor * (i.e. the Hermon Queue Pair handle). If we fail here, we must 141217a2b317SBill Taylor * undo the reference counts and the previous resource allocation. 141317a2b317SBill Taylor */ 141417a2b317SBill Taylor status = hermon_rsrc_alloc(state, HERMON_QPHDL, 1, sleepflag, &rsrc); 141517a2b317SBill Taylor if (status != DDI_SUCCESS) { 141617a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 141717a2b317SBill Taylor goto qpalloc_fail4; 141817a2b317SBill Taylor } 141917a2b317SBill Taylor qp = (hermon_qphdl_t)rsrc->hr_addr; 142017a2b317SBill Taylor bzero(qp, sizeof (struct hermon_sw_qp_s)); 142117a2b317SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp)) 142217a2b317SBill Taylor qp->qp_alloc_flags = attr_p->qp_alloc_flags; 142317a2b317SBill Taylor 142417a2b317SBill Taylor /* 142517a2b317SBill Taylor * Calculate the QP number from QPC index. This routine handles 142617a2b317SBill Taylor * all of the operations necessary to keep track of used, unused, 142717a2b317SBill Taylor * and released QP numbers. 142817a2b317SBill Taylor */ 142917a2b317SBill Taylor qp->qp_qpnum = qpc->hr_indx + ii; 143017a2b317SBill Taylor qp->qp_ring = qp->qp_qpnum << 8; 143117a2b317SBill Taylor qp->qp_qpn_hdl = NULL; 143217a2b317SBill Taylor 143317a2b317SBill Taylor /* 143417a2b317SBill Taylor * Allocate the doorbell record. Hermon just needs one for the RQ, 143517a2b317SBill Taylor * if the QP is not associated with an SRQ, and use uarpg (above) as 143617a2b317SBill Taylor * the uar index 143717a2b317SBill Taylor */ 143817a2b317SBill Taylor 143917a2b317SBill Taylor if (!qp_srq_en) { 144017a2b317SBill Taylor status = hermon_dbr_alloc(state, uarpg, &qp->qp_rq_dbr_acchdl, 144117a2b317SBill Taylor &qp->qp_rq_vdbr, &qp->qp_rq_pdbr, &qp->qp_rdbr_mapoffset); 144217a2b317SBill Taylor if (status != DDI_SUCCESS) { 144317a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 144417a2b317SBill Taylor goto qpalloc_fail6; 144517a2b317SBill Taylor } 144617a2b317SBill Taylor } 144717a2b317SBill Taylor 144817a2b317SBill Taylor qp->qp_uses_lso = (attr_p->qp_flags & IBT_USES_LSO); 144917a2b317SBill Taylor 145017a2b317SBill Taylor /* 145117a2b317SBill Taylor * We verify that the requested number of SGL is valid (i.e. 145217a2b317SBill Taylor * consistent with the device limits and/or software-configured 145317a2b317SBill Taylor * limits). If not, then obviously the same cleanup needs to be done. 145417a2b317SBill Taylor */ 145517a2b317SBill Taylor max_sgl = state->hs_ibtfinfo.hca_attr->hca_ud_send_sgl_sz; 145617a2b317SBill Taylor swq_type = HERMON_QP_WQ_TYPE_SENDQ_UD; 145717a2b317SBill Taylor max_recv_sgl = state->hs_ibtfinfo.hca_attr->hca_recv_sgl_sz; 145817a2b317SBill Taylor if ((attr_p->qp_sizes.cs_sq_sgl > max_sgl) || 145917a2b317SBill Taylor (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_recv_sgl))) { 146017a2b317SBill Taylor status = IBT_HCA_SGL_EXCEEDED; 146117a2b317SBill Taylor goto qpalloc_fail7; 146217a2b317SBill Taylor } 146317a2b317SBill Taylor 146417a2b317SBill Taylor /* 146517a2b317SBill Taylor * Determine this QP's WQE stride (for both the Send and Recv WQEs). 146617a2b317SBill Taylor * This will depend on the requested number of SGLs. Note: this 146717a2b317SBill Taylor * has the side-effect of also calculating the real number of SGLs 146817a2b317SBill Taylor * (for the calculated WQE size). 146917a2b317SBill Taylor * 147017a2b317SBill Taylor * For QP's on an SRQ, we set these to 0. 147117a2b317SBill Taylor */ 147217a2b317SBill Taylor if (qp_srq_en) { 147317a2b317SBill Taylor qp->qp_rq_log_wqesz = 0; 147417a2b317SBill Taylor qp->qp_rq_sgl = 0; 147517a2b317SBill Taylor } else { 147617a2b317SBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl, 147717a2b317SBill Taylor max_recv_sgl, HERMON_QP_WQ_TYPE_RECVQ, 147817a2b317SBill Taylor &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl); 147917a2b317SBill Taylor } 148017a2b317SBill Taylor hermon_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl, 148117a2b317SBill Taylor max_sgl, swq_type, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl); 148217a2b317SBill Taylor 148317a2b317SBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 148417a2b317SBill Taylor 148517a2b317SBill Taylor /* NOTE: currently policy in driver, later maybe IBTF interface */ 148617a2b317SBill Taylor qp->qp_no_prefetch = 0; 148717a2b317SBill Taylor 148817a2b317SBill Taylor /* 148917a2b317SBill Taylor * for prefetching, we need to add the number of wqes in 149017a2b317SBill Taylor * the 2k area plus one to the number requested, but 149117a2b317SBill Taylor * ONLY for send queue. If no_prefetch == 1 (prefetch off) 149217a2b317SBill Taylor * it's exactly TWO wqes for the headroom 149317a2b317SBill Taylor */ 149417a2b317SBill Taylor if (qp->qp_no_prefetch) 149517a2b317SBill Taylor qp->qp_sq_headroom = 2 * sq_wqe_size; 149617a2b317SBill Taylor else 149717a2b317SBill Taylor qp->qp_sq_headroom = sq_wqe_size + HERMON_QP_OH_SIZE; 149817a2b317SBill Taylor /* 149917a2b317SBill Taylor * hdrm wqes must be integral since both sq_wqe_size & 150017a2b317SBill Taylor * HERMON_QP_OH_SIZE are power of 2 150117a2b317SBill Taylor */ 150217a2b317SBill Taylor qp->qp_sq_hdrmwqes = (qp->qp_sq_headroom / sq_wqe_size); 150317a2b317SBill Taylor 150417a2b317SBill Taylor 150517a2b317SBill Taylor /* 150617a2b317SBill Taylor * Calculate the appropriate size for the work queues. 150717a2b317SBill Taylor * For send queue, add in the headroom wqes to the calculation. 150817a2b317SBill Taylor * Note: All Hermon QP work queues must be a power-of-2 in size. Also 150917a2b317SBill Taylor * they may not be any smaller than HERMON_QP_MIN_SIZE. This step is 151017a2b317SBill Taylor * to round the requested size up to the next highest power-of-2 151117a2b317SBill Taylor */ 151217a2b317SBill Taylor /* first, adjust to a minimum and tell the caller the change */ 151317a2b317SBill Taylor attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq, 151417a2b317SBill Taylor HERMON_QP_MIN_SIZE); 151517a2b317SBill Taylor attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq, 151617a2b317SBill Taylor HERMON_QP_MIN_SIZE); 151717a2b317SBill Taylor /* 151817a2b317SBill Taylor * now, calculate the alloc size, taking into account 151917a2b317SBill Taylor * the headroom for the sq 152017a2b317SBill Taylor */ 152117a2b317SBill Taylor log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes); 152217a2b317SBill Taylor /* if the total is a power of two, reduce it */ 1523*de710d24SJosef 'Jeff' Sipek if (ISP2(attr_p->qp_sizes.cs_sq + qp->qp_sq_hdrmwqes)) { 152417a2b317SBill Taylor log_qp_sq_size = log_qp_sq_size - 1; 152517a2b317SBill Taylor } 152617a2b317SBill Taylor 152717a2b317SBill Taylor log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq); 1528*de710d24SJosef 'Jeff' Sipek if (ISP2(attr_p->qp_sizes.cs_rq)) { 152917a2b317SBill Taylor log_qp_rq_size = log_qp_rq_size - 1; 153017a2b317SBill Taylor } 153117a2b317SBill Taylor 153217a2b317SBill Taylor /* 153317a2b317SBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent 153417a2b317SBill Taylor * with the device limits and/or software-configured limits). If not, 153517a2b317SBill Taylor * then obviously we have a lot of cleanup to do before returning. 153617a2b317SBill Taylor * 153717a2b317SBill Taylor * NOTE: the first condition deals with the (test) case of cs_sq 153817a2b317SBill Taylor * being just less than 2^32. In this case, the headroom addition 153917a2b317SBill Taylor * to the requested cs_sq will pass the test when it should not. 154017a2b317SBill Taylor * This test no longer lets that case slip through the check. 154117a2b317SBill Taylor */ 154217a2b317SBill Taylor if ((attr_p->qp_sizes.cs_sq > 154317a2b317SBill Taylor (1 << state->hs_cfg_profile->cp_log_max_qp_sz)) || 154417a2b317SBill Taylor (log_qp_sq_size > state->hs_cfg_profile->cp_log_max_qp_sz) || 154517a2b317SBill Taylor (!qp_srq_en && (log_qp_rq_size > 154617a2b317SBill Taylor state->hs_cfg_profile->cp_log_max_qp_sz))) { 154717a2b317SBill Taylor status = IBT_HCA_WR_EXCEEDED; 154817a2b317SBill Taylor goto qpalloc_fail7; 154917a2b317SBill Taylor } 155017a2b317SBill Taylor 155117a2b317SBill Taylor /* 155217a2b317SBill Taylor * Allocate the memory for QP work queues. Since Hermon work queues 155317a2b317SBill Taylor * are not allowed to cross a 32-bit (4GB) boundary, the alignment of 155417a2b317SBill Taylor * the work queue memory is very important. We used to allocate 155517a2b317SBill Taylor * work queues (the combined receive and send queues) so that they 155617a2b317SBill Taylor * would be aligned on their combined size. That alignment guaranteed 155717a2b317SBill Taylor * that they would never cross the 4GB boundary (Hermon work queues 155817a2b317SBill Taylor * are on the order of MBs at maximum). Now we are able to relax 155917a2b317SBill Taylor * this alignment constraint by ensuring that the IB address assigned 156017a2b317SBill Taylor * to the queue memory (as a result of the hermon_mr_register() call) 156117a2b317SBill Taylor * is offset from zero. 156217a2b317SBill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to 156317a2b317SBill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass 156417a2b317SBill Taylor * mode we found that we were not allowed to specify any alignment 156517a2b317SBill Taylor * that was more restrictive than the system page size. 156617a2b317SBill Taylor * So we avoided this constraint by passing two alignment values, 156717a2b317SBill Taylor * one for the memory allocation itself and the other for the DMA 156817a2b317SBill Taylor * handle (for later bind). This used to cause more memory than 156917a2b317SBill Taylor * necessary to be allocated (in order to guarantee the more 157017a2b317SBill Taylor * restrictive alignment contraint). But by guaranteeing the 157117a2b317SBill Taylor * zero-based IB virtual address for the queue, we are able to 157217a2b317SBill Taylor * conserve this memory. 157317a2b317SBill Taylor */ 157417a2b317SBill Taylor sq_wqe_size = 1 << qp->qp_sq_log_wqesz; 157517a2b317SBill Taylor sq_depth = 1 << log_qp_sq_size; 157617a2b317SBill Taylor sq_size = sq_depth * sq_wqe_size; 157717a2b317SBill Taylor 157817a2b317SBill Taylor /* QP on SRQ sets these to 0 */ 157917a2b317SBill Taylor if (qp_srq_en) { 158017a2b317SBill Taylor rq_wqe_size = 0; 158117a2b317SBill Taylor rq_size = 0; 158217a2b317SBill Taylor } else { 158317a2b317SBill Taylor rq_wqe_size = 1 << qp->qp_rq_log_wqesz; 158417a2b317SBill Taylor rq_depth = 1 << log_qp_rq_size; 158517a2b317SBill Taylor rq_size = rq_depth * rq_wqe_size; 158617a2b317SBill Taylor } 158717a2b317SBill Taylor 158817a2b317SBill Taylor qp->qp_wqinfo.qa_size = sq_size + rq_size; 158917a2b317SBill Taylor qp->qp_wqinfo.qa_alloc_align = PAGESIZE; 159017a2b317SBill Taylor qp->qp_wqinfo.qa_bind_align = PAGESIZE; 159117a2b317SBill Taylor qp->qp_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL; 159217a2b317SBill Taylor status = hermon_queue_alloc(state, &qp->qp_wqinfo, sleepflag); 159317a2b317SBill Taylor if (status != DDI_SUCCESS) { 159417a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 159517a2b317SBill Taylor goto qpalloc_fail7; 159617a2b317SBill Taylor } 159717a2b317SBill Taylor 159817a2b317SBill Taylor /* 159917a2b317SBill Taylor * Sort WQs in memory according to stride (*q_wqe_size), largest first 160017a2b317SBill Taylor * If they are equal, still put the SQ first 160117a2b317SBill Taylor */ 160217a2b317SBill Taylor qp->qp_sq_baseaddr = 0; 160317a2b317SBill Taylor qp->qp_rq_baseaddr = 0; 160417a2b317SBill Taylor if ((sq_wqe_size > rq_wqe_size) || (sq_wqe_size == rq_wqe_size)) { 160517a2b317SBill Taylor sq_buf = qp->qp_wqinfo.qa_buf_aligned; 160617a2b317SBill Taylor 160717a2b317SBill Taylor /* if this QP is on an SRQ, set the rq_buf to NULL */ 160817a2b317SBill Taylor if (qp_srq_en) { 160917a2b317SBill Taylor rq_buf = NULL; 161017a2b317SBill Taylor } else { 161117a2b317SBill Taylor rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size); 161217a2b317SBill Taylor qp->qp_rq_baseaddr = sq_size; 161317a2b317SBill Taylor } 161417a2b317SBill Taylor } else { 161517a2b317SBill Taylor rq_buf = qp->qp_wqinfo.qa_buf_aligned; 161617a2b317SBill Taylor sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size); 161717a2b317SBill Taylor qp->qp_sq_baseaddr = rq_size; 161817a2b317SBill Taylor } 161917a2b317SBill Taylor 162017a2b317SBill Taylor qp->qp_sq_wqhdr = hermon_wrid_wqhdr_create(sq_depth); 162117a2b317SBill Taylor if (qp->qp_sq_wqhdr == NULL) { 162217a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 162317a2b317SBill Taylor goto qpalloc_fail8; 162417a2b317SBill Taylor } 162517a2b317SBill Taylor if (qp_srq_en) { 162617a2b317SBill Taylor qp->qp_rq_wqavl.wqa_wq = srq->srq_wq_wqhdr; 162717a2b317SBill Taylor qp->qp_rq_wqavl.wqa_srq_en = 1; 162817a2b317SBill Taylor qp->qp_rq_wqavl.wqa_srq = srq; 162917a2b317SBill Taylor } else { 163017a2b317SBill Taylor qp->qp_rq_wqhdr = hermon_wrid_wqhdr_create(rq_depth); 163117a2b317SBill Taylor if (qp->qp_rq_wqhdr == NULL) { 163217a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 163317a2b317SBill Taylor goto qpalloc_fail8; 163417a2b317SBill Taylor } 163517a2b317SBill Taylor qp->qp_rq_wqavl.wqa_wq = qp->qp_rq_wqhdr; 163617a2b317SBill Taylor } 163717a2b317SBill Taylor qp->qp_sq_wqavl.wqa_qpn = qp->qp_qpnum; 163817a2b317SBill Taylor qp->qp_sq_wqavl.wqa_type = HERMON_WR_SEND; 163917a2b317SBill Taylor qp->qp_sq_wqavl.wqa_wq = qp->qp_sq_wqhdr; 164017a2b317SBill Taylor qp->qp_rq_wqavl.wqa_qpn = qp->qp_qpnum; 164117a2b317SBill Taylor qp->qp_rq_wqavl.wqa_type = HERMON_WR_RECV; 164217a2b317SBill Taylor 164317a2b317SBill Taylor /* 164417a2b317SBill Taylor * Register the memory for the QP work queues. The memory for the 164517a2b317SBill Taylor * QP must be registered in the Hermon cMPT tables. This gives us the 164617a2b317SBill Taylor * LKey to specify in the QP context later. Note: The memory for 164717a2b317SBill Taylor * Hermon work queues (both Send and Recv) must be contiguous and 164817a2b317SBill Taylor * registered as a single memory region. Note: If the QP memory is 164917a2b317SBill Taylor * user-mappable, force DDI_DMA_CONSISTENT mapping. Also, in order to 165017a2b317SBill Taylor * meet the alignment restriction, we pass the "mro_bind_override_addr" 165117a2b317SBill Taylor * flag in the call to hermon_mr_register(). This guarantees that the 165217a2b317SBill Taylor * resulting IB vaddr will be zero-based (modulo the offset into the 165317a2b317SBill Taylor * first page). If we fail here, we still have the bunch of resource 165417a2b317SBill Taylor * and reference count cleanup to do. 165517a2b317SBill Taylor */ 165617a2b317SBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : 165717a2b317SBill Taylor IBT_MR_NOSLEEP; 165817a2b317SBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned; 165917a2b317SBill Taylor mr_attr.mr_len = qp->qp_wqinfo.qa_size; 166017a2b317SBill Taylor mr_attr.mr_as = NULL; 166117a2b317SBill Taylor mr_attr.mr_flags = flag; 166217a2b317SBill Taylor /* HERMON_QUEUE_LOCATION_NORMAL */ 166317a2b317SBill Taylor mr_op.mro_bind_type = 166417a2b317SBill Taylor state->hs_cfg_profile->cp_iommu_bypass; 166517a2b317SBill Taylor mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl; 166617a2b317SBill Taylor mr_op.mro_bind_override_addr = 1; 166717a2b317SBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, 166817a2b317SBill Taylor &mr_op, HERMON_QP_CMPT); 166917a2b317SBill Taylor if (status != DDI_SUCCESS) { 167017a2b317SBill Taylor status = IBT_INSUFF_RESOURCE; 167117a2b317SBill Taylor goto qpalloc_fail9; 167217a2b317SBill Taylor } 167317a2b317SBill Taylor 167417a2b317SBill Taylor /* 167517a2b317SBill Taylor * Calculate the offset between the kernel virtual address space 167617a2b317SBill Taylor * and the IB virtual address space. This will be used when 167717a2b317SBill Taylor * posting work requests to properly initialize each WQE. 167817a2b317SBill Taylor */ 167917a2b317SBill Taylor qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned - 168017a2b317SBill Taylor (uint64_t)mr->mr_bindinfo.bi_addr; 168117a2b317SBill Taylor 168217a2b317SBill Taylor /* 168317a2b317SBill Taylor * Fill in all the return arguments (if necessary). This includes 168417a2b317SBill Taylor * real work queue sizes (in wqes), real SGLs, and QP number 168517a2b317SBill Taylor */ 168617a2b317SBill Taylor if (queuesz_p != NULL) { 168717a2b317SBill Taylor queuesz_p->cs_sq = 168817a2b317SBill Taylor (1 << log_qp_sq_size) - qp->qp_sq_hdrmwqes; 168917a2b317SBill Taylor queuesz_p->cs_sq_sgl = qp->qp_sq_sgl; 169017a2b317SBill Taylor 169117a2b317SBill Taylor /* if this QP is on an SRQ, set these to 0 */ 169217a2b317SBill Taylor if (qp_srq_en) { 169317a2b317SBill Taylor queuesz_p->cs_rq = 0; 169417a2b317SBill Taylor queuesz_p->cs_rq_sgl = 0; 169517a2b317SBill Taylor } else { 169617a2b317SBill Taylor queuesz_p->cs_rq = (1 << log_qp_rq_size); 169717a2b317SBill Taylor queuesz_p->cs_rq_sgl = qp->qp_rq_sgl; 169817a2b317SBill Taylor } 169917a2b317SBill Taylor } 170017a2b317SBill Taylor 170117a2b317SBill Taylor /* 170217a2b317SBill Taylor * Fill in the rest of the Hermon Queue Pair handle. 170317a2b317SBill Taylor */ 170417a2b317SBill Taylor qp->qp_qpcrsrcp = NULL; 170517a2b317SBill Taylor qp->qp_rsrcp = rsrc; 170617a2b317SBill Taylor qp->qp_state = HERMON_QP_RESET; 170717a2b317SBill Taylor HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET); 170817a2b317SBill Taylor qp->qp_pdhdl = pd; 170917a2b317SBill Taylor qp->qp_mrhdl = mr; 171017a2b317SBill Taylor qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ? 171117a2b317SBill Taylor HERMON_QP_SQ_WR_SIGNALED : HERMON_QP_SQ_ALL_SIGNALED; 171217a2b317SBill Taylor qp->qp_is_special = 0; 171317a2b317SBill Taylor qp->qp_uarpg = uarpg; 171417a2b317SBill Taylor qp->qp_umap_dhp = (devmap_cookie_t)NULL; 171517a2b317SBill Taylor qp->qp_sq_cqhdl = sq_cq; 171617a2b317SBill Taylor qp->qp_sq_bufsz = (1 << log_qp_sq_size); 171717a2b317SBill Taylor qp->qp_sq_logqsz = log_qp_sq_size; 171817a2b317SBill Taylor qp->qp_sq_buf = sq_buf; 171917a2b317SBill Taylor qp->qp_desc_off = qp_desc_off; 172017a2b317SBill Taylor qp->qp_rq_cqhdl = rq_cq; 172117a2b317SBill Taylor qp->qp_rq_buf = rq_buf; 172217a2b317SBill Taylor qp->qp_rlky = (attr_p->qp_flags & IBT_FAST_REG_RES_LKEY) != 172317a2b317SBill Taylor 0; 172417a2b317SBill Taylor 172517a2b317SBill Taylor /* if this QP is on an SRQ, set rq_bufsz to 0 */ 172617a2b317SBill Taylor if (qp_srq_en) { 172717a2b317SBill Taylor qp->qp_rq_bufsz = 0; 172817a2b317SBill Taylor qp->qp_rq_logqsz = 0; 172917a2b317SBill Taylor } else { 173017a2b317SBill Taylor qp->qp_rq_bufsz = (1 << log_qp_rq_size); 173117a2b317SBill Taylor qp->qp_rq_logqsz = log_qp_rq_size; 173217a2b317SBill Taylor } 173317a2b317SBill Taylor 173417a2b317SBill Taylor qp->qp_forward_sqd_event = 0; 173517a2b317SBill Taylor qp->qp_sqd_still_draining = 0; 173617a2b317SBill Taylor qp->qp_hdlrarg = (void *)ibt_qphdl[ii]; 173717a2b317SBill Taylor qp->qp_mcg_refcnt = 0; 173817a2b317SBill Taylor 173917a2b317SBill Taylor /* 174017a2b317SBill Taylor * If this QP is to be associated with an SRQ, set the SRQ handle 174117a2b317SBill Taylor */ 174217a2b317SBill Taylor if (qp_srq_en) { 174317a2b317SBill Taylor qp->qp_srqhdl = srq; 174417a2b317SBill Taylor hermon_srq_refcnt_inc(qp->qp_srqhdl); 174517a2b317SBill Taylor } else { 174617a2b317SBill Taylor qp->qp_srqhdl = NULL; 174717a2b317SBill Taylor } 174817a2b317SBill Taylor 174917a2b317SBill Taylor qp->qp_type = IBT_UD_RQP; 175017a2b317SBill Taylor qp->qp_serv_type = serv_type; 175117a2b317SBill Taylor 175217a2b317SBill Taylor /* 175317a2b317SBill Taylor * Initialize the RQ WQEs - unlike Arbel, no Rcv init is needed 175417a2b317SBill Taylor */ 175517a2b317SBill Taylor 175617a2b317SBill Taylor /* 175717a2b317SBill Taylor * Initialize the SQ WQEs - all that needs to be done is every 64 bytes 175817a2b317SBill Taylor * set the quadword to all F's - high-order bit is owner (init to one) 175917a2b317SBill Taylor * and the rest for the headroom definition of prefetching. 176017a2b317SBill Taylor */ 176117a2b317SBill Taylor if ((attr_p->qp_alloc_flags & IBT_QP_USES_FEXCH) == 0) { 176217a2b317SBill Taylor wqesz_shift = qp->qp_sq_log_wqesz; 176317a2b317SBill Taylor thewqesz = 1 << wqesz_shift; 176417a2b317SBill Taylor thewqe = (uint64_t *)(void *)(qp->qp_sq_buf); 176517a2b317SBill Taylor for (i = 0; i < sq_depth; i++) { 176617a2b317SBill Taylor /* 176717a2b317SBill Taylor * for each stride, go through and every 64 bytes 176817a2b317SBill Taylor * write the init value - having set the address 176917a2b317SBill Taylor * once, just keep incrementing it 177017a2b317SBill Taylor */ 177117a2b317SBill Taylor for (j = 0; j < thewqesz; j += 64, thewqe += 8) { 177217a2b317SBill Taylor *(uint32_t *)thewqe = 0xFFFFFFFF; 177317a2b317SBill Taylor } 177417a2b317SBill Taylor } 177517a2b317SBill Taylor } 177617a2b317SBill Taylor 177717a2b317SBill Taylor /* Zero out the QP context */ 177817a2b317SBill Taylor bzero(&qp->qpc, sizeof (hermon_hw_qpc_t)); 177917a2b317SBill Taylor 178017a2b317SBill Taylor /* 178117a2b317SBill Taylor * Put QP handle in Hermon QPNum-to-QPHdl list. Then fill in the 178217a2b317SBill Taylor * "qphdl" and return success 178317a2b317SBill Taylor */ 178417a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_QPC, qpc->hr_indx + ii, qp); 178517a2b317SBill Taylor 178617a2b317SBill Taylor mutex_init(&qp->qp_sq_lock, NULL, MUTEX_DRIVER, 178717a2b317SBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 178817a2b317SBill Taylor 178917a2b317SBill Taylor qp->qp_rangep = qp_range_p; 179017a2b317SBill Taylor 179117a2b317SBill Taylor qphdl[ii] = qp; 179217a2b317SBill Taylor 179317a2b317SBill Taylor if (++ii < (1 << log2)) 179417a2b317SBill Taylor goto for_each_qp; 179517a2b317SBill Taylor 179617a2b317SBill Taylor return (DDI_SUCCESS); 179717a2b317SBill Taylor 179817a2b317SBill Taylor /* 179917a2b317SBill Taylor * The following is cleanup for all possible failure cases in this routine 180017a2b317SBill Taylor */ 180117a2b317SBill Taylor qpalloc_fail9: 180217a2b317SBill Taylor hermon_queue_free(&qp->qp_wqinfo); 180317a2b317SBill Taylor qpalloc_fail8: 180417a2b317SBill Taylor if (qp->qp_sq_wqhdr) 180517a2b317SBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr); 180617a2b317SBill Taylor if (qp->qp_rq_wqhdr) 180717a2b317SBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr); 180817a2b317SBill Taylor qpalloc_fail7: 180917a2b317SBill Taylor if (!qp_srq_en) { 181017a2b317SBill Taylor hermon_dbr_free(state, uarpg, qp->qp_rq_vdbr); 181117a2b317SBill Taylor } 181217a2b317SBill Taylor 181317a2b317SBill Taylor qpalloc_fail6: 181417a2b317SBill Taylor hermon_rsrc_free(state, &rsrc); 181517a2b317SBill Taylor qpalloc_fail4: 181617a2b317SBill Taylor hermon_cq_refcnt_dec(rq_cq); 181717a2b317SBill Taylor qpalloc_fail2: 181817a2b317SBill Taylor hermon_cq_refcnt_dec(sq_cq); 181917a2b317SBill Taylor qpalloc_fail1: 182017a2b317SBill Taylor hermon_pd_refcnt_dec(pd); 182117a2b317SBill Taylor qpalloc_fail0: 182217a2b317SBill Taylor if (ii == 0) { 182317a2b317SBill Taylor if (qp_range_p) 182417a2b317SBill Taylor kmem_free(qp_range_p, sizeof (*qp_range_p)); 182517a2b317SBill Taylor hermon_rsrc_free(state, &qpc); 182617a2b317SBill Taylor } else { 182717a2b317SBill Taylor /* qp_range_p and qpc rsrc will be freed in hermon_qp_free */ 182817a2b317SBill Taylor 182917a2b317SBill Taylor mutex_enter(&qp->qp_rangep->hqpr_lock); 183017a2b317SBill Taylor qp_range_p->hqpr_refcnt = ii; 183117a2b317SBill Taylor mutex_exit(&qp->qp_rangep->hqpr_lock); 183217a2b317SBill Taylor while (--ii >= 0) { 183317a2b317SBill Taylor ibc_qpn_hdl_t qpn_hdl; 183417a2b317SBill Taylor int free_status; 183517a2b317SBill Taylor 183617a2b317SBill Taylor free_status = hermon_qp_free(state, &qphdl[ii], 183717a2b317SBill Taylor IBC_FREE_QP_AND_QPN, &qpn_hdl, sleepflag); 183817a2b317SBill Taylor if (free_status != DDI_SUCCESS) 183917a2b317SBill Taylor cmn_err(CE_CONT, "!qp_range: status 0x%x: " 184017a2b317SBill Taylor "error status %x during free", 184117a2b317SBill Taylor status, free_status); 184217a2b317SBill Taylor } 184317a2b317SBill Taylor } 184417a2b317SBill Taylor 184517a2b317SBill Taylor return (status); 184617a2b317SBill Taylor } 184717a2b317SBill Taylor 184817a2b317SBill Taylor 184917a2b317SBill Taylor /* 185017a2b317SBill Taylor * hermon_qp_free() 185117a2b317SBill Taylor * This function frees up the QP resources. Depending on the value 185217a2b317SBill Taylor * of the "free_qp_flags", the QP number may not be released until 185317a2b317SBill Taylor * a subsequent call to hermon_qp_release_qpn(). 185417a2b317SBill Taylor * 185517a2b317SBill Taylor * Context: Can be called only from user or kernel context. 185617a2b317SBill Taylor */ 185717a2b317SBill Taylor /* ARGSUSED */ 185817a2b317SBill Taylor int 185917a2b317SBill Taylor hermon_qp_free(hermon_state_t *state, hermon_qphdl_t *qphdl, 186017a2b317SBill Taylor ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh, 186117a2b317SBill Taylor uint_t sleepflag) 186217a2b317SBill Taylor { 186317a2b317SBill Taylor hermon_rsrc_t *qpc, *rsrc; 186417a2b317SBill Taylor hermon_umap_db_entry_t *umapdb; 186517a2b317SBill Taylor hermon_qpn_entry_t *entry; 186617a2b317SBill Taylor hermon_pdhdl_t pd; 186717a2b317SBill Taylor hermon_mrhdl_t mr; 186817a2b317SBill Taylor hermon_cqhdl_t sq_cq, rq_cq; 186917a2b317SBill Taylor hermon_srqhdl_t srq; 187017a2b317SBill Taylor hermon_qphdl_t qp; 187117a2b317SBill Taylor uint64_t value; 187217a2b317SBill Taylor uint_t type, port; 187317a2b317SBill Taylor uint_t maxprot; 187417a2b317SBill Taylor uint_t qp_srq_en; 187517a2b317SBill Taylor int status; 187617a2b317SBill Taylor 187717a2b317SBill Taylor /* 187817a2b317SBill Taylor * Pull all the necessary information from the Hermon Queue Pair 187917a2b317SBill Taylor * handle. This is necessary here because the resource for the 188017a2b317SBill Taylor * QP handle is going to be freed up as part of this operation. 188117a2b317SBill Taylor */ 188217a2b317SBill Taylor qp = *qphdl; 188317a2b317SBill Taylor mutex_enter(&qp->qp_lock); 188417a2b317SBill Taylor qpc = qp->qp_qpcrsrcp; /* NULL if part of a "range" */ 188517a2b317SBill Taylor rsrc = qp->qp_rsrcp; 188617a2b317SBill Taylor pd = qp->qp_pdhdl; 188717a2b317SBill Taylor srq = qp->qp_srqhdl; 188817a2b317SBill Taylor mr = qp->qp_mrhdl; 188917a2b317SBill Taylor rq_cq = qp->qp_rq_cqhdl; 189017a2b317SBill Taylor sq_cq = qp->qp_sq_cqhdl; 189117a2b317SBill Taylor port = qp->qp_portnum; 189217a2b317SBill Taylor qp_srq_en = qp->qp_alloc_flags & IBT_QP_USES_SRQ; 189317a2b317SBill Taylor 189417a2b317SBill Taylor /* 189517a2b317SBill Taylor * If the QP is part of an MCG, then we fail the qp_free 189617a2b317SBill Taylor */ 189717a2b317SBill Taylor if (qp->qp_mcg_refcnt != 0) { 189817a2b317SBill Taylor mutex_exit(&qp->qp_lock); 189917a2b317SBill Taylor status = ibc_get_ci_failure(0); 190017a2b317SBill Taylor goto qpfree_fail; 190117a2b317SBill Taylor } 190217a2b317SBill Taylor 190317a2b317SBill Taylor /* 190417a2b317SBill Taylor * If the QP is not already in "Reset" state, then transition to 190517a2b317SBill Taylor * "Reset". This is necessary because software does not reclaim 190617a2b317SBill Taylor * ownership of the QP context until the QP is in the "Reset" state. 190717a2b317SBill Taylor * If the ownership transfer fails for any reason, then it is an 190817a2b317SBill Taylor * indication that something (either in HW or SW) has gone seriously 190917a2b317SBill Taylor * wrong. So we print a warning message and return. 191017a2b317SBill Taylor */ 191117a2b317SBill Taylor if (qp->qp_state != HERMON_QP_RESET) { 191217a2b317SBill Taylor if (hermon_qp_to_reset(state, qp) != DDI_SUCCESS) { 191317a2b317SBill Taylor mutex_exit(&qp->qp_lock); 191417a2b317SBill Taylor HERMON_WARNING(state, "failed to reset QP context"); 191517a2b317SBill Taylor status = ibc_get_ci_failure(0); 191617a2b317SBill Taylor goto qpfree_fail; 191717a2b317SBill Taylor } 191817a2b317SBill Taylor qp->qp_state = HERMON_QP_RESET; 191917a2b317SBill Taylor HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_RESET); 192017a2b317SBill Taylor 192117a2b317SBill Taylor /* 192217a2b317SBill Taylor * Do any additional handling necessary for the transition 192317a2b317SBill Taylor * to the "Reset" state (e.g. update the WRID lists) 192417a2b317SBill Taylor */ 192517a2b317SBill Taylor if (hermon_wrid_to_reset_handling(state, qp) != DDI_SUCCESS) { 192617a2b317SBill Taylor mutex_exit(&qp->qp_lock); 192717a2b317SBill Taylor HERMON_WARNING(state, "failed to reset QP WRID list"); 192817a2b317SBill Taylor status = ibc_get_ci_failure(0); 192917a2b317SBill Taylor goto qpfree_fail; 193017a2b317SBill Taylor } 193117a2b317SBill Taylor } 193217a2b317SBill Taylor 193317a2b317SBill Taylor /* 193417a2b317SBill Taylor * If this was a user-mappable QP, then we need to remove its entry 193517a2b317SBill Taylor * from the "userland resources database". If it is also currently 193617a2b317SBill Taylor * mmap()'d out to a user process, then we need to call 193717a2b317SBill Taylor * devmap_devmem_remap() to remap the QP memory to an invalid mapping. 193817a2b317SBill Taylor * We also need to invalidate the QP tracking information for the 193917a2b317SBill Taylor * user mapping. 194017a2b317SBill Taylor */ 194117a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USER_MAP) { 194217a2b317SBill Taylor status = hermon_umap_db_find(state->hs_instance, qp->qp_qpnum, 194317a2b317SBill Taylor MLNX_UMAP_QPMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE, 194417a2b317SBill Taylor &umapdb); 194517a2b317SBill Taylor if (status != DDI_SUCCESS) { 194617a2b317SBill Taylor mutex_exit(&qp->qp_lock); 194717a2b317SBill Taylor HERMON_WARNING(state, "failed to find in database"); 194817a2b317SBill Taylor return (ibc_get_ci_failure(0)); 194917a2b317SBill Taylor } 195017a2b317SBill Taylor hermon_umap_db_free(umapdb); 195117a2b317SBill Taylor if (qp->qp_umap_dhp != NULL) { 195217a2b317SBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 195317a2b317SBill Taylor status = devmap_devmem_remap(qp->qp_umap_dhp, 195417a2b317SBill Taylor state->hs_dip, 0, 0, qp->qp_wqinfo.qa_size, 195517a2b317SBill Taylor maxprot, DEVMAP_MAPPING_INVALID, NULL); 195617a2b317SBill Taylor if (status != DDI_SUCCESS) { 195717a2b317SBill Taylor mutex_exit(&qp->qp_lock); 195817a2b317SBill Taylor HERMON_WARNING(state, "failed in QP memory " 195917a2b317SBill Taylor "devmap_devmem_remap()"); 196017a2b317SBill Taylor return (ibc_get_ci_failure(0)); 196117a2b317SBill Taylor } 196217a2b317SBill Taylor qp->qp_umap_dhp = (devmap_cookie_t)NULL; 196317a2b317SBill Taylor } 196417a2b317SBill Taylor } 196517a2b317SBill Taylor 196617a2b317SBill Taylor 196717a2b317SBill Taylor /* 196817a2b317SBill Taylor * Put NULL into the Hermon QPNum-to-QPHdl list. This will allow any 196917a2b317SBill Taylor * in-progress events to detect that the QP corresponding to this 197017a2b317SBill Taylor * number has been freed. Note: it does depend in whether we are 197117a2b317SBill Taylor * freeing a special QP or not. 197217a2b317SBill Taylor */ 197317a2b317SBill Taylor if (qpc == NULL) { 197417a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_QPC, 197517a2b317SBill Taylor qp->qp_qpnum, NULL); 197617a2b317SBill Taylor } else if (qp->qp_is_special) { 197717a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_QPC, 197817a2b317SBill Taylor qpc->hr_indx + port, NULL); 197917a2b317SBill Taylor } else { 198017a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_QPC, 198117a2b317SBill Taylor qpc->hr_indx, NULL); 198217a2b317SBill Taylor } 198317a2b317SBill Taylor 198417a2b317SBill Taylor /* 198517a2b317SBill Taylor * Drop the QP lock 198617a2b317SBill Taylor * At this point the lock is no longer necessary. We cannot 198717a2b317SBill Taylor * protect from multiple simultaneous calls to free the same QP. 198817a2b317SBill Taylor * In addition, since the QP lock is contained in the QP "software 198917a2b317SBill Taylor * handle" resource, which we will free (see below), it is 199017a2b317SBill Taylor * important that we have no further references to that memory. 199117a2b317SBill Taylor */ 199217a2b317SBill Taylor mutex_exit(&qp->qp_lock); 199317a2b317SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qp)) 199417a2b317SBill Taylor 199517a2b317SBill Taylor /* 199617a2b317SBill Taylor * Free the QP resources 199717a2b317SBill Taylor * Start by deregistering and freeing the memory for work queues. 199817a2b317SBill Taylor * Next free any previously allocated context information 199917a2b317SBill Taylor * (depending on QP type) 200017a2b317SBill Taylor * Finally, decrement the necessary reference counts. 200117a2b317SBill Taylor * If this fails for any reason, then it is an indication that 200217a2b317SBill Taylor * something (either in HW or SW) has gone seriously wrong. So we 200317a2b317SBill Taylor * print a warning message and return. 200417a2b317SBill Taylor */ 200517a2b317SBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL, 200617a2b317SBill Taylor sleepflag); 200717a2b317SBill Taylor if (status != DDI_SUCCESS) { 200817a2b317SBill Taylor HERMON_WARNING(state, "failed to deregister QP memory"); 200917a2b317SBill Taylor status = ibc_get_ci_failure(0); 201017a2b317SBill Taylor goto qpfree_fail; 201117a2b317SBill Taylor } 201217a2b317SBill Taylor 201317a2b317SBill Taylor /* Free the memory for the QP */ 201417a2b317SBill Taylor hermon_queue_free(&qp->qp_wqinfo); 201517a2b317SBill Taylor 201617a2b317SBill Taylor if (qp->qp_sq_wqhdr) 201717a2b317SBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_sq_wqhdr); 201817a2b317SBill Taylor if (qp->qp_rq_wqhdr) 201917a2b317SBill Taylor hermon_wrid_wqhdr_destroy(qp->qp_rq_wqhdr); 202017a2b317SBill Taylor 202117a2b317SBill Taylor /* Free the dbr */ 202217a2b317SBill Taylor if (!qp_srq_en) { 202317a2b317SBill Taylor hermon_dbr_free(state, qp->qp_uarpg, qp->qp_rq_vdbr); 202417a2b317SBill Taylor } 202517a2b317SBill Taylor 202617a2b317SBill Taylor /* 202717a2b317SBill Taylor * Free up the remainder of the QP resources. Note: we have a few 202817a2b317SBill Taylor * different resources to free up depending on whether the QP is a 202917a2b317SBill Taylor * special QP or not. As described above, if any of these fail for 203017a2b317SBill Taylor * any reason it is an indication that something (either in HW or SW) 203117a2b317SBill Taylor * has gone seriously wrong. So we print a warning message and 203217a2b317SBill Taylor * return. 203317a2b317SBill Taylor */ 203417a2b317SBill Taylor if (qp->qp_is_special) { 203517a2b317SBill Taylor type = (qp->qp_is_special == HERMON_QP_SMI) ? 203617a2b317SBill Taylor IBT_SMI_SQP : IBT_GSI_SQP; 203717a2b317SBill Taylor 203817a2b317SBill Taylor /* Free up resources for the special QP */ 203917a2b317SBill Taylor status = hermon_special_qp_rsrc_free(state, type, port); 204017a2b317SBill Taylor if (status != DDI_SUCCESS) { 204117a2b317SBill Taylor HERMON_WARNING(state, "failed to free special QP rsrc"); 204217a2b317SBill Taylor status = ibc_get_ci_failure(0); 204317a2b317SBill Taylor goto qpfree_fail; 204417a2b317SBill Taylor } 204517a2b317SBill Taylor 204617a2b317SBill Taylor } else if (qp->qp_rangep) { 204717a2b317SBill Taylor int refcnt; 204817a2b317SBill Taylor mutex_enter(&qp->qp_rangep->hqpr_lock); 204917a2b317SBill Taylor refcnt = --qp->qp_rangep->hqpr_refcnt; 205017a2b317SBill Taylor mutex_exit(&qp->qp_rangep->hqpr_lock); 205117a2b317SBill Taylor if (refcnt == 0) { 205217a2b317SBill Taylor mutex_destroy(&qp->qp_rangep->hqpr_lock); 205317a2b317SBill Taylor hermon_rsrc_free(state, &qp->qp_rangep->hqpr_qpcrsrc); 205417a2b317SBill Taylor kmem_free(qp->qp_rangep, sizeof (*qp->qp_rangep)); 205517a2b317SBill Taylor } 205617a2b317SBill Taylor qp->qp_rangep = NULL; 205717a2b317SBill Taylor } else if (qp->qp_qpn_hdl == NULL) { 205817a2b317SBill Taylor hermon_rsrc_free(state, &qpc); 205917a2b317SBill Taylor } else { 206017a2b317SBill Taylor /* 206117a2b317SBill Taylor * Check the flags and determine whether to release the 206217a2b317SBill Taylor * QPN or not, based on their value. 206317a2b317SBill Taylor */ 206417a2b317SBill Taylor if (free_qp_flags == IBC_FREE_QP_ONLY) { 206517a2b317SBill Taylor entry = qp->qp_qpn_hdl; 206617a2b317SBill Taylor hermon_qp_release_qpn(state, qp->qp_qpn_hdl, 206717a2b317SBill Taylor HERMON_QPN_FREE_ONLY); 206817a2b317SBill Taylor *qpnh = (ibc_qpn_hdl_t)entry; 206917a2b317SBill Taylor } else { 207017a2b317SBill Taylor hermon_qp_release_qpn(state, qp->qp_qpn_hdl, 207117a2b317SBill Taylor HERMON_QPN_RELEASE); 207217a2b317SBill Taylor } 207317a2b317SBill Taylor } 207417a2b317SBill Taylor 207517a2b317SBill Taylor mutex_destroy(&qp->qp_sq_lock); 207617a2b317SBill Taylor 207717a2b317SBill Taylor /* Free the Hermon Queue Pair handle */ 207817a2b317SBill Taylor hermon_rsrc_free(state, &rsrc); 207917a2b317SBill Taylor 208017a2b317SBill Taylor /* Decrement the reference counts on CQs, PD and SRQ (if needed) */ 208117a2b317SBill Taylor hermon_cq_refcnt_dec(rq_cq); 208217a2b317SBill Taylor hermon_cq_refcnt_dec(sq_cq); 208317a2b317SBill Taylor hermon_pd_refcnt_dec(pd); 20849e39c5baSBill Taylor if (qp_srq_en == HERMON_QP_SRQ_ENABLED) { 20859e39c5baSBill Taylor hermon_srq_refcnt_dec(srq); 20869e39c5baSBill Taylor } 20879e39c5baSBill Taylor 20889e39c5baSBill Taylor /* Set the qphdl pointer to NULL and return success */ 20899e39c5baSBill Taylor *qphdl = NULL; 20909e39c5baSBill Taylor 20919e39c5baSBill Taylor return (DDI_SUCCESS); 20929e39c5baSBill Taylor 20939e39c5baSBill Taylor qpfree_fail: 20949e39c5baSBill Taylor return (status); 20959e39c5baSBill Taylor } 20969e39c5baSBill Taylor 20979e39c5baSBill Taylor 20989e39c5baSBill Taylor /* 20999e39c5baSBill Taylor * hermon_qp_query() 21009e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 21019e39c5baSBill Taylor */ 21029e39c5baSBill Taylor int 21039e39c5baSBill Taylor hermon_qp_query(hermon_state_t *state, hermon_qphdl_t qp, 21049e39c5baSBill Taylor ibt_qp_query_attr_t *attr_p) 21059e39c5baSBill Taylor { 21069e39c5baSBill Taylor ibt_cep_state_t qp_state; 21079e39c5baSBill Taylor ibt_qp_ud_attr_t *ud; 21089e39c5baSBill Taylor ibt_qp_rc_attr_t *rc; 21099e39c5baSBill Taylor ibt_qp_uc_attr_t *uc; 21109e39c5baSBill Taylor ibt_cep_flags_t enable_flags; 21119e39c5baSBill Taylor hermon_hw_addr_path_t *qpc_path, *qpc_alt_path; 21129e39c5baSBill Taylor ibt_cep_path_t *path_ptr, *alt_path_ptr; 21139e39c5baSBill Taylor hermon_hw_qpc_t *qpc; 21149e39c5baSBill Taylor int status; 21159e39c5baSBill Taylor uint_t tmp_sched_q, tmp_alt_sched_q; 21169e39c5baSBill Taylor 21179e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 21189e39c5baSBill Taylor 21199e39c5baSBill Taylor /* 21209e39c5baSBill Taylor * Grab the temporary QPC entry from QP software state 21219e39c5baSBill Taylor */ 21229e39c5baSBill Taylor qpc = &qp->qpc; 21239e39c5baSBill Taylor 21249e39c5baSBill Taylor /* Convert the current Hermon QP state to IBTF QP state */ 21259e39c5baSBill Taylor switch (qp->qp_state) { 21269e39c5baSBill Taylor case HERMON_QP_RESET: 21279e39c5baSBill Taylor qp_state = IBT_STATE_RESET; /* "Reset" */ 21289e39c5baSBill Taylor break; 21299e39c5baSBill Taylor case HERMON_QP_INIT: 21309e39c5baSBill Taylor qp_state = IBT_STATE_INIT; /* Initialized */ 21319e39c5baSBill Taylor break; 21329e39c5baSBill Taylor case HERMON_QP_RTR: 21339e39c5baSBill Taylor qp_state = IBT_STATE_RTR; /* Ready to Receive */ 21349e39c5baSBill Taylor break; 21359e39c5baSBill Taylor case HERMON_QP_RTS: 21369e39c5baSBill Taylor qp_state = IBT_STATE_RTS; /* Ready to Send */ 21379e39c5baSBill Taylor break; 21389e39c5baSBill Taylor case HERMON_QP_SQERR: 21399e39c5baSBill Taylor qp_state = IBT_STATE_SQE; /* Send Queue Error */ 21409e39c5baSBill Taylor break; 21419e39c5baSBill Taylor case HERMON_QP_SQD: 21429e39c5baSBill Taylor if (qp->qp_sqd_still_draining) { 21439e39c5baSBill Taylor qp_state = IBT_STATE_SQDRAIN; /* SQ Draining */ 21449e39c5baSBill Taylor } else { 21459e39c5baSBill Taylor qp_state = IBT_STATE_SQD; /* SQ Drained */ 21469e39c5baSBill Taylor } 21479e39c5baSBill Taylor break; 21489e39c5baSBill Taylor case HERMON_QP_ERR: 21499e39c5baSBill Taylor qp_state = IBT_STATE_ERROR; /* Error */ 21509e39c5baSBill Taylor break; 21519e39c5baSBill Taylor default: 21529e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 21539e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 21549e39c5baSBill Taylor } 21559e39c5baSBill Taylor attr_p->qp_info.qp_state = qp_state; 21569e39c5baSBill Taylor 21579e39c5baSBill Taylor /* SRQ Hook. */ 21589e39c5baSBill Taylor attr_p->qp_srq = NULL; 21599e39c5baSBill Taylor 21609e39c5baSBill Taylor /* 21619e39c5baSBill Taylor * The following QP information is always returned, regardless of 21629e39c5baSBill Taylor * the current QP state. Note: Some special handling is necessary 21639e39c5baSBill Taylor * for calculating the QP number on special QP (QP0 and QP1). 21649e39c5baSBill Taylor */ 216517a2b317SBill Taylor attr_p->qp_sq_cq = 216617a2b317SBill Taylor (qp->qp_sq_cqhdl == NULL) ? NULL : qp->qp_sq_cqhdl->cq_hdlrarg; 216717a2b317SBill Taylor attr_p->qp_rq_cq = 216817a2b317SBill Taylor (qp->qp_rq_cqhdl == NULL) ? NULL : qp->qp_rq_cqhdl->cq_hdlrarg; 21699e39c5baSBill Taylor if (qp->qp_is_special) { 21709e39c5baSBill Taylor attr_p->qp_qpn = (qp->qp_is_special == HERMON_QP_SMI) ? 0 : 1; 21719e39c5baSBill Taylor } else { 21729e39c5baSBill Taylor attr_p->qp_qpn = (ib_qpn_t)qp->qp_qpnum; 21739e39c5baSBill Taylor } 21749e39c5baSBill Taylor attr_p->qp_sq_sgl = qp->qp_sq_sgl; 21759e39c5baSBill Taylor attr_p->qp_rq_sgl = qp->qp_rq_sgl; 21769e39c5baSBill Taylor attr_p->qp_info.qp_sq_sz = qp->qp_sq_bufsz - qp->qp_sq_hdrmwqes; 21779e39c5baSBill Taylor attr_p->qp_info.qp_rq_sz = qp->qp_rq_bufsz; 21789e39c5baSBill Taylor 21799e39c5baSBill Taylor /* 21809e39c5baSBill Taylor * If QP is currently in the "Reset" state, then only the above are 21819e39c5baSBill Taylor * returned 21829e39c5baSBill Taylor */ 21839e39c5baSBill Taylor if (qp_state == IBT_STATE_RESET) { 21849e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 21859e39c5baSBill Taylor return (DDI_SUCCESS); 21869e39c5baSBill Taylor } 21879e39c5baSBill Taylor 21889e39c5baSBill Taylor /* 21899e39c5baSBill Taylor * Post QUERY_QP command to firmware 21909e39c5baSBill Taylor * 21919e39c5baSBill Taylor * We do a HERMON_NOSLEEP here because we are holding the "qp_lock". 21929e39c5baSBill Taylor * Since we may be in the interrupt context (or subsequently raised 21939e39c5baSBill Taylor * to interrupt level by priority inversion), we do not want to block 21949e39c5baSBill Taylor * in this routine waiting for success. 21959e39c5baSBill Taylor */ 21969e39c5baSBill Taylor tmp_sched_q = qpc->pri_addr_path.sched_q; 21979e39c5baSBill Taylor tmp_alt_sched_q = qpc->alt_addr_path.sched_q; 21989e39c5baSBill Taylor status = hermon_cmn_query_cmd_post(state, QUERY_QP, 0, qp->qp_qpnum, 21999e39c5baSBill Taylor qpc, sizeof (hermon_hw_qpc_t), HERMON_CMD_NOSLEEP_SPIN); 22009e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 22019e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 22029e39c5baSBill Taylor cmn_err(CE_WARN, "hermon%d: hermon_qp_query: QUERY_QP " 22039e39c5baSBill Taylor "command failed: %08x\n", state->hs_instance, status); 22049e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 22059e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 22069e39c5baSBill Taylor } 22079e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 22089e39c5baSBill Taylor } 22099e39c5baSBill Taylor qpc->pri_addr_path.sched_q = tmp_sched_q; 22109e39c5baSBill Taylor qpc->alt_addr_path.sched_q = tmp_alt_sched_q; 22119e39c5baSBill Taylor 22129e39c5baSBill Taylor /* 22139e39c5baSBill Taylor * Fill in the additional QP info based on the QP's transport type. 22149e39c5baSBill Taylor */ 221517a2b317SBill Taylor if (qp->qp_type == IBT_UD_RQP) { 22169e39c5baSBill Taylor 22179e39c5baSBill Taylor /* Fill in the UD-specific info */ 22189e39c5baSBill Taylor ud = &attr_p->qp_info.qp_transport.ud; 22199e39c5baSBill Taylor ud->ud_qkey = (ib_qkey_t)qpc->qkey; 22209e39c5baSBill Taylor ud->ud_sq_psn = qpc->next_snd_psn; 22219e39c5baSBill Taylor ud->ud_pkey_ix = qpc->pri_addr_path.pkey_indx; 22229e39c5baSBill Taylor /* port+1 for port 1/2 */ 22239e39c5baSBill Taylor ud->ud_port = 22249e39c5baSBill Taylor (uint8_t)(((qpc->pri_addr_path.sched_q >> 6) & 0x01) + 1); 22259e39c5baSBill Taylor 22269e39c5baSBill Taylor attr_p->qp_info.qp_trans = IBT_UD_SRV; 22279e39c5baSBill Taylor 222817a2b317SBill Taylor if (qp->qp_serv_type == HERMON_QP_FEXCH) { 222917a2b317SBill Taylor ibt_pmr_desc_t *pmr; 223017a2b317SBill Taylor uint64_t heart_beat; 223117a2b317SBill Taylor 223217a2b317SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr)) 223317a2b317SBill Taylor pmr = &attr_p->qp_query_fexch.fq_uni_mem_desc; 223417a2b317SBill Taylor pmr->pmd_iova = 0; 223517a2b317SBill Taylor pmr->pmd_lkey = pmr->pmd_rkey = 223617a2b317SBill Taylor hermon_fcoib_qpn_to_mkey(state, qp->qp_qpnum); 223717a2b317SBill Taylor pmr->pmd_phys_buf_list_sz = 223817a2b317SBill Taylor state->hs_fcoib.hfc_mtts_per_mpt; 223917a2b317SBill Taylor pmr->pmd_sync_required = 0; 224017a2b317SBill Taylor 224117a2b317SBill Taylor pmr = &attr_p->qp_query_fexch.fq_bi_mem_desc; 224217a2b317SBill Taylor pmr->pmd_iova = 0; 224317a2b317SBill Taylor pmr->pmd_lkey = 0; 224417a2b317SBill Taylor pmr->pmd_rkey = 0; 224517a2b317SBill Taylor pmr->pmd_phys_buf_list_sz = 0; 224617a2b317SBill Taylor pmr->pmd_sync_required = 0; 224717a2b317SBill Taylor 224817a2b317SBill Taylor attr_p->qp_query_fexch.fq_flags = 224917a2b317SBill Taylor ((hermon_get_heart_beat_rq_cmd_post(state, 225017a2b317SBill Taylor qp->qp_qpnum, &heart_beat) == HERMON_CMD_SUCCESS) && 225117a2b317SBill Taylor (heart_beat == 0)) ? IBT_FEXCH_HEART_BEAT_OK : 225217a2b317SBill Taylor IBT_FEXCH_NO_FLAGS; 225317a2b317SBill Taylor 225417a2b317SBill Taylor ud->ud_fc = qp->qp_fc_attr; 225517a2b317SBill Taylor } else if (qp->qp_serv_type == HERMON_QP_FCMND || 225617a2b317SBill Taylor qp->qp_serv_type == HERMON_QP_RFCI) { 225717a2b317SBill Taylor ud->ud_fc = qp->qp_fc_attr; 225817a2b317SBill Taylor } 225917a2b317SBill Taylor 22609e39c5baSBill Taylor } else if (qp->qp_serv_type == HERMON_QP_RC) { 22619e39c5baSBill Taylor 22629e39c5baSBill Taylor /* Fill in the RC-specific info */ 22639e39c5baSBill Taylor rc = &attr_p->qp_info.qp_transport.rc; 22649e39c5baSBill Taylor rc->rc_sq_psn = qpc->next_snd_psn; 22659e39c5baSBill Taylor rc->rc_rq_psn = qpc->next_rcv_psn; 22669e39c5baSBill Taylor rc->rc_dst_qpn = qpc->rem_qpn; 22679e39c5baSBill Taylor 22689e39c5baSBill Taylor /* Grab the path migration state information */ 22699e39c5baSBill Taylor if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) { 22709e39c5baSBill Taylor rc->rc_mig_state = IBT_STATE_MIGRATED; 22719e39c5baSBill Taylor } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) { 22729e39c5baSBill Taylor rc->rc_mig_state = IBT_STATE_REARMED; 22739e39c5baSBill Taylor } else { 22749e39c5baSBill Taylor rc->rc_mig_state = IBT_STATE_ARMED; 22759e39c5baSBill Taylor } 22769e39c5baSBill Taylor rc->rc_rdma_ra_out = (1 << qpc->sra_max); 22779e39c5baSBill Taylor rc->rc_rdma_ra_in = (1 << qpc->rra_max); 22789e39c5baSBill Taylor rc->rc_min_rnr_nak = qpc->min_rnr_nak; 22799e39c5baSBill Taylor rc->rc_path_mtu = qpc->mtu; 22809e39c5baSBill Taylor rc->rc_retry_cnt = qpc->retry_cnt; 22819e39c5baSBill Taylor 22829e39c5baSBill Taylor /* Get the common primary address path fields */ 22839e39c5baSBill Taylor qpc_path = &qpc->pri_addr_path; 22849e39c5baSBill Taylor path_ptr = &rc->rc_path; 22859e39c5baSBill Taylor hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect, 22869e39c5baSBill Taylor HERMON_ADDRPATH_QP); 22879e39c5baSBill Taylor 22889e39c5baSBill Taylor /* Fill in the additional primary address path fields */ 22899e39c5baSBill Taylor path_ptr->cep_pkey_ix = qpc_path->pkey_indx; 22909e39c5baSBill Taylor path_ptr->cep_hca_port_num = 22919e39c5baSBill Taylor path_ptr->cep_adds_vect.av_port_num = 22929e39c5baSBill Taylor (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1); 22939e39c5baSBill Taylor path_ptr->cep_timeout = qpc_path->ack_timeout; 22949e39c5baSBill Taylor 22959e39c5baSBill Taylor /* Get the common alternate address path fields */ 22969e39c5baSBill Taylor qpc_alt_path = &qpc->alt_addr_path; 22979e39c5baSBill Taylor alt_path_ptr = &rc->rc_alt_path; 22989e39c5baSBill Taylor hermon_get_addr_path(state, qpc_alt_path, 22999e39c5baSBill Taylor &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP); 23009e39c5baSBill Taylor 23019e39c5baSBill Taylor /* Fill in the additional alternate address path fields */ 23029e39c5baSBill Taylor alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx; 23039e39c5baSBill Taylor alt_path_ptr->cep_hca_port_num = 23049e39c5baSBill Taylor alt_path_ptr->cep_adds_vect.av_port_num = 23059e39c5baSBill Taylor (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1); 23069e39c5baSBill Taylor alt_path_ptr->cep_timeout = qpc_alt_path->ack_timeout; 23079e39c5baSBill Taylor 23089e39c5baSBill Taylor /* Get the RNR retry time from primary path */ 23099e39c5baSBill Taylor rc->rc_rnr_retry_cnt = qpc->rnr_retry; 23109e39c5baSBill Taylor 23119e39c5baSBill Taylor /* Set the enable flags based on RDMA/Atomic enable bits */ 23129e39c5baSBill Taylor enable_flags = IBT_CEP_NO_FLAGS; 23139e39c5baSBill Taylor enable_flags |= ((qpc->rre == 0) ? 0 : IBT_CEP_RDMA_RD); 23149e39c5baSBill Taylor enable_flags |= ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR); 23159e39c5baSBill Taylor enable_flags |= ((qpc->rae == 0) ? 0 : IBT_CEP_ATOMIC); 23169e39c5baSBill Taylor attr_p->qp_info.qp_flags = enable_flags; 23179e39c5baSBill Taylor 23189e39c5baSBill Taylor attr_p->qp_info.qp_trans = IBT_RC_SRV; 23199e39c5baSBill Taylor 23209e39c5baSBill Taylor } else if (qp->qp_serv_type == HERMON_QP_UC) { 23219e39c5baSBill Taylor 23229e39c5baSBill Taylor /* Fill in the UC-specific info */ 23239e39c5baSBill Taylor uc = &attr_p->qp_info.qp_transport.uc; 23249e39c5baSBill Taylor uc->uc_sq_psn = qpc->next_snd_psn; 23259e39c5baSBill Taylor uc->uc_rq_psn = qpc->next_rcv_psn; 23269e39c5baSBill Taylor uc->uc_dst_qpn = qpc->rem_qpn; 23279e39c5baSBill Taylor 23289e39c5baSBill Taylor /* Grab the path migration state information */ 23299e39c5baSBill Taylor if (qpc->pm_state == HERMON_QP_PMSTATE_MIGRATED) { 23309e39c5baSBill Taylor uc->uc_mig_state = IBT_STATE_MIGRATED; 23319e39c5baSBill Taylor } else if (qpc->pm_state == HERMON_QP_PMSTATE_REARM) { 23329e39c5baSBill Taylor uc->uc_mig_state = IBT_STATE_REARMED; 23339e39c5baSBill Taylor } else { 23349e39c5baSBill Taylor uc->uc_mig_state = IBT_STATE_ARMED; 23359e39c5baSBill Taylor } 23369e39c5baSBill Taylor uc->uc_path_mtu = qpc->mtu; 23379e39c5baSBill Taylor 23389e39c5baSBill Taylor /* Get the common primary address path fields */ 23399e39c5baSBill Taylor qpc_path = &qpc->pri_addr_path; 23409e39c5baSBill Taylor path_ptr = &uc->uc_path; 23419e39c5baSBill Taylor hermon_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect, 23429e39c5baSBill Taylor HERMON_ADDRPATH_QP); 23439e39c5baSBill Taylor 23449e39c5baSBill Taylor /* Fill in the additional primary address path fields */ 23459e39c5baSBill Taylor path_ptr->cep_pkey_ix = qpc_path->pkey_indx; 23469e39c5baSBill Taylor path_ptr->cep_hca_port_num = 23479e39c5baSBill Taylor path_ptr->cep_adds_vect.av_port_num = 23489e39c5baSBill Taylor (uint8_t)(((qpc_path->sched_q >> 6) & 0x01) + 1); 23499e39c5baSBill Taylor 23509e39c5baSBill Taylor /* Get the common alternate address path fields */ 23519e39c5baSBill Taylor qpc_alt_path = &qpc->alt_addr_path; 23529e39c5baSBill Taylor alt_path_ptr = &uc->uc_alt_path; 23539e39c5baSBill Taylor hermon_get_addr_path(state, qpc_alt_path, 23549e39c5baSBill Taylor &alt_path_ptr->cep_adds_vect, HERMON_ADDRPATH_QP); 23559e39c5baSBill Taylor 23569e39c5baSBill Taylor /* Fill in the additional alternate address path fields */ 23579e39c5baSBill Taylor alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx; 23589e39c5baSBill Taylor alt_path_ptr->cep_hca_port_num = 23599e39c5baSBill Taylor alt_path_ptr->cep_adds_vect.av_port_num = 23609e39c5baSBill Taylor (uint8_t)(((qpc_alt_path->sched_q >> 6) & 0x01) + 1); 23619e39c5baSBill Taylor 23629e39c5baSBill Taylor /* 23639e39c5baSBill Taylor * Set the enable flags based on RDMA enable bits (by 23649e39c5baSBill Taylor * definition UC doesn't support Atomic or RDMA Read) 23659e39c5baSBill Taylor */ 23669e39c5baSBill Taylor enable_flags = ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR); 23679e39c5baSBill Taylor attr_p->qp_info.qp_flags = enable_flags; 23689e39c5baSBill Taylor 23699e39c5baSBill Taylor attr_p->qp_info.qp_trans = IBT_UC_SRV; 23709e39c5baSBill Taylor 23719e39c5baSBill Taylor } else { 23729e39c5baSBill Taylor HERMON_WARNING(state, "unexpected QP transport type"); 23739e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 23749e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 23759e39c5baSBill Taylor } 23769e39c5baSBill Taylor 23779e39c5baSBill Taylor /* 23789e39c5baSBill Taylor * Under certain circumstances it is possible for the Hermon hardware 23799e39c5baSBill Taylor * to transition to one of the error states without software directly 23809e39c5baSBill Taylor * knowing about it. The QueryQP() call is the one place where we 23819e39c5baSBill Taylor * have an opportunity to sample and update our view of the QP state. 23829e39c5baSBill Taylor */ 23839e39c5baSBill Taylor if (qpc->state == HERMON_QP_SQERR) { 23849e39c5baSBill Taylor attr_p->qp_info.qp_state = IBT_STATE_SQE; 23859e39c5baSBill Taylor qp->qp_state = HERMON_QP_SQERR; 238617a2b317SBill Taylor HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_SQERR); 23879e39c5baSBill Taylor } 23889e39c5baSBill Taylor if (qpc->state == HERMON_QP_ERR) { 23899e39c5baSBill Taylor attr_p->qp_info.qp_state = IBT_STATE_ERROR; 23909e39c5baSBill Taylor qp->qp_state = HERMON_QP_ERR; 239117a2b317SBill Taylor HERMON_SET_QP_POST_SEND_STATE(qp, HERMON_QP_ERR); 23929e39c5baSBill Taylor } 23939e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 23949e39c5baSBill Taylor 23959e39c5baSBill Taylor return (DDI_SUCCESS); 23969e39c5baSBill Taylor } 23979e39c5baSBill Taylor 23989e39c5baSBill Taylor 23999e39c5baSBill Taylor /* 24009e39c5baSBill Taylor * hermon_qp_create_qpn() 24019e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 24029e39c5baSBill Taylor */ 24039e39c5baSBill Taylor static int 24049e39c5baSBill Taylor hermon_qp_create_qpn(hermon_state_t *state, hermon_qphdl_t qp, 24059e39c5baSBill Taylor hermon_rsrc_t *qpc) 24069e39c5baSBill Taylor { 24079e39c5baSBill Taylor hermon_qpn_entry_t query; 24089e39c5baSBill Taylor hermon_qpn_entry_t *entry; 24099e39c5baSBill Taylor avl_index_t where; 24109e39c5baSBill Taylor 24119e39c5baSBill Taylor /* 24129e39c5baSBill Taylor * Build a query (for the AVL tree lookup) and attempt to find 24139e39c5baSBill Taylor * a previously added entry that has a matching QPC index. If 24149e39c5baSBill Taylor * no matching entry is found, then allocate, initialize, and 24159e39c5baSBill Taylor * add an entry to the AVL tree. 24169e39c5baSBill Taylor * If a matching entry is found, then increment its QPN counter 24179e39c5baSBill Taylor * and reference counter. 24189e39c5baSBill Taylor */ 24199e39c5baSBill Taylor query.qpn_indx = qpc->hr_indx; 24209e39c5baSBill Taylor mutex_enter(&state->hs_qpn_avl_lock); 24219e39c5baSBill Taylor entry = (hermon_qpn_entry_t *)avl_find(&state->hs_qpn_avl, 24229e39c5baSBill Taylor &query, &where); 24239e39c5baSBill Taylor if (entry == NULL) { 24249e39c5baSBill Taylor /* 24259e39c5baSBill Taylor * Allocate and initialize a QPN entry, then insert 24269e39c5baSBill Taylor * it into the AVL tree. 24279e39c5baSBill Taylor */ 24289e39c5baSBill Taylor entry = (hermon_qpn_entry_t *)kmem_zalloc( 24299e39c5baSBill Taylor sizeof (hermon_qpn_entry_t), KM_NOSLEEP); 24309e39c5baSBill Taylor if (entry == NULL) { 24319e39c5baSBill Taylor mutex_exit(&state->hs_qpn_avl_lock); 24329e39c5baSBill Taylor return (DDI_FAILURE); 24339e39c5baSBill Taylor } 24349e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry)) 24359e39c5baSBill Taylor 24369e39c5baSBill Taylor entry->qpn_indx = qpc->hr_indx; 24379e39c5baSBill Taylor entry->qpn_refcnt = 0; 24389e39c5baSBill Taylor entry->qpn_counter = 0; 24399e39c5baSBill Taylor 24409e39c5baSBill Taylor avl_insert(&state->hs_qpn_avl, entry, where); 24419e39c5baSBill Taylor } 24429e39c5baSBill Taylor 24439e39c5baSBill Taylor /* 24449e39c5baSBill Taylor * Make the AVL tree entry point to the QP context resource that 24459e39c5baSBill Taylor * it will be responsible for tracking 24469e39c5baSBill Taylor */ 24479e39c5baSBill Taylor entry->qpn_qpc = qpc; 24489e39c5baSBill Taylor 24499e39c5baSBill Taylor /* 24509e39c5baSBill Taylor * Setup the QP handle to point to the AVL tree entry. Then 24519e39c5baSBill Taylor * generate the new QP number from the entry's QPN counter value 24529e39c5baSBill Taylor * and the hardware's QP context table index. 24539e39c5baSBill Taylor */ 24549e39c5baSBill Taylor qp->qp_qpn_hdl = entry; 24559e39c5baSBill Taylor qp->qp_qpnum = ((entry->qpn_counter << 24569e39c5baSBill Taylor state->hs_cfg_profile->cp_log_num_qp) | qpc->hr_indx) & 24579e39c5baSBill Taylor HERMON_QP_MAXNUMBER_MSK; 24589e39c5baSBill Taylor qp->qp_ring = qp->qp_qpnum << 8; 24599e39c5baSBill Taylor 24609e39c5baSBill Taylor /* 24619e39c5baSBill Taylor * Increment the reference counter and QPN counter. The QPN 24629e39c5baSBill Taylor * counter always indicates the next available number for use. 24639e39c5baSBill Taylor */ 24649e39c5baSBill Taylor entry->qpn_counter++; 24659e39c5baSBill Taylor entry->qpn_refcnt++; 24669e39c5baSBill Taylor 24679e39c5baSBill Taylor mutex_exit(&state->hs_qpn_avl_lock); 24689e39c5baSBill Taylor 24699e39c5baSBill Taylor return (DDI_SUCCESS); 24709e39c5baSBill Taylor } 24719e39c5baSBill Taylor 24729e39c5baSBill Taylor 24739e39c5baSBill Taylor /* 24749e39c5baSBill Taylor * hermon_qp_release_qpn() 24759e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 24769e39c5baSBill Taylor */ 24779e39c5baSBill Taylor void 24789e39c5baSBill Taylor hermon_qp_release_qpn(hermon_state_t *state, hermon_qpn_entry_t *entry, 24799e39c5baSBill Taylor int flags) 24809e39c5baSBill Taylor { 24819e39c5baSBill Taylor ASSERT(entry != NULL); 24829e39c5baSBill Taylor 24839e39c5baSBill Taylor mutex_enter(&state->hs_qpn_avl_lock); 24849e39c5baSBill Taylor 24859e39c5baSBill Taylor /* 24869e39c5baSBill Taylor * If we are releasing the QP number here, then we decrement the 24879e39c5baSBill Taylor * reference count and check for zero references. If there are 24889e39c5baSBill Taylor * zero references, then we free the QPC context (if it hadn't 24899e39c5baSBill Taylor * already been freed during a HERMON_QPN_FREE_ONLY free, i.e. for 24909e39c5baSBill Taylor * reuse with another similar QP number) and remove the tracking 24919e39c5baSBill Taylor * structure from the QP number AVL tree and free the structure. 24929e39c5baSBill Taylor * If we are not releasing the QP number here, then, as long as we 24939e39c5baSBill Taylor * have not exhausted the usefulness of the QPC context (that is, 24949e39c5baSBill Taylor * re-used it too many times without the reference count having 24959e39c5baSBill Taylor * gone to zero), we free up the QPC context for use by another 24969e39c5baSBill Taylor * thread (which will use it to construct a different QP number 24979e39c5baSBill Taylor * from the same QPC table index). 24989e39c5baSBill Taylor */ 24999e39c5baSBill Taylor if (flags == HERMON_QPN_RELEASE) { 25009e39c5baSBill Taylor entry->qpn_refcnt--; 25019e39c5baSBill Taylor 25029e39c5baSBill Taylor /* 25039e39c5baSBill Taylor * If the reference count is zero, then we free the QPC 25049e39c5baSBill Taylor * context (if it hadn't already been freed in an early 25059e39c5baSBill Taylor * step, e.g. HERMON_QPN_FREE_ONLY) and remove/free the 25069e39c5baSBill Taylor * tracking structure from the QP number AVL tree. 25079e39c5baSBill Taylor */ 25089e39c5baSBill Taylor if (entry->qpn_refcnt == 0) { 25099e39c5baSBill Taylor if (entry->qpn_qpc != NULL) { 25109e39c5baSBill Taylor hermon_rsrc_free(state, &entry->qpn_qpc); 25119e39c5baSBill Taylor } 25129e39c5baSBill Taylor 25139e39c5baSBill Taylor /* 25149e39c5baSBill Taylor * If the current entry has served it's useful 25159e39c5baSBill Taylor * purpose (i.e. been reused the maximum allowable 25169e39c5baSBill Taylor * number of times), then remove it from QP number 25179e39c5baSBill Taylor * AVL tree and free it up. 25189e39c5baSBill Taylor */ 25199e39c5baSBill Taylor if (entry->qpn_counter >= (1 << 25209e39c5baSBill Taylor (24 - state->hs_cfg_profile->cp_log_num_qp))) { 25219e39c5baSBill Taylor avl_remove(&state->hs_qpn_avl, entry); 25229e39c5baSBill Taylor kmem_free(entry, sizeof (hermon_qpn_entry_t)); 25239e39c5baSBill Taylor } 25249e39c5baSBill Taylor } 25259e39c5baSBill Taylor 25269e39c5baSBill Taylor } else if (flags == HERMON_QPN_FREE_ONLY) { 25279e39c5baSBill Taylor /* 25289e39c5baSBill Taylor * Even if we are not freeing the QP number, that will not 25299e39c5baSBill Taylor * always prevent us from releasing the QPC context. In fact, 25309e39c5baSBill Taylor * since the QPC context only forms part of the whole QPN, 25319e39c5baSBill Taylor * we want to free it up for use by other consumers. But 25329e39c5baSBill Taylor * if the reference count is non-zero (which it will always 25339e39c5baSBill Taylor * be when we are doing HERMON_QPN_FREE_ONLY) and the counter 25349e39c5baSBill Taylor * has reached its maximum value, then we cannot reuse the 25359e39c5baSBill Taylor * QPC context until the reference count eventually reaches 25369e39c5baSBill Taylor * zero (in HERMON_QPN_RELEASE, above). 25379e39c5baSBill Taylor */ 25389e39c5baSBill Taylor if (entry->qpn_counter < (1 << 25399e39c5baSBill Taylor (24 - state->hs_cfg_profile->cp_log_num_qp))) { 25409e39c5baSBill Taylor hermon_rsrc_free(state, &entry->qpn_qpc); 25419e39c5baSBill Taylor } 25429e39c5baSBill Taylor } 25439e39c5baSBill Taylor mutex_exit(&state->hs_qpn_avl_lock); 25449e39c5baSBill Taylor } 25459e39c5baSBill Taylor 25469e39c5baSBill Taylor 25479e39c5baSBill Taylor /* 25489e39c5baSBill Taylor * hermon_qpn_avl_compare() 25499e39c5baSBill Taylor * Context: Can be called from user or kernel context. 25509e39c5baSBill Taylor */ 25519e39c5baSBill Taylor static int 25529e39c5baSBill Taylor hermon_qpn_avl_compare(const void *q, const void *e) 25539e39c5baSBill Taylor { 25549e39c5baSBill Taylor hermon_qpn_entry_t *entry, *query; 25559e39c5baSBill Taylor 25569e39c5baSBill Taylor entry = (hermon_qpn_entry_t *)e; 25579e39c5baSBill Taylor query = (hermon_qpn_entry_t *)q; 25589e39c5baSBill Taylor 25599e39c5baSBill Taylor if (query->qpn_indx < entry->qpn_indx) { 25609e39c5baSBill Taylor return (-1); 25619e39c5baSBill Taylor } else if (query->qpn_indx > entry->qpn_indx) { 25629e39c5baSBill Taylor return (+1); 25639e39c5baSBill Taylor } else { 25649e39c5baSBill Taylor return (0); 25659e39c5baSBill Taylor } 25669e39c5baSBill Taylor } 25679e39c5baSBill Taylor 25689e39c5baSBill Taylor 25699e39c5baSBill Taylor /* 25709e39c5baSBill Taylor * hermon_qpn_avl_init() 25719e39c5baSBill Taylor * Context: Only called from attach() path context 25729e39c5baSBill Taylor */ 25739e39c5baSBill Taylor void 25749e39c5baSBill Taylor hermon_qpn_avl_init(hermon_state_t *state) 25759e39c5baSBill Taylor { 25769e39c5baSBill Taylor /* Initialize the lock used for QP number (QPN) AVL tree access */ 25779e39c5baSBill Taylor mutex_init(&state->hs_qpn_avl_lock, NULL, MUTEX_DRIVER, 25789e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 25799e39c5baSBill Taylor 25809e39c5baSBill Taylor /* Initialize the AVL tree for the QP number (QPN) storage */ 25819e39c5baSBill Taylor avl_create(&state->hs_qpn_avl, hermon_qpn_avl_compare, 25829e39c5baSBill Taylor sizeof (hermon_qpn_entry_t), 25839e39c5baSBill Taylor offsetof(hermon_qpn_entry_t, qpn_avlnode)); 25849e39c5baSBill Taylor } 25859e39c5baSBill Taylor 25869e39c5baSBill Taylor 25879e39c5baSBill Taylor /* 25889e39c5baSBill Taylor * hermon_qpn_avl_fini() 25899e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts 25909e39c5baSBill Taylor */ 25919e39c5baSBill Taylor void 25929e39c5baSBill Taylor hermon_qpn_avl_fini(hermon_state_t *state) 25939e39c5baSBill Taylor { 25949e39c5baSBill Taylor hermon_qpn_entry_t *entry; 25959e39c5baSBill Taylor void *cookie; 25969e39c5baSBill Taylor 25979e39c5baSBill Taylor /* 25989e39c5baSBill Taylor * Empty all entries (if necessary) and destroy the AVL tree 25999e39c5baSBill Taylor * that was used for QP number (QPN) tracking. 26009e39c5baSBill Taylor */ 26019e39c5baSBill Taylor cookie = NULL; 26029e39c5baSBill Taylor while ((entry = (hermon_qpn_entry_t *)avl_destroy_nodes( 26039e39c5baSBill Taylor &state->hs_qpn_avl, &cookie)) != NULL) { 26049e39c5baSBill Taylor kmem_free(entry, sizeof (hermon_qpn_entry_t)); 26059e39c5baSBill Taylor } 26069e39c5baSBill Taylor avl_destroy(&state->hs_qpn_avl); 26079e39c5baSBill Taylor 26089e39c5baSBill Taylor /* Destroy the lock used for QP number (QPN) AVL tree access */ 26099e39c5baSBill Taylor mutex_destroy(&state->hs_qpn_avl_lock); 26109e39c5baSBill Taylor } 26119e39c5baSBill Taylor 26129e39c5baSBill Taylor 26139e39c5baSBill Taylor /* 26149e39c5baSBill Taylor * hermon_qphdl_from_qpnum() 26159e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 26169e39c5baSBill Taylor * 26179e39c5baSBill Taylor * This routine is important because changing the unconstrained 26189e39c5baSBill Taylor * portion of the QP number is critical to the detection of a 26199e39c5baSBill Taylor * potential race condition in the QP event handler code (i.e. the case 26209e39c5baSBill Taylor * where a QP is freed and alloc'd again before an event for the 26219e39c5baSBill Taylor * "old" QP can be handled). 26229e39c5baSBill Taylor * 26239e39c5baSBill Taylor * While this is not a perfect solution (not sure that one exists) 26249e39c5baSBill Taylor * it does help to mitigate the chance that this race condition will 26259e39c5baSBill Taylor * cause us to deliver a "stale" event to the new QP owner. Note: 26269e39c5baSBill Taylor * this solution does not scale well because the number of constrained 26279e39c5baSBill Taylor * bits increases (and, hence, the number of unconstrained bits 26289e39c5baSBill Taylor * decreases) as the number of supported QPs grows. For small and 26299e39c5baSBill Taylor * intermediate values, it should hopefully provide sufficient 26309e39c5baSBill Taylor * protection. 26319e39c5baSBill Taylor */ 26329e39c5baSBill Taylor hermon_qphdl_t 26339e39c5baSBill Taylor hermon_qphdl_from_qpnum(hermon_state_t *state, uint_t qpnum) 26349e39c5baSBill Taylor { 26359e39c5baSBill Taylor uint_t qpindx, qpmask; 26369e39c5baSBill Taylor 26379e39c5baSBill Taylor /* Calculate the QP table index from the qpnum */ 26389e39c5baSBill Taylor qpmask = (1 << state->hs_cfg_profile->cp_log_num_qp) - 1; 26399e39c5baSBill Taylor qpindx = qpnum & qpmask; 264017a2b317SBill Taylor return (hermon_icm_num_to_hdl(state, HERMON_QPC, qpindx)); 26419e39c5baSBill Taylor } 26429e39c5baSBill Taylor 26439e39c5baSBill Taylor 26449e39c5baSBill Taylor /* 26459e39c5baSBill Taylor * hermon_special_qp_rsrc_alloc 26469e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 26479e39c5baSBill Taylor */ 26489e39c5baSBill Taylor static int 26499e39c5baSBill Taylor hermon_special_qp_rsrc_alloc(hermon_state_t *state, ibt_sqp_type_t type, 26509e39c5baSBill Taylor uint_t port, hermon_rsrc_t **qp_rsrc) 26519e39c5baSBill Taylor { 26529e39c5baSBill Taylor uint_t mask, flags; 26539e39c5baSBill Taylor int status; 26549e39c5baSBill Taylor 26559e39c5baSBill Taylor mutex_enter(&state->hs_spec_qplock); 26569e39c5baSBill Taylor flags = state->hs_spec_qpflags; 26579e39c5baSBill Taylor if (type == IBT_SMI_SQP) { 26589e39c5baSBill Taylor /* 26599e39c5baSBill Taylor * Check here to see if the driver has been configured 26609e39c5baSBill Taylor * to instruct the Hermon firmware to handle all incoming 26619e39c5baSBill Taylor * SMP messages (i.e. messages sent to SMA). If so, 26629e39c5baSBill Taylor * then we will treat QP0 as if it has already been 26639e39c5baSBill Taylor * allocated (for internal use). Otherwise, if we allow 26649e39c5baSBill Taylor * the allocation to happen, it will cause unexpected 26659e39c5baSBill Taylor * behaviors (e.g. Hermon SMA becomes unresponsive). 26669e39c5baSBill Taylor */ 26679e39c5baSBill Taylor if (state->hs_cfg_profile->cp_qp0_agents_in_fw != 0) { 26689e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 26699e39c5baSBill Taylor return (IBT_QP_IN_USE); 26709e39c5baSBill Taylor } 26719e39c5baSBill Taylor 26729e39c5baSBill Taylor /* 26739e39c5baSBill Taylor * If this is the first QP0 allocation, then post 26749e39c5baSBill Taylor * a CONF_SPECIAL_QP firmware command 26759e39c5baSBill Taylor */ 26769e39c5baSBill Taylor if ((flags & HERMON_SPECIAL_QP0_RSRC_MASK) == 0) { 26779e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 26789e39c5baSBill Taylor state->hs_spec_qp0->hr_indx, HERMON_CMD_QP_SMI, 26799e39c5baSBill Taylor HERMON_CMD_NOSLEEP_SPIN, 26809e39c5baSBill Taylor HERMON_CMD_SPEC_QP_OPMOD( 26819e39c5baSBill Taylor state->hs_cfg_profile->cp_qp0_agents_in_fw, 26829e39c5baSBill Taylor state->hs_cfg_profile->cp_qp1_agents_in_fw)); 26839e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 26849e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 26859e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 26869e39c5baSBill Taylor "command failed: %08x\n", 26879e39c5baSBill Taylor state->hs_instance, status); 26889e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 26899e39c5baSBill Taylor } 26909e39c5baSBill Taylor } 26919e39c5baSBill Taylor 26929e39c5baSBill Taylor /* 26939e39c5baSBill Taylor * Now check (and, if necessary, modify) the flags to indicate 26949e39c5baSBill Taylor * whether the allocation was successful 26959e39c5baSBill Taylor */ 26969e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port)); 26979e39c5baSBill Taylor if (flags & mask) { 26989e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 26999e39c5baSBill Taylor return (IBT_QP_IN_USE); 27009e39c5baSBill Taylor } 27019e39c5baSBill Taylor state->hs_spec_qpflags |= mask; 27029e39c5baSBill Taylor *qp_rsrc = state->hs_spec_qp0; 27039e39c5baSBill Taylor 27049e39c5baSBill Taylor } else { 27059e39c5baSBill Taylor /* 27069e39c5baSBill Taylor * If this is the first QP1 allocation, then post 27079e39c5baSBill Taylor * a CONF_SPECIAL_QP firmware command 27089e39c5baSBill Taylor */ 27099e39c5baSBill Taylor if ((flags & HERMON_SPECIAL_QP1_RSRC_MASK) == 0) { 27109e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 27119e39c5baSBill Taylor state->hs_spec_qp1->hr_indx, HERMON_CMD_QP_GSI, 27129e39c5baSBill Taylor HERMON_CMD_NOSLEEP_SPIN, 27139e39c5baSBill Taylor HERMON_CMD_SPEC_QP_OPMOD( 27149e39c5baSBill Taylor state->hs_cfg_profile->cp_qp0_agents_in_fw, 27159e39c5baSBill Taylor state->hs_cfg_profile->cp_qp1_agents_in_fw)); 27169e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 27179e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 27189e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 27199e39c5baSBill Taylor "command failed: %08x\n", 27209e39c5baSBill Taylor state->hs_instance, status); 27219e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 27229e39c5baSBill Taylor } 27239e39c5baSBill Taylor } 27249e39c5baSBill Taylor 27259e39c5baSBill Taylor /* 27269e39c5baSBill Taylor * Now check (and, if necessary, modify) the flags to indicate 27279e39c5baSBill Taylor * whether the allocation was successful 27289e39c5baSBill Taylor */ 27299e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port)); 27309e39c5baSBill Taylor if (flags & mask) { 27319e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 27329e39c5baSBill Taylor return (IBT_QP_IN_USE); 27339e39c5baSBill Taylor } 27349e39c5baSBill Taylor state->hs_spec_qpflags |= mask; 27359e39c5baSBill Taylor *qp_rsrc = state->hs_spec_qp1; 27369e39c5baSBill Taylor } 27379e39c5baSBill Taylor 27389e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 27399e39c5baSBill Taylor return (DDI_SUCCESS); 27409e39c5baSBill Taylor } 27419e39c5baSBill Taylor 27429e39c5baSBill Taylor 27439e39c5baSBill Taylor /* 27449e39c5baSBill Taylor * hermon_special_qp_rsrc_free 27459e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 27469e39c5baSBill Taylor */ 27479e39c5baSBill Taylor static int 27489e39c5baSBill Taylor hermon_special_qp_rsrc_free(hermon_state_t *state, ibt_sqp_type_t type, 27499e39c5baSBill Taylor uint_t port) 27509e39c5baSBill Taylor { 27519e39c5baSBill Taylor uint_t mask, flags; 27529e39c5baSBill Taylor int status; 27539e39c5baSBill Taylor 27549e39c5baSBill Taylor mutex_enter(&state->hs_spec_qplock); 27559e39c5baSBill Taylor if (type == IBT_SMI_SQP) { 27569e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP0_RSRC + port)); 27579e39c5baSBill Taylor state->hs_spec_qpflags &= ~mask; 27589e39c5baSBill Taylor flags = state->hs_spec_qpflags; 27599e39c5baSBill Taylor 27609e39c5baSBill Taylor /* 27619e39c5baSBill Taylor * If this is the last QP0 free, then post a CONF_SPECIAL_QP 27629e39c5baSBill Taylor * NOW, If this is the last Special QP free, then post a 27639e39c5baSBill Taylor * CONF_SPECIAL_QP firmware command - it'll stop them all 27649e39c5baSBill Taylor */ 27659e39c5baSBill Taylor if (flags) { 27669e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 0, 27679e39c5baSBill Taylor HERMON_CMD_QP_SMI, HERMON_CMD_NOSLEEP_SPIN, 0); 27689e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 27699e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 27709e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 27719e39c5baSBill Taylor "command failed: %08x\n", 27729e39c5baSBill Taylor state->hs_instance, status); 27739e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 27749e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 27759e39c5baSBill Taylor HCA_ERR_SRV_LOST); 27769e39c5baSBill Taylor } 27779e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 27789e39c5baSBill Taylor } 27799e39c5baSBill Taylor } 27809e39c5baSBill Taylor } else { 27819e39c5baSBill Taylor mask = (1 << (HERMON_SPECIAL_QP1_RSRC + port)); 27829e39c5baSBill Taylor state->hs_spec_qpflags &= ~mask; 27839e39c5baSBill Taylor flags = state->hs_spec_qpflags; 27849e39c5baSBill Taylor 27859e39c5baSBill Taylor /* 27869e39c5baSBill Taylor * If this is the last QP1 free, then post a CONF_SPECIAL_QP 27879e39c5baSBill Taylor * NOW, if this is the last special QP free, then post a 27889e39c5baSBill Taylor * CONF_SPECIAL_QP firmware command - it'll stop them all 27899e39c5baSBill Taylor */ 27909e39c5baSBill Taylor if (flags) { 27919e39c5baSBill Taylor status = hermon_conf_special_qp_cmd_post(state, 0, 27929e39c5baSBill Taylor HERMON_CMD_QP_GSI, HERMON_CMD_NOSLEEP_SPIN, 0); 27939e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 27949e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 27959e39c5baSBill Taylor cmn_err(CE_NOTE, "hermon%d: CONF_SPECIAL_QP " 27969e39c5baSBill Taylor "command failed: %08x\n", 27979e39c5baSBill Taylor state->hs_instance, status); 27989e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 27999e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 28009e39c5baSBill Taylor HCA_ERR_SRV_LOST); 28019e39c5baSBill Taylor } 28029e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 28039e39c5baSBill Taylor } 28049e39c5baSBill Taylor } 28059e39c5baSBill Taylor } 28069e39c5baSBill Taylor 28079e39c5baSBill Taylor mutex_exit(&state->hs_spec_qplock); 28089e39c5baSBill Taylor return (DDI_SUCCESS); 28099e39c5baSBill Taylor } 28109e39c5baSBill Taylor 28119e39c5baSBill Taylor 28129e39c5baSBill Taylor /* 28139e39c5baSBill Taylor * hermon_qp_sgl_to_logwqesz() 28149e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 28159e39c5baSBill Taylor */ 28169e39c5baSBill Taylor static void 28179e39c5baSBill Taylor hermon_qp_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl, 28189e39c5baSBill Taylor uint_t real_max_sgl, hermon_qp_wq_type_t wq_type, 28199e39c5baSBill Taylor uint_t *logwqesz, uint_t *max_sgl) 28209e39c5baSBill Taylor { 28219e39c5baSBill Taylor uint_t max_size, log2, actual_sgl; 28229e39c5baSBill Taylor 28239e39c5baSBill Taylor switch (wq_type) { 28249e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDQ_UD: 28259e39c5baSBill Taylor /* 28269e39c5baSBill Taylor * Use requested maximum SGL to calculate max descriptor size 28279e39c5baSBill Taylor * (while guaranteeing that the descriptor size is a 28289e39c5baSBill Taylor * power-of-2 cachelines). 28299e39c5baSBill Taylor */ 28309e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4)); 28319e39c5baSBill Taylor log2 = highbit(max_size); 2832*de710d24SJosef 'Jeff' Sipek if (ISP2(max_size)) { 28339e39c5baSBill Taylor log2 = log2 - 1; 28349e39c5baSBill Taylor } 28359e39c5baSBill Taylor 28369e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 28379e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 28389e39c5baSBill Taylor 28399e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 28409e39c5baSBill Taylor actual_sgl = ((1 << log2) - 28419e39c5baSBill Taylor sizeof (hermon_hw_snd_wqe_ctrl_t)) >> 4; 28429e39c5baSBill Taylor break; 28439e39c5baSBill Taylor 28449e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDQ_CONN: 28459e39c5baSBill Taylor /* 28469e39c5baSBill Taylor * Use requested maximum SGL to calculate max descriptor size 28479e39c5baSBill Taylor * (while guaranteeing that the descriptor size is a 28489e39c5baSBill Taylor * power-of-2 cachelines). 28499e39c5baSBill Taylor */ 28509e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_SND_HDRS + (num_sgl << 4)); 28519e39c5baSBill Taylor log2 = highbit(max_size); 2852*de710d24SJosef 'Jeff' Sipek if (ISP2(max_size)) { 28539e39c5baSBill Taylor log2 = log2 - 1; 28549e39c5baSBill Taylor } 28559e39c5baSBill Taylor 28569e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 28579e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 28589e39c5baSBill Taylor 28599e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 28609e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SND_HDRS) >> 4; 28619e39c5baSBill Taylor break; 28629e39c5baSBill Taylor 28639e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_RECVQ: 28649e39c5baSBill Taylor /* 28659e39c5baSBill Taylor * Same as above (except for Recv WQEs) 28669e39c5baSBill Taylor */ 28679e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4)); 28689e39c5baSBill Taylor log2 = highbit(max_size); 2869*de710d24SJosef 'Jeff' Sipek if (ISP2(max_size)) { 28709e39c5baSBill Taylor log2 = log2 - 1; 28719e39c5baSBill Taylor } 28729e39c5baSBill Taylor 28739e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 28749e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 28759e39c5baSBill Taylor 28769e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 28779e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_RCV_HDRS) >> 4; 28789e39c5baSBill Taylor break; 28799e39c5baSBill Taylor 28809e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDMLX_QP0: 28819e39c5baSBill Taylor /* 28829e39c5baSBill Taylor * Same as above (except for MLX transport WQEs). For these 28839e39c5baSBill Taylor * WQEs we have to account for the space consumed by the 28849e39c5baSBill Taylor * "inline" packet headers. (This is smaller than for QP1 28859e39c5baSBill Taylor * below because QP0 is not allowed to send packets with a GRH. 28869e39c5baSBill Taylor */ 28879e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_QP0_HDRS + (num_sgl << 4)); 28889e39c5baSBill Taylor log2 = highbit(max_size); 2889*de710d24SJosef 'Jeff' Sipek if (ISP2(max_size)) { 28909e39c5baSBill Taylor log2 = log2 - 1; 28919e39c5baSBill Taylor } 28929e39c5baSBill Taylor 28939e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 28949e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 28959e39c5baSBill Taylor 28969e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 28979e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP0_HDRS) >> 4; 28989e39c5baSBill Taylor break; 28999e39c5baSBill Taylor 29009e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_SENDMLX_QP1: 29019e39c5baSBill Taylor /* 29029e39c5baSBill Taylor * Same as above. For these WQEs we again have to account for 29039e39c5baSBill Taylor * the space consumed by the "inline" packet headers. (This 29049e39c5baSBill Taylor * is larger than for QP0 above because we have to account for 29059e39c5baSBill Taylor * the possibility of a GRH in each packet - and this 29069e39c5baSBill Taylor * introduces an alignment issue that causes us to consume 29079e39c5baSBill Taylor * an additional 8 bytes). 29089e39c5baSBill Taylor */ 29099e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (num_sgl << 4)); 29109e39c5baSBill Taylor log2 = highbit(max_size); 2911*de710d24SJosef 'Jeff' Sipek if (ISP2(max_size)) { 29129e39c5baSBill Taylor log2 = log2 - 1; 29139e39c5baSBill Taylor } 29149e39c5baSBill Taylor 29159e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */ 29169e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM); 29179e39c5baSBill Taylor 29189e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */ 29199e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4; 29209e39c5baSBill Taylor break; 29219e39c5baSBill Taylor 29229e39c5baSBill Taylor default: 29239e39c5baSBill Taylor HERMON_WARNING(state, "unexpected work queue type"); 29249e39c5baSBill Taylor break; 29259e39c5baSBill Taylor } 29269e39c5baSBill Taylor 29279e39c5baSBill Taylor /* Fill in the return values */ 29289e39c5baSBill Taylor *logwqesz = log2; 29299e39c5baSBill Taylor *max_sgl = min(real_max_sgl, actual_sgl); 29309e39c5baSBill Taylor } 2931