19e39c5baSBill Taylor /*
29e39c5baSBill Taylor * CDDL HEADER START
39e39c5baSBill Taylor *
49e39c5baSBill Taylor * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor *
89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor * See the License for the specific language governing permissions
119e39c5baSBill Taylor * and limitations under the License.
129e39c5baSBill Taylor *
139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor *
199e39c5baSBill Taylor * CDDL HEADER END
209e39c5baSBill Taylor */
219e39c5baSBill Taylor
229e39c5baSBill Taylor /*
2317a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor */
259e39c5baSBill Taylor
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor * hermon_srq.c
289e39c5baSBill Taylor * Hermon Shared Receive Queue Processing Routines
299e39c5baSBill Taylor *
309e39c5baSBill Taylor * Implements all the routines necessary for allocating, freeing, querying,
319e39c5baSBill Taylor * modifying and posting shared receive queues.
329e39c5baSBill Taylor */
339e39c5baSBill Taylor
34*de710d24SJosef 'Jeff' Sipek #include <sys/sysmacros.h>
359e39c5baSBill Taylor #include <sys/types.h>
369e39c5baSBill Taylor #include <sys/conf.h>
379e39c5baSBill Taylor #include <sys/ddi.h>
389e39c5baSBill Taylor #include <sys/sunddi.h>
399e39c5baSBill Taylor #include <sys/modctl.h>
409e39c5baSBill Taylor #include <sys/bitmap.h>
419e39c5baSBill Taylor
429e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
439e39c5baSBill Taylor
449e39c5baSBill Taylor static void hermon_srq_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
459e39c5baSBill Taylor hermon_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl);
469e39c5baSBill Taylor
479e39c5baSBill Taylor /*
489e39c5baSBill Taylor * hermon_srq_alloc()
499e39c5baSBill Taylor * Context: Can be called only from user or kernel context.
509e39c5baSBill Taylor */
519e39c5baSBill Taylor int
hermon_srq_alloc(hermon_state_t * state,hermon_srq_info_t * srqinfo,uint_t sleepflag)529e39c5baSBill Taylor hermon_srq_alloc(hermon_state_t *state, hermon_srq_info_t *srqinfo,
539e39c5baSBill Taylor uint_t sleepflag)
549e39c5baSBill Taylor {
559e39c5baSBill Taylor ibt_srq_hdl_t ibt_srqhdl;
569e39c5baSBill Taylor hermon_pdhdl_t pd;
579e39c5baSBill Taylor ibt_srq_sizes_t *sizes;
589e39c5baSBill Taylor ibt_srq_sizes_t *real_sizes;
599e39c5baSBill Taylor hermon_srqhdl_t *srqhdl;
609e39c5baSBill Taylor ibt_srq_flags_t flags;
619e39c5baSBill Taylor hermon_rsrc_t *srqc, *rsrc;
629e39c5baSBill Taylor hermon_hw_srqc_t srqc_entry;
639e39c5baSBill Taylor uint32_t *buf;
649e39c5baSBill Taylor hermon_srqhdl_t srq;
659e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb;
669e39c5baSBill Taylor ibt_mr_attr_t mr_attr;
679e39c5baSBill Taylor hermon_mr_options_t mr_op;
689e39c5baSBill Taylor hermon_mrhdl_t mr;
699e39c5baSBill Taylor uint64_t value, srq_desc_off;
709e39c5baSBill Taylor uint32_t log_srq_size;
719e39c5baSBill Taylor uint32_t uarpg;
729e39c5baSBill Taylor uint_t srq_is_umap;
739e39c5baSBill Taylor int flag, status;
749e39c5baSBill Taylor uint_t max_sgl;
759e39c5baSBill Taylor uint_t wqesz;
769e39c5baSBill Taylor uint_t srq_wr_sz;
779e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sizes))
789e39c5baSBill Taylor
799e39c5baSBill Taylor /*
809e39c5baSBill Taylor * options-->wq_location used to be for location, now explicitly
819e39c5baSBill Taylor * LOCATION_NORMAL
829e39c5baSBill Taylor */
839e39c5baSBill Taylor
849e39c5baSBill Taylor /*
859e39c5baSBill Taylor * Extract the necessary info from the hermon_srq_info_t structure
869e39c5baSBill Taylor */
879e39c5baSBill Taylor real_sizes = srqinfo->srqi_real_sizes;
889e39c5baSBill Taylor sizes = srqinfo->srqi_sizes;
899e39c5baSBill Taylor pd = srqinfo->srqi_pd;
909e39c5baSBill Taylor ibt_srqhdl = srqinfo->srqi_ibt_srqhdl;
919e39c5baSBill Taylor flags = srqinfo->srqi_flags;
929e39c5baSBill Taylor srqhdl = srqinfo->srqi_srqhdl;
939e39c5baSBill Taylor
949e39c5baSBill Taylor /*
959e39c5baSBill Taylor * Determine whether SRQ is being allocated for userland access or
969e39c5baSBill Taylor * whether it is being allocated for kernel access. If the SRQ is
979e39c5baSBill Taylor * being allocated for userland access, then lookup the UAR doorbell
989e39c5baSBill Taylor * page number for the current process. Note: If this is not found
999e39c5baSBill Taylor * (e.g. if the process has not previously open()'d the Hermon driver),
1009e39c5baSBill Taylor * then an error is returned.
1019e39c5baSBill Taylor */
1029e39c5baSBill Taylor srq_is_umap = (flags & IBT_SRQ_USER_MAP) ? 1 : 0;
1039e39c5baSBill Taylor if (srq_is_umap) {
1049e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
1059e39c5baSBill Taylor MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
1069e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1079e39c5baSBill Taylor status = IBT_INVALID_PARAM;
1089e39c5baSBill Taylor goto srqalloc_fail3;
1099e39c5baSBill Taylor }
1109e39c5baSBill Taylor uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
1119e39c5baSBill Taylor } else {
1129e39c5baSBill Taylor uarpg = state->hs_kernel_uar_index;
1139e39c5baSBill Taylor }
1149e39c5baSBill Taylor
1159e39c5baSBill Taylor /* Increase PD refcnt */
1169e39c5baSBill Taylor hermon_pd_refcnt_inc(pd);
1179e39c5baSBill Taylor
1189e39c5baSBill Taylor /* Allocate an SRQ context entry */
1199e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_SRQC, 1, sleepflag, &srqc);
1209e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1219e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1229e39c5baSBill Taylor goto srqalloc_fail1;
1239e39c5baSBill Taylor }
1249e39c5baSBill Taylor
1259e39c5baSBill Taylor /* Allocate the SRQ Handle entry */
1269e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_SRQHDL, 1, sleepflag, &rsrc);
1279e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1289e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1299e39c5baSBill Taylor goto srqalloc_fail2;
1309e39c5baSBill Taylor }
1319e39c5baSBill Taylor
1329e39c5baSBill Taylor srq = (hermon_srqhdl_t)rsrc->hr_addr;
1339e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq))
1349e39c5baSBill Taylor
1359e39c5baSBill Taylor bzero(srq, sizeof (struct hermon_sw_srq_s));
1369e39c5baSBill Taylor /* Calculate the SRQ number */
1379e39c5baSBill Taylor
1389e39c5baSBill Taylor /* just use the index, implicit in Hermon */
1399e39c5baSBill Taylor srq->srq_srqnum = srqc->hr_indx;
1409e39c5baSBill Taylor
1419e39c5baSBill Taylor /*
1429e39c5baSBill Taylor * If this will be a user-mappable SRQ, then allocate an entry for
1439e39c5baSBill Taylor * the "userland resources database". This will later be added to
1449e39c5baSBill Taylor * the database (after all further SRQ operations are successful).
1459e39c5baSBill Taylor * If we fail here, we must undo the reference counts and the
1469e39c5baSBill Taylor * previous resource allocation.
1479e39c5baSBill Taylor */
1489e39c5baSBill Taylor if (srq_is_umap) {
1499e39c5baSBill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance,
1509e39c5baSBill Taylor srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC,
1519e39c5baSBill Taylor (uint64_t)(uintptr_t)rsrc);
1529e39c5baSBill Taylor if (umapdb == NULL) {
1539e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1549e39c5baSBill Taylor goto srqalloc_fail3;
1559e39c5baSBill Taylor }
1569e39c5baSBill Taylor }
1579e39c5baSBill Taylor
1589e39c5baSBill Taylor /*
1599e39c5baSBill Taylor * Allocate the doorbell record. Hermon just needs one for the
1609e39c5baSBill Taylor * SRQ, and use uarpg (above) as the uar index
1619e39c5baSBill Taylor */
1629e39c5baSBill Taylor
1639e39c5baSBill Taylor status = hermon_dbr_alloc(state, uarpg, &srq->srq_wq_dbr_acchdl,
1649e39c5baSBill Taylor &srq->srq_wq_vdbr, &srq->srq_wq_pdbr, &srq->srq_rdbr_mapoffset);
1659e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1669e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1679e39c5baSBill Taylor goto srqalloc_fail4;
1689e39c5baSBill Taylor }
1699e39c5baSBill Taylor
1709e39c5baSBill Taylor /*
1719e39c5baSBill Taylor * Calculate the appropriate size for the SRQ.
1729e39c5baSBill Taylor * Note: All Hermon SRQs must be a power-of-2 in size. Also
1739e39c5baSBill Taylor * they may not be any smaller than HERMON_SRQ_MIN_SIZE. This step
1749e39c5baSBill Taylor * is to round the requested size up to the next highest power-of-2
1759e39c5baSBill Taylor */
1769e39c5baSBill Taylor srq_wr_sz = max(sizes->srq_wr_sz + 1, HERMON_SRQ_MIN_SIZE);
1779e39c5baSBill Taylor log_srq_size = highbit(srq_wr_sz);
178*de710d24SJosef 'Jeff' Sipek if (ISP2(srq_wr_sz)) {
1799e39c5baSBill Taylor log_srq_size = log_srq_size - 1;
1809e39c5baSBill Taylor }
1819e39c5baSBill Taylor
1829e39c5baSBill Taylor /*
1839e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
1849e39c5baSBill Taylor * with the device limits and/or software-configured limits). If not,
1859e39c5baSBill Taylor * then obviously we have a lot of cleanup to do before returning.
1869e39c5baSBill Taylor */
1879e39c5baSBill Taylor if (log_srq_size > state->hs_cfg_profile->cp_log_max_srq_sz) {
1889e39c5baSBill Taylor status = IBT_HCA_WR_EXCEEDED;
1899e39c5baSBill Taylor goto srqalloc_fail4a;
1909e39c5baSBill Taylor }
1919e39c5baSBill Taylor
1929e39c5baSBill Taylor /*
1939e39c5baSBill Taylor * Next we verify that the requested number of SGL is valid (i.e.
1949e39c5baSBill Taylor * consistent with the device limits and/or software-configured
1959e39c5baSBill Taylor * limits). If not, then obviously the same cleanup needs to be done.
1969e39c5baSBill Taylor */
1979e39c5baSBill Taylor max_sgl = state->hs_ibtfinfo.hca_attr->hca_max_srq_sgl;
1989e39c5baSBill Taylor if (sizes->srq_sgl_sz > max_sgl) {
1999e39c5baSBill Taylor status = IBT_HCA_SGL_EXCEEDED;
2009e39c5baSBill Taylor goto srqalloc_fail4a;
2019e39c5baSBill Taylor }
2029e39c5baSBill Taylor
2039e39c5baSBill Taylor /*
2049e39c5baSBill Taylor * Determine the SRQ's WQE sizes. This depends on the requested
2059e39c5baSBill Taylor * number of SGLs. Note: This also has the side-effect of
2069e39c5baSBill Taylor * calculating the real number of SGLs (for the calculated WQE size)
2079e39c5baSBill Taylor */
2089e39c5baSBill Taylor hermon_srq_sgl_to_logwqesz(state, sizes->srq_sgl_sz,
2099e39c5baSBill Taylor HERMON_QP_WQ_TYPE_RECVQ, &srq->srq_wq_log_wqesz,
2109e39c5baSBill Taylor &srq->srq_wq_sgl);
2119e39c5baSBill Taylor
2129e39c5baSBill Taylor /*
2139e39c5baSBill Taylor * Allocate the memory for SRQ work queues. Note: The location from
2149e39c5baSBill Taylor * which we will allocate these work queues is always
2159e39c5baSBill Taylor * QUEUE_LOCATION_NORMAL. Since Hermon work queues are not
2169e39c5baSBill Taylor * allowed to cross a 32-bit (4GB) boundary, the alignment of the work
2179e39c5baSBill Taylor * queue memory is very important. We used to allocate work queues
2189e39c5baSBill Taylor * (the combined receive and send queues) so that they would be aligned
2199e39c5baSBill Taylor * on their combined size. That alignment guaranteed that they would
2209e39c5baSBill Taylor * never cross the 4GB boundary (Hermon work queues are on the order of
2219e39c5baSBill Taylor * MBs at maximum). Now we are able to relax this alignment constraint
2229e39c5baSBill Taylor * by ensuring that the IB address assigned to the queue memory (as a
2239e39c5baSBill Taylor * result of the hermon_mr_register() call) is offset from zero.
2249e39c5baSBill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
2259e39c5baSBill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass
2269e39c5baSBill Taylor * mode we found that we were not allowed to specify any alignment that
2279e39c5baSBill Taylor * was more restrictive than the system page size. So we avoided this
2289e39c5baSBill Taylor * constraint by passing two alignment values, one for the memory
2299e39c5baSBill Taylor * allocation itself and the other for the DMA handle (for later bind).
2309e39c5baSBill Taylor * This used to cause more memory than necessary to be allocated (in
2319e39c5baSBill Taylor * order to guarantee the more restrictive alignment contraint). But
2329e39c5baSBill Taylor * be guaranteeing the zero-based IB virtual address for the queue, we
2339e39c5baSBill Taylor * are able to conserve this memory.
2349e39c5baSBill Taylor *
2359e39c5baSBill Taylor * Note: If SRQ is not user-mappable, then it may come from either
2369e39c5baSBill Taylor * kernel system memory or from HCA-attached local DDR memory.
2379e39c5baSBill Taylor *
2389e39c5baSBill Taylor * Note2: We align this queue on a pagesize boundary. This is required
2399e39c5baSBill Taylor * to make sure that all the resulting IB addresses will start at 0, for
2409e39c5baSBill Taylor * a zero-based queue. By making sure we are aligned on at least a
2419e39c5baSBill Taylor * page, any offset we use into our queue will be the same as when we
2429e39c5baSBill Taylor * perform hermon_srq_modify() operations later.
2439e39c5baSBill Taylor */
2449e39c5baSBill Taylor wqesz = (1 << srq->srq_wq_log_wqesz);
2459e39c5baSBill Taylor srq->srq_wqinfo.qa_size = (1 << log_srq_size) * wqesz;
2469e39c5baSBill Taylor srq->srq_wqinfo.qa_alloc_align = PAGESIZE;
2479e39c5baSBill Taylor srq->srq_wqinfo.qa_bind_align = PAGESIZE;
2489e39c5baSBill Taylor if (srq_is_umap) {
2499e39c5baSBill Taylor srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
2509e39c5baSBill Taylor } else {
2519e39c5baSBill Taylor srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
2529e39c5baSBill Taylor }
2539e39c5baSBill Taylor status = hermon_queue_alloc(state, &srq->srq_wqinfo, sleepflag);
2549e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2559e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
2569e39c5baSBill Taylor goto srqalloc_fail4a;
2579e39c5baSBill Taylor }
2589e39c5baSBill Taylor buf = (uint32_t *)srq->srq_wqinfo.qa_buf_aligned;
2599e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
2609e39c5baSBill Taylor
2619e39c5baSBill Taylor /*
2629e39c5baSBill Taylor * Register the memory for the SRQ work queues. The memory for the SRQ
2639e39c5baSBill Taylor * must be registered in the Hermon cMPT tables. This gives us the LKey
2649e39c5baSBill Taylor * to specify in the SRQ context later. Note: If the work queue is to
2659e39c5baSBill Taylor * be allocated from DDR memory, then only a "bypass" mapping is
2669e39c5baSBill Taylor * appropriate. And if the SRQ memory is user-mappable, then we force
2679e39c5baSBill Taylor * DDI_DMA_CONSISTENT mapping. Also, in order to meet the alignment
2689e39c5baSBill Taylor * restriction, we pass the "mro_bind_override_addr" flag in the call
2699e39c5baSBill Taylor * to hermon_mr_register(). This guarantees that the resulting IB vaddr
2709e39c5baSBill Taylor * will be zero-based (modulo the offset into the first page). If we
2719e39c5baSBill Taylor * fail here, we still have the bunch of resource and reference count
2729e39c5baSBill Taylor * cleanup to do.
2739e39c5baSBill Taylor */
2749e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
2759e39c5baSBill Taylor IBT_MR_NOSLEEP;
2769e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
2779e39c5baSBill Taylor mr_attr.mr_len = srq->srq_wqinfo.qa_size;
2789e39c5baSBill Taylor mr_attr.mr_as = NULL;
2799e39c5baSBill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
2809e39c5baSBill Taylor mr_op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
2819e39c5baSBill Taylor mr_op.mro_bind_dmahdl = srq->srq_wqinfo.qa_dmahdl;
2829e39c5baSBill Taylor mr_op.mro_bind_override_addr = 1;
2839e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr,
2849e39c5baSBill Taylor &mr_op, HERMON_SRQ_CMPT);
2859e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2869e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
2879e39c5baSBill Taylor goto srqalloc_fail5;
2889e39c5baSBill Taylor }
2899e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2909e39c5baSBill Taylor
2919e39c5baSBill Taylor /*
2929e39c5baSBill Taylor * Calculate the offset between the kernel virtual address space
2939e39c5baSBill Taylor * and the IB virtual address space. This will be used when
2949e39c5baSBill Taylor * posting work requests to properly initialize each WQE.
2959e39c5baSBill Taylor */
2969e39c5baSBill Taylor srq_desc_off = (uint64_t)(uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
2979e39c5baSBill Taylor (uint64_t)mr->mr_bindinfo.bi_addr;
2989e39c5baSBill Taylor
2999e39c5baSBill Taylor srq->srq_wq_wqhdr = hermon_wrid_wqhdr_create(1 << log_srq_size);
3009e39c5baSBill Taylor
3019e39c5baSBill Taylor /*
3029e39c5baSBill Taylor * Fill in all the return arguments (if necessary). This includes
3039e39c5baSBill Taylor * real queue size and real SGLs.
3049e39c5baSBill Taylor */
3059e39c5baSBill Taylor if (real_sizes != NULL) {
3069e39c5baSBill Taylor real_sizes->srq_wr_sz = (1 << log_srq_size) - 1;
3079e39c5baSBill Taylor real_sizes->srq_sgl_sz = srq->srq_wq_sgl;
3089e39c5baSBill Taylor }
3099e39c5baSBill Taylor
3109e39c5baSBill Taylor /*
3119e39c5baSBill Taylor * Fill in the SRQC entry. This is the final step before passing
3129e39c5baSBill Taylor * ownership of the SRQC entry to the Hermon hardware. We use all of
3139e39c5baSBill Taylor * the information collected/calculated above to fill in the
3149e39c5baSBill Taylor * requisite portions of the SRQC. Note: If this SRQ is going to be
3159e39c5baSBill Taylor * used for userland access, then we need to set the UAR page number
3169e39c5baSBill Taylor * appropriately (otherwise it's a "don't care")
3179e39c5baSBill Taylor */
3189e39c5baSBill Taylor bzero(&srqc_entry, sizeof (hermon_hw_srqc_t));
3199e39c5baSBill Taylor srqc_entry.state = HERMON_SRQ_STATE_HW_OWNER;
3209e39c5baSBill Taylor srqc_entry.log_srq_size = log_srq_size;
3219e39c5baSBill Taylor srqc_entry.srqn = srq->srq_srqnum;
3229e39c5baSBill Taylor srqc_entry.log_rq_stride = srq->srq_wq_log_wqesz - 4;
3239e39c5baSBill Taylor /* 16-byte chunks */
3249e39c5baSBill Taylor
3259e39c5baSBill Taylor srqc_entry.page_offs = srq->srq_wqinfo.qa_pgoffs >> 6;
3269e39c5baSBill Taylor srqc_entry.log2_pgsz = mr->mr_log2_pgsz;
3279e39c5baSBill Taylor srqc_entry.mtt_base_addrh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
3289e39c5baSBill Taylor srqc_entry.mtt_base_addrl = mr->mr_mttaddr >> 3;
3299e39c5baSBill Taylor srqc_entry.pd = pd->pd_pdnum;
3309e39c5baSBill Taylor srqc_entry.dbr_addrh = (uint32_t)((uint64_t)srq->srq_wq_pdbr >> 32);
3319e39c5baSBill Taylor srqc_entry.dbr_addrl = (uint32_t)((uint64_t)srq->srq_wq_pdbr >> 2);
3329e39c5baSBill Taylor
3339e39c5baSBill Taylor /*
3349e39c5baSBill Taylor * all others - specifically, xrcd, cqn_xrc, lwm, wqe_cnt, and wqe_cntr
3359e39c5baSBill Taylor * are zero thanks to the bzero of the structure
3369e39c5baSBill Taylor */
3379e39c5baSBill Taylor
3389e39c5baSBill Taylor /*
3399e39c5baSBill Taylor * Write the SRQC entry to hardware. Lastly, we pass ownership of
3409e39c5baSBill Taylor * the entry to the hardware (using the Hermon SW2HW_SRQ firmware
3419e39c5baSBill Taylor * command). Note: In general, this operation shouldn't fail. But
3429e39c5baSBill Taylor * if it does, we have to undo everything we've done above before
3439e39c5baSBill Taylor * returning error.
3449e39c5baSBill Taylor */
3459e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_SRQ, &srqc_entry,
3469e39c5baSBill Taylor sizeof (hermon_hw_srqc_t), srq->srq_srqnum,
3479e39c5baSBill Taylor sleepflag);
3489e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) {
3499e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_SRQ command failed: %08x\n",
3509e39c5baSBill Taylor status);
3519e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
3529e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
3539e39c5baSBill Taylor }
3549e39c5baSBill Taylor status = ibc_get_ci_failure(0);
3559e39c5baSBill Taylor goto srqalloc_fail8;
3569e39c5baSBill Taylor }
3579e39c5baSBill Taylor
3589e39c5baSBill Taylor /*
3599e39c5baSBill Taylor * Fill in the rest of the Hermon SRQ handle. We can update
3609e39c5baSBill Taylor * the following fields for use in further operations on the SRQ.
3619e39c5baSBill Taylor */
3629e39c5baSBill Taylor srq->srq_srqcrsrcp = srqc;
3639e39c5baSBill Taylor srq->srq_rsrcp = rsrc;
3649e39c5baSBill Taylor srq->srq_mrhdl = mr;
3659e39c5baSBill Taylor srq->srq_refcnt = 0;
3669e39c5baSBill Taylor srq->srq_is_umap = srq_is_umap;
3679e39c5baSBill Taylor srq->srq_uarpg = uarpg;
3689e39c5baSBill Taylor srq->srq_umap_dhp = (devmap_cookie_t)NULL;
3699e39c5baSBill Taylor srq->srq_pdhdl = pd;
3709e39c5baSBill Taylor srq->srq_wq_bufsz = (1 << log_srq_size);
3719e39c5baSBill Taylor srq->srq_wq_buf = buf;
3729e39c5baSBill Taylor srq->srq_desc_off = srq_desc_off;
3739e39c5baSBill Taylor srq->srq_hdlrarg = (void *)ibt_srqhdl;
3749e39c5baSBill Taylor srq->srq_state = 0;
3759e39c5baSBill Taylor srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
3769e39c5baSBill Taylor srq->srq_real_sizes.srq_sgl_sz = srq->srq_wq_sgl;
3779e39c5baSBill Taylor
3789e39c5baSBill Taylor /*
3799e39c5baSBill Taylor * Put SRQ handle in Hermon SRQNum-to-SRQhdl list. Then fill in the
3809e39c5baSBill Taylor * "srqhdl" and return success
3819e39c5baSBill Taylor */
38217a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, srq);
3839e39c5baSBill Taylor
3849e39c5baSBill Taylor /*
3859e39c5baSBill Taylor * If this is a user-mappable SRQ, then we need to insert the
3869e39c5baSBill Taylor * previously allocated entry into the "userland resources database".
3879e39c5baSBill Taylor * This will allow for later lookup during devmap() (i.e. mmap())
3889e39c5baSBill Taylor * calls.
3899e39c5baSBill Taylor */
3909e39c5baSBill Taylor if (srq->srq_is_umap) {
3919e39c5baSBill Taylor hermon_umap_db_add(umapdb);
3929e39c5baSBill Taylor } else { /* initialize work queue for kernel SRQs */
3939e39c5baSBill Taylor int i, len, last;
3949e39c5baSBill Taylor uint16_t *desc;
3959e39c5baSBill Taylor
3969e39c5baSBill Taylor desc = (uint16_t *)buf;
3979e39c5baSBill Taylor len = wqesz / sizeof (*desc);
3989e39c5baSBill Taylor last = srq->srq_wq_bufsz - 1;
3999e39c5baSBill Taylor for (i = 0; i < last; i++) {
4009e39c5baSBill Taylor desc[1] = htons(i + 1);
4019e39c5baSBill Taylor desc += len;
4029e39c5baSBill Taylor }
4039e39c5baSBill Taylor srq->srq_wq_wqhdr->wq_tail = last;
4049e39c5baSBill Taylor srq->srq_wq_wqhdr->wq_head = 0;
4059e39c5baSBill Taylor }
4069e39c5baSBill Taylor
4079e39c5baSBill Taylor *srqhdl = srq;
4089e39c5baSBill Taylor
4099e39c5baSBill Taylor return (status);
4109e39c5baSBill Taylor
4119e39c5baSBill Taylor /*
4129e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine
4139e39c5baSBill Taylor */
4149e39c5baSBill Taylor srqalloc_fail8:
4159e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(srq->srq_wq_wqhdr);
4169e39c5baSBill Taylor srqalloc_fail7:
4179e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
4189e39c5baSBill Taylor HERMON_SLEEPFLAG_FOR_CONTEXT()) != DDI_SUCCESS) {
4199e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister SRQ memory");
4209e39c5baSBill Taylor }
4219e39c5baSBill Taylor srqalloc_fail5:
4229e39c5baSBill Taylor hermon_queue_free(&srq->srq_wqinfo);
4239e39c5baSBill Taylor srqalloc_fail4a:
4249e39c5baSBill Taylor hermon_dbr_free(state, uarpg, srq->srq_wq_vdbr);
4259e39c5baSBill Taylor srqalloc_fail4:
4269e39c5baSBill Taylor if (srq_is_umap) {
4279e39c5baSBill Taylor hermon_umap_db_free(umapdb);
4289e39c5baSBill Taylor }
4299e39c5baSBill Taylor srqalloc_fail3:
4309e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc);
4319e39c5baSBill Taylor srqalloc_fail2:
4329e39c5baSBill Taylor hermon_rsrc_free(state, &srqc);
4339e39c5baSBill Taylor srqalloc_fail1:
4349e39c5baSBill Taylor hermon_pd_refcnt_dec(pd);
4359e39c5baSBill Taylor srqalloc_fail:
4369e39c5baSBill Taylor return (status);
4379e39c5baSBill Taylor }
4389e39c5baSBill Taylor
4399e39c5baSBill Taylor
4409e39c5baSBill Taylor /*
4419e39c5baSBill Taylor * hermon_srq_free()
4429e39c5baSBill Taylor * Context: Can be called only from user or kernel context.
4439e39c5baSBill Taylor */
4449e39c5baSBill Taylor /* ARGSUSED */
4459e39c5baSBill Taylor int
hermon_srq_free(hermon_state_t * state,hermon_srqhdl_t * srqhdl,uint_t sleepflag)4469e39c5baSBill Taylor hermon_srq_free(hermon_state_t *state, hermon_srqhdl_t *srqhdl,
4479e39c5baSBill Taylor uint_t sleepflag)
4489e39c5baSBill Taylor {
4499e39c5baSBill Taylor hermon_rsrc_t *srqc, *rsrc;
4509e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb;
4519e39c5baSBill Taylor uint64_t value;
4529e39c5baSBill Taylor hermon_srqhdl_t srq;
4539e39c5baSBill Taylor hermon_mrhdl_t mr;
4549e39c5baSBill Taylor hermon_pdhdl_t pd;
4559e39c5baSBill Taylor hermon_hw_srqc_t srqc_entry;
4569e39c5baSBill Taylor uint32_t srqnum;
4579e39c5baSBill Taylor uint_t maxprot;
4589e39c5baSBill Taylor int status;
4599e39c5baSBill Taylor
4609e39c5baSBill Taylor /*
4619e39c5baSBill Taylor * Pull all the necessary information from the Hermon Shared Receive
4629e39c5baSBill Taylor * Queue handle. This is necessary here because the resource for the
4639e39c5baSBill Taylor * SRQ handle is going to be freed up as part of this operation.
4649e39c5baSBill Taylor */
4659e39c5baSBill Taylor srq = *srqhdl;
4669e39c5baSBill Taylor mutex_enter(&srq->srq_lock);
4679e39c5baSBill Taylor srqc = srq->srq_srqcrsrcp;
4689e39c5baSBill Taylor rsrc = srq->srq_rsrcp;
4699e39c5baSBill Taylor pd = srq->srq_pdhdl;
4709e39c5baSBill Taylor mr = srq->srq_mrhdl;
4719e39c5baSBill Taylor srqnum = srq->srq_srqnum;
4729e39c5baSBill Taylor
4739e39c5baSBill Taylor /*
4749e39c5baSBill Taylor * If there are work queues still associated with the SRQ, then return
4759e39c5baSBill Taylor * an error. Otherwise, we will be holding the SRQ lock.
4769e39c5baSBill Taylor */
4779e39c5baSBill Taylor if (srq->srq_refcnt != 0) {
4789e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
4799e39c5baSBill Taylor return (IBT_SRQ_IN_USE);
4809e39c5baSBill Taylor }
4819e39c5baSBill Taylor
4829e39c5baSBill Taylor /*
4839e39c5baSBill Taylor * If this was a user-mappable SRQ, then we need to remove its entry
4849e39c5baSBill Taylor * from the "userland resources database". If it is also currently
4859e39c5baSBill Taylor * mmap()'d out to a user process, then we need to call
4869e39c5baSBill Taylor * devmap_devmem_remap() to remap the SRQ memory to an invalid mapping.
4879e39c5baSBill Taylor * We also need to invalidate the SRQ tracking information for the
4889e39c5baSBill Taylor * user mapping.
4899e39c5baSBill Taylor */
4909e39c5baSBill Taylor if (srq->srq_is_umap) {
4919e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance,
4929e39c5baSBill Taylor srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC, &value,
4939e39c5baSBill Taylor HERMON_UMAP_DB_REMOVE, &umapdb);
4949e39c5baSBill Taylor if (status != DDI_SUCCESS) {
4959e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
4969e39c5baSBill Taylor HERMON_WARNING(state, "failed to find in database");
4979e39c5baSBill Taylor return (ibc_get_ci_failure(0));
4989e39c5baSBill Taylor }
4999e39c5baSBill Taylor hermon_umap_db_free(umapdb);
5009e39c5baSBill Taylor if (srq->srq_umap_dhp != NULL) {
5019e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
5029e39c5baSBill Taylor status = devmap_devmem_remap(srq->srq_umap_dhp,
5039e39c5baSBill Taylor state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size,
5049e39c5baSBill Taylor maxprot, DEVMAP_MAPPING_INVALID, NULL);
5059e39c5baSBill Taylor if (status != DDI_SUCCESS) {
5069e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
5079e39c5baSBill Taylor HERMON_WARNING(state, "failed in SRQ memory "
5089e39c5baSBill Taylor "devmap_devmem_remap()");
5099e39c5baSBill Taylor return (ibc_get_ci_failure(0));
5109e39c5baSBill Taylor }
5119e39c5baSBill Taylor srq->srq_umap_dhp = (devmap_cookie_t)NULL;
5129e39c5baSBill Taylor }
5139e39c5baSBill Taylor }
5149e39c5baSBill Taylor
5159e39c5baSBill Taylor /*
5169e39c5baSBill Taylor * Put NULL into the Hermon SRQNum-to-SRQHdl list. This will allow any
5179e39c5baSBill Taylor * in-progress events to detect that the SRQ corresponding to this
5189e39c5baSBill Taylor * number has been freed.
5199e39c5baSBill Taylor */
52017a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, NULL);
5219e39c5baSBill Taylor
5229e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
5239e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq));
5249e39c5baSBill Taylor
5259e39c5baSBill Taylor /*
5269e39c5baSBill Taylor * Reclaim SRQC entry from hardware (using the Hermon HW2SW_SRQ
5279e39c5baSBill Taylor * firmware command). If the ownership transfer fails for any reason,
5289e39c5baSBill Taylor * then it is an indication that something (either in HW or SW) has
5299e39c5baSBill Taylor * gone seriously wrong.
5309e39c5baSBill Taylor */
5319e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, HW2SW_SRQ, &srqc_entry,
5329e39c5baSBill Taylor sizeof (hermon_hw_srqc_t), srqnum, sleepflag);
5339e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) {
5349e39c5baSBill Taylor HERMON_WARNING(state, "failed to reclaim SRQC ownership");
5359e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: HW2SW_SRQ command failed: %08x\n",
5369e39c5baSBill Taylor status);
5379e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
5389e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
5399e39c5baSBill Taylor }
5409e39c5baSBill Taylor return (ibc_get_ci_failure(0));
5419e39c5baSBill Taylor }
5429e39c5baSBill Taylor
5439e39c5baSBill Taylor /*
5449e39c5baSBill Taylor * Deregister the memory for the Shared Receive Queue. If this fails
5459e39c5baSBill Taylor * for any reason, then it is an indication that something (either
5469e39c5baSBill Taylor * in HW or SW) has gone seriously wrong. So we print a warning
5479e39c5baSBill Taylor * message and return.
5489e39c5baSBill Taylor */
5499e39c5baSBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
5509e39c5baSBill Taylor sleepflag);
5519e39c5baSBill Taylor if (status != DDI_SUCCESS) {
5529e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister SRQ memory");
5539e39c5baSBill Taylor return (IBT_FAILURE);
5549e39c5baSBill Taylor }
5559e39c5baSBill Taylor
5569e39c5baSBill Taylor hermon_wrid_wqhdr_destroy(srq->srq_wq_wqhdr);
5579e39c5baSBill Taylor
5589e39c5baSBill Taylor /* Free the memory for the SRQ */
5599e39c5baSBill Taylor hermon_queue_free(&srq->srq_wqinfo);
5609e39c5baSBill Taylor
5619e39c5baSBill Taylor /* Free the dbr */
5629e39c5baSBill Taylor hermon_dbr_free(state, srq->srq_uarpg, srq->srq_wq_vdbr);
5639e39c5baSBill Taylor
5649e39c5baSBill Taylor /* Free the Hermon SRQ Handle */
5659e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc);
5669e39c5baSBill Taylor
5679e39c5baSBill Taylor /* Free the SRQC entry resource */
5689e39c5baSBill Taylor hermon_rsrc_free(state, &srqc);
5699e39c5baSBill Taylor
5709e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */
5719e39c5baSBill Taylor hermon_pd_refcnt_dec(pd);
5729e39c5baSBill Taylor
5739e39c5baSBill Taylor /* Set the srqhdl pointer to NULL and return success */
5749e39c5baSBill Taylor *srqhdl = NULL;
5759e39c5baSBill Taylor
5769e39c5baSBill Taylor return (DDI_SUCCESS);
5779e39c5baSBill Taylor }
5789e39c5baSBill Taylor
5799e39c5baSBill Taylor
5809e39c5baSBill Taylor /*
5819e39c5baSBill Taylor * hermon_srq_modify()
5829e39c5baSBill Taylor * Context: Can be called only from user or kernel context.
5839e39c5baSBill Taylor */
5849e39c5baSBill Taylor int
hermon_srq_modify(hermon_state_t * state,hermon_srqhdl_t srq,uint_t size,uint_t * real_size,uint_t sleepflag)5859e39c5baSBill Taylor hermon_srq_modify(hermon_state_t *state, hermon_srqhdl_t srq, uint_t size,
5869e39c5baSBill Taylor uint_t *real_size, uint_t sleepflag)
5879e39c5baSBill Taylor {
5889e39c5baSBill Taylor hermon_qalloc_info_t new_srqinfo, old_srqinfo;
5899e39c5baSBill Taylor hermon_rsrc_t *mtt, *old_mtt;
5909e39c5baSBill Taylor hermon_bind_info_t bind;
5919e39c5baSBill Taylor hermon_bind_info_t old_bind;
5929e39c5baSBill Taylor hermon_mrhdl_t mr;
5939e39c5baSBill Taylor hermon_hw_srqc_t srqc_entry;
5949e39c5baSBill Taylor hermon_hw_dmpt_t mpt_entry;
5959e39c5baSBill Taylor uint64_t *wre_new, *wre_old;
5969e39c5baSBill Taylor uint64_t mtt_addr;
5979e39c5baSBill Taylor uint64_t srq_pgoffs;
5989e39c5baSBill Taylor uint64_t srq_desc_off;
5999e39c5baSBill Taylor uint32_t *buf, srq_old_bufsz;
6009e39c5baSBill Taylor uint32_t wqesz;
6019e39c5baSBill Taylor uint_t max_srq_size;
6029e39c5baSBill Taylor uint_t mtt_pgsize_bits;
6039e39c5baSBill Taylor uint_t log_srq_size, maxprot;
6049e39c5baSBill Taylor int status;
6059e39c5baSBill Taylor
6069e39c5baSBill Taylor if ((state->hs_devlim.mod_wr_srq == 0) ||
6079e39c5baSBill Taylor (state->hs_cfg_profile->cp_srq_resize_enabled == 0))
6089e39c5baSBill Taylor return (IBT_NOT_SUPPORTED);
6099e39c5baSBill Taylor
6109e39c5baSBill Taylor /*
6119e39c5baSBill Taylor * If size requested is larger than device capability, return
6129e39c5baSBill Taylor * Insufficient Resources
6139e39c5baSBill Taylor */
6149e39c5baSBill Taylor max_srq_size = (1 << state->hs_cfg_profile->cp_log_max_srq_sz);
6159e39c5baSBill Taylor if (size > max_srq_size) {
6169e39c5baSBill Taylor return (IBT_HCA_WR_EXCEEDED);
6179e39c5baSBill Taylor }
6189e39c5baSBill Taylor
6199e39c5baSBill Taylor /*
6209e39c5baSBill Taylor * Calculate the appropriate size for the SRQ.
6219e39c5baSBill Taylor * Note: All Hermon SRQs must be a power-of-2 in size. Also
6229e39c5baSBill Taylor * they may not be any smaller than HERMON_SRQ_MIN_SIZE. This step
6239e39c5baSBill Taylor * is to round the requested size up to the next highest power-of-2
6249e39c5baSBill Taylor */
6259e39c5baSBill Taylor size = max(size, HERMON_SRQ_MIN_SIZE);
6269e39c5baSBill Taylor log_srq_size = highbit(size);
627*de710d24SJosef 'Jeff' Sipek if (ISP2(size)) {
6289e39c5baSBill Taylor log_srq_size = log_srq_size - 1;
6299e39c5baSBill Taylor }
6309e39c5baSBill Taylor
6319e39c5baSBill Taylor /*
6329e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
6339e39c5baSBill Taylor * with the device limits and/or software-configured limits).
6349e39c5baSBill Taylor */
6359e39c5baSBill Taylor if (log_srq_size > state->hs_cfg_profile->cp_log_max_srq_sz) {
6369e39c5baSBill Taylor status = IBT_HCA_WR_EXCEEDED;
6379e39c5baSBill Taylor goto srqmodify_fail;
6389e39c5baSBill Taylor }
6399e39c5baSBill Taylor
6409e39c5baSBill Taylor /*
6419e39c5baSBill Taylor * Allocate the memory for newly resized Shared Receive Queue.
6429e39c5baSBill Taylor *
6439e39c5baSBill Taylor * Note: If SRQ is not user-mappable, then it may come from either
6449e39c5baSBill Taylor * kernel system memory or from HCA-attached local DDR memory.
6459e39c5baSBill Taylor *
6469e39c5baSBill Taylor * Note2: We align this queue on a pagesize boundary. This is required
6479e39c5baSBill Taylor * to make sure that all the resulting IB addresses will start at 0,
6489e39c5baSBill Taylor * for a zero-based queue. By making sure we are aligned on at least a
6499e39c5baSBill Taylor * page, any offset we use into our queue will be the same as it was
6509e39c5baSBill Taylor * when we allocated it at hermon_srq_alloc() time.
6519e39c5baSBill Taylor */
6529e39c5baSBill Taylor wqesz = (1 << srq->srq_wq_log_wqesz);
6539e39c5baSBill Taylor new_srqinfo.qa_size = (1 << log_srq_size) * wqesz;
6549e39c5baSBill Taylor new_srqinfo.qa_alloc_align = PAGESIZE;
6559e39c5baSBill Taylor new_srqinfo.qa_bind_align = PAGESIZE;
6569e39c5baSBill Taylor if (srq->srq_is_umap) {
6579e39c5baSBill Taylor new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
6589e39c5baSBill Taylor } else {
6599e39c5baSBill Taylor new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
6609e39c5baSBill Taylor }
6619e39c5baSBill Taylor status = hermon_queue_alloc(state, &new_srqinfo, sleepflag);
6629e39c5baSBill Taylor if (status != DDI_SUCCESS) {
6639e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
6649e39c5baSBill Taylor goto srqmodify_fail;
6659e39c5baSBill Taylor }
6669e39c5baSBill Taylor buf = (uint32_t *)new_srqinfo.qa_buf_aligned;
6679e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
6689e39c5baSBill Taylor
6699e39c5baSBill Taylor /*
6709e39c5baSBill Taylor * Allocate the memory for the new WRE list. This will be used later
6719e39c5baSBill Taylor * when we resize the wridlist based on the new SRQ size.
6729e39c5baSBill Taylor */
6739e39c5baSBill Taylor wre_new = kmem_zalloc((1 << log_srq_size) * sizeof (uint64_t),
6749e39c5baSBill Taylor sleepflag);
6759e39c5baSBill Taylor if (wre_new == NULL) {
6769e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
6779e39c5baSBill Taylor goto srqmodify_fail;
6789e39c5baSBill Taylor }
6799e39c5baSBill Taylor
6809e39c5baSBill Taylor /*
6819e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority
6829e39c5baSBill Taylor * of the information that will be used to distinguish between an
6839e39c5baSBill Taylor * "addr" binding (as is the case here) and a "buf" binding (see
6849e39c5baSBill Taylor * below). The "bind" struct is later passed to hermon_mr_mem_bind()
6859e39c5baSBill Taylor * which does most of the "heavy lifting" for the Hermon memory
6869e39c5baSBill Taylor * registration routines.
6879e39c5baSBill Taylor */
6889e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(bind))
6899e39c5baSBill Taylor bzero(&bind, sizeof (hermon_bind_info_t));
6909e39c5baSBill Taylor bind.bi_type = HERMON_BINDHDL_VADDR;
6919e39c5baSBill Taylor bind.bi_addr = (uint64_t)(uintptr_t)buf;
6929e39c5baSBill Taylor bind.bi_len = new_srqinfo.qa_size;
6939e39c5baSBill Taylor bind.bi_as = NULL;
6949e39c5baSBill Taylor bind.bi_flags = sleepflag == HERMON_SLEEP ? IBT_MR_SLEEP :
6959e39c5baSBill Taylor IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
6969e39c5baSBill Taylor bind.bi_bypass = state->hs_cfg_profile->cp_iommu_bypass;
6979e39c5baSBill Taylor
6989e39c5baSBill Taylor status = hermon_mr_mtt_bind(state, &bind, new_srqinfo.qa_dmahdl, &mtt,
6999e39c5baSBill Taylor &mtt_pgsize_bits, 0); /* no relaxed ordering */
7009e39c5baSBill Taylor if (status != DDI_SUCCESS) {
7019e39c5baSBill Taylor status = status;
7029e39c5baSBill Taylor kmem_free(wre_new, (1 << log_srq_size) *
7039e39c5baSBill Taylor sizeof (uint64_t));
7049e39c5baSBill Taylor hermon_queue_free(&new_srqinfo);
7059e39c5baSBill Taylor goto srqmodify_fail;
7069e39c5baSBill Taylor }
7079e39c5baSBill Taylor
7089e39c5baSBill Taylor /*
7099e39c5baSBill Taylor * Calculate the offset between the kernel virtual address space
7109e39c5baSBill Taylor * and the IB virtual address space. This will be used when
7119e39c5baSBill Taylor * posting work requests to properly initialize each WQE.
7129e39c5baSBill Taylor *
7139e39c5baSBill Taylor * Note: bind addr is zero-based (from alloc) so we calculate the
7149e39c5baSBill Taylor * correct new offset here.
7159e39c5baSBill Taylor */
7169e39c5baSBill Taylor bind.bi_addr = bind.bi_addr & ((1 << mtt_pgsize_bits) - 1);
7179e39c5baSBill Taylor srq_desc_off = (uint64_t)(uintptr_t)new_srqinfo.qa_buf_aligned -
7189e39c5baSBill Taylor (uint64_t)bind.bi_addr;
7199e39c5baSBill Taylor srq_pgoffs = (uint_t)
720c7facc54SBill Taylor ((uintptr_t)new_srqinfo.qa_buf_aligned & HERMON_PAGEOFFSET);
7219e39c5baSBill Taylor
7229e39c5baSBill Taylor /*
7239e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing
7249e39c5baSBill Taylor * ownership of the MPT entry to the Hermon hardware. We use all of
7259e39c5baSBill Taylor * the information collected/calculated above to fill in the
7269e39c5baSBill Taylor * requisite portions of the MPT.
7279e39c5baSBill Taylor */
7289e39c5baSBill Taylor bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
7299e39c5baSBill Taylor mpt_entry.reg_win_len = bind.bi_len;
7309e39c5baSBill Taylor mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
7319e39c5baSBill Taylor mpt_entry.mtt_addr_h = mtt_addr >> 32;
7329e39c5baSBill Taylor mpt_entry.mtt_addr_l = mtt_addr >> 3;
7339e39c5baSBill Taylor
7349e39c5baSBill Taylor /*
7359e39c5baSBill Taylor * for hermon we build up a new srqc and pass that (partially filled
7369e39c5baSBill Taylor * to resize SRQ instead of modifying the (d)mpt directly
7379e39c5baSBill Taylor */
7389e39c5baSBill Taylor
7399e39c5baSBill Taylor
7409e39c5baSBill Taylor
7419e39c5baSBill Taylor /*
7429e39c5baSBill Taylor * Now we grab the SRQ lock. Since we will be updating the actual
7439e39c5baSBill Taylor * SRQ location and the producer/consumer indexes, we should hold
7449e39c5baSBill Taylor * the lock.
7459e39c5baSBill Taylor *
7469e39c5baSBill Taylor * We do a HERMON_NOSLEEP here (and below), though, because we are
7479e39c5baSBill Taylor * holding the "srq_lock" and if we got raised to interrupt level
7489e39c5baSBill Taylor * by priority inversion, we would not want to block in this routine
7499e39c5baSBill Taylor * waiting for success.
7509e39c5baSBill Taylor */
7519e39c5baSBill Taylor mutex_enter(&srq->srq_lock);
7529e39c5baSBill Taylor
7539e39c5baSBill Taylor /*
7549e39c5baSBill Taylor * Copy old entries to new buffer
7559e39c5baSBill Taylor */
7569e39c5baSBill Taylor srq_old_bufsz = srq->srq_wq_bufsz;
7579e39c5baSBill Taylor bcopy(srq->srq_wq_buf, buf, srq_old_bufsz * wqesz);
7589e39c5baSBill Taylor
7599e39c5baSBill Taylor /*
7609e39c5baSBill Taylor * Setup MPT information for use in the MODIFY_MPT command
7619e39c5baSBill Taylor */
7629e39c5baSBill Taylor mr = srq->srq_mrhdl;
7639e39c5baSBill Taylor mutex_enter(&mr->mr_lock);
7649e39c5baSBill Taylor
7659e39c5baSBill Taylor /*
7669e39c5baSBill Taylor * now, setup the srqc information needed for resize - limit the
7679e39c5baSBill Taylor * values, but use the same structure as the srqc
7689e39c5baSBill Taylor */
7699e39c5baSBill Taylor
7709e39c5baSBill Taylor srqc_entry.log_srq_size = log_srq_size;
7719e39c5baSBill Taylor srqc_entry.page_offs = srq_pgoffs >> 6;
7729e39c5baSBill Taylor srqc_entry.log2_pgsz = mr->mr_log2_pgsz;
7739e39c5baSBill Taylor srqc_entry.mtt_base_addrl = (uint64_t)mtt_addr >> 32;
7749e39c5baSBill Taylor srqc_entry.mtt_base_addrh = mtt_addr >> 3;
7759e39c5baSBill Taylor
7769e39c5baSBill Taylor /*
7779e39c5baSBill Taylor * RESIZE_SRQ
7789e39c5baSBill Taylor *
7799e39c5baSBill Taylor * If this fails for any reason, then it is an indication that
7809e39c5baSBill Taylor * something (either in HW or SW) has gone seriously wrong. So we
7819e39c5baSBill Taylor * print a warning message and return.
7829e39c5baSBill Taylor */
7839e39c5baSBill Taylor status = hermon_resize_srq_cmd_post(state, &srqc_entry,
7849e39c5baSBill Taylor srq->srq_srqnum, sleepflag);
7859e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) {
7869e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: RESIZE_SRQ command failed: %08x\n",
7879e39c5baSBill Taylor status);
7889e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
7899e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
7909e39c5baSBill Taylor }
7919e39c5baSBill Taylor (void) hermon_mr_mtt_unbind(state, &bind, mtt);
7929e39c5baSBill Taylor kmem_free(wre_new, (1 << log_srq_size) *
7939e39c5baSBill Taylor sizeof (uint64_t));
7949e39c5baSBill Taylor hermon_queue_free(&new_srqinfo);
7959e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
7969e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
7979e39c5baSBill Taylor return (ibc_get_ci_failure(0));
7989e39c5baSBill Taylor }
7999e39c5baSBill Taylor /*
8009e39c5baSBill Taylor * Update the Hermon Shared Receive Queue handle with all the new
8019e39c5baSBill Taylor * information. At the same time, save away all the necessary
8029e39c5baSBill Taylor * information for freeing up the old resources
8039e39c5baSBill Taylor */
8049e39c5baSBill Taylor old_srqinfo = srq->srq_wqinfo;
8059e39c5baSBill Taylor old_mtt = srq->srq_mrhdl->mr_mttrsrcp;
8069e39c5baSBill Taylor bcopy(&srq->srq_mrhdl->mr_bindinfo, &old_bind,
8079e39c5baSBill Taylor sizeof (hermon_bind_info_t));
8089e39c5baSBill Taylor
8099e39c5baSBill Taylor /* Now set the new info */
8109e39c5baSBill Taylor srq->srq_wqinfo = new_srqinfo;
8119e39c5baSBill Taylor srq->srq_wq_buf = buf;
8129e39c5baSBill Taylor srq->srq_wq_bufsz = (1 << log_srq_size);
8139e39c5baSBill Taylor bcopy(&bind, &srq->srq_mrhdl->mr_bindinfo, sizeof (hermon_bind_info_t));
8149e39c5baSBill Taylor srq->srq_mrhdl->mr_mttrsrcp = mtt;
8159e39c5baSBill Taylor srq->srq_desc_off = srq_desc_off;
8169e39c5baSBill Taylor srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
8179e39c5baSBill Taylor
8189e39c5baSBill Taylor /* Update MR mtt pagesize */
8199e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits;
8209e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
8219e39c5baSBill Taylor
8229e39c5baSBill Taylor /*
8239e39c5baSBill Taylor * Initialize new wridlist, if needed.
8249e39c5baSBill Taylor *
8259e39c5baSBill Taylor * If a wridlist already is setup on an SRQ (the QP associated with an
8269e39c5baSBill Taylor * SRQ has moved "from_reset") then we must update this wridlist based
8279e39c5baSBill Taylor * on the new SRQ size. We allocate the new size of Work Request ID
8289e39c5baSBill Taylor * Entries, copy over the old entries to the new list, and
8299e39c5baSBill Taylor * re-initialize the srq wridlist in non-umap case
8309e39c5baSBill Taylor */
8319e39c5baSBill Taylor wre_old = srq->srq_wq_wqhdr->wq_wrid;
8329e39c5baSBill Taylor
8339e39c5baSBill Taylor bcopy(wre_old, wre_new, srq_old_bufsz * sizeof (uint64_t));
8349e39c5baSBill Taylor
8359e39c5baSBill Taylor /* Setup new sizes in wre */
8369e39c5baSBill Taylor srq->srq_wq_wqhdr->wq_wrid = wre_new;
8379e39c5baSBill Taylor
8389e39c5baSBill Taylor /*
8399e39c5baSBill Taylor * If "old" SRQ was a user-mappable SRQ that is currently mmap()'d out
8409e39c5baSBill Taylor * to a user process, then we need to call devmap_devmem_remap() to
8419e39c5baSBill Taylor * invalidate the mapping to the SRQ memory. We also need to
8429e39c5baSBill Taylor * invalidate the SRQ tracking information for the user mapping.
8439e39c5baSBill Taylor *
8449e39c5baSBill Taylor * Note: On failure, the remap really shouldn't ever happen. So, if it
8459e39c5baSBill Taylor * does, it is an indication that something has gone seriously wrong.
8469e39c5baSBill Taylor * So we print a warning message and return error (knowing, of course,
8479e39c5baSBill Taylor * that the "old" SRQ memory will be leaked)
8489e39c5baSBill Taylor */
8499e39c5baSBill Taylor if ((srq->srq_is_umap) && (srq->srq_umap_dhp != NULL)) {
8509e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
8519e39c5baSBill Taylor status = devmap_devmem_remap(srq->srq_umap_dhp,
8529e39c5baSBill Taylor state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size, maxprot,
8539e39c5baSBill Taylor DEVMAP_MAPPING_INVALID, NULL);
8549e39c5baSBill Taylor if (status != DDI_SUCCESS) {
8559e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
8569e39c5baSBill Taylor HERMON_WARNING(state, "failed in SRQ memory "
8579e39c5baSBill Taylor "devmap_devmem_remap()");
8589e39c5baSBill Taylor /* We can, however, free the memory for old wre */
8599e39c5baSBill Taylor kmem_free(wre_old, srq_old_bufsz * sizeof (uint64_t));
8609e39c5baSBill Taylor return (ibc_get_ci_failure(0));
8619e39c5baSBill Taylor }
8629e39c5baSBill Taylor srq->srq_umap_dhp = (devmap_cookie_t)NULL;
8639e39c5baSBill Taylor }
8649e39c5baSBill Taylor
8659e39c5baSBill Taylor /*
8669e39c5baSBill Taylor * Drop the SRQ lock now. The only thing left to do is to free up
8679e39c5baSBill Taylor * the old resources.
8689e39c5baSBill Taylor */
8699e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
8709e39c5baSBill Taylor
8719e39c5baSBill Taylor /*
8729e39c5baSBill Taylor * Unbind the MTT entries.
8739e39c5baSBill Taylor */
8749e39c5baSBill Taylor status = hermon_mr_mtt_unbind(state, &old_bind, old_mtt);
8759e39c5baSBill Taylor if (status != DDI_SUCCESS) {
8769e39c5baSBill Taylor HERMON_WARNING(state, "failed to unbind old SRQ memory");
8779e39c5baSBill Taylor status = ibc_get_ci_failure(0);
8789e39c5baSBill Taylor goto srqmodify_fail;
8799e39c5baSBill Taylor }
8809e39c5baSBill Taylor
8819e39c5baSBill Taylor /* Free the memory for old wre */
8829e39c5baSBill Taylor kmem_free(wre_old, srq_old_bufsz * sizeof (uint64_t));
8839e39c5baSBill Taylor
8849e39c5baSBill Taylor /* Free the memory for the old SRQ */
8859e39c5baSBill Taylor hermon_queue_free(&old_srqinfo);
8869e39c5baSBill Taylor
8879e39c5baSBill Taylor /*
8889e39c5baSBill Taylor * Fill in the return arguments (if necessary). This includes the
8899e39c5baSBill Taylor * real new completion queue size.
8909e39c5baSBill Taylor */
8919e39c5baSBill Taylor if (real_size != NULL) {
8929e39c5baSBill Taylor *real_size = (1 << log_srq_size);
8939e39c5baSBill Taylor }
8949e39c5baSBill Taylor
8959e39c5baSBill Taylor return (DDI_SUCCESS);
8969e39c5baSBill Taylor
8979e39c5baSBill Taylor srqmodify_fail:
8989e39c5baSBill Taylor return (status);
8999e39c5baSBill Taylor }
9009e39c5baSBill Taylor
9019e39c5baSBill Taylor
9029e39c5baSBill Taylor /*
9039e39c5baSBill Taylor * hermon_srq_refcnt_inc()
9049e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
9059e39c5baSBill Taylor */
9069e39c5baSBill Taylor void
hermon_srq_refcnt_inc(hermon_srqhdl_t srq)9079e39c5baSBill Taylor hermon_srq_refcnt_inc(hermon_srqhdl_t srq)
9089e39c5baSBill Taylor {
9099e39c5baSBill Taylor mutex_enter(&srq->srq_lock);
9109e39c5baSBill Taylor srq->srq_refcnt++;
9119e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
9129e39c5baSBill Taylor }
9139e39c5baSBill Taylor
9149e39c5baSBill Taylor
9159e39c5baSBill Taylor /*
9169e39c5baSBill Taylor * hermon_srq_refcnt_dec()
9179e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
9189e39c5baSBill Taylor */
9199e39c5baSBill Taylor void
hermon_srq_refcnt_dec(hermon_srqhdl_t srq)9209e39c5baSBill Taylor hermon_srq_refcnt_dec(hermon_srqhdl_t srq)
9219e39c5baSBill Taylor {
9229e39c5baSBill Taylor mutex_enter(&srq->srq_lock);
9239e39c5baSBill Taylor srq->srq_refcnt--;
9249e39c5baSBill Taylor mutex_exit(&srq->srq_lock);
9259e39c5baSBill Taylor }
9269e39c5baSBill Taylor
9279e39c5baSBill Taylor
9289e39c5baSBill Taylor /*
9299e39c5baSBill Taylor * hermon_srqhdl_from_srqnum()
9309e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
9319e39c5baSBill Taylor *
9329e39c5baSBill Taylor * This routine is important because changing the unconstrained
9339e39c5baSBill Taylor * portion of the SRQ number is critical to the detection of a
9349e39c5baSBill Taylor * potential race condition in the SRQ handler code (i.e. the case
9359e39c5baSBill Taylor * where a SRQ is freed and alloc'd again before an event for the
9369e39c5baSBill Taylor * "old" SRQ can be handled).
9379e39c5baSBill Taylor *
9389e39c5baSBill Taylor * While this is not a perfect solution (not sure that one exists)
9399e39c5baSBill Taylor * it does help to mitigate the chance that this race condition will
9409e39c5baSBill Taylor * cause us to deliver a "stale" event to the new SRQ owner. Note:
9419e39c5baSBill Taylor * this solution does not scale well because the number of constrained
9429e39c5baSBill Taylor * bits increases (and, hence, the number of unconstrained bits
9439e39c5baSBill Taylor * decreases) as the number of supported SRQ grows. For small and
9449e39c5baSBill Taylor * intermediate values, it should hopefully provide sufficient
9459e39c5baSBill Taylor * protection.
9469e39c5baSBill Taylor */
9479e39c5baSBill Taylor hermon_srqhdl_t
hermon_srqhdl_from_srqnum(hermon_state_t * state,uint_t srqnum)9489e39c5baSBill Taylor hermon_srqhdl_from_srqnum(hermon_state_t *state, uint_t srqnum)
9499e39c5baSBill Taylor {
9509e39c5baSBill Taylor uint_t srqindx, srqmask;
9519e39c5baSBill Taylor
9529e39c5baSBill Taylor /* Calculate the SRQ table index from the srqnum */
9539e39c5baSBill Taylor srqmask = (1 << state->hs_cfg_profile->cp_log_num_srq) - 1;
9549e39c5baSBill Taylor srqindx = srqnum & srqmask;
95517a2b317SBill Taylor return (hermon_icm_num_to_hdl(state, HERMON_SRQC, srqindx));
9569e39c5baSBill Taylor }
9579e39c5baSBill Taylor
9589e39c5baSBill Taylor
9599e39c5baSBill Taylor /*
9609e39c5baSBill Taylor * hermon_srq_sgl_to_logwqesz()
9619e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
9629e39c5baSBill Taylor */
9639e39c5baSBill Taylor static void
hermon_srq_sgl_to_logwqesz(hermon_state_t * state,uint_t num_sgl,hermon_qp_wq_type_t wq_type,uint_t * logwqesz,uint_t * max_sgl)9649e39c5baSBill Taylor hermon_srq_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
9659e39c5baSBill Taylor hermon_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl)
9669e39c5baSBill Taylor {
9679e39c5baSBill Taylor uint_t max_size, log2, actual_sgl;
9689e39c5baSBill Taylor
9699e39c5baSBill Taylor switch (wq_type) {
9709e39c5baSBill Taylor case HERMON_QP_WQ_TYPE_RECVQ:
9719e39c5baSBill Taylor /*
9729e39c5baSBill Taylor * Use requested maximum SGL to calculate max descriptor size
9739e39c5baSBill Taylor * (while guaranteeing that the descriptor size is a
9749e39c5baSBill Taylor * power-of-2 cachelines).
9759e39c5baSBill Taylor */
9769e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_SRQ_HDRS + (num_sgl << 4));
9779e39c5baSBill Taylor log2 = highbit(max_size);
978*de710d24SJosef 'Jeff' Sipek if (ISP2(max_size)) {
9799e39c5baSBill Taylor log2 = log2 - 1;
9809e39c5baSBill Taylor }
9819e39c5baSBill Taylor
9829e39c5baSBill Taylor /* Make sure descriptor is at least the minimum size */
9839e39c5baSBill Taylor log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
9849e39c5baSBill Taylor
9859e39c5baSBill Taylor /* Calculate actual number of SGL (given WQE size) */
9869e39c5baSBill Taylor actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SRQ_HDRS) >> 4;
9879e39c5baSBill Taylor break;
9889e39c5baSBill Taylor
9899e39c5baSBill Taylor default:
9909e39c5baSBill Taylor HERMON_WARNING(state, "unexpected work queue type");
9919e39c5baSBill Taylor break;
9929e39c5baSBill Taylor }
9939e39c5baSBill Taylor
9949e39c5baSBill Taylor /* Fill in the return values */
9959e39c5baSBill Taylor *logwqesz = log2;
9969e39c5baSBill Taylor *max_sgl = min(state->hs_cfg_profile->cp_srq_max_sgl, actual_sgl);
9979e39c5baSBill Taylor }
998