19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
2336e3c695SPramod Gunjikar  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
249e39c5baSBill Taylor  * Use is subject to license terms.
259e39c5baSBill Taylor  */
269e39c5baSBill Taylor 
279e39c5baSBill Taylor /*
289e39c5baSBill Taylor  * tavor_srq.c
299e39c5baSBill Taylor  *    Tavor Shared Receive Queue Processing Routines
309e39c5baSBill Taylor  *
319e39c5baSBill Taylor  *    Implements all the routines necessary for allocating, freeing, querying,
329e39c5baSBill Taylor  *    modifying and posting shared receive queues.
339e39c5baSBill Taylor  */
349e39c5baSBill Taylor 
35*de710d24SJosef 'Jeff' Sipek #include <sys/sysmacros.h>
369e39c5baSBill Taylor #include <sys/types.h>
379e39c5baSBill Taylor #include <sys/conf.h>
389e39c5baSBill Taylor #include <sys/ddi.h>
399e39c5baSBill Taylor #include <sys/sunddi.h>
409e39c5baSBill Taylor #include <sys/modctl.h>
419e39c5baSBill Taylor #include <sys/bitmap.h>
429e39c5baSBill Taylor 
439e39c5baSBill Taylor #include <sys/ib/adapters/tavor/tavor.h>
449e39c5baSBill Taylor 
459e39c5baSBill Taylor static void tavor_srq_sgl_to_logwqesz(tavor_state_t *state, uint_t num_sgl,
469e39c5baSBill Taylor     tavor_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl);
479e39c5baSBill Taylor 
489e39c5baSBill Taylor /*
499e39c5baSBill Taylor  * tavor_srq_alloc()
509e39c5baSBill Taylor  *    Context: Can be called only from user or kernel context.
519e39c5baSBill Taylor  */
529e39c5baSBill Taylor int
tavor_srq_alloc(tavor_state_t * state,tavor_srq_info_t * srqinfo,uint_t sleepflag,tavor_srq_options_t * op)539e39c5baSBill Taylor tavor_srq_alloc(tavor_state_t *state, tavor_srq_info_t *srqinfo,
549e39c5baSBill Taylor     uint_t sleepflag, tavor_srq_options_t *op)
559e39c5baSBill Taylor {
569e39c5baSBill Taylor 	ibt_srq_hdl_t		ibt_srqhdl;
579e39c5baSBill Taylor 	tavor_pdhdl_t		pd;
589e39c5baSBill Taylor 	ibt_srq_sizes_t		*sizes;
599e39c5baSBill Taylor 	ibt_srq_sizes_t		*real_sizes;
609e39c5baSBill Taylor 	tavor_srqhdl_t		*srqhdl;
619e39c5baSBill Taylor 	ibt_srq_flags_t		flags;
629e39c5baSBill Taylor 	tavor_rsrc_t		*srqc, *rsrc;
639e39c5baSBill Taylor 	tavor_hw_srqc_t		srqc_entry;
649e39c5baSBill Taylor 	uint32_t		*buf;
659e39c5baSBill Taylor 	tavor_srqhdl_t		srq;
669e39c5baSBill Taylor 	tavor_umap_db_entry_t	*umapdb;
679e39c5baSBill Taylor 	ibt_mr_attr_t		mr_attr;
689e39c5baSBill Taylor 	tavor_mr_options_t	mr_op;
699e39c5baSBill Taylor 	tavor_mrhdl_t		mr;
709e39c5baSBill Taylor 	uint64_t		addr;
719e39c5baSBill Taylor 	uint64_t		value, srq_desc_off;
729e39c5baSBill Taylor 	uint32_t		lkey;
739e39c5baSBill Taylor 	uint32_t		log_srq_size;
749e39c5baSBill Taylor 	uint32_t		uarpg;
759e39c5baSBill Taylor 	uint_t			wq_location, dma_xfer_mode, srq_is_umap;
769e39c5baSBill Taylor 	int			flag, status;
779e39c5baSBill Taylor 	uint_t			max_sgl;
789e39c5baSBill Taylor 	uint_t			wqesz;
799e39c5baSBill Taylor 
809e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sizes))
819e39c5baSBill Taylor 
829e39c5baSBill Taylor 	/*
839e39c5baSBill Taylor 	 * Check the "options" flag.  Currently this flag tells the driver
849e39c5baSBill Taylor 	 * whether or not the SRQ's work queues should be come from normal
859e39c5baSBill Taylor 	 * system memory or whether they should be allocated from DDR memory.
869e39c5baSBill Taylor 	 */
879e39c5baSBill Taylor 	if (op == NULL) {
889e39c5baSBill Taylor 		wq_location = TAVOR_QUEUE_LOCATION_NORMAL;
899e39c5baSBill Taylor 	} else {
909e39c5baSBill Taylor 		wq_location = op->srqo_wq_loc;
919e39c5baSBill Taylor 	}
929e39c5baSBill Taylor 
939e39c5baSBill Taylor 	/*
949e39c5baSBill Taylor 	 * Extract the necessary info from the tavor_srq_info_t structure
959e39c5baSBill Taylor 	 */
969e39c5baSBill Taylor 	real_sizes = srqinfo->srqi_real_sizes;
979e39c5baSBill Taylor 	sizes	   = srqinfo->srqi_sizes;
989e39c5baSBill Taylor 	pd	   = srqinfo->srqi_pd;
999e39c5baSBill Taylor 	ibt_srqhdl = srqinfo->srqi_ibt_srqhdl;
1009e39c5baSBill Taylor 	flags	   = srqinfo->srqi_flags;
1019e39c5baSBill Taylor 	srqhdl	   = srqinfo->srqi_srqhdl;
1029e39c5baSBill Taylor 
1039e39c5baSBill Taylor 	/*
1049e39c5baSBill Taylor 	 * Determine whether SRQ is being allocated for userland access or
1059e39c5baSBill Taylor 	 * whether it is being allocated for kernel access.  If the SRQ is
1069e39c5baSBill Taylor 	 * being allocated for userland access, then lookup the UAR doorbell
1079e39c5baSBill Taylor 	 * page number for the current process.  Note:  If this is not found
1089e39c5baSBill Taylor 	 * (e.g. if the process has not previously open()'d the Tavor driver),
1099e39c5baSBill Taylor 	 * then an error is returned.
1109e39c5baSBill Taylor 	 */
1119e39c5baSBill Taylor 	srq_is_umap = (flags & IBT_SRQ_USER_MAP) ? 1 : 0;
1129e39c5baSBill Taylor 	if (srq_is_umap) {
1139e39c5baSBill Taylor 		status = tavor_umap_db_find(state->ts_instance, ddi_get_pid(),
1149e39c5baSBill Taylor 		    MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
1159e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
1169e39c5baSBill Taylor 			goto srqalloc_fail3;
1179e39c5baSBill Taylor 		}
1189e39c5baSBill Taylor 		uarpg = ((tavor_rsrc_t *)(uintptr_t)value)->tr_indx;
1199e39c5baSBill Taylor 	}
1209e39c5baSBill Taylor 
1219e39c5baSBill Taylor 	/* Increase PD refcnt */
1229e39c5baSBill Taylor 	tavor_pd_refcnt_inc(pd);
1239e39c5baSBill Taylor 
1249e39c5baSBill Taylor 	/* Allocate an SRQ context entry */
1259e39c5baSBill Taylor 	status = tavor_rsrc_alloc(state, TAVOR_SRQC, 1, sleepflag, &srqc);
1269e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
1279e39c5baSBill Taylor 		goto srqalloc_fail1;
1289e39c5baSBill Taylor 	}
1299e39c5baSBill Taylor 
1309e39c5baSBill Taylor 	/* Allocate the SRQ Handle entry */
1319e39c5baSBill Taylor 	status = tavor_rsrc_alloc(state, TAVOR_SRQHDL, 1, sleepflag, &rsrc);
1329e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
1339e39c5baSBill Taylor 		goto srqalloc_fail2;
1349e39c5baSBill Taylor 	}
1359e39c5baSBill Taylor 
1369e39c5baSBill Taylor 	srq = (tavor_srqhdl_t)rsrc->tr_addr;
1379e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq))
1389e39c5baSBill Taylor 
13936e3c695SPramod Gunjikar 	srq->srq_srqnum = srqc->tr_indx;	/* just use index */
1409e39c5baSBill Taylor 
1419e39c5baSBill Taylor 	/*
1429e39c5baSBill Taylor 	 * If this will be a user-mappable SRQ, then allocate an entry for
1439e39c5baSBill Taylor 	 * the "userland resources database".  This will later be added to
1449e39c5baSBill Taylor 	 * the database (after all further SRQ operations are successful).
1459e39c5baSBill Taylor 	 * If we fail here, we must undo the reference counts and the
1469e39c5baSBill Taylor 	 * previous resource allocation.
1479e39c5baSBill Taylor 	 */
1489e39c5baSBill Taylor 	if (srq_is_umap) {
1499e39c5baSBill Taylor 		umapdb = tavor_umap_db_alloc(state->ts_instance,
1509e39c5baSBill Taylor 		    srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC,
1519e39c5baSBill Taylor 		    (uint64_t)(uintptr_t)rsrc);
1529e39c5baSBill Taylor 		if (umapdb == NULL) {
1539e39c5baSBill Taylor 			goto srqalloc_fail3;
1549e39c5baSBill Taylor 		}
1559e39c5baSBill Taylor 	}
1569e39c5baSBill Taylor 
1579e39c5baSBill Taylor 	/*
1589e39c5baSBill Taylor 	 * Calculate the appropriate size for the SRQ.
1599e39c5baSBill Taylor 	 * Note:  All Tavor SRQs must be a power-of-2 in size.  Also
1609e39c5baSBill Taylor 	 * they may not be any smaller than TAVOR_SRQ_MIN_SIZE.  This step
1619e39c5baSBill Taylor 	 * is to round the requested size up to the next highest power-of-2
1629e39c5baSBill Taylor 	 */
1639e39c5baSBill Taylor 	sizes->srq_wr_sz = max(sizes->srq_wr_sz, TAVOR_SRQ_MIN_SIZE);
1649e39c5baSBill Taylor 	log_srq_size = highbit(sizes->srq_wr_sz);
165*de710d24SJosef 'Jeff' Sipek 	if (ISP2(sizes->srq_wr_sz)) {
1669e39c5baSBill Taylor 		log_srq_size = log_srq_size - 1;
1679e39c5baSBill Taylor 	}
1689e39c5baSBill Taylor 
1699e39c5baSBill Taylor 	/*
1709e39c5baSBill Taylor 	 * Next we verify that the rounded-up size is valid (i.e. consistent
1719e39c5baSBill Taylor 	 * with the device limits and/or software-configured limits).  If not,
1729e39c5baSBill Taylor 	 * then obviously we have a lot of cleanup to do before returning.
1739e39c5baSBill Taylor 	 */
1749e39c5baSBill Taylor 	if (log_srq_size > state->ts_cfg_profile->cp_log_max_srq_sz) {
1759e39c5baSBill Taylor 		goto srqalloc_fail4;
1769e39c5baSBill Taylor 	}
1779e39c5baSBill Taylor 
1789e39c5baSBill Taylor 	/*
1799e39c5baSBill Taylor 	 * Next we verify that the requested number of SGL is valid (i.e.
1809e39c5baSBill Taylor 	 * consistent with the device limits and/or software-configured
1819e39c5baSBill Taylor 	 * limits).  If not, then obviously the same cleanup needs to be done.
1829e39c5baSBill Taylor 	 */
1839e39c5baSBill Taylor 	max_sgl = state->ts_cfg_profile->cp_srq_max_sgl;
1849e39c5baSBill Taylor 	if (sizes->srq_sgl_sz > max_sgl) {
1859e39c5baSBill Taylor 		goto srqalloc_fail4;
1869e39c5baSBill Taylor 	}
1879e39c5baSBill Taylor 
1889e39c5baSBill Taylor 	/*
1899e39c5baSBill Taylor 	 * Determine the SRQ's WQE sizes.  This depends on the requested
1909e39c5baSBill Taylor 	 * number of SGLs.  Note: This also has the side-effect of
1919e39c5baSBill Taylor 	 * calculating the real number of SGLs (for the calculated WQE size)
1929e39c5baSBill Taylor 	 */
1939e39c5baSBill Taylor 	tavor_srq_sgl_to_logwqesz(state, sizes->srq_sgl_sz,
1949e39c5baSBill Taylor 	    TAVOR_QP_WQ_TYPE_RECVQ, &srq->srq_wq_log_wqesz,
1959e39c5baSBill Taylor 	    &srq->srq_wq_sgl);
1969e39c5baSBill Taylor 
1979e39c5baSBill Taylor 	/*
1989e39c5baSBill Taylor 	 * Allocate the memory for SRQ work queues.  Note:  The location from
1999e39c5baSBill Taylor 	 * which we will allocate these work queues has been passed in through
2009e39c5baSBill Taylor 	 * the tavor_qp_options_t structure.  Since Tavor work queues are not
2019e39c5baSBill Taylor 	 * allowed to cross a 32-bit (4GB) boundary, the alignment of the work
2029e39c5baSBill Taylor 	 * queue memory is very important.  We used to allocate work queues
2039e39c5baSBill Taylor 	 * (the combined receive and send queues) so that they would be aligned
2049e39c5baSBill Taylor 	 * on their combined size.  That alignment guaranteed that they would
2059e39c5baSBill Taylor 	 * never cross the 4GB boundary (Tavor work queues are on the order of
2069e39c5baSBill Taylor 	 * MBs at maximum).  Now we are able to relax this alignment constraint
2079e39c5baSBill Taylor 	 * by ensuring that the IB address assigned to the queue memory (as a
2089e39c5baSBill Taylor 	 * result of the tavor_mr_register() call) is offset from zero.
2099e39c5baSBill Taylor 	 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
2109e39c5baSBill Taylor 	 * guarantee the alignment, but when attempting to use IOMMU bypass
2119e39c5baSBill Taylor 	 * mode we found that we were not allowed to specify any alignment that
2129e39c5baSBill Taylor 	 * was more restrictive than the system page size.  So we avoided this
2139e39c5baSBill Taylor 	 * constraint by passing two alignment values, one for the memory
2149e39c5baSBill Taylor 	 * allocation itself and the other for the DMA handle (for later bind).
2159e39c5baSBill Taylor 	 * This used to cause more memory than necessary to be allocated (in
2169e39c5baSBill Taylor 	 * order to guarantee the more restrictive alignment contraint).  But
2179e39c5baSBill Taylor 	 * be guaranteeing the zero-based IB virtual address for the queue, we
2189e39c5baSBill Taylor 	 * are able to conserve this memory.
2199e39c5baSBill Taylor 	 *
2209e39c5baSBill Taylor 	 * Note: If SRQ is not user-mappable, then it may come from either
2219e39c5baSBill Taylor 	 * kernel system memory or from HCA-attached local DDR memory.
2229e39c5baSBill Taylor 	 *
2239e39c5baSBill Taylor 	 * Note2: We align this queue on a pagesize boundary.  This is required
2249e39c5baSBill Taylor 	 * to make sure that all the resulting IB addresses will start at 0, for
2259e39c5baSBill Taylor 	 * a zero-based queue.  By making sure we are aligned on at least a
2269e39c5baSBill Taylor 	 * page, any offset we use into our queue will be the same as when we
2279e39c5baSBill Taylor 	 * perform tavor_srq_modify() operations later.
2289e39c5baSBill Taylor 	 */
2299e39c5baSBill Taylor 	wqesz = (1 << srq->srq_wq_log_wqesz);
2309e39c5baSBill Taylor 	srq->srq_wqinfo.qa_size = (1 << log_srq_size) * wqesz;
2319e39c5baSBill Taylor 	srq->srq_wqinfo.qa_alloc_align = PAGESIZE;
2329e39c5baSBill Taylor 	srq->srq_wqinfo.qa_bind_align = PAGESIZE;
2339e39c5baSBill Taylor 	if (srq_is_umap) {
2349e39c5baSBill Taylor 		srq->srq_wqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
2359e39c5baSBill Taylor 	} else {
2369e39c5baSBill Taylor 		srq->srq_wqinfo.qa_location = wq_location;
2379e39c5baSBill Taylor 	}
2389e39c5baSBill Taylor 	status = tavor_queue_alloc(state, &srq->srq_wqinfo, sleepflag);
2399e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2409e39c5baSBill Taylor 		goto srqalloc_fail4;
2419e39c5baSBill Taylor 	}
2429e39c5baSBill Taylor 	buf = (uint32_t *)srq->srq_wqinfo.qa_buf_aligned;
2439e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
2449e39c5baSBill Taylor 
2459e39c5baSBill Taylor 	/*
2469e39c5baSBill Taylor 	 * Register the memory for the SRQ work queues.  The memory for the SRQ
2479e39c5baSBill Taylor 	 * must be registered in the Tavor TPT tables.  This gives us the LKey
2489e39c5baSBill Taylor 	 * to specify in the SRQ context later.  Note: If the work queue is to
2499e39c5baSBill Taylor 	 * be allocated from DDR memory, then only a "bypass" mapping is
2509e39c5baSBill Taylor 	 * appropriate.  And if the SRQ memory is user-mappable, then we force
2519e39c5baSBill Taylor 	 * DDI_DMA_CONSISTENT mapping.  Also, in order to meet the alignment
2529e39c5baSBill Taylor 	 * restriction, we pass the "mro_bind_override_addr" flag in the call
2539e39c5baSBill Taylor 	 * to tavor_mr_register().  This guarantees that the resulting IB vaddr
2549e39c5baSBill Taylor 	 * will be zero-based (modulo the offset into the first page).  If we
2559e39c5baSBill Taylor 	 * fail here, we still have the bunch of resource and reference count
2569e39c5baSBill Taylor 	 * cleanup to do.
2579e39c5baSBill Taylor 	 */
2589e39c5baSBill Taylor 	flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP :
2599e39c5baSBill Taylor 	    IBT_MR_NOSLEEP;
2609e39c5baSBill Taylor 	mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
2619e39c5baSBill Taylor 	mr_attr.mr_len   = srq->srq_wqinfo.qa_size;
2629e39c5baSBill Taylor 	mr_attr.mr_as    = NULL;
2639e39c5baSBill Taylor 	mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
2649e39c5baSBill Taylor 	if (srq_is_umap) {
2659e39c5baSBill Taylor 		mr_op.mro_bind_type   = state->ts_cfg_profile->cp_iommu_bypass;
2669e39c5baSBill Taylor 	} else {
2679e39c5baSBill Taylor 		if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
2689e39c5baSBill Taylor 			mr_op.mro_bind_type =
2699e39c5baSBill Taylor 			    state->ts_cfg_profile->cp_iommu_bypass;
2709e39c5baSBill Taylor 			dma_xfer_mode =
2719e39c5baSBill Taylor 			    state->ts_cfg_profile->cp_streaming_consistent;
2729e39c5baSBill Taylor 			if (dma_xfer_mode == DDI_DMA_STREAMING) {
2739e39c5baSBill Taylor 				mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
2749e39c5baSBill Taylor 			}
2759e39c5baSBill Taylor 		} else {
2769e39c5baSBill Taylor 			mr_op.mro_bind_type = TAVOR_BINDMEM_BYPASS;
2779e39c5baSBill Taylor 		}
2789e39c5baSBill Taylor 	}
2799e39c5baSBill Taylor 	mr_op.mro_bind_dmahdl = srq->srq_wqinfo.qa_dmahdl;
2809e39c5baSBill Taylor 	mr_op.mro_bind_override_addr = 1;
2819e39c5baSBill Taylor 	status = tavor_mr_register(state, pd, &mr_attr, &mr, &mr_op);
2829e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2839e39c5baSBill Taylor 		goto srqalloc_fail5;
2849e39c5baSBill Taylor 	}
2859e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2869e39c5baSBill Taylor 	addr = mr->mr_bindinfo.bi_addr;
2879e39c5baSBill Taylor 	lkey = mr->mr_lkey;
2889e39c5baSBill Taylor 
2899e39c5baSBill Taylor 	/*
2909e39c5baSBill Taylor 	 * Calculate the offset between the kernel virtual address space
2919e39c5baSBill Taylor 	 * and the IB virtual address space.  This will be used when
2929e39c5baSBill Taylor 	 * posting work requests to properly initialize each WQE.
2939e39c5baSBill Taylor 	 */
2949e39c5baSBill Taylor 	srq_desc_off = (uint64_t)(uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
2959e39c5baSBill Taylor 	    (uint64_t)mr->mr_bindinfo.bi_addr;
2969e39c5baSBill Taylor 
2979e39c5baSBill Taylor 	/*
2989e39c5baSBill Taylor 	 * Create WQL and Wridlist for use by this SRQ
2999e39c5baSBill Taylor 	 */
3009e39c5baSBill Taylor 	srq->srq_wrid_wql = tavor_wrid_wql_create(state);
3019e39c5baSBill Taylor 	if (srq->srq_wrid_wql == NULL) {
3029e39c5baSBill Taylor 		goto srqalloc_fail6;
3039e39c5baSBill Taylor 	}
3049e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(srq->srq_wrid_wql)))
3059e39c5baSBill Taylor 
3069e39c5baSBill Taylor 	srq->srq_wridlist = tavor_wrid_get_list(1 << log_srq_size);
3079e39c5baSBill Taylor 	if (srq->srq_wridlist == NULL) {
3089e39c5baSBill Taylor 		goto srqalloc_fail7;
3099e39c5baSBill Taylor 	}
3109e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(srq->srq_wridlist)))
3119e39c5baSBill Taylor 
3129e39c5baSBill Taylor 	srq->srq_wridlist->wl_srq_en = 1;
3139e39c5baSBill Taylor 	srq->srq_wridlist->wl_free_list_indx = -1;
3149e39c5baSBill Taylor 
3159e39c5baSBill Taylor 	/*
3169e39c5baSBill Taylor 	 * Fill in all the return arguments (if necessary).  This includes
3179e39c5baSBill Taylor 	 * real queue size and real SGLs.
3189e39c5baSBill Taylor 	 */
3199e39c5baSBill Taylor 	if (real_sizes != NULL) {
3209e39c5baSBill Taylor 		real_sizes->srq_wr_sz = (1 << log_srq_size);
3219e39c5baSBill Taylor 		real_sizes->srq_sgl_sz = srq->srq_wq_sgl;
3229e39c5baSBill Taylor 	}
3239e39c5baSBill Taylor 
3249e39c5baSBill Taylor 	/*
3259e39c5baSBill Taylor 	 * Fill in the SRQC entry.  This is the final step before passing
3269e39c5baSBill Taylor 	 * ownership of the SRQC entry to the Tavor hardware.  We use all of
3279e39c5baSBill Taylor 	 * the information collected/calculated above to fill in the
3289e39c5baSBill Taylor 	 * requisite portions of the SRQC.  Note: If this SRQ is going to be
3299e39c5baSBill Taylor 	 * used for userland access, then we need to set the UAR page number
3309e39c5baSBill Taylor 	 * appropriately (otherwise it's a "don't care")
3319e39c5baSBill Taylor 	 */
3329e39c5baSBill Taylor 	bzero(&srqc_entry, sizeof (tavor_hw_srqc_t));
3339e39c5baSBill Taylor 	srqc_entry.wqe_addr_h	   = (addr >> 32);
3349e39c5baSBill Taylor 	srqc_entry.next_wqe_addr_l = 0;
3359e39c5baSBill Taylor 	srqc_entry.ds		   = (wqesz >> 4);
3369e39c5baSBill Taylor 	srqc_entry.state	   = TAVOR_SRQ_STATE_HW_OWNER;
3379e39c5baSBill Taylor 	srqc_entry.pd		   = pd->pd_pdnum;
3389e39c5baSBill Taylor 	srqc_entry.lkey		   = lkey;
3399e39c5baSBill Taylor 	srqc_entry.wqe_cnt	   = 0;
3409e39c5baSBill Taylor 	if (srq_is_umap) {
3419e39c5baSBill Taylor 		srqc_entry.uar	   = uarpg;
3429e39c5baSBill Taylor 	} else {
3439e39c5baSBill Taylor 		srqc_entry.uar	   = 0;
3449e39c5baSBill Taylor 	}
3459e39c5baSBill Taylor 
3469e39c5baSBill Taylor 	/*
3479e39c5baSBill Taylor 	 * Write the SRQC entry to hardware.  Lastly, we pass ownership of
3489e39c5baSBill Taylor 	 * the entry to the hardware (using the Tavor SW2HW_SRQ firmware
3499e39c5baSBill Taylor 	 * command).  Note: In general, this operation shouldn't fail.  But
3509e39c5baSBill Taylor 	 * if it does, we have to undo everything we've done above before
3519e39c5baSBill Taylor 	 * returning error.
3529e39c5baSBill Taylor 	 */
3539e39c5baSBill Taylor 	status = tavor_cmn_ownership_cmd_post(state, SW2HW_SRQ, &srqc_entry,
3549e39c5baSBill Taylor 	    sizeof (tavor_hw_srqc_t), srq->srq_srqnum,
3559e39c5baSBill Taylor 	    sleepflag);
3569e39c5baSBill Taylor 	if (status != TAVOR_CMD_SUCCESS) {
3579e39c5baSBill Taylor 		cmn_err(CE_CONT, "Tavor: SW2HW_SRQ command failed: %08x\n",
3589e39c5baSBill Taylor 		    status);
3599e39c5baSBill Taylor 		goto srqalloc_fail8;
3609e39c5baSBill Taylor 	}
3619e39c5baSBill Taylor 
3629e39c5baSBill Taylor 	/*
3639e39c5baSBill Taylor 	 * Fill in the rest of the Tavor SRQ handle.  We can update
3649e39c5baSBill Taylor 	 * the following fields for use in further operations on the SRQ.
3659e39c5baSBill Taylor 	 */
3669e39c5baSBill Taylor 	srq->srq_srqcrsrcp = srqc;
3679e39c5baSBill Taylor 	srq->srq_rsrcp	   = rsrc;
3689e39c5baSBill Taylor 	srq->srq_mrhdl	   = mr;
3699e39c5baSBill Taylor 	srq->srq_refcnt	   = 0;
3709e39c5baSBill Taylor 	srq->srq_is_umap   = srq_is_umap;
3719e39c5baSBill Taylor 	srq->srq_uarpg	   = (srq->srq_is_umap) ? uarpg : 0;
3729e39c5baSBill Taylor 	srq->srq_umap_dhp  = (devmap_cookie_t)NULL;
3739e39c5baSBill Taylor 	srq->srq_pdhdl	   = pd;
3749e39c5baSBill Taylor 	srq->srq_wq_lastwqeindx = -1;
3759e39c5baSBill Taylor 	srq->srq_wq_bufsz  = (1 << log_srq_size);
3769e39c5baSBill Taylor 	srq->srq_wq_buf	   = buf;
3779e39c5baSBill Taylor 	srq->srq_desc_off  = srq_desc_off;
3789e39c5baSBill Taylor 	srq->srq_hdlrarg   = (void *)ibt_srqhdl;
3799e39c5baSBill Taylor 	srq->srq_state	   = 0;
3809e39c5baSBill Taylor 	srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
3819e39c5baSBill Taylor 	srq->srq_real_sizes.srq_sgl_sz = srq->srq_wq_sgl;
3829e39c5baSBill Taylor 
3839e39c5baSBill Taylor 	/* Determine if later ddi_dma_sync will be necessary */
3849e39c5baSBill Taylor 	srq->srq_sync = TAVOR_SRQ_IS_SYNC_REQ(state, srq->srq_wqinfo);
3859e39c5baSBill Taylor 
3869e39c5baSBill Taylor 	/*
3879e39c5baSBill Taylor 	 * Put SRQ handle in Tavor SRQNum-to-SRQhdl list.  Then fill in the
3889e39c5baSBill Taylor 	 * "srqhdl" and return success
3899e39c5baSBill Taylor 	 */
3909e39c5baSBill Taylor 	ASSERT(state->ts_srqhdl[srqc->tr_indx] == NULL);
3919e39c5baSBill Taylor 	state->ts_srqhdl[srqc->tr_indx] = srq;
3929e39c5baSBill Taylor 
3939e39c5baSBill Taylor 	/*
3949e39c5baSBill Taylor 	 * If this is a user-mappable SRQ, then we need to insert the
3959e39c5baSBill Taylor 	 * previously allocated entry into the "userland resources database".
3969e39c5baSBill Taylor 	 * This will allow for later lookup during devmap() (i.e. mmap())
3979e39c5baSBill Taylor 	 * calls.
3989e39c5baSBill Taylor 	 */
3999e39c5baSBill Taylor 	if (srq->srq_is_umap) {
4009e39c5baSBill Taylor 		tavor_umap_db_add(umapdb);
4019e39c5baSBill Taylor 	} else {
4029e39c5baSBill Taylor 		mutex_enter(&srq->srq_wrid_wql->wql_lock);
4039e39c5baSBill Taylor 		tavor_wrid_list_srq_init(srq->srq_wridlist, srq, 0);
4049e39c5baSBill Taylor 		mutex_exit(&srq->srq_wrid_wql->wql_lock);
4059e39c5baSBill Taylor 	}
4069e39c5baSBill Taylor 
4079e39c5baSBill Taylor 	*srqhdl = srq;
4089e39c5baSBill Taylor 
4099e39c5baSBill Taylor 	return (status);
4109e39c5baSBill Taylor 
4119e39c5baSBill Taylor /*
4129e39c5baSBill Taylor  * The following is cleanup for all possible failure cases in this routine
4139e39c5baSBill Taylor  */
4149e39c5baSBill Taylor srqalloc_fail8:
4159e39c5baSBill Taylor 	kmem_free(srq->srq_wridlist->wl_wre, srq->srq_wridlist->wl_size *
4169e39c5baSBill Taylor 	    sizeof (tavor_wrid_entry_t));
4179e39c5baSBill Taylor 	kmem_free(srq->srq_wridlist, sizeof (tavor_wrid_list_hdr_t));
4189e39c5baSBill Taylor srqalloc_fail7:
4199e39c5baSBill Taylor 	tavor_wql_refcnt_dec(srq->srq_wrid_wql);
4209e39c5baSBill Taylor srqalloc_fail6:
4219e39c5baSBill Taylor 	if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
4229e39c5baSBill Taylor 	    TAVOR_SLEEPFLAG_FOR_CONTEXT()) != DDI_SUCCESS) {
4239e39c5baSBill Taylor 		TAVOR_WARNING(state, "failed to deregister SRQ memory");
4249e39c5baSBill Taylor 	}
4259e39c5baSBill Taylor srqalloc_fail5:
4269e39c5baSBill Taylor 	tavor_queue_free(state, &srq->srq_wqinfo);
4279e39c5baSBill Taylor srqalloc_fail4:
4289e39c5baSBill Taylor 	if (srq_is_umap) {
4299e39c5baSBill Taylor 		tavor_umap_db_free(umapdb);
4309e39c5baSBill Taylor 	}
4319e39c5baSBill Taylor srqalloc_fail3:
4329e39c5baSBill Taylor 	tavor_rsrc_free(state, &rsrc);
4339e39c5baSBill Taylor srqalloc_fail2:
4349e39c5baSBill Taylor 	tavor_rsrc_free(state, &srqc);
4359e39c5baSBill Taylor srqalloc_fail1:
4369e39c5baSBill Taylor 	tavor_pd_refcnt_dec(pd);
4379e39c5baSBill Taylor srqalloc_fail:
4389e39c5baSBill Taylor 	return (status);
4399e39c5baSBill Taylor }
4409e39c5baSBill Taylor 
4419e39c5baSBill Taylor 
4429e39c5baSBill Taylor /*
4439e39c5baSBill Taylor  * tavor_srq_free()
4449e39c5baSBill Taylor  *    Context: Can be called only from user or kernel context.
4459e39c5baSBill Taylor  */
4469e39c5baSBill Taylor /* ARGSUSED */
4479e39c5baSBill Taylor int
tavor_srq_free(tavor_state_t * state,tavor_srqhdl_t * srqhdl,uint_t sleepflag)4489e39c5baSBill Taylor tavor_srq_free(tavor_state_t *state, tavor_srqhdl_t *srqhdl, uint_t sleepflag)
4499e39c5baSBill Taylor {
4509e39c5baSBill Taylor 	tavor_rsrc_t		*srqc, *rsrc;
4519e39c5baSBill Taylor 	tavor_umap_db_entry_t	*umapdb;
4529e39c5baSBill Taylor 	uint64_t		value;
4539e39c5baSBill Taylor 	tavor_srqhdl_t		srq;
4549e39c5baSBill Taylor 	tavor_mrhdl_t		mr;
4559e39c5baSBill Taylor 	tavor_pdhdl_t		pd;
4569e39c5baSBill Taylor 	tavor_hw_srqc_t		srqc_entry;
4579e39c5baSBill Taylor 	uint32_t		srqnum;
4589e39c5baSBill Taylor 	uint32_t		size;
4599e39c5baSBill Taylor 	uint_t			maxprot;
4609e39c5baSBill Taylor 	int			status;
4619e39c5baSBill Taylor 
4629e39c5baSBill Taylor 	/*
4639e39c5baSBill Taylor 	 * Pull all the necessary information from the Tavor Shared Receive
4649e39c5baSBill Taylor 	 * Queue handle.  This is necessary here because the resource for the
4659e39c5baSBill Taylor 	 * SRQ handle is going to be freed up as part of this operation.
4669e39c5baSBill Taylor 	 */
4679e39c5baSBill Taylor 	srq	= *srqhdl;
4689e39c5baSBill Taylor 	mutex_enter(&srq->srq_lock);
4699e39c5baSBill Taylor 	srqc	= srq->srq_srqcrsrcp;
4709e39c5baSBill Taylor 	rsrc	= srq->srq_rsrcp;
4719e39c5baSBill Taylor 	pd	= srq->srq_pdhdl;
4729e39c5baSBill Taylor 	mr	= srq->srq_mrhdl;
4739e39c5baSBill Taylor 	srqnum	= srq->srq_srqnum;
4749e39c5baSBill Taylor 
4759e39c5baSBill Taylor 	/*
4769e39c5baSBill Taylor 	 * If there are work queues still associated with the SRQ, then return
4779e39c5baSBill Taylor 	 * an error.  Otherwise, we will be holding the SRQ lock.
4789e39c5baSBill Taylor 	 */
4799e39c5baSBill Taylor 	if (srq->srq_refcnt != 0) {
4809e39c5baSBill Taylor 		mutex_exit(&srq->srq_lock);
4819e39c5baSBill Taylor 		return (IBT_SRQ_IN_USE);
4829e39c5baSBill Taylor 	}
4839e39c5baSBill Taylor 
4849e39c5baSBill Taylor 	/*
4859e39c5baSBill Taylor 	 * If this was a user-mappable SRQ, then we need to remove its entry
4869e39c5baSBill Taylor 	 * from the "userland resources database".  If it is also currently
4879e39c5baSBill Taylor 	 * mmap()'d out to a user process, then we need to call
4889e39c5baSBill Taylor 	 * devmap_devmem_remap() to remap the SRQ memory to an invalid mapping.
4899e39c5baSBill Taylor 	 * We also need to invalidate the SRQ tracking information for the
4909e39c5baSBill Taylor 	 * user mapping.
4919e39c5baSBill Taylor 	 */
4929e39c5baSBill Taylor 	if (srq->srq_is_umap) {
4939e39c5baSBill Taylor 		status = tavor_umap_db_find(state->ts_instance, srq->srq_srqnum,
4949e39c5baSBill Taylor 		    MLNX_UMAP_SRQMEM_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
4959e39c5baSBill Taylor 		    &umapdb);
4969e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
4979e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
4989e39c5baSBill Taylor 			TAVOR_WARNING(state, "failed to find in database");
4999e39c5baSBill Taylor 			return (ibc_get_ci_failure(0));
5009e39c5baSBill Taylor 		}
5019e39c5baSBill Taylor 		tavor_umap_db_free(umapdb);
5029e39c5baSBill Taylor 		if (srq->srq_umap_dhp != NULL) {
5039e39c5baSBill Taylor 			maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
5049e39c5baSBill Taylor 			status = devmap_devmem_remap(srq->srq_umap_dhp,
5059e39c5baSBill Taylor 			    state->ts_dip, 0, 0, srq->srq_wqinfo.qa_size,
5069e39c5baSBill Taylor 			    maxprot, DEVMAP_MAPPING_INVALID, NULL);
5079e39c5baSBill Taylor 			if (status != DDI_SUCCESS) {
5089e39c5baSBill Taylor 				mutex_exit(&srq->srq_lock);
5099e39c5baSBill Taylor 				TAVOR_WARNING(state, "failed in SRQ memory "
5109e39c5baSBill Taylor 				    "devmap_devmem_remap()");
5119e39c5baSBill Taylor 				return (ibc_get_ci_failure(0));
5129e39c5baSBill Taylor 			}
5139e39c5baSBill Taylor 			srq->srq_umap_dhp = (devmap_cookie_t)NULL;
5149e39c5baSBill Taylor 		}
5159e39c5baSBill Taylor 	}
5169e39c5baSBill Taylor 
5179e39c5baSBill Taylor 	/*
5189e39c5baSBill Taylor 	 * Put NULL into the Tavor SRQNum-to-SRQHdl list.  This will allow any
5199e39c5baSBill Taylor 	 * in-progress events to detect that the SRQ corresponding to this
5209e39c5baSBill Taylor 	 * number has been freed.
5219e39c5baSBill Taylor 	 */
5229e39c5baSBill Taylor 	state->ts_srqhdl[srqc->tr_indx] = NULL;
5239e39c5baSBill Taylor 
5249e39c5baSBill Taylor 	mutex_exit(&srq->srq_lock);
5259e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq));
5269e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq->srq_wridlist));
5279e39c5baSBill Taylor 
5289e39c5baSBill Taylor 	/*
5299e39c5baSBill Taylor 	 * Reclaim SRQC entry from hardware (using the Tavor HW2SW_SRQ
5309e39c5baSBill Taylor 	 * firmware command).  If the ownership transfer fails for any reason,
5319e39c5baSBill Taylor 	 * then it is an indication that something (either in HW or SW) has
5329e39c5baSBill Taylor 	 * gone seriously wrong.
5339e39c5baSBill Taylor 	 */
5349e39c5baSBill Taylor 	status = tavor_cmn_ownership_cmd_post(state, HW2SW_SRQ, &srqc_entry,
5359e39c5baSBill Taylor 	    sizeof (tavor_hw_srqc_t), srqnum, sleepflag);
5369e39c5baSBill Taylor 	if (status != TAVOR_CMD_SUCCESS) {
5379e39c5baSBill Taylor 		TAVOR_WARNING(state, "failed to reclaim SRQC ownership");
5389e39c5baSBill Taylor 		cmn_err(CE_CONT, "Tavor: HW2SW_SRQ command failed: %08x\n",
5399e39c5baSBill Taylor 		    status);
5409e39c5baSBill Taylor 		return (IBT_FAILURE);
5419e39c5baSBill Taylor 	}
5429e39c5baSBill Taylor 
5439e39c5baSBill Taylor 	/*
5449e39c5baSBill Taylor 	 * Deregister the memory for the Shared Receive Queue.  If this fails
5459e39c5baSBill Taylor 	 * for any reason, then it is an indication that something (either
5469e39c5baSBill Taylor 	 * in HW or SW) has gone seriously wrong.  So we print a warning
5479e39c5baSBill Taylor 	 * message and return.
5489e39c5baSBill Taylor 	 */
5499e39c5baSBill Taylor 	status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
5509e39c5baSBill Taylor 	    sleepflag);
5519e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
5529e39c5baSBill Taylor 		TAVOR_WARNING(state, "failed to deregister SRQ memory");
5539e39c5baSBill Taylor 		return (IBT_FAILURE);
5549e39c5baSBill Taylor 	}
5559e39c5baSBill Taylor 
5569e39c5baSBill Taylor 	/* Calculate the size and free the wridlist container */
5579e39c5baSBill Taylor 	if (srq->srq_wridlist != NULL) {
5589e39c5baSBill Taylor 		size = (srq->srq_wridlist->wl_size *
5599e39c5baSBill Taylor 		    sizeof (tavor_wrid_entry_t));
5609e39c5baSBill Taylor 		kmem_free(srq->srq_wridlist->wl_wre, size);
5619e39c5baSBill Taylor 		kmem_free(srq->srq_wridlist, sizeof (tavor_wrid_list_hdr_t));
5629e39c5baSBill Taylor 
5639e39c5baSBill Taylor 		/*
5649e39c5baSBill Taylor 		 * Release reference to WQL; If this is the last reference,
5659e39c5baSBill Taylor 		 * this call also has the side effect of freeing up the
5669e39c5baSBill Taylor 		 * 'srq_wrid_wql' memory.
5679e39c5baSBill Taylor 		 */
5689e39c5baSBill Taylor 		tavor_wql_refcnt_dec(srq->srq_wrid_wql);
5699e39c5baSBill Taylor 	}
5709e39c5baSBill Taylor 
5719e39c5baSBill Taylor 	/* Free the memory for the SRQ */
5729e39c5baSBill Taylor 	tavor_queue_free(state, &srq->srq_wqinfo);
5739e39c5baSBill Taylor 
5749e39c5baSBill Taylor 	/* Free the Tavor SRQ Handle */
5759e39c5baSBill Taylor 	tavor_rsrc_free(state, &rsrc);
5769e39c5baSBill Taylor 
5779e39c5baSBill Taylor 	/* Free the SRQC entry resource */
5789e39c5baSBill Taylor 	tavor_rsrc_free(state, &srqc);
5799e39c5baSBill Taylor 
5809e39c5baSBill Taylor 	/* Decrement the reference count on the protection domain (PD) */
5819e39c5baSBill Taylor 	tavor_pd_refcnt_dec(pd);
5829e39c5baSBill Taylor 
5839e39c5baSBill Taylor 	/* Set the srqhdl pointer to NULL and return success */
5849e39c5baSBill Taylor 	*srqhdl = NULL;
5859e39c5baSBill Taylor 
5869e39c5baSBill Taylor 	return (DDI_SUCCESS);
5879e39c5baSBill Taylor }
5889e39c5baSBill Taylor 
5899e39c5baSBill Taylor 
5909e39c5baSBill Taylor /*
5919e39c5baSBill Taylor  * tavor_srq_modify()
5929e39c5baSBill Taylor  *    Context: Can be called only from user or kernel context.
5939e39c5baSBill Taylor  */
5949e39c5baSBill Taylor int
tavor_srq_modify(tavor_state_t * state,tavor_srqhdl_t srq,uint_t size,uint_t * real_size,uint_t sleepflag)5959e39c5baSBill Taylor tavor_srq_modify(tavor_state_t *state, tavor_srqhdl_t srq, uint_t size,
5969e39c5baSBill Taylor     uint_t *real_size, uint_t sleepflag)
5979e39c5baSBill Taylor {
5989e39c5baSBill Taylor 	tavor_qalloc_info_t	new_srqinfo, old_srqinfo;
5999e39c5baSBill Taylor 	tavor_rsrc_t		*mtt, *mpt, *old_mtt;
6009e39c5baSBill Taylor 	tavor_bind_info_t	bind;
6019e39c5baSBill Taylor 	tavor_bind_info_t	old_bind;
6029e39c5baSBill Taylor 	tavor_rsrc_pool_info_t	*rsrc_pool;
6039e39c5baSBill Taylor 	tavor_mrhdl_t		mr;
6049e39c5baSBill Taylor 	tavor_hw_mpt_t		mpt_entry;
6059e39c5baSBill Taylor 	tavor_wrid_entry_t	*wre_new, *wre_old;
6069e39c5baSBill Taylor 	uint64_t		mtt_ddrbaseaddr, mtt_addr;
6079e39c5baSBill Taylor 	uint64_t		srq_desc_off;
6089e39c5baSBill Taylor 	uint32_t		*buf, srq_old_bufsz;
6099e39c5baSBill Taylor 	uint32_t		wqesz;
6109e39c5baSBill Taylor 	uint_t			max_srq_size;
6119e39c5baSBill Taylor 	uint_t			dma_xfer_mode, mtt_pgsize_bits;
6129e39c5baSBill Taylor 	uint_t			srq_sync, log_srq_size, maxprot;
6139e39c5baSBill Taylor 	uint_t			wq_location;
6149e39c5baSBill Taylor 	int			status;
6159e39c5baSBill Taylor 
6169e39c5baSBill Taylor 	/*
6179e39c5baSBill Taylor 	 * Check the "inddr" flag.  This flag tells the driver whether or not
6189e39c5baSBill Taylor 	 * the SRQ's work queues should be come from normal system memory or
6199e39c5baSBill Taylor 	 * whether they should be allocated from DDR memory.
6209e39c5baSBill Taylor 	 */
6219e39c5baSBill Taylor 	wq_location = state->ts_cfg_profile->cp_srq_wq_inddr;
6229e39c5baSBill Taylor 
6239e39c5baSBill Taylor 	/*
6249e39c5baSBill Taylor 	 * If size requested is larger than device capability, return
6259e39c5baSBill Taylor 	 * Insufficient Resources
6269e39c5baSBill Taylor 	 */
6279e39c5baSBill Taylor 	max_srq_size = (1 << state->ts_cfg_profile->cp_log_max_srq_sz);
6289e39c5baSBill Taylor 	if (size > max_srq_size) {
6299e39c5baSBill Taylor 		return (IBT_HCA_WR_EXCEEDED);
6309e39c5baSBill Taylor 	}
6319e39c5baSBill Taylor 
6329e39c5baSBill Taylor 	/*
6339e39c5baSBill Taylor 	 * Calculate the appropriate size for the SRQ.
6349e39c5baSBill Taylor 	 * Note:  All Tavor SRQs must be a power-of-2 in size.  Also
6359e39c5baSBill Taylor 	 * they may not be any smaller than TAVOR_SRQ_MIN_SIZE.  This step
6369e39c5baSBill Taylor 	 * is to round the requested size up to the next highest power-of-2
6379e39c5baSBill Taylor 	 */
6389e39c5baSBill Taylor 	size = max(size, TAVOR_SRQ_MIN_SIZE);
6399e39c5baSBill Taylor 	log_srq_size = highbit(size);
640*de710d24SJosef 'Jeff' Sipek 	if (ISP2(size)) {
6419e39c5baSBill Taylor 		log_srq_size = log_srq_size - 1;
6429e39c5baSBill Taylor 	}
6439e39c5baSBill Taylor 
6449e39c5baSBill Taylor 	/*
6459e39c5baSBill Taylor 	 * Next we verify that the rounded-up size is valid (i.e. consistent
6469e39c5baSBill Taylor 	 * with the device limits and/or software-configured limits).
6479e39c5baSBill Taylor 	 */
6489e39c5baSBill Taylor 	if (log_srq_size > state->ts_cfg_profile->cp_log_max_srq_sz) {
6499e39c5baSBill Taylor 		goto srqmodify_fail;
6509e39c5baSBill Taylor 	}
6519e39c5baSBill Taylor 
6529e39c5baSBill Taylor 	/*
6539e39c5baSBill Taylor 	 * Allocate the memory for newly resized Shared Receive Queue.
6549e39c5baSBill Taylor 	 *
6559e39c5baSBill Taylor 	 * Note: If SRQ is not user-mappable, then it may come from either
6569e39c5baSBill Taylor 	 * kernel system memory or from HCA-attached local DDR memory.
6579e39c5baSBill Taylor 	 *
6589e39c5baSBill Taylor 	 * Note2: We align this queue on a pagesize boundary.  This is required
6599e39c5baSBill Taylor 	 * to make sure that all the resulting IB addresses will start at 0,
6609e39c5baSBill Taylor 	 * for a zero-based queue.  By making sure we are aligned on at least a
6619e39c5baSBill Taylor 	 * page, any offset we use into our queue will be the same as it was
6629e39c5baSBill Taylor 	 * when we allocated it at tavor_srq_alloc() time.
6639e39c5baSBill Taylor 	 */
6649e39c5baSBill Taylor 	wqesz = (1 << srq->srq_wq_log_wqesz);
6659e39c5baSBill Taylor 	new_srqinfo.qa_size = (1 << log_srq_size) * wqesz;
6669e39c5baSBill Taylor 	new_srqinfo.qa_alloc_align = PAGESIZE;
6679e39c5baSBill Taylor 	new_srqinfo.qa_bind_align  = PAGESIZE;
6689e39c5baSBill Taylor 	if (srq->srq_is_umap) {
6699e39c5baSBill Taylor 		new_srqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
6709e39c5baSBill Taylor 	} else {
6719e39c5baSBill Taylor 		new_srqinfo.qa_location = wq_location;
6729e39c5baSBill Taylor 	}
6739e39c5baSBill Taylor 	status = tavor_queue_alloc(state, &new_srqinfo, sleepflag);
6749e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
6759e39c5baSBill Taylor 		goto srqmodify_fail;
6769e39c5baSBill Taylor 	}
6779e39c5baSBill Taylor 	buf = (uint32_t *)new_srqinfo.qa_buf_aligned;
6789e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
6799e39c5baSBill Taylor 
6809e39c5baSBill Taylor 	/*
6819e39c5baSBill Taylor 	 * Allocate the memory for the new WRE list.  This will be used later
6829e39c5baSBill Taylor 	 * when we resize the wridlist based on the new SRQ size.
6839e39c5baSBill Taylor 	 */
6849e39c5baSBill Taylor 	wre_new = (tavor_wrid_entry_t *)kmem_zalloc((1 << log_srq_size) *
6859e39c5baSBill Taylor 	    sizeof (tavor_wrid_entry_t), sleepflag);
6869e39c5baSBill Taylor 	if (wre_new == NULL) {
6879e39c5baSBill Taylor 		goto srqmodify_fail;
6889e39c5baSBill Taylor 	}
6899e39c5baSBill Taylor 
6909e39c5baSBill Taylor 	/*
6919e39c5baSBill Taylor 	 * Fill in the "bind" struct.  This struct provides the majority
6929e39c5baSBill Taylor 	 * of the information that will be used to distinguish between an
6939e39c5baSBill Taylor 	 * "addr" binding (as is the case here) and a "buf" binding (see
6949e39c5baSBill Taylor 	 * below).  The "bind" struct is later passed to tavor_mr_mem_bind()
6959e39c5baSBill Taylor 	 * which does most of the "heavy lifting" for the Tavor memory
6969e39c5baSBill Taylor 	 * registration routines.
6979e39c5baSBill Taylor 	 */
6989e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(bind))
6999e39c5baSBill Taylor 	bzero(&bind, sizeof (tavor_bind_info_t));
7009e39c5baSBill Taylor 	bind.bi_type  = TAVOR_BINDHDL_VADDR;
7019e39c5baSBill Taylor 	bind.bi_addr  = (uint64_t)(uintptr_t)buf;
7029e39c5baSBill Taylor 	bind.bi_len   = new_srqinfo.qa_size;
7039e39c5baSBill Taylor 	bind.bi_as    = NULL;
7049e39c5baSBill Taylor 	bind.bi_flags = sleepflag == TAVOR_SLEEP ? IBT_MR_SLEEP :
7059e39c5baSBill Taylor 	    IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
7069e39c5baSBill Taylor 	if (srq->srq_is_umap) {
7079e39c5baSBill Taylor 		bind.bi_bypass = state->ts_cfg_profile->cp_iommu_bypass;
7089e39c5baSBill Taylor 	} else {
7099e39c5baSBill Taylor 		if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
7109e39c5baSBill Taylor 			bind.bi_bypass =
7119e39c5baSBill Taylor 			    state->ts_cfg_profile->cp_iommu_bypass;
7129e39c5baSBill Taylor 			dma_xfer_mode =
7139e39c5baSBill Taylor 			    state->ts_cfg_profile->cp_streaming_consistent;
7149e39c5baSBill Taylor 			if (dma_xfer_mode == DDI_DMA_STREAMING) {
7159e39c5baSBill Taylor 				bind.bi_flags |= IBT_MR_NONCOHERENT;
7169e39c5baSBill Taylor 			}
7179e39c5baSBill Taylor 		} else {
7189e39c5baSBill Taylor 			bind.bi_bypass = TAVOR_BINDMEM_BYPASS;
7199e39c5baSBill Taylor 		}
7209e39c5baSBill Taylor 	}
7219e39c5baSBill Taylor 	status = tavor_mr_mtt_bind(state, &bind, new_srqinfo.qa_dmahdl, &mtt,
7229e39c5baSBill Taylor 	    &mtt_pgsize_bits);
7239e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
7249e39c5baSBill Taylor 		kmem_free(wre_new, srq->srq_wq_bufsz *
7259e39c5baSBill Taylor 		    sizeof (tavor_wrid_entry_t));
7269e39c5baSBill Taylor 		tavor_queue_free(state, &new_srqinfo);
7279e39c5baSBill Taylor 		goto srqmodify_fail;
7289e39c5baSBill Taylor 	}
7299e39c5baSBill Taylor 
7309e39c5baSBill Taylor 	/*
7319e39c5baSBill Taylor 	 * Calculate the offset between the kernel virtual address space
7329e39c5baSBill Taylor 	 * and the IB virtual address space.  This will be used when
7339e39c5baSBill Taylor 	 * posting work requests to properly initialize each WQE.
7349e39c5baSBill Taylor 	 *
7359e39c5baSBill Taylor 	 * Note: bind addr is zero-based (from alloc) so we calculate the
7369e39c5baSBill Taylor 	 * correct new offset here.
7379e39c5baSBill Taylor 	 */
7389e39c5baSBill Taylor 	bind.bi_addr = bind.bi_addr & ((1 << mtt_pgsize_bits) - 1);
7399e39c5baSBill Taylor 	srq_desc_off = (uint64_t)(uintptr_t)new_srqinfo.qa_buf_aligned -
7409e39c5baSBill Taylor 	    (uint64_t)bind.bi_addr;
7419e39c5baSBill Taylor 
7429e39c5baSBill Taylor 	/*
7439e39c5baSBill Taylor 	 * Get the base address for the MTT table.  This will be necessary
7449e39c5baSBill Taylor 	 * below when we are modifying the MPT entry.
7459e39c5baSBill Taylor 	 */
7469e39c5baSBill Taylor 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
7479e39c5baSBill Taylor 	mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
7489e39c5baSBill Taylor 
7499e39c5baSBill Taylor 	/*
7509e39c5baSBill Taylor 	 * Fill in the MPT entry.  This is the final step before passing
7519e39c5baSBill Taylor 	 * ownership of the MPT entry to the Tavor hardware.  We use all of
7529e39c5baSBill Taylor 	 * the information collected/calculated above to fill in the
7539e39c5baSBill Taylor 	 * requisite portions of the MPT.
7549e39c5baSBill Taylor 	 */
7559e39c5baSBill Taylor 	bzero(&mpt_entry, sizeof (tavor_hw_mpt_t));
7569e39c5baSBill Taylor 	mpt_entry.reg_win_len	= bind.bi_len;
7579e39c5baSBill Taylor 	mtt_addr = mtt_ddrbaseaddr + (mtt->tr_indx << TAVOR_MTT_SIZE_SHIFT);
7589e39c5baSBill Taylor 	mpt_entry.mttseg_addr_h = mtt_addr >> 32;
7599e39c5baSBill Taylor 	mpt_entry.mttseg_addr_l = mtt_addr >> 6;
7609e39c5baSBill Taylor 
7619e39c5baSBill Taylor 	/*
7629e39c5baSBill Taylor 	 * Now we grab the SRQ lock.  Since we will be updating the actual
7639e39c5baSBill Taylor 	 * SRQ location and the producer/consumer indexes, we should hold
7649e39c5baSBill Taylor 	 * the lock.
7659e39c5baSBill Taylor 	 *
7669e39c5baSBill Taylor 	 * We do a TAVOR_NOSLEEP here (and below), though, because we are
7679e39c5baSBill Taylor 	 * holding the "srq_lock" and if we got raised to interrupt level
7689e39c5baSBill Taylor 	 * by priority inversion, we would not want to block in this routine
7699e39c5baSBill Taylor 	 * waiting for success.
7709e39c5baSBill Taylor 	 */
7719e39c5baSBill Taylor 	mutex_enter(&srq->srq_lock);
7729e39c5baSBill Taylor 
7739e39c5baSBill Taylor 	/*
7749e39c5baSBill Taylor 	 * Copy old entries to new buffer
7759e39c5baSBill Taylor 	 */
7769e39c5baSBill Taylor 	srq_old_bufsz = srq->srq_wq_bufsz;
7779e39c5baSBill Taylor 	bcopy(srq->srq_wq_buf, buf, srq_old_bufsz * wqesz);
7789e39c5baSBill Taylor 
7799e39c5baSBill Taylor 	/* Determine if later ddi_dma_sync will be necessary */
7809e39c5baSBill Taylor 	srq_sync = TAVOR_SRQ_IS_SYNC_REQ(state, srq->srq_wqinfo);
7819e39c5baSBill Taylor 
7829e39c5baSBill Taylor 	/* Sync entire "new" SRQ for use by hardware (if necessary) */
7839e39c5baSBill Taylor 	if (srq_sync) {
7849e39c5baSBill Taylor 		(void) ddi_dma_sync(bind.bi_dmahdl, 0,
7859e39c5baSBill Taylor 		    new_srqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
7869e39c5baSBill Taylor 	}
7879e39c5baSBill Taylor 
7889e39c5baSBill Taylor 	/*
7899e39c5baSBill Taylor 	 * Setup MPT information for use in the MODIFY_MPT command
7909e39c5baSBill Taylor 	 */
7919e39c5baSBill Taylor 	mr = srq->srq_mrhdl;
7929e39c5baSBill Taylor 	mutex_enter(&mr->mr_lock);
7939e39c5baSBill Taylor 	mpt = srq->srq_mrhdl->mr_mptrsrcp;
7949e39c5baSBill Taylor 
7959e39c5baSBill Taylor 	/*
7969e39c5baSBill Taylor 	 * MODIFY_MPT
7979e39c5baSBill Taylor 	 *
7989e39c5baSBill Taylor 	 * If this fails for any reason, then it is an indication that
7999e39c5baSBill Taylor 	 * something (either in HW or SW) has gone seriously wrong.  So we
8009e39c5baSBill Taylor 	 * print a warning message and return.
8019e39c5baSBill Taylor 	 */
8029e39c5baSBill Taylor 	status = tavor_modify_mpt_cmd_post(state, &mpt_entry, mpt->tr_indx,
8039e39c5baSBill Taylor 	    TAVOR_CMD_MODIFY_MPT_RESIZESRQ, sleepflag);
8049e39c5baSBill Taylor 	if (status != TAVOR_CMD_SUCCESS) {
8059e39c5baSBill Taylor 		cmn_err(CE_CONT, "Tavor: MODIFY_MPT command failed: %08x\n",
8069e39c5baSBill Taylor 		    status);
8079e39c5baSBill Taylor 		(void) tavor_mr_mtt_unbind(state, &srq->srq_mrhdl->mr_bindinfo,
8089e39c5baSBill Taylor 		    srq->srq_mrhdl->mr_mttrsrcp);
8099e39c5baSBill Taylor 		kmem_free(wre_new, srq->srq_wq_bufsz *
8109e39c5baSBill Taylor 		    sizeof (tavor_wrid_entry_t));
8119e39c5baSBill Taylor 		tavor_queue_free(state, &new_srqinfo);
8129e39c5baSBill Taylor 		mutex_exit(&mr->mr_lock);
8139e39c5baSBill Taylor 		mutex_exit(&srq->srq_lock);
8149e39c5baSBill Taylor 		return (ibc_get_ci_failure(0));
8159e39c5baSBill Taylor 	}
8169e39c5baSBill Taylor 
8179e39c5baSBill Taylor 	/*
8189e39c5baSBill Taylor 	 * Update the Tavor Shared Receive Queue handle with all the new
8199e39c5baSBill Taylor 	 * information.  At the same time, save away all the necessary
8209e39c5baSBill Taylor 	 * information for freeing up the old resources
8219e39c5baSBill Taylor 	 */
8229e39c5baSBill Taylor 	old_srqinfo	   = srq->srq_wqinfo;
8239e39c5baSBill Taylor 	old_mtt		   = srq->srq_mrhdl->mr_mttrsrcp;
8249e39c5baSBill Taylor 	bcopy(&srq->srq_mrhdl->mr_bindinfo, &old_bind,
8259e39c5baSBill Taylor 	    sizeof (tavor_bind_info_t));
8269e39c5baSBill Taylor 
8279e39c5baSBill Taylor 	/* Now set the new info */
8289e39c5baSBill Taylor 	srq->srq_wqinfo	   = new_srqinfo;
8299e39c5baSBill Taylor 	srq->srq_wq_buf	   = buf;
8309e39c5baSBill Taylor 	srq->srq_wq_bufsz  = (1 << log_srq_size);
8319e39c5baSBill Taylor 	bcopy(&bind, &srq->srq_mrhdl->mr_bindinfo, sizeof (tavor_bind_info_t));
8329e39c5baSBill Taylor 	srq->srq_mrhdl->mr_mttrsrcp = mtt;
8339e39c5baSBill Taylor 	srq->srq_desc_off  = srq_desc_off;
8349e39c5baSBill Taylor 	srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
8359e39c5baSBill Taylor 
8369e39c5baSBill Taylor 	/* Update MR mtt pagesize */
8379e39c5baSBill Taylor 	mr->mr_logmttpgsz = mtt_pgsize_bits;
8389e39c5baSBill Taylor 	mutex_exit(&mr->mr_lock);
8399e39c5baSBill Taylor 
8409e39c5baSBill Taylor #ifdef __lock_lint
8419e39c5baSBill Taylor 	mutex_enter(&srq->srq_wrid_wql->wql_lock);
8429e39c5baSBill Taylor #else
8439e39c5baSBill Taylor 	if (srq->srq_wrid_wql != NULL) {
8449e39c5baSBill Taylor 		mutex_enter(&srq->srq_wrid_wql->wql_lock);
8459e39c5baSBill Taylor 	}
8469e39c5baSBill Taylor #endif
8479e39c5baSBill Taylor 
8489e39c5baSBill Taylor 	/*
8499e39c5baSBill Taylor 	 * Initialize new wridlist, if needed.
8509e39c5baSBill Taylor 	 *
8519e39c5baSBill Taylor 	 * If a wridlist already is setup on an SRQ (the QP associated with an
8529e39c5baSBill Taylor 	 * SRQ has moved "from_reset") then we must update this wridlist based
8539e39c5baSBill Taylor 	 * on the new SRQ size.  We allocate the new size of Work Request ID
8549e39c5baSBill Taylor 	 * Entries, copy over the old entries to the new list, and
8559e39c5baSBill Taylor 	 * re-initialize the srq wridlist in non-umap case
8569e39c5baSBill Taylor 	 */
8579e39c5baSBill Taylor 	wre_old = NULL;
8589e39c5baSBill Taylor 	if (srq->srq_wridlist != NULL) {
8599e39c5baSBill Taylor 		wre_old = srq->srq_wridlist->wl_wre;
8609e39c5baSBill Taylor 
8619e39c5baSBill Taylor 		bcopy(wre_old, wre_new, srq_old_bufsz *
8629e39c5baSBill Taylor 		    sizeof (tavor_wrid_entry_t));
8639e39c5baSBill Taylor 
8649e39c5baSBill Taylor 		/* Setup new sizes in wre */
8659e39c5baSBill Taylor 		srq->srq_wridlist->wl_wre = wre_new;
8669e39c5baSBill Taylor 		srq->srq_wridlist->wl_size = srq->srq_wq_bufsz;
8679e39c5baSBill Taylor 
8689e39c5baSBill Taylor 		if (!srq->srq_is_umap) {
8699e39c5baSBill Taylor 			tavor_wrid_list_srq_init(srq->srq_wridlist, srq,
8709e39c5baSBill Taylor 			    srq_old_bufsz);
8719e39c5baSBill Taylor 		}
8729e39c5baSBill Taylor 	}
8739e39c5baSBill Taylor 
8749e39c5baSBill Taylor #ifdef __lock_lint
8759e39c5baSBill Taylor 	mutex_exit(&srq->srq_wrid_wql->wql_lock);
8769e39c5baSBill Taylor #else
8779e39c5baSBill Taylor 	if (srq->srq_wrid_wql != NULL) {
8789e39c5baSBill Taylor 		mutex_exit(&srq->srq_wrid_wql->wql_lock);
8799e39c5baSBill Taylor 	}
8809e39c5baSBill Taylor #endif
8819e39c5baSBill Taylor 
8829e39c5baSBill Taylor 	/*
8839e39c5baSBill Taylor 	 * If "old" SRQ was a user-mappable SRQ that is currently mmap()'d out
8849e39c5baSBill Taylor 	 * to a user process, then we need to call devmap_devmem_remap() to
8859e39c5baSBill Taylor 	 * invalidate the mapping to the SRQ memory.  We also need to
8869e39c5baSBill Taylor 	 * invalidate the SRQ tracking information for the user mapping.
8879e39c5baSBill Taylor 	 *
8889e39c5baSBill Taylor 	 * Note: On failure, the remap really shouldn't ever happen.  So, if it
8899e39c5baSBill Taylor 	 * does, it is an indication that something has gone seriously wrong.
8909e39c5baSBill Taylor 	 * So we print a warning message and return error (knowing, of course,
8919e39c5baSBill Taylor 	 * that the "old" SRQ memory will be leaked)
8929e39c5baSBill Taylor 	 */
8939e39c5baSBill Taylor 	if ((srq->srq_is_umap) && (srq->srq_umap_dhp != NULL)) {
8949e39c5baSBill Taylor 		maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
8959e39c5baSBill Taylor 		status = devmap_devmem_remap(srq->srq_umap_dhp,
8969e39c5baSBill Taylor 		    state->ts_dip, 0, 0, srq->srq_wqinfo.qa_size, maxprot,
8979e39c5baSBill Taylor 		    DEVMAP_MAPPING_INVALID, NULL);
8989e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
8999e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
9009e39c5baSBill Taylor 			TAVOR_WARNING(state, "failed in SRQ memory "
9019e39c5baSBill Taylor 			    "devmap_devmem_remap()");
9029e39c5baSBill Taylor 			/* We can, however, free the memory for old wre */
9039e39c5baSBill Taylor 			if (wre_old != NULL) {
9049e39c5baSBill Taylor 				kmem_free(wre_old, srq_old_bufsz *
9059e39c5baSBill Taylor 				    sizeof (tavor_wrid_entry_t));
9069e39c5baSBill Taylor 			}
9079e39c5baSBill Taylor 			return (ibc_get_ci_failure(0));
9089e39c5baSBill Taylor 		}
9099e39c5baSBill Taylor 		srq->srq_umap_dhp = (devmap_cookie_t)NULL;
9109e39c5baSBill Taylor 	}
9119e39c5baSBill Taylor 
9129e39c5baSBill Taylor 	/*
9139e39c5baSBill Taylor 	 * Drop the SRQ lock now.  The only thing left to do is to free up
9149e39c5baSBill Taylor 	 * the old resources.
9159e39c5baSBill Taylor 	 */
9169e39c5baSBill Taylor 	mutex_exit(&srq->srq_lock);
9179e39c5baSBill Taylor 
9189e39c5baSBill Taylor 	/*
9199e39c5baSBill Taylor 	 * Unbind the MTT entries.
9209e39c5baSBill Taylor 	 */
9219e39c5baSBill Taylor 	status = tavor_mr_mtt_unbind(state, &old_bind, old_mtt);
9229e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
9239e39c5baSBill Taylor 		TAVOR_WARNING(state, "failed to unbind old SRQ memory");
9249e39c5baSBill Taylor 		goto srqmodify_fail;
9259e39c5baSBill Taylor 	}
9269e39c5baSBill Taylor 
9279e39c5baSBill Taylor 	/* Free the memory for old wre */
9289e39c5baSBill Taylor 	if (wre_old != NULL) {
9299e39c5baSBill Taylor 		kmem_free(wre_old, srq_old_bufsz *
9309e39c5baSBill Taylor 		    sizeof (tavor_wrid_entry_t));
9319e39c5baSBill Taylor 	}
9329e39c5baSBill Taylor 
9339e39c5baSBill Taylor 	/* Free the memory for the old SRQ */
9349e39c5baSBill Taylor 	tavor_queue_free(state, &old_srqinfo);
9359e39c5baSBill Taylor 
9369e39c5baSBill Taylor 	/*
9379e39c5baSBill Taylor 	 * Fill in the return arguments (if necessary).  This includes the
9389e39c5baSBill Taylor 	 * real new completion queue size.
9399e39c5baSBill Taylor 	 */
9409e39c5baSBill Taylor 	if (real_size != NULL) {
9419e39c5baSBill Taylor 		*real_size = (1 << log_srq_size);
9429e39c5baSBill Taylor 	}
9439e39c5baSBill Taylor 
9449e39c5baSBill Taylor 	return (DDI_SUCCESS);
9459e39c5baSBill Taylor 
9469e39c5baSBill Taylor srqmodify_fail:
9479e39c5baSBill Taylor 	return (status);
9489e39c5baSBill Taylor }
9499e39c5baSBill Taylor 
9509e39c5baSBill Taylor 
9519e39c5baSBill Taylor /*
9529e39c5baSBill Taylor  * tavor_srq_refcnt_inc()
9539e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
9549e39c5baSBill Taylor  */
9559e39c5baSBill Taylor void
tavor_srq_refcnt_inc(tavor_srqhdl_t srq)9569e39c5baSBill Taylor tavor_srq_refcnt_inc(tavor_srqhdl_t srq)
9579e39c5baSBill Taylor {
9589e39c5baSBill Taylor 	mutex_enter(&srq->srq_lock);
9599e39c5baSBill Taylor 	srq->srq_refcnt++;
9609e39c5baSBill Taylor 	mutex_exit(&srq->srq_lock);
9619e39c5baSBill Taylor }
9629e39c5baSBill Taylor 
9639e39c5baSBill Taylor 
9649e39c5baSBill Taylor /*
9659e39c5baSBill Taylor  * tavor_srq_refcnt_dec()
9669e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
9679e39c5baSBill Taylor  */
9689e39c5baSBill Taylor void
tavor_srq_refcnt_dec(tavor_srqhdl_t srq)9699e39c5baSBill Taylor tavor_srq_refcnt_dec(tavor_srqhdl_t srq)
9709e39c5baSBill Taylor {
9719e39c5baSBill Taylor 	mutex_enter(&srq->srq_lock);
9729e39c5baSBill Taylor 	srq->srq_refcnt--;
9739e39c5baSBill Taylor 	mutex_exit(&srq->srq_lock);
9749e39c5baSBill Taylor }
9759e39c5baSBill Taylor 
9769e39c5baSBill Taylor 
9779e39c5baSBill Taylor /*
9789e39c5baSBill Taylor  * tavor_srqhdl_from_srqnum()
9799e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
9809e39c5baSBill Taylor  *
9819e39c5baSBill Taylor  *    This routine is important because changing the unconstrained
9829e39c5baSBill Taylor  *    portion of the SRQ number is critical to the detection of a
9839e39c5baSBill Taylor  *    potential race condition in the SRQ handler code (i.e. the case
9849e39c5baSBill Taylor  *    where a SRQ is freed and alloc'd again before an event for the
9859e39c5baSBill Taylor  *    "old" SRQ can be handled).
9869e39c5baSBill Taylor  *
9879e39c5baSBill Taylor  *    While this is not a perfect solution (not sure that one exists)
9889e39c5baSBill Taylor  *    it does help to mitigate the chance that this race condition will
9899e39c5baSBill Taylor  *    cause us to deliver a "stale" event to the new SRQ owner.  Note:
9909e39c5baSBill Taylor  *    this solution does not scale well because the number of constrained
9919e39c5baSBill Taylor  *    bits increases (and, hence, the number of unconstrained bits
9929e39c5baSBill Taylor  *    decreases) as the number of supported SRQ grows.  For small and
9939e39c5baSBill Taylor  *    intermediate values, it should hopefully provide sufficient
9949e39c5baSBill Taylor  *    protection.
9959e39c5baSBill Taylor  */
9969e39c5baSBill Taylor tavor_srqhdl_t
tavor_srqhdl_from_srqnum(tavor_state_t * state,uint_t srqnum)9979e39c5baSBill Taylor tavor_srqhdl_from_srqnum(tavor_state_t *state, uint_t srqnum)
9989e39c5baSBill Taylor {
9999e39c5baSBill Taylor 	uint_t	srqindx, srqmask;
10009e39c5baSBill Taylor 
10019e39c5baSBill Taylor 	/* Calculate the SRQ table index from the srqnum */
10029e39c5baSBill Taylor 	srqmask = (1 << state->ts_cfg_profile->cp_log_num_srq) - 1;
10039e39c5baSBill Taylor 	srqindx = srqnum & srqmask;
10049e39c5baSBill Taylor 	return (state->ts_srqhdl[srqindx]);
10059e39c5baSBill Taylor }
10069e39c5baSBill Taylor 
10079e39c5baSBill Taylor 
10089e39c5baSBill Taylor /*
10099e39c5baSBill Taylor  * tavor_srq_sgl_to_logwqesz()
10109e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
10119e39c5baSBill Taylor  */
10129e39c5baSBill Taylor static void
tavor_srq_sgl_to_logwqesz(tavor_state_t * state,uint_t num_sgl,tavor_qp_wq_type_t wq_type,uint_t * logwqesz,uint_t * max_sgl)10139e39c5baSBill Taylor tavor_srq_sgl_to_logwqesz(tavor_state_t *state, uint_t num_sgl,
10149e39c5baSBill Taylor     tavor_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl)
10159e39c5baSBill Taylor {
10169e39c5baSBill Taylor 	uint_t	max_size, log2, actual_sgl;
10179e39c5baSBill Taylor 
10189e39c5baSBill Taylor 	switch (wq_type) {
10199e39c5baSBill Taylor 	case TAVOR_QP_WQ_TYPE_RECVQ:
10209e39c5baSBill Taylor 		/*
10219e39c5baSBill Taylor 		 * Use requested maximum SGL to calculate max descriptor size
10229e39c5baSBill Taylor 		 * (while guaranteeing that the descriptor size is a
10239e39c5baSBill Taylor 		 * power-of-2 cachelines).
10249e39c5baSBill Taylor 		 */
10259e39c5baSBill Taylor 		max_size = (TAVOR_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4));
10269e39c5baSBill Taylor 		log2 = highbit(max_size);
1027*de710d24SJosef 'Jeff' Sipek 		if (ISP2(max_size)) {
10289e39c5baSBill Taylor 			log2 = log2 - 1;
10299e39c5baSBill Taylor 		}
10309e39c5baSBill Taylor 
10319e39c5baSBill Taylor 		/* Make sure descriptor is at least the minimum size */
10329e39c5baSBill Taylor 		log2 = max(log2, TAVOR_QP_WQE_LOG_MINIMUM);
10339e39c5baSBill Taylor 
10349e39c5baSBill Taylor 		/* Calculate actual number of SGL (given WQE size) */
10359e39c5baSBill Taylor 		actual_sgl = ((1 << log2) - TAVOR_QP_WQE_MLX_RCV_HDRS) >> 4;
10369e39c5baSBill Taylor 		break;
10379e39c5baSBill Taylor 
10389e39c5baSBill Taylor 	default:
10399e39c5baSBill Taylor 		TAVOR_WARNING(state, "unexpected work queue type");
10409e39c5baSBill Taylor 		break;
10419e39c5baSBill Taylor 	}
10429e39c5baSBill Taylor 
10439e39c5baSBill Taylor 	/* Fill in the return values */
10449e39c5baSBill Taylor 	*logwqesz = log2;
10459e39c5baSBill Taylor 	*max_sgl  = min(state->ts_cfg_profile->cp_srq_max_sgl, actual_sgl);
10469e39c5baSBill Taylor }
1047