19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
23*17a2b317SBill Taylor  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor  */
259e39c5baSBill Taylor 
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor  * hermon_mr.c
289e39c5baSBill Taylor  *    Hermon Memory Region/Window Routines
299e39c5baSBill Taylor  *
309e39c5baSBill Taylor  *    Implements all the routines necessary to provide the requisite memory
319e39c5baSBill Taylor  *    registration verbs.  These include operations like RegisterMemRegion(),
329e39c5baSBill Taylor  *    DeregisterMemRegion(), ReregisterMemRegion, RegisterSharedMemRegion,
339e39c5baSBill Taylor  *    etc., that affect Memory Regions.  It also includes the verbs that
349e39c5baSBill Taylor  *    affect Memory Windows, including AllocMemWindow(), FreeMemWindow(),
359e39c5baSBill Taylor  *    and QueryMemWindow().
369e39c5baSBill Taylor  */
379e39c5baSBill Taylor 
389e39c5baSBill Taylor #include <sys/types.h>
399e39c5baSBill Taylor #include <sys/conf.h>
409e39c5baSBill Taylor #include <sys/ddi.h>
419e39c5baSBill Taylor #include <sys/sunddi.h>
429e39c5baSBill Taylor #include <sys/modctl.h>
439e39c5baSBill Taylor #include <sys/esunddi.h>
449e39c5baSBill Taylor 
459e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
469e39c5baSBill Taylor 
479e39c5baSBill Taylor extern uint32_t hermon_kernel_data_ro;
489e39c5baSBill Taylor extern uint32_t hermon_user_data_ro;
49c7facc54SBill Taylor extern int hermon_rdma_debug;
509e39c5baSBill Taylor 
519e39c5baSBill Taylor /*
529e39c5baSBill Taylor  * Used by hermon_mr_keycalc() below to fill in the "unconstrained" portion
539e39c5baSBill Taylor  * of Hermon memory keys (LKeys and RKeys)
549e39c5baSBill Taylor  */
559e39c5baSBill Taylor static	uint_t hermon_memkey_cnt = 0x00;
56c7facc54SBill Taylor #define	HERMON_MEMKEY_SHIFT	24
57c7facc54SBill Taylor 
58c7facc54SBill Taylor /* initial state of an MPT */
59c7facc54SBill Taylor #define	HERMON_MPT_SW_OWNERSHIP	0xF	/* memory regions */
60c7facc54SBill Taylor #define	HERMON_MPT_FREE		0x3	/* allocate lkey */
619e39c5baSBill Taylor 
629e39c5baSBill Taylor static int hermon_mr_common_reg(hermon_state_t *state, hermon_pdhdl_t pd,
639e39c5baSBill Taylor     hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op,
649e39c5baSBill Taylor     hermon_mpt_rsrc_type_t mpt_type);
659e39c5baSBill Taylor static int hermon_mr_common_rereg(hermon_state_t *state, hermon_mrhdl_t mr,
669e39c5baSBill Taylor     hermon_pdhdl_t pd, hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl_new,
679e39c5baSBill Taylor     hermon_mr_options_t *op);
689e39c5baSBill Taylor static int hermon_mr_rereg_xlat_helper(hermon_state_t *state, hermon_mrhdl_t mr,
699e39c5baSBill Taylor     hermon_bind_info_t *bind, hermon_mr_options_t *op, uint64_t *mtt_addr,
709e39c5baSBill Taylor     uint_t sleep, uint_t *dereg_level);
719e39c5baSBill Taylor static uint64_t hermon_mr_nummtt_needed(hermon_state_t *state,
729e39c5baSBill Taylor     hermon_bind_info_t *bind, uint_t *mtt_pgsize);
739e39c5baSBill Taylor static int hermon_mr_mem_bind(hermon_state_t *state, hermon_bind_info_t *bind,
749e39c5baSBill Taylor     ddi_dma_handle_t dmahdl, uint_t sleep, uint_t is_buffer);
759e39c5baSBill Taylor static void hermon_mr_mem_unbind(hermon_state_t *state,
769e39c5baSBill Taylor     hermon_bind_info_t *bind);
779e39c5baSBill Taylor static int hermon_mr_fast_mtt_write(hermon_state_t *state, hermon_rsrc_t *mtt,
789e39c5baSBill Taylor     hermon_bind_info_t *bind, uint32_t mtt_pgsize_bits);
79c7facc54SBill Taylor static int hermon_mr_fast_mtt_write_fmr(hermon_state_t *state,
80c7facc54SBill Taylor     hermon_rsrc_t *mtt, ibt_pmr_attr_t *mem_pattr, uint32_t mtt_pgsize_bits);
819e39c5baSBill Taylor static uint_t hermon_mtt_refcnt_inc(hermon_rsrc_t *rsrc);
829e39c5baSBill Taylor static uint_t hermon_mtt_refcnt_dec(hermon_rsrc_t *rsrc);
839e39c5baSBill Taylor 
849e39c5baSBill Taylor 
859e39c5baSBill Taylor /*
869e39c5baSBill Taylor  * The Hermon umem_lockmemory() callback ops.  When userland memory is
879e39c5baSBill Taylor  * registered, these callback ops are specified.  The hermon_umap_umemlock_cb()
889e39c5baSBill Taylor  * callback will be called whenever the memory for the corresponding
899e39c5baSBill Taylor  * ddi_umem_cookie_t is being freed.
909e39c5baSBill Taylor  */
919e39c5baSBill Taylor static struct umem_callback_ops hermon_umem_cbops = {
929e39c5baSBill Taylor 	UMEM_CALLBACK_VERSION,
939e39c5baSBill Taylor 	hermon_umap_umemlock_cb,
949e39c5baSBill Taylor };
959e39c5baSBill Taylor 
969e39c5baSBill Taylor 
979e39c5baSBill Taylor 
989e39c5baSBill Taylor /*
999e39c5baSBill Taylor  * hermon_mr_register()
1009e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
1019e39c5baSBill Taylor  */
1029e39c5baSBill Taylor int
hermon_mr_register(hermon_state_t * state,hermon_pdhdl_t pd,ibt_mr_attr_t * mr_attr,hermon_mrhdl_t * mrhdl,hermon_mr_options_t * op,hermon_mpt_rsrc_type_t mpt_type)1039e39c5baSBill Taylor hermon_mr_register(hermon_state_t *state, hermon_pdhdl_t pd,
1049e39c5baSBill Taylor     ibt_mr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op,
1059e39c5baSBill Taylor     hermon_mpt_rsrc_type_t mpt_type)
1069e39c5baSBill Taylor {
1079e39c5baSBill Taylor 	hermon_bind_info_t	bind;
1089e39c5baSBill Taylor 	int			status;
1099e39c5baSBill Taylor 
1109e39c5baSBill Taylor 	/*
1119e39c5baSBill Taylor 	 * Fill in the "bind" struct.  This struct provides the majority
1129e39c5baSBill Taylor 	 * of the information that will be used to distinguish between an
1139e39c5baSBill Taylor 	 * "addr" binding (as is the case here) and a "buf" binding (see
1149e39c5baSBill Taylor 	 * below).  The "bind" struct is later passed to hermon_mr_mem_bind()
1159e39c5baSBill Taylor 	 * which does most of the "heavy lifting" for the Hermon memory
1169e39c5baSBill Taylor 	 * registration routines.
1179e39c5baSBill Taylor 	 */
1189e39c5baSBill Taylor 	bind.bi_type  = HERMON_BINDHDL_VADDR;
1199e39c5baSBill Taylor 	bind.bi_addr  = mr_attr->mr_vaddr;
1209e39c5baSBill Taylor 	bind.bi_len   = mr_attr->mr_len;
1219e39c5baSBill Taylor 	bind.bi_as    = mr_attr->mr_as;
1229e39c5baSBill Taylor 	bind.bi_flags = mr_attr->mr_flags;
1239e39c5baSBill Taylor 	status = hermon_mr_common_reg(state, pd, &bind, mrhdl, op,
1249e39c5baSBill Taylor 	    mpt_type);
1259e39c5baSBill Taylor 	return (status);
1269e39c5baSBill Taylor }
1279e39c5baSBill Taylor 
1289e39c5baSBill Taylor 
1299e39c5baSBill Taylor /*
1309e39c5baSBill Taylor  * hermon_mr_register_buf()
1319e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
1329e39c5baSBill Taylor  */
1339e39c5baSBill Taylor int
hermon_mr_register_buf(hermon_state_t * state,hermon_pdhdl_t pd,ibt_smr_attr_t * mr_attr,struct buf * buf,hermon_mrhdl_t * mrhdl,hermon_mr_options_t * op,hermon_mpt_rsrc_type_t mpt_type)1349e39c5baSBill Taylor hermon_mr_register_buf(hermon_state_t *state, hermon_pdhdl_t pd,
1359e39c5baSBill Taylor     ibt_smr_attr_t *mr_attr, struct buf *buf, hermon_mrhdl_t *mrhdl,
1369e39c5baSBill Taylor     hermon_mr_options_t *op, hermon_mpt_rsrc_type_t mpt_type)
1379e39c5baSBill Taylor {
1389e39c5baSBill Taylor 	hermon_bind_info_t	bind;
1399e39c5baSBill Taylor 	int			status;
1409e39c5baSBill Taylor 
1419e39c5baSBill Taylor 	/*
1429e39c5baSBill Taylor 	 * Fill in the "bind" struct.  This struct provides the majority
1439e39c5baSBill Taylor 	 * of the information that will be used to distinguish between an
1449e39c5baSBill Taylor 	 * "addr" binding (see above) and a "buf" binding (as is the case
1459e39c5baSBill Taylor 	 * here).  The "bind" struct is later passed to hermon_mr_mem_bind()
1469e39c5baSBill Taylor 	 * which does most of the "heavy lifting" for the Hermon memory
1479e39c5baSBill Taylor 	 * registration routines.  Note: We have chosen to provide
1489e39c5baSBill Taylor 	 * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is
1499e39c5baSBill Taylor 	 * not set).  It is not critical what value we choose here as it need
1509e39c5baSBill Taylor 	 * only be unique for the given RKey (which will happen by default),
1519e39c5baSBill Taylor 	 * so the choice here is somewhat arbitrary.
1529e39c5baSBill Taylor 	 */
1539e39c5baSBill Taylor 	bind.bi_type  = HERMON_BINDHDL_BUF;
1549e39c5baSBill Taylor 	bind.bi_buf   = buf;
1559e39c5baSBill Taylor 	if (mr_attr->mr_flags & IBT_MR_PHYS_IOVA) {
1569e39c5baSBill Taylor 		bind.bi_addr  = mr_attr->mr_vaddr;
1579e39c5baSBill Taylor 	} else {
1589e39c5baSBill Taylor 		bind.bi_addr  = (uint64_t)(uintptr_t)buf->b_un.b_addr;
1599e39c5baSBill Taylor 	}
1609e39c5baSBill Taylor 	bind.bi_as    = NULL;
1619e39c5baSBill Taylor 	bind.bi_len   = (uint64_t)buf->b_bcount;
1629e39c5baSBill Taylor 	bind.bi_flags = mr_attr->mr_flags;
1639e39c5baSBill Taylor 	status = hermon_mr_common_reg(state, pd, &bind, mrhdl, op, mpt_type);
1649e39c5baSBill Taylor 	return (status);
1659e39c5baSBill Taylor }
1669e39c5baSBill Taylor 
1679e39c5baSBill Taylor 
1689e39c5baSBill Taylor /*
1699e39c5baSBill Taylor  * hermon_mr_register_shared()
1709e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
1719e39c5baSBill Taylor  */
1729e39c5baSBill Taylor int
hermon_mr_register_shared(hermon_state_t * state,hermon_mrhdl_t mrhdl,hermon_pdhdl_t pd,ibt_smr_attr_t * mr_attr,hermon_mrhdl_t * mrhdl_new)1739e39c5baSBill Taylor hermon_mr_register_shared(hermon_state_t *state, hermon_mrhdl_t mrhdl,
1749e39c5baSBill Taylor     hermon_pdhdl_t pd, ibt_smr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl_new)
1759e39c5baSBill Taylor {
1769e39c5baSBill Taylor 	hermon_rsrc_t		*mpt, *mtt, *rsrc;
1779e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
1789e39c5baSBill Taylor 	hermon_hw_dmpt_t	mpt_entry;
1799e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
1809e39c5baSBill Taylor 	hermon_bind_info_t	*bind;
1819e39c5baSBill Taylor 	ddi_umem_cookie_t	umem_cookie;
1829e39c5baSBill Taylor 	size_t			umem_len;
1839e39c5baSBill Taylor 	caddr_t			umem_addr;
1849e39c5baSBill Taylor 	uint64_t		mtt_addr, pgsize_msk;
1859e39c5baSBill Taylor 	uint_t			sleep, mr_is_umem;
1869e39c5baSBill Taylor 	int			status, umem_flags;
1879e39c5baSBill Taylor 
1889e39c5baSBill Taylor 	/*
1899e39c5baSBill Taylor 	 * Check the sleep flag.  Ensure that it is consistent with the
1909e39c5baSBill Taylor 	 * current thread context (i.e. if we are currently in the interrupt
1919e39c5baSBill Taylor 	 * context, then we shouldn't be attempting to sleep).
1929e39c5baSBill Taylor 	 */
1939e39c5baSBill Taylor 	sleep = (mr_attr->mr_flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP :
1949e39c5baSBill Taylor 	    HERMON_SLEEP;
1959e39c5baSBill Taylor 	if ((sleep == HERMON_SLEEP) &&
1969e39c5baSBill Taylor 	    (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
1979e39c5baSBill Taylor 		status = IBT_INVALID_PARAM;
1989e39c5baSBill Taylor 		goto mrshared_fail;
1999e39c5baSBill Taylor 	}
2009e39c5baSBill Taylor 
2019e39c5baSBill Taylor 	/* Increment the reference count on the protection domain (PD) */
2029e39c5baSBill Taylor 	hermon_pd_refcnt_inc(pd);
2039e39c5baSBill Taylor 
2049e39c5baSBill Taylor 	/*
2059e39c5baSBill Taylor 	 * Allocate an MPT entry.  This will be filled in with all the
2069e39c5baSBill Taylor 	 * necessary parameters to define the shared memory region.
2079e39c5baSBill Taylor 	 * Specifically, it will be made to reference the currently existing
2089e39c5baSBill Taylor 	 * MTT entries and ownership of the MPT will be passed to the hardware
2099e39c5baSBill Taylor 	 * in the last step below.  If we fail here, we must undo the
2109e39c5baSBill Taylor 	 * protection domain reference count.
2119e39c5baSBill Taylor 	 */
2129e39c5baSBill Taylor 	status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
2139e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2149e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
2159e39c5baSBill Taylor 		goto mrshared_fail1;
2169e39c5baSBill Taylor 	}
2179e39c5baSBill Taylor 
2189e39c5baSBill Taylor 	/*
2199e39c5baSBill Taylor 	 * Allocate the software structure for tracking the shared memory
2209e39c5baSBill Taylor 	 * region (i.e. the Hermon Memory Region handle).  If we fail here, we
2219e39c5baSBill Taylor 	 * must undo the protection domain reference count and the previous
2229e39c5baSBill Taylor 	 * resource allocation.
2239e39c5baSBill Taylor 	 */
2249e39c5baSBill Taylor 	status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
2259e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2269e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
2279e39c5baSBill Taylor 		goto mrshared_fail2;
2289e39c5baSBill Taylor 	}
2299e39c5baSBill Taylor 	mr = (hermon_mrhdl_t)rsrc->hr_addr;
2309e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2319e39c5baSBill Taylor 
2329e39c5baSBill Taylor 	/*
2339e39c5baSBill Taylor 	 * Setup and validate the memory region access flags.  This means
2349e39c5baSBill Taylor 	 * translating the IBTF's enable flags into the access flags that
2359e39c5baSBill Taylor 	 * will be used in later operations.
2369e39c5baSBill Taylor 	 */
2379e39c5baSBill Taylor 	mr->mr_accflag = 0;
2389e39c5baSBill Taylor 	if (mr_attr->mr_flags & IBT_MR_ENABLE_WINDOW_BIND)
2399e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_WINDOW_BIND;
2409e39c5baSBill Taylor 	if (mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)
2419e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
2429e39c5baSBill Taylor 	if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)
2439e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_REMOTE_READ;
2449e39c5baSBill Taylor 	if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE)
2459e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
2469e39c5baSBill Taylor 	if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
2479e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
2489e39c5baSBill Taylor 
2499e39c5baSBill Taylor 	/*
2509e39c5baSBill Taylor 	 * Calculate keys (Lkey, Rkey) from MPT index.  Each key is formed
2519e39c5baSBill Taylor 	 * from a certain number of "constrained" bits (the least significant
2529e39c5baSBill Taylor 	 * bits) and some number of "unconstrained" bits.  The constrained
2539e39c5baSBill Taylor 	 * bits must be set to the index of the entry in the MPT table, but
2549e39c5baSBill Taylor 	 * the unconstrained bits can be set to any value we wish.  Note:
2559e39c5baSBill Taylor 	 * if no remote access is required, then the RKey value is not filled
2569e39c5baSBill Taylor 	 * in.  Otherwise both Rkey and LKey are given the same value.
2579e39c5baSBill Taylor 	 */
258c7facc54SBill Taylor 	mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx);
2599e39c5baSBill Taylor 
2609e39c5baSBill Taylor 	/* Grab the MR lock for the current memory region */
2619e39c5baSBill Taylor 	mutex_enter(&mrhdl->mr_lock);
2629e39c5baSBill Taylor 
2639e39c5baSBill Taylor 	/*
2649e39c5baSBill Taylor 	 * Check here to see if the memory region has already been partially
2659e39c5baSBill Taylor 	 * deregistered as a result of a hermon_umap_umemlock_cb() callback.
2669e39c5baSBill Taylor 	 * If so, this is an error, return failure.
2679e39c5baSBill Taylor 	 */
2689e39c5baSBill Taylor 	if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) {
2699e39c5baSBill Taylor 		mutex_exit(&mrhdl->mr_lock);
2709e39c5baSBill Taylor 		status = IBT_MR_HDL_INVALID;
2719e39c5baSBill Taylor 		goto mrshared_fail3;
2729e39c5baSBill Taylor 	}
2739e39c5baSBill Taylor 
2749e39c5baSBill Taylor 	/*
2759e39c5baSBill Taylor 	 * Determine if the original memory was from userland and, if so, pin
2769e39c5baSBill Taylor 	 * the pages (again) with umem_lockmemory().  This will guarantee a
2779e39c5baSBill Taylor 	 * separate callback for each of this shared region's MR handles.
2789e39c5baSBill Taylor 	 * If this is userland memory, then allocate an entry in the
2799e39c5baSBill Taylor 	 * "userland resources database".  This will later be added to
2809e39c5baSBill Taylor 	 * the database (after all further memory registration operations are
2819e39c5baSBill Taylor 	 * successful).  If we fail here, we must undo all the above setup.
2829e39c5baSBill Taylor 	 */
2839e39c5baSBill Taylor 	mr_is_umem = mrhdl->mr_is_umem;
2849e39c5baSBill Taylor 	if (mr_is_umem) {
2859e39c5baSBill Taylor 		umem_len   = ptob(btopr(mrhdl->mr_bindinfo.bi_len));
2869e39c5baSBill Taylor 		umem_addr  = (caddr_t)((uintptr_t)mrhdl->mr_bindinfo.bi_addr &
2879e39c5baSBill Taylor 		    ~PAGEOFFSET);
2889e39c5baSBill Taylor 		umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ |
2899e39c5baSBill Taylor 		    DDI_UMEMLOCK_LONGTERM);
2909e39c5baSBill Taylor 		status = umem_lockmemory(umem_addr, umem_len, umem_flags,
291d863b343SBill Taylor 		    &umem_cookie, &hermon_umem_cbops, NULL);
2929e39c5baSBill Taylor 		if (status != 0) {
2939e39c5baSBill Taylor 			mutex_exit(&mrhdl->mr_lock);
2949e39c5baSBill Taylor 			status = IBT_INSUFF_RESOURCE;
2959e39c5baSBill Taylor 			goto mrshared_fail3;
2969e39c5baSBill Taylor 		}
2979e39c5baSBill Taylor 
2989e39c5baSBill Taylor 		umapdb = hermon_umap_db_alloc(state->hs_instance,
2999e39c5baSBill Taylor 		    (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC,
3009e39c5baSBill Taylor 		    (uint64_t)(uintptr_t)rsrc);
3019e39c5baSBill Taylor 		if (umapdb == NULL) {
3029e39c5baSBill Taylor 			mutex_exit(&mrhdl->mr_lock);
3039e39c5baSBill Taylor 			status = IBT_INSUFF_RESOURCE;
3049e39c5baSBill Taylor 			goto mrshared_fail4;
3059e39c5baSBill Taylor 		}
3069e39c5baSBill Taylor 	}
3079e39c5baSBill Taylor 
3089e39c5baSBill Taylor 	/*
3099e39c5baSBill Taylor 	 * Copy the MTT resource pointer (and additional parameters) from
3109e39c5baSBill Taylor 	 * the original Hermon Memory Region handle.  Note: this is normally
3119e39c5baSBill Taylor 	 * where the hermon_mr_mem_bind() routine would be called, but because
3129e39c5baSBill Taylor 	 * we already have bound and filled-in MTT entries it is simply a
3139e39c5baSBill Taylor 	 * matter here of managing the MTT reference count and grabbing the
3149e39c5baSBill Taylor 	 * address of the MTT table entries (for filling in the shared region's
3159e39c5baSBill Taylor 	 * MPT entry).
3169e39c5baSBill Taylor 	 */
3179e39c5baSBill Taylor 	mr->mr_mttrsrcp	  = mrhdl->mr_mttrsrcp;
3189e39c5baSBill Taylor 	mr->mr_logmttpgsz = mrhdl->mr_logmttpgsz;
3199e39c5baSBill Taylor 	mr->mr_bindinfo	  = mrhdl->mr_bindinfo;
3209e39c5baSBill Taylor 	mr->mr_mttrefcntp = mrhdl->mr_mttrefcntp;
3219e39c5baSBill Taylor 	mutex_exit(&mrhdl->mr_lock);
3229e39c5baSBill Taylor 	bind = &mr->mr_bindinfo;
3239e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
3249e39c5baSBill Taylor 	mtt = mr->mr_mttrsrcp;
3259e39c5baSBill Taylor 
3269e39c5baSBill Taylor 	/*
3279e39c5baSBill Taylor 	 * Increment the MTT reference count (to reflect the fact that
3289e39c5baSBill Taylor 	 * the MTT is now shared)
3299e39c5baSBill Taylor 	 */
3309e39c5baSBill Taylor 	(void) hermon_mtt_refcnt_inc(mr->mr_mttrefcntp);
3319e39c5baSBill Taylor 
3329e39c5baSBill Taylor 	/*
3339e39c5baSBill Taylor 	 * Update the new "bind" virtual address.  Do some extra work here
3349e39c5baSBill Taylor 	 * to ensure proper alignment.  That is, make sure that the page
3359e39c5baSBill Taylor 	 * offset for the beginning of the old range is the same as the
3369e39c5baSBill Taylor 	 * offset for this new mapping
3379e39c5baSBill Taylor 	 */
3389e39c5baSBill Taylor 	pgsize_msk = (((uint64_t)1 << mr->mr_logmttpgsz) - 1);
3399e39c5baSBill Taylor 	bind->bi_addr = ((mr_attr->mr_vaddr & ~pgsize_msk) |
3409e39c5baSBill Taylor 	    (mr->mr_bindinfo.bi_addr & pgsize_msk));
3419e39c5baSBill Taylor 
3429e39c5baSBill Taylor 	/*
3439e39c5baSBill Taylor 	 * Fill in the MPT entry.  This is the final step before passing
3449e39c5baSBill Taylor 	 * ownership of the MPT entry to the Hermon hardware.  We use all of
3459e39c5baSBill Taylor 	 * the information collected/calculated above to fill in the
3469e39c5baSBill Taylor 	 * requisite portions of the MPT.
3479e39c5baSBill Taylor 	 */
3489e39c5baSBill Taylor 	bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
3499e39c5baSBill Taylor 	mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND)   ? 1 : 0;
3509e39c5baSBill Taylor 	mpt_entry.atomic  = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
3519e39c5baSBill Taylor 	mpt_entry.rw	  = (mr->mr_accflag & IBT_MR_REMOTE_WRITE)  ? 1 : 0;
3529e39c5baSBill Taylor 	mpt_entry.rr	  = (mr->mr_accflag & IBT_MR_REMOTE_READ)   ? 1 : 0;
3539e39c5baSBill Taylor 	mpt_entry.lw	  = (mr->mr_accflag & IBT_MR_LOCAL_WRITE)   ? 1 : 0;
3549e39c5baSBill Taylor 	mpt_entry.lr	  = 1;
3559e39c5baSBill Taylor 	mpt_entry.reg_win = HERMON_MPT_IS_REGION;
3569e39c5baSBill Taylor 	mpt_entry.entity_sz	= mr->mr_logmttpgsz;
3579e39c5baSBill Taylor 	mpt_entry.mem_key	= mr->mr_lkey;
3589e39c5baSBill Taylor 	mpt_entry.pd		= pd->pd_pdnum;
3599e39c5baSBill Taylor 	mpt_entry.start_addr	= bind->bi_addr;
3609e39c5baSBill Taylor 	mpt_entry.reg_win_len	= bind->bi_len;
3619e39c5baSBill Taylor 	mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
3629e39c5baSBill Taylor 	mpt_entry.mtt_addr_h = mtt_addr >> 32;
3639e39c5baSBill Taylor 	mpt_entry.mtt_addr_l = mtt_addr >> 3;
3649e39c5baSBill Taylor 
3659e39c5baSBill Taylor 	/*
3669e39c5baSBill Taylor 	 * Write the MPT entry to hardware.  Lastly, we pass ownership of
3679e39c5baSBill Taylor 	 * the entry to the hardware.  Note: in general, this operation
3689e39c5baSBill Taylor 	 * shouldn't fail.  But if it does, we have to undo everything we've
3699e39c5baSBill Taylor 	 * done above before returning error.
3709e39c5baSBill Taylor 	 */
3719e39c5baSBill Taylor 	status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
3729e39c5baSBill Taylor 	    sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
3739e39c5baSBill Taylor 	if (status != HERMON_CMD_SUCCESS) {
3749e39c5baSBill Taylor 		cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
3759e39c5baSBill Taylor 		    status);
3769e39c5baSBill Taylor 		if (status == HERMON_CMD_INVALID_STATUS) {
3779e39c5baSBill Taylor 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
3789e39c5baSBill Taylor 		}
3799e39c5baSBill Taylor 		status = ibc_get_ci_failure(0);
3809e39c5baSBill Taylor 		goto mrshared_fail5;
3819e39c5baSBill Taylor 	}
3829e39c5baSBill Taylor 
3839e39c5baSBill Taylor 	/*
3849e39c5baSBill Taylor 	 * Fill in the rest of the Hermon Memory Region handle.  Having
3859e39c5baSBill Taylor 	 * successfully transferred ownership of the MPT, we can update the
3869e39c5baSBill Taylor 	 * following fields for use in further operations on the MR.
3879e39c5baSBill Taylor 	 */
3889e39c5baSBill Taylor 	mr->mr_mptrsrcp	  = mpt;
3899e39c5baSBill Taylor 	mr->mr_mttrsrcp	  = mtt;
3909e39c5baSBill Taylor 	mr->mr_mpt_type	  = HERMON_MPT_DMPT;
3919e39c5baSBill Taylor 	mr->mr_pdhdl	  = pd;
3929e39c5baSBill Taylor 	mr->mr_rsrcp	  = rsrc;
3939e39c5baSBill Taylor 	mr->mr_is_umem	  = mr_is_umem;
3949e39c5baSBill Taylor 	mr->mr_is_fmr	  = 0;
3959e39c5baSBill Taylor 	mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
3969e39c5baSBill Taylor 	mr->mr_umem_cbfunc = NULL;
3979e39c5baSBill Taylor 	mr->mr_umem_cbarg1 = NULL;
3989e39c5baSBill Taylor 	mr->mr_umem_cbarg2 = NULL;
3999e39c5baSBill Taylor 	mr->mr_lkey	   = hermon_mr_key_swap(mr->mr_lkey);
4009e39c5baSBill Taylor 	mr->mr_rkey	   = hermon_mr_key_swap(mr->mr_rkey);
4019e39c5baSBill Taylor 
4029e39c5baSBill Taylor 	/*
4039e39c5baSBill Taylor 	 * If this is userland memory, then we need to insert the previously
4049e39c5baSBill Taylor 	 * allocated entry into the "userland resources database".  This will
4059e39c5baSBill Taylor 	 * allow for later coordination between the hermon_umap_umemlock_cb()
4069e39c5baSBill Taylor 	 * callback and hermon_mr_deregister().
4079e39c5baSBill Taylor 	 */
4089e39c5baSBill Taylor 	if (mr_is_umem) {
4099e39c5baSBill Taylor 		hermon_umap_db_add(umapdb);
4109e39c5baSBill Taylor 	}
4119e39c5baSBill Taylor 
4129e39c5baSBill Taylor 	*mrhdl_new = mr;
4139e39c5baSBill Taylor 
4149e39c5baSBill Taylor 	return (DDI_SUCCESS);
4159e39c5baSBill Taylor 
4169e39c5baSBill Taylor /*
4179e39c5baSBill Taylor  * The following is cleanup for all possible failure cases in this routine
4189e39c5baSBill Taylor  */
4199e39c5baSBill Taylor mrshared_fail5:
4209e39c5baSBill Taylor 	(void) hermon_mtt_refcnt_dec(mr->mr_mttrefcntp);
4219e39c5baSBill Taylor 	if (mr_is_umem) {
4229e39c5baSBill Taylor 		hermon_umap_db_free(umapdb);
4239e39c5baSBill Taylor 	}
4249e39c5baSBill Taylor mrshared_fail4:
4259e39c5baSBill Taylor 	if (mr_is_umem) {
4269e39c5baSBill Taylor 		ddi_umem_unlock(umem_cookie);
4279e39c5baSBill Taylor 	}
4289e39c5baSBill Taylor mrshared_fail3:
4299e39c5baSBill Taylor 	hermon_rsrc_free(state, &rsrc);
4309e39c5baSBill Taylor mrshared_fail2:
4319e39c5baSBill Taylor 	hermon_rsrc_free(state, &mpt);
4329e39c5baSBill Taylor mrshared_fail1:
4339e39c5baSBill Taylor 	hermon_pd_refcnt_dec(pd);
4349e39c5baSBill Taylor mrshared_fail:
4359e39c5baSBill Taylor 	return (status);
4369e39c5baSBill Taylor }
4379e39c5baSBill Taylor 
4389e39c5baSBill Taylor /*
4399e39c5baSBill Taylor  * hermon_mr_alloc_fmr()
4409e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
4419e39c5baSBill Taylor  */
4429e39c5baSBill Taylor int
hermon_mr_alloc_fmr(hermon_state_t * state,hermon_pdhdl_t pd,hermon_fmrhdl_t fmr_pool,hermon_mrhdl_t * mrhdl)4439e39c5baSBill Taylor hermon_mr_alloc_fmr(hermon_state_t *state, hermon_pdhdl_t pd,
4449e39c5baSBill Taylor     hermon_fmrhdl_t fmr_pool, hermon_mrhdl_t *mrhdl)
4459e39c5baSBill Taylor {
4469e39c5baSBill Taylor 	hermon_rsrc_t		*mpt, *mtt, *rsrc;
447c7facc54SBill Taylor 	hermon_hw_dmpt_t	mpt_entry;
4489e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
4499e39c5baSBill Taylor 	hermon_bind_info_t	bind;
4509e39c5baSBill Taylor 	uint64_t		mtt_addr;
4519e39c5baSBill Taylor 	uint64_t		nummtt;
4529e39c5baSBill Taylor 	uint_t			sleep, mtt_pgsize_bits;
4539e39c5baSBill Taylor 	int			status;
454c7facc54SBill Taylor 	offset_t		i;
455c7facc54SBill Taylor 	hermon_icm_table_t	*icm_table;
456c7facc54SBill Taylor 	hermon_dma_info_t	*dma_info;
457c7facc54SBill Taylor 	uint32_t		index1, index2, rindx;
4589e39c5baSBill Taylor 
4599e39c5baSBill Taylor 	/*
4609e39c5baSBill Taylor 	 * Check the sleep flag.  Ensure that it is consistent with the
4619e39c5baSBill Taylor 	 * current thread context (i.e. if we are currently in the interrupt
4629e39c5baSBill Taylor 	 * context, then we shouldn't be attempting to sleep).
4639e39c5baSBill Taylor 	 */
4649e39c5baSBill Taylor 	sleep = (fmr_pool->fmr_flags & IBT_MR_SLEEP) ? HERMON_SLEEP :
4659e39c5baSBill Taylor 	    HERMON_NOSLEEP;
4669e39c5baSBill Taylor 	if ((sleep == HERMON_SLEEP) &&
4679e39c5baSBill Taylor 	    (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
4689e39c5baSBill Taylor 		return (IBT_INVALID_PARAM);
4699e39c5baSBill Taylor 	}
4709e39c5baSBill Taylor 
4719e39c5baSBill Taylor 	/* Increment the reference count on the protection domain (PD) */
4729e39c5baSBill Taylor 	hermon_pd_refcnt_inc(pd);
4739e39c5baSBill Taylor 
4749e39c5baSBill Taylor 	/*
4759e39c5baSBill Taylor 	 * Allocate an MPT entry.  This will be filled in with all the
4769e39c5baSBill Taylor 	 * necessary parameters to define the FMR.  Specifically, it will be
4779e39c5baSBill Taylor 	 * made to reference the currently existing MTT entries and ownership
4789e39c5baSBill Taylor 	 * of the MPT will be passed to the hardware in the last step below.
4799e39c5baSBill Taylor 	 * If we fail here, we must undo the protection domain reference count.
4809e39c5baSBill Taylor 	 */
4819e39c5baSBill Taylor 
4829e39c5baSBill Taylor 	status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt);
4839e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
4849e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
4859e39c5baSBill Taylor 		goto fmralloc_fail1;
4869e39c5baSBill Taylor 	}
4879e39c5baSBill Taylor 
4889e39c5baSBill Taylor 	/*
4899e39c5baSBill Taylor 	 * Allocate the software structure for tracking the fmr memory
4909e39c5baSBill Taylor 	 * region (i.e. the Hermon Memory Region handle).  If we fail here, we
4919e39c5baSBill Taylor 	 * must undo the protection domain reference count and the previous
4929e39c5baSBill Taylor 	 * resource allocation.
4939e39c5baSBill Taylor 	 */
4949e39c5baSBill Taylor 	status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc);
4959e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
4969e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
4979e39c5baSBill Taylor 		goto fmralloc_fail2;
4989e39c5baSBill Taylor 	}
4999e39c5baSBill Taylor 	mr = (hermon_mrhdl_t)rsrc->hr_addr;
5009e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
5019e39c5baSBill Taylor 
5029e39c5baSBill Taylor 	/*
5039e39c5baSBill Taylor 	 * Setup and validate the memory region access flags.  This means
5049e39c5baSBill Taylor 	 * translating the IBTF's enable flags into the access flags that
5059e39c5baSBill Taylor 	 * will be used in later operations.
5069e39c5baSBill Taylor 	 */
5079e39c5baSBill Taylor 	mr->mr_accflag = 0;
5089e39c5baSBill Taylor 	if (fmr_pool->fmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)
5099e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
5109e39c5baSBill Taylor 	if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_READ)
5119e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_REMOTE_READ;
5129e39c5baSBill Taylor 	if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_WRITE)
5139e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
5149e39c5baSBill Taylor 	if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
5159e39c5baSBill Taylor 		mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
5169e39c5baSBill Taylor 
5179e39c5baSBill Taylor 	/*
5189e39c5baSBill Taylor 	 * Calculate keys (Lkey, Rkey) from MPT index.  Each key is formed
5199e39c5baSBill Taylor 	 * from a certain number of "constrained" bits (the least significant
5209e39c5baSBill Taylor 	 * bits) and some number of "unconstrained" bits.  The constrained
5219e39c5baSBill Taylor 	 * bits must be set to the index of the entry in the MPT table, but
5229e39c5baSBill Taylor 	 * the unconstrained bits can be set to any value we wish.  Note:
5239e39c5baSBill Taylor 	 * if no remote access is required, then the RKey value is not filled
5249e39c5baSBill Taylor 	 * in.  Otherwise both Rkey and LKey are given the same value.
5259e39c5baSBill Taylor 	 */
526c7facc54SBill Taylor 	mr->mr_fmr_key = 1;	/* ready for the next reload */
527c7facc54SBill Taylor 	mr->mr_rkey = mr->mr_lkey = mpt->hr_indx;
5289e39c5baSBill Taylor 
5299e39c5baSBill Taylor 	/*
5309e39c5baSBill Taylor 	 * Determine number of pages spanned.  This routine uses the
5319e39c5baSBill Taylor 	 * information in the "bind" struct to determine the required
5329e39c5baSBill Taylor 	 * number of MTT entries needed (and returns the suggested page size -
5339e39c5baSBill Taylor 	 * as a "power-of-2" - for each MTT entry).
5349e39c5baSBill Taylor 	 */
5359e39c5baSBill Taylor 	/* Assume address will be page aligned later */
5369e39c5baSBill Taylor 	bind.bi_addr = 0;
5379e39c5baSBill Taylor 	/* Calculate size based on given max pages */
5389e39c5baSBill Taylor 	bind.bi_len = fmr_pool->fmr_max_pages << PAGESHIFT;
5399e39c5baSBill Taylor 	nummtt = hermon_mr_nummtt_needed(state, &bind, &mtt_pgsize_bits);
5409e39c5baSBill Taylor 
5419e39c5baSBill Taylor 	/*
5429e39c5baSBill Taylor 	 * Allocate the MTT entries.  Use the calculations performed above to
5439e39c5baSBill Taylor 	 * allocate the required number of MTT entries.  If we fail here, we
5449e39c5baSBill Taylor 	 * must not only undo all the previous resource allocation (and PD
5459e39c5baSBill Taylor 	 * reference count), but we must also unbind the memory.
5469e39c5baSBill Taylor 	 */
5479e39c5baSBill Taylor 	status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt, sleep, &mtt);
5489e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
549*17a2b317SBill Taylor 		IBTF_DPRINTF_L2("FMR", "FATAL: too few MTTs");
5509e39c5baSBill Taylor 		status = IBT_INSUFF_RESOURCE;
5519e39c5baSBill Taylor 		goto fmralloc_fail3;
5529e39c5baSBill Taylor 	}
5539e39c5baSBill Taylor 	mr->mr_logmttpgsz = mtt_pgsize_bits;
5549e39c5baSBill Taylor 
5559e39c5baSBill Taylor 	/*
5569e39c5baSBill Taylor 	 * Fill in the MPT entry.  This is the final step before passing
5579e39c5baSBill Taylor 	 * ownership of the MPT entry to the Hermon hardware.  We use all of
5589e39c5baSBill Taylor 	 * the information collected/calculated above to fill in the
5599e39c5baSBill Taylor 	 * requisite portions of the MPT.
5609e39c5baSBill Taylor 	 */
5619e39c5baSBill Taylor 	bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
5629e39c5baSBill Taylor 	mpt_entry.en_bind = 0;
5639e39c5baSBill Taylor 	mpt_entry.atomic  = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
5649e39c5baSBill Taylor 	mpt_entry.rw	  = (mr->mr_accflag & IBT_MR_REMOTE_WRITE)  ? 1 : 0;
5659e39c5baSBill Taylor 	mpt_entry.rr	  = (mr->mr_accflag & IBT_MR_REMOTE_READ)   ? 1 : 0;
5669e39c5baSBill Taylor 	mpt_entry.lw	  = (mr->mr_accflag & IBT_MR_LOCAL_WRITE)   ? 1 : 0;
5679e39c5baSBill Taylor 	mpt_entry.lr	  = 1;
5689e39c5baSBill Taylor 	mpt_entry.reg_win = HERMON_MPT_IS_REGION;
5699e39c5baSBill Taylor 	mpt_entry.pd		= pd->pd_pdnum;
5709e39c5baSBill Taylor 
5719e39c5baSBill Taylor 	mpt_entry.entity_sz	= mr->mr_logmttpgsz;
5729e39c5baSBill Taylor 	mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
573c7facc54SBill Taylor 	mpt_entry.fast_reg_en = 1;
574c7facc54SBill Taylor 	mpt_entry.mtt_size = (uint_t)nummtt;
5759e39c5baSBill Taylor 	mpt_entry.mtt_addr_h = mtt_addr >> 32;
5769e39c5baSBill Taylor 	mpt_entry.mtt_addr_l = mtt_addr >> 3;
5779e39c5baSBill Taylor 	mpt_entry.mem_key = mr->mr_lkey;
5789e39c5baSBill Taylor 
5799e39c5baSBill Taylor 	/*
5809e39c5baSBill Taylor 	 * FMR sets these to 0 for now.  Later during actual fmr registration
5819e39c5baSBill Taylor 	 * these values are filled in.
5829e39c5baSBill Taylor 	 */
5839e39c5baSBill Taylor 	mpt_entry.start_addr	= 0;
5849e39c5baSBill Taylor 	mpt_entry.reg_win_len	= 0;
5859e39c5baSBill Taylor 
5869e39c5baSBill Taylor 	/*
5879e39c5baSBill Taylor 	 * Write the MPT entry to hardware.  Lastly, we pass ownership of
5889e39c5baSBill Taylor 	 * the entry to the hardware.  Note: in general, this operation
5899e39c5baSBill Taylor 	 * shouldn't fail.  But if it does, we have to undo everything we've
5909e39c5baSBill Taylor 	 * done above before returning error.
5919e39c5baSBill Taylor 	 */
5929e39c5baSBill Taylor 	status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
5939e39c5baSBill Taylor 	    sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep);
5949e39c5baSBill Taylor 	if (status != HERMON_CMD_SUCCESS) {
5959e39c5baSBill Taylor 		cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n",
5969e39c5baSBill Taylor 		    status);
5979e39c5baSBill Taylor 		if (status == HERMON_CMD_INVALID_STATUS) {
5989e39c5baSBill Taylor 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
5999e39c5baSBill Taylor 		}
6009e39c5baSBill Taylor 		status = ibc_get_ci_failure(0);
6019e39c5baSBill Taylor 		goto fmralloc_fail4;
6029e39c5baSBill Taylor 	}
6039e39c5baSBill Taylor 
6049e39c5baSBill Taylor 	/*
6059e39c5baSBill Taylor 	 * Fill in the rest of the Hermon Memory Region handle.  Having
6069e39c5baSBill Taylor 	 * successfully transferred ownership of the MPT, we can update the
6079e39c5baSBill Taylor 	 * following fields for use in further operations on the MR.  Also, set
6089e39c5baSBill Taylor 	 * that this is an FMR region.
6099e39c5baSBill Taylor 	 */
6109e39c5baSBill Taylor 	mr->mr_mptrsrcp	  = mpt;
6119e39c5baSBill Taylor 	mr->mr_mttrsrcp	  = mtt;
612c7facc54SBill Taylor 
6139e39c5baSBill Taylor 	mr->mr_mpt_type   = HERMON_MPT_DMPT;
6149e39c5baSBill Taylor 	mr->mr_pdhdl	  = pd;
6159e39c5baSBill Taylor 	mr->mr_rsrcp	  = rsrc;
6169e39c5baSBill Taylor 	mr->mr_is_fmr	  = 1;
6179e39c5baSBill Taylor 	mr->mr_lkey	   = hermon_mr_key_swap(mr->mr_lkey);
6189e39c5baSBill Taylor 	mr->mr_rkey	   = hermon_mr_key_swap(mr->mr_rkey);
619c7facc54SBill Taylor 	mr->mr_mttaddr	   = mtt_addr;
6209e39c5baSBill Taylor 	(void) memcpy(&mr->mr_bindinfo, &bind, sizeof (hermon_bind_info_t));
6219e39c5baSBill Taylor 
622c7facc54SBill Taylor 	/* initialize hr_addr for use during register/deregister/invalidate */
623c7facc54SBill Taylor 	icm_table = &state->hs_icm[HERMON_DMPT];
624c7facc54SBill Taylor 	rindx = mpt->hr_indx;
625c7facc54SBill Taylor 	hermon_index(index1, index2, rindx, icm_table, i);
626c7facc54SBill Taylor 	dma_info = icm_table->icm_dma[index1] + index2;
627c7facc54SBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mpt))
628c7facc54SBill Taylor 	mpt->hr_addr = (void *)((uintptr_t)(dma_info->vaddr + i * mpt->hr_len));
629c7facc54SBill Taylor 
6309e39c5baSBill Taylor 	*mrhdl = mr;
6319e39c5baSBill Taylor 
6329e39c5baSBill Taylor 	return (DDI_SUCCESS);
6339e39c5baSBill Taylor 
6349e39c5baSBill Taylor /*
6359e39c5baSBill Taylor  * The following is cleanup for all possible failure cases in this routine
6369e39c5baSBill Taylor  */
6379e39c5baSBill Taylor fmralloc_fail4:
6389e39c5baSBill Taylor 	kmem_free(mtt, sizeof (hermon_rsrc_t) * nummtt);
6399e39c5baSBill Taylor fmralloc_fail3:
6409e39c5baSBill Taylor 	hermon_rsrc_free(state, &rsrc);
6419e39c5baSBill Taylor fmralloc_fail2:
6429e39c5baSBill Taylor 	hermon_rsrc_free(state, &mpt);
6439e39c5baSBill Taylor fmralloc_fail1:
6449e39c5baSBill Taylor 	hermon_pd_refcnt_dec(pd);
6459e39c5baSBill Taylor fmralloc_fail:
6469e39c5baSBill Taylor 	return (status);
6479e39c5baSBill Taylor }
6489e39c5baSBill Taylor 
649c7facc54SBill Taylor 
6509e39c5baSBill Taylor /*
6519e39c5baSBill Taylor  * hermon_mr_register_physical_fmr()
6529e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
6539e39c5baSBill Taylor  */
6549e39c5baSBill Taylor /*ARGSUSED*/
6559e39c5baSBill Taylor int
hermon_mr_register_physical_fmr(hermon_state_t * state,ibt_pmr_attr_t * mem_pattr_p,hermon_mrhdl_t mr,ibt_pmr_desc_t * mem_desc_p)6569e39c5baSBill Taylor hermon_mr_register_physical_fmr(hermon_state_t *state,
6579e39c5baSBill Taylor     ibt_pmr_attr_t *mem_pattr_p, hermon_mrhdl_t mr, ibt_pmr_desc_t *mem_desc_p)
6589e39c5baSBill Taylor {
6599e39c5baSBill Taylor 	hermon_rsrc_t		*mpt;
6609e39c5baSBill Taylor 	uint64_t		*mpt_table;
6619e39c5baSBill Taylor 	int			status;
662c7facc54SBill Taylor 	uint32_t		key;
6639e39c5baSBill Taylor 
6649e39c5baSBill Taylor 	mutex_enter(&mr->mr_lock);
6659e39c5baSBill Taylor 	mpt = mr->mr_mptrsrcp;
6669e39c5baSBill Taylor 	mpt_table = (uint64_t *)mpt->hr_addr;
6679e39c5baSBill Taylor 
6689e39c5baSBill Taylor 	/* Write MPT status to SW bit */
669c7facc54SBill Taylor 	*(uint8_t *)mpt_table = 0xF0;
670c7facc54SBill Taylor 
671c7facc54SBill Taylor 	membar_producer();
6729e39c5baSBill Taylor 
6739e39c5baSBill Taylor 	/*
6749e39c5baSBill Taylor 	 * Write the mapped addresses into the MTT entries.  FMR needs to do
6759e39c5baSBill Taylor 	 * this a little differently, so we call the fmr specific fast mtt
6769e39c5baSBill Taylor 	 * write here.
6779e39c5baSBill Taylor 	 */
678c7facc54SBill Taylor 	status = hermon_mr_fast_mtt_write_fmr(state, mr->mr_mttrsrcp,
679c7facc54SBill Taylor 	    mem_pattr_p, mr->mr_logmttpgsz);
6809e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
6819e39c5baSBill Taylor 		mutex_exit(&mr->mr_lock);
6829e39c5baSBill Taylor 		status = ibc_get_ci_failure(0);
6839e39c5baSBill Taylor 		goto fmr_reg_fail1;
6849e39c5baSBill Taylor 	}
6859e39c5baSBill Taylor 
6869e39c5baSBill Taylor 	/*
6879e39c5baSBill Taylor 	 * Calculate keys (Lkey, Rkey) from MPT index.  Each key is formed
6889e39c5baSBill Taylor 	 * from a certain number of "constrained" bits (the least significant
6899e39c5baSBill Taylor 	 * bits) and some number of "unconstrained" bits.  The constrained
6909e39c5baSBill Taylor 	 * bits must be set to the index of the entry in the MPT table, but
6919e39c5baSBill Taylor 	 * the unconstrained bits can be set to any value we wish.  Note:
6929e39c5baSBill Taylor 	 * if no remote access is required, then the RKey value is not filled
6939e39c5baSBill Taylor 	 * in.  Otherwise both Rkey and LKey are given the same value.
6949e39c5baSBill Taylor 	 */
695c7facc54SBill Taylor 	key = mpt->hr_indx | (mr->mr_fmr_key++ << HERMON_MEMKEY_SHIFT);
696c7facc54SBill Taylor 	mr->mr_lkey = mr->mr_rkey = hermon_mr_key_swap(key);
6979e39c5baSBill Taylor 
6989e39c5baSBill Taylor 	/* write mem key value */
699c7facc54SBill Taylor 	*(uint32_t *)&mpt_table[1] = htonl(key);
7009e39c5baSBill Taylor 
7019e39c5baSBill Taylor 	/* write length value */
702c7facc54SBill Taylor 	mpt_table[3] = htonll(mem_pattr_p->pmr_len);
7039e39c5baSBill Taylor 
7049e39c5baSBill Taylor 	/* write start addr value */
705c7facc54SBill Taylor 	mpt_table[2] = htonll(mem_pattr_p->pmr_iova);
7069e39c5baSBill Taylor 
7079e39c5baSBill Taylor 	/* write lkey value */
708c7facc54SBill Taylor 	*(uint32_t *)&mpt_table[4] = htonl(key);
709c7facc54SBill Taylor 
710c7facc54SBill Taylor 	membar_producer();
7119e39c5baSBill Taylor 
7129e39c5baSBill Taylor 	/* Write MPT status to HW bit */
713c7facc54SBill Taylor 	*(uint8_t *)mpt_table = 0x00;
7149e39c5baSBill Taylor 
7159e39c5baSBill Taylor 	/* Fill in return parameters */
7169e39c5baSBill Taylor 	mem_desc_p->pmd_lkey = mr->mr_lkey;
7179e39c5baSBill Taylor 	mem_desc_p->pmd_rkey = mr->mr_rkey;
7189e39c5baSBill Taylor 	mem_desc_p->pmd_iova = mem_pattr_p->pmr_iova;
7199e39c5baSBill Taylor 	mem_desc_p->pmd_phys_buf_list_sz = mem_pattr_p->pmr_len;
7209e39c5baSBill Taylor 
7219e39c5baSBill Taylor 	/* Fill in MR bindinfo struct for later sync or query operations */
7229e39c5baSBill Taylor 	mr->mr_bindinfo.bi_addr = mem_pattr_p->pmr_iova;
7239e39c5baSBill Taylor 	mr->mr_bindinfo.bi_flags = mem_pattr_p->pmr_flags & IBT_MR_NONCOHERENT;
7249e39c5baSBill Taylor 
7259e39c5baSBill Taylor 	mutex_exit(&mr->mr_lock);
7269e39c5baSBill Taylor 
7279e39c5baSBill Taylor 	return (DDI_SUCCESS);
7289e39c5baSBill Taylor 
7299e39c5baSBill Taylor fmr_reg_fail1:
7309e39c5baSBill Taylor 	/*
7319e39c5baSBill Taylor 	 * Note, we fail here, and purposely leave the memory ownership in
7329e39c5baSBill Taylor 	 * software.  The memory tables may be corrupt, so we leave the region
7339e39c5baSBill Taylor 	 * unregistered.
7349e39c5baSBill Taylor 	 */
735c7facc54SBill Taylor 	return (status);
7369e39c5baSBill Taylor }
7379e39c5baSBill Taylor 
7389e39c5baSBill Taylor 
7399e39c5baSBill Taylor /*
7409e39c5baSBill Taylor  * hermon_mr_deregister()
7419e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
7429e39c5baSBill Taylor  */
7439e39c5baSBill Taylor /* ARGSUSED */
7449e39c5baSBill Taylor int
hermon_mr_deregister(hermon_state_t * state,hermon_mrhdl_t * mrhdl,uint_t level,uint_t sleep)7459e39c5baSBill Taylor hermon_mr_deregister(hermon_state_t *state, hermon_mrhdl_t *mrhdl, uint_t level,
7469e39c5baSBill Taylor     uint_t sleep)
7479e39c5baSBill Taylor {
7489e39c5baSBill Taylor 	hermon_rsrc_t		*mpt, *mtt, *rsrc, *mtt_refcnt;
7499e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
7509e39c5baSBill Taylor 	hermon_pdhdl_t		pd;
7519e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
7529e39c5baSBill Taylor 	hermon_bind_info_t	*bind;
7539e39c5baSBill Taylor 	uint64_t		value;
7549e39c5baSBill Taylor 	int			status;
7559e39c5baSBill Taylor 	uint_t			shared_mtt;
7569e39c5baSBill Taylor 
7579e39c5baSBill Taylor 	/*
7589e39c5baSBill Taylor 	 * Check the sleep flag.  Ensure that it is consistent with the
7599e39c5baSBill Taylor 	 * current thread context (i.e. if we are currently in the interrupt
7609e39c5baSBill Taylor 	 * context, then we shouldn't be attempting to sleep).
7619e39c5baSBill Taylor 	 */
7629e39c5baSBill Taylor 	if ((sleep == HERMON_SLEEP) &&
7639e39c5baSBill Taylor 	    (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
7649e39c5baSBill Taylor 		status = IBT_INVALID_PARAM;
7659e39c5baSBill Taylor 		return (status);
7669e39c5baSBill Taylor 	}
7679e39c5baSBill Taylor 
7689e39c5baSBill Taylor 	/*
7699e39c5baSBill Taylor 	 * Pull all the necessary information from the Hermon Memory Region
7709e39c5baSBill Taylor 	 * handle.  This is necessary here because the resource for the
7719e39c5baSBill Taylor 	 * MR handle is going to be freed up as part of the this
7729e39c5baSBill Taylor 	 * deregistration
7739e39c5baSBill Taylor 	 */
7749e39c5baSBill Taylor 	mr	= *mrhdl;
7759e39c5baSBill Taylor 	mutex_enter(&mr->mr_lock);
7769e39c5baSBill Taylor 	mpt	= mr->mr_mptrsrcp;
7779e39c5baSBill Taylor 	mtt	= mr->mr_mttrsrcp;
7789e39c5baSBill Taylor 	mtt_refcnt = mr->mr_mttrefcntp;
7799e39c5baSBill Taylor 	rsrc	= mr->mr_rsrcp;
7809e39c5baSBill Taylor 	pd	= mr->mr_pdhdl;
7819e39c5baSBill Taylor 	bind	= &mr->mr_bindinfo;
7829e39c5baSBill Taylor 
7839e39c5baSBill Taylor 	/*
7849e39c5baSBill Taylor 	 * Check here if the memory region is really an FMR.  If so, this is a
7859e39c5baSBill Taylor 	 * bad thing and we shouldn't be here.  Return failure.
7869e39c5baSBill Taylor 	 */
7879e39c5baSBill Taylor 	if (mr->mr_is_fmr) {
7889e39c5baSBill Taylor 		mutex_exit(&mr->mr_lock);
7899e39c5baSBill Taylor 		return (IBT_INVALID_PARAM);
7909e39c5baSBill Taylor 	}
7919e39c5baSBill Taylor 
7929e39c5baSBill Taylor 	/*
7939e39c5baSBill Taylor 	 * Check here to see if the memory region has already been partially
7949e39c5baSBill Taylor 	 * deregistered as a result of the hermon_umap_umemlock_cb() callback.
7959e39c5baSBill Taylor 	 * If so, then jump to the end and free the remaining resources.
7969e39c5baSBill Taylor 	 */
7979e39c5baSBill Taylor 	if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
7989e39c5baSBill Taylor 		goto mrdereg_finish_cleanup;
7999e39c5baSBill Taylor 	}
800c7facc54SBill Taylor 	if (hermon_rdma_debug & 0x4)
801c7facc54SBill Taylor 		IBTF_DPRINTF_L2("mr", "dereg: mr %p  key %x",
802c7facc54SBill Taylor 		    mr, mr->mr_rkey);
8039e39c5baSBill Taylor 
8049e39c5baSBill Taylor 	/*
8059e39c5baSBill Taylor 	 * We must drop the "mr_lock" here to ensure that both SLEEP and
8069e39c5baSBill Taylor 	 * NOSLEEP calls into the firmware work as expected.  Also, if two
8079e39c5baSBill Taylor 	 * threads are attemping to access this MR (via de-register,
8089e39c5baSBill Taylor 	 * re-register, or otherwise), then we allow the firmware to enforce
8099e39c5baSBill Taylor 	 * the checking, that only one deregister is valid.
8109e39c5baSBill Taylor 	 */
8119e39c5baSBill Taylor 	mutex_exit(&mr->mr_lock);
8129e39c5baSBill Taylor 
8139e39c5baSBill Taylor 	/*
8149e39c5baSBill Taylor 	 * Reclaim MPT entry from hardware (if necessary).  Since the
8159e39c5baSBill Taylor 	 * hermon_mr_deregister() routine is used in the memory region
8169e39c5baSBill Taylor 	 * reregistration process as well, it is possible that we will
8179e39c5baSBill Taylor 	 * not always wish to reclaim ownership of the MPT.  Check the
8189e39c5baSBill Taylor 	 * "level" arg and, if necessary, attempt to reclaim it.  If
8199e39c5baSBill Taylor 	 * the ownership transfer fails for any reason, we check to see
8209e39c5baSBill Taylor 	 * what command status was returned from the hardware.  The only
8219e39c5baSBill Taylor 	 * "expected" error status is the one that indicates an attempt to
8229e39c5baSBill Taylor 	 * deregister a memory region that has memory windows bound to it
8239e39c5baSBill Taylor 	 */
8249e39c5baSBill Taylor 	if (level >= HERMON_MR_DEREG_ALL) {
8259e39c5baSBill Taylor 		if (mr->mr_mpt_type >= HERMON_MPT_DMPT) {
8269e39c5baSBill Taylor 			status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT,
8279e39c5baSBill Taylor 			    NULL, 0, mpt->hr_indx, sleep);
8289e39c5baSBill Taylor 			if (status != HERMON_CMD_SUCCESS) {
8299e39c5baSBill Taylor 				if (status == HERMON_CMD_REG_BOUND) {
8309e39c5baSBill Taylor 					return (IBT_MR_IN_USE);
8319e39c5baSBill Taylor 				} else {
8329e39c5baSBill Taylor 					cmn_err(CE_CONT, "Hermon: HW2SW_MPT "
8339e39c5baSBill Taylor 					    "command failed: %08x\n", status);
8349e39c5baSBill Taylor 					if (status ==
8359e39c5baSBill Taylor 					    HERMON_CMD_INVALID_STATUS) {
8369e39c5baSBill Taylor 						hermon_fm_ereport(state,
8379e39c5baSBill Taylor 						    HCA_SYS_ERR,
8389e39c5baSBill Taylor 						    DDI_SERVICE_LOST);
8399e39c5baSBill Taylor 					}
8409e39c5baSBill Taylor 					return (IBT_INVALID_PARAM);
8419e39c5baSBill Taylor 				}
8429e39c5baSBill Taylor 			}
8439e39c5baSBill Taylor 		}
8449e39c5baSBill Taylor 	}
8459e39c5baSBill Taylor 
8469e39c5baSBill Taylor 	/*
8479e39c5baSBill Taylor 	 * Re-grab the mr_lock here.  Since further access to the protected
8489e39c5baSBill Taylor 	 * 'mr' structure is needed, and we would have returned previously for
8499e39c5baSBill Taylor 	 * the multiple deregistration case, we can safely grab the lock here.
8509e39c5baSBill Taylor 	 */
8519e39c5baSBill Taylor 	mutex_enter(&mr->mr_lock);
8529e39c5baSBill Taylor 
8539e39c5baSBill Taylor 	/*
8549e39c5baSBill Taylor 	 * If the memory had come from userland, then we do a lookup in the
8559e39c5baSBill Taylor 	 * "userland resources database".  On success, we free the entry, call
8569e39c5baSBill Taylor 	 * ddi_umem_unlock(), and continue the cleanup.  On failure (which is
8579e39c5baSBill Taylor 	 * an indication that the umem_lockmemory() callback has called
8589e39c5baSBill Taylor 	 * hermon_mr_deregister()), we call ddi_umem_unlock() and invalidate
8599e39c5baSBill Taylor 	 * the "mr_umemcookie" field in the MR handle (this will be used
8609e39c5baSBill Taylor 	 * later to detect that only partial cleaup still remains to be done
8619e39c5baSBill Taylor 	 * on the MR handle).
8629e39c5baSBill Taylor 	 */
8639e39c5baSBill Taylor 	if (mr->mr_is_umem) {
8649e39c5baSBill Taylor 		status = hermon_umap_db_find(state->hs_instance,
8659e39c5baSBill Taylor 		    (uint64_t)(uintptr_t)mr->mr_umemcookie,
8669e39c5baSBill Taylor 		    MLNX_UMAP_MRMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
8679e39c5baSBill Taylor 		    &umapdb);
8689e39c5baSBill Taylor 		if (status == DDI_SUCCESS) {
8699e39c5baSBill Taylor 			hermon_umap_db_free(umapdb);
8709e39c5baSBill Taylor 			ddi_umem_unlock(mr->mr_umemcookie);
8719e39c5baSBill Taylor 		} else {
8729e39c5baSBill Taylor 			ddi_umem_unlock(mr->mr_umemcookie);
8739e39c5baSBill Taylor 			mr->mr_umemcookie = NULL;
8749e39c5baSBill Taylor 		}
8759e39c5baSBill Taylor 	}
8769e39c5baSBill Taylor 
877*17a2b317SBill Taylor 	/* mtt_refcnt is NULL in the case of hermon_dma_mr_register() */
878*17a2b317SBill Taylor 	if (mtt_refcnt != NULL) {
879*17a2b317SBill Taylor 		/*
880*17a2b317SBill Taylor 		 * Decrement the MTT reference count.  Since the MTT resource
881*17a2b317SBill Taylor 		 * may be shared between multiple memory regions (as a result
882*17a2b317SBill Taylor 		 * of a "RegisterSharedMR" verb) it is important that we not
883*17a2b317SBill Taylor 		 * free up or unbind resources prematurely.  If it's not shared
884*17a2b317SBill Taylor 		 * (as indicated by the return status), then free the resource.
885*17a2b317SBill Taylor 		 */
886*17a2b317SBill Taylor 		shared_mtt = hermon_mtt_refcnt_dec(mtt_refcnt);
887*17a2b317SBill Taylor 		if (!shared_mtt) {
888*17a2b317SBill Taylor 			hermon_rsrc_free(state, &mtt_refcnt);
889*17a2b317SBill Taylor 		}
8909e39c5baSBill Taylor 
891*17a2b317SBill Taylor 		/*
892*17a2b317SBill Taylor 		 * Free up the MTT entries and unbind the memory.  Here,
893*17a2b317SBill Taylor 		 * as above, we attempt to free these resources only if
894*17a2b317SBill Taylor 		 * it is appropriate to do so.
895*17a2b317SBill Taylor 		 * Note, 'bind' is NULL in the alloc_lkey case.
896*17a2b317SBill Taylor 		 */
897*17a2b317SBill Taylor 		if (!shared_mtt) {
898*17a2b317SBill Taylor 			if (level >= HERMON_MR_DEREG_NO_HW2SW_MPT) {
899*17a2b317SBill Taylor 				hermon_mr_mem_unbind(state, bind);
900*17a2b317SBill Taylor 			}
901*17a2b317SBill Taylor 			hermon_rsrc_free(state, &mtt);
9029e39c5baSBill Taylor 		}
9039e39c5baSBill Taylor 	}
9049e39c5baSBill Taylor 
9059e39c5baSBill Taylor 	/*
9069e39c5baSBill Taylor 	 * If the MR handle has been invalidated, then drop the
9079e39c5baSBill Taylor 	 * lock and return success.  Note: This only happens because
9089e39c5baSBill Taylor 	 * the umem_lockmemory() callback has been triggered.  The
9099e39c5baSBill Taylor 	 * cleanup here is partial, and further cleanup (in a
9109e39c5baSBill Taylor 	 * subsequent hermon_mr_deregister() call) will be necessary.
9119e39c5baSBill Taylor 	 */
9129e39c5baSBill Taylor 	if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
9139e39c5baSBill Taylor 		mutex_exit(&mr->mr_lock);
9149e39c5baSBill Taylor 		return (DDI_SUCCESS);
9159e39c5baSBill Taylor 	}
9169e39c5baSBill Taylor 
9179e39c5baSBill Taylor mrdereg_finish_cleanup:
9189e39c5baSBill Taylor 	mutex_exit(&mr->mr_lock);
9199e39c5baSBill Taylor 
9209e39c5baSBill Taylor 	/* Free the Hermon Memory Region handle */
9219e39c5baSBill Taylor 	hermon_rsrc_free(state, &rsrc);
9229e39c5baSBill Taylor 
9239e39c5baSBill Taylor 	/* Free up the MPT entry resource */
9249e39c5baSBill Taylor 	if (mpt != NULL)
9259e39c5baSBill Taylor 		hermon_rsrc_free(state, &mpt);
9269e39c5baSBill Taylor 
9279e39c5baSBill Taylor 	/* Decrement the reference count on the protection domain (PD) */
9289e39c5baSBill Taylor 	hermon_pd_refcnt_dec(pd);
9299e39c5baSBill Taylor 
9309e39c5baSBill Taylor 	/* Set the mrhdl pointer to NULL and return success */
9319e39c5baSBill Taylor 	*mrhdl = NULL;
9329e39c5baSBill Taylor 
9339e39c5baSBill Taylor 	return (DDI_SUCCESS);
9349e39c5baSBill Taylor }
9359e39c5baSBill Taylor 
9369e39c5baSBill Taylor /*
9379e39c5baSBill Taylor  * hermon_mr_dealloc_fmr()
9389e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
9399e39c5baSBill Taylor  */
9409e39c5baSBill Taylor /* ARGSUSED */
9419e39c5baSBill Taylor int
hermon_mr_dealloc_fmr(hermon_state_t * state,hermon_mrhdl_t * mrhdl)9429e39c5baSBill Taylor hermon_mr_dealloc_fmr(hermon_state_t *state, hermon_mrhdl_t *mrhdl)
9439e39c5baSBill Taylor {
9449e39c5baSBill Taylor 	hermon_rsrc_t		*mpt, *mtt, *rsrc;
9459e39c5baSBill Taylor 	hermon_pdhdl_t		pd;
9469e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
9479e39c5baSBill Taylor 
9489e39c5baSBill Taylor 	/*
9499e39c5baSBill Taylor 	 * Pull all the necessary information from the Hermon Memory Region
9509e39c5baSBill Taylor 	 * handle.  This is necessary here because the resource for the
9519e39c5baSBill Taylor 	 * MR handle is going to be freed up as part of the this
9529e39c5baSBill Taylor 	 * deregistration
9539e39c5baSBill Taylor 	 */
9549e39c5baSBill Taylor 	mr	= *mrhdl;
9559e39c5baSBill Taylor 	mutex_enter(&mr->mr_lock);
9569e39c5baSBill Taylor 	mpt	= mr->mr_mptrsrcp;
9579e39c5baSBill Taylor 	mtt	= mr->mr_mttrsrcp;
9589e39c5baSBill Taylor 	rsrc	= mr->mr_rsrcp;
9599e39c5baSBill Taylor 	pd	= mr->mr_pdhdl;
9609e39c5baSBill Taylor 	mutex_exit(&mr->mr_lock);
9619e39c5baSBill Taylor 
9629e39c5baSBill Taylor 	/* Free the MTT entries */
9639e39c5baSBill Taylor 	hermon_rsrc_free(state, &mtt);
9649e39c5baSBill Taylor 
9659e39c5baSBill Taylor 	/* Free the Hermon Memory Region handle */
9669e39c5baSBill Taylor 	hermon_rsrc_free(state, &rsrc);
9679e39c5baSBill Taylor 
9689e39c5baSBill Taylor 	/* Free up the MPT entry resource */
9699e39c5baSBill Taylor 	hermon_rsrc_free(state, &mpt);
9709e39c5baSBill Taylor 
9719e39c5baSBill Taylor 	/* Decrement the reference count on the protection domain (PD) */
9729e39c5baSBill Taylor 	hermon_pd_refcnt_dec(pd);
9739e39c5baSBill Taylor 
9749e39c5baSBill Taylor 	/* Set the mrhdl pointer to NULL and return success */
9759e39c5baSBill Taylor 	*mrhdl = NULL;
9769e39c5baSBill Taylor 
9779e39c5baSBill Taylor 	return (DDI_SUCCESS);
9789e39c5baSBill Taylor }
9799e39c5baSBill Taylor 
9809e39c5baSBill Taylor 
9819e39c5baSBill Taylor /*
9829e39c5baSBill Taylor  * hermon_mr_query()
9839e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
9849e39c5baSBill Taylor  */
9859e39c5baSBill Taylor /* ARGSUSED */
9869e39c5baSBill Taylor int
hermon_mr_query(hermon_state_t * state,hermon_mrhdl_t mr,ibt_mr_query_attr_t * attr)9879e39c5baSBill Taylor hermon_mr_query(hermon_state_t *state, hermon_mrhdl_t mr,
9889e39c5baSBill Taylor     ibt_mr_query_attr_t *attr)
9899e39c5baSBill Taylor {
990c7facc54SBill Taylor 	int			status;
991c7facc54SBill Taylor 	hermon_hw_dmpt_t	mpt_entry;
992c7facc54SBill Taylor 	uint32_t		lkey;
993c7facc54SBill Taylor 
9949e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr))
9959e39c5baSBill Taylor 
9969e39c5baSBill Taylor 	mutex_enter(&mr->mr_lock);
9979e39c5baSBill Taylor 
9989e39c5baSBill Taylor 	/*
9999e39c5baSBill Taylor 	 * Check here to see if the memory region has already been partially
10009e39c5baSBill Taylor 	 * deregistered as a result of a hermon_umap_umemlock_cb() callback.
10019e39c5baSBill Taylor 	 * If so, this is an error, return failure.
10029e39c5baSBill Taylor 	 */
10039e39c5baSBill Taylor 	if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
10049e39c5baSBill Taylor 		mutex_exit(&mr->mr_lock);
10059e39c5baSBill Taylor 		return (IBT_MR_HDL_INVALID);
10069e39c5baSBill Taylor 	}
10079e39c5baSBill Taylor 
1008c7facc54SBill Taylor 	status = hermon_cmn_query_cmd_post(