19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
23*17a2b317SBill Taylor  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor  */
259e39c5baSBill Taylor 
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor  * hermon_umap.c
289e39c5baSBill Taylor  *    Hermon Userland Mapping Routines
299e39c5baSBill Taylor  *
309e39c5baSBill Taylor  *    Implements all the routines necessary for enabling direct userland
319e39c5baSBill Taylor  *    access to the Hermon hardware.  This includes all routines necessary for
329e39c5baSBill Taylor  *    maintaining the "userland resources database" and all the support routines
339e39c5baSBill Taylor  *    for the devmap calls.
349e39c5baSBill Taylor  */
359e39c5baSBill Taylor 
369e39c5baSBill Taylor #include <sys/types.h>
379e39c5baSBill Taylor #include <sys/conf.h>
389e39c5baSBill Taylor #include <sys/ddi.h>
399e39c5baSBill Taylor #include <sys/sunddi.h>
409e39c5baSBill Taylor #include <sys/modctl.h>
419e39c5baSBill Taylor #include <sys/file.h>
429e39c5baSBill Taylor #include <sys/avl.h>
439e39c5baSBill Taylor #include <sys/sysmacros.h>
449e39c5baSBill Taylor 
459e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
469e39c5baSBill Taylor 
479e39c5baSBill Taylor /* Hermon HCA state pointer (extern) */
489e39c5baSBill Taylor extern void *hermon_statep;
499e39c5baSBill Taylor 
509e39c5baSBill Taylor /* Hermon HCA Userland Resource Database (extern) */
519e39c5baSBill Taylor extern hermon_umap_db_t hermon_userland_rsrc_db;
529e39c5baSBill Taylor 
539e39c5baSBill Taylor static int hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp,
549e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err);
559e39c5baSBill Taylor static int hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp,
569e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
579e39c5baSBill Taylor static int hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp,
589e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
599e39c5baSBill Taylor static int hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp,
609e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
619e39c5baSBill Taylor static int hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp,
629e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err);
639e39c5baSBill Taylor static int hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
649e39c5baSBill Taylor     offset_t off, size_t len, void **pvtp);
659e39c5baSBill Taylor static int hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp,
669e39c5baSBill Taylor     devmap_cookie_t new_dhp, void **new_pvtp);
679e39c5baSBill Taylor static void hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp,
689e39c5baSBill Taylor     offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
699e39c5baSBill Taylor     devmap_cookie_t new_dhp2, void **pvtp2);
709e39c5baSBill Taylor static int hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev,
719e39c5baSBill Taylor     uint_t flags, offset_t off, size_t len, void **pvtp);
729e39c5baSBill Taylor static int hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
739e39c5baSBill Taylor     devmap_cookie_t new_dhp, void **new_pvtp);
749e39c5baSBill Taylor static void hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp,
759e39c5baSBill Taylor     offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
769e39c5baSBill Taylor     devmap_cookie_t new_dhp2, void **pvtp2);
779e39c5baSBill Taylor static int hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev,
789e39c5baSBill Taylor     uint_t flags, offset_t off, size_t len, void **pvtp);
799e39c5baSBill Taylor static int hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
809e39c5baSBill Taylor     devmap_cookie_t new_dhp, void **new_pvtp);
819e39c5baSBill Taylor static void hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp,
829e39c5baSBill Taylor     offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
839e39c5baSBill Taylor     devmap_cookie_t new_dhp2, void **pvtp2);
849e39c5baSBill Taylor static ibt_status_t hermon_umap_mr_data_in(hermon_mrhdl_t mr,
859e39c5baSBill Taylor     ibt_mr_data_in_t *data, size_t data_sz);
869e39c5baSBill Taylor static ibt_status_t hermon_umap_cq_data_out(hermon_cqhdl_t cq,
879e39c5baSBill Taylor     mlnx_umap_cq_data_out_t *data, size_t data_sz);
889e39c5baSBill Taylor static ibt_status_t hermon_umap_qp_data_out(hermon_qphdl_t qp,
899e39c5baSBill Taylor     mlnx_umap_qp_data_out_t *data, size_t data_sz);
909e39c5baSBill Taylor static ibt_status_t hermon_umap_srq_data_out(hermon_srqhdl_t srq,
919e39c5baSBill Taylor     mlnx_umap_srq_data_out_t *data, size_t data_sz);
929e39c5baSBill Taylor static ibt_status_t hermon_umap_pd_data_out(hermon_pdhdl_t pd,
939e39c5baSBill Taylor     mlnx_umap_pd_data_out_t *data, size_t data_sz);
949e39c5baSBill Taylor static int hermon_umap_db_compare(const void *query, const void *entry);
959e39c5baSBill Taylor 
969e39c5baSBill Taylor 
979e39c5baSBill Taylor /*
989e39c5baSBill Taylor  * These callbacks are passed to devmap_umem_setup() and devmap_devmem_setup(),
999e39c5baSBill Taylor  * respectively.  They are used to handle (among other things) partial
1009e39c5baSBill Taylor  * unmappings and to provide a method for invalidating mappings inherited
1019e39c5baSBill Taylor  * as a result of a fork(2) system call.
1029e39c5baSBill Taylor  */
1039e39c5baSBill Taylor static struct devmap_callback_ctl hermon_devmap_umem_cbops = {
1049e39c5baSBill Taylor 	DEVMAP_OPS_REV,
1059e39c5baSBill Taylor 	hermon_devmap_umem_map,
1069e39c5baSBill Taylor 	NULL,
1079e39c5baSBill Taylor 	hermon_devmap_umem_dup,
1089e39c5baSBill Taylor 	hermon_devmap_umem_unmap
1099e39c5baSBill Taylor };
1109e39c5baSBill Taylor static struct devmap_callback_ctl hermon_devmap_devmem_cbops = {
1119e39c5baSBill Taylor 	DEVMAP_OPS_REV,
1129e39c5baSBill Taylor 	hermon_devmap_devmem_map,
1139e39c5baSBill Taylor 	NULL,
1149e39c5baSBill Taylor 	hermon_devmap_devmem_dup,
1159e39c5baSBill Taylor 	hermon_devmap_devmem_unmap
1169e39c5baSBill Taylor };
1179e39c5baSBill Taylor static struct devmap_callback_ctl hermon_devmap_dbrecmem_cbops = {
1189e39c5baSBill Taylor 	DEVMAP_OPS_REV,
1199e39c5baSBill Taylor 	hermon_devmap_dbrecmem_map,
1209e39c5baSBill Taylor 	NULL,
1219e39c5baSBill Taylor 	hermon_devmap_dbrecmem_dup,
1229e39c5baSBill Taylor 	hermon_devmap_dbrecmem_unmap
1239e39c5baSBill Taylor };
1249e39c5baSBill Taylor 
1259e39c5baSBill Taylor /*
1269e39c5baSBill Taylor  * hermon_devmap()
1279e39c5baSBill Taylor  *    Context: Can be called from user context.
1289e39c5baSBill Taylor  */
1299e39c5baSBill Taylor /* ARGSUSED */
1309e39c5baSBill Taylor int
1319e39c5baSBill Taylor hermon_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
1329e39c5baSBill Taylor     size_t *maplen, uint_t model)
1339e39c5baSBill Taylor {
1349e39c5baSBill Taylor 	hermon_state_t	*state;
1359e39c5baSBill Taylor 	hermon_rsrc_t 	*rsrcp;
1369e39c5baSBill Taylor 	minor_t		instance;
1379e39c5baSBill Taylor 	uint64_t	key, value;
1389e39c5baSBill Taylor 	uint64_t	bf_offset = 0;
1399e39c5baSBill Taylor 	uint_t		type;
1409e39c5baSBill Taylor 	int		err, status;
1419e39c5baSBill Taylor 
1429e39c5baSBill Taylor 	/* Get Hermon softstate structure from instance */
1439e39c5baSBill Taylor 	instance = HERMON_DEV_INSTANCE(dev);
1449e39c5baSBill Taylor 	state = ddi_get_soft_state(hermon_statep, instance);
1459e39c5baSBill Taylor 	if (state == NULL) {
1469e39c5baSBill Taylor 		return (ENXIO);
1479e39c5baSBill Taylor 	}
1489e39c5baSBill Taylor 
1499e39c5baSBill Taylor 	/*
1509e39c5baSBill Taylor 	 * Access to Hermon devmap interface is not allowed in
1519e39c5baSBill Taylor 	 * "maintenance mode".
1529e39c5baSBill Taylor 	 */
1539e39c5baSBill Taylor 	if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
1549e39c5baSBill Taylor 		return (EFAULT);
1559e39c5baSBill Taylor 	}
1569e39c5baSBill Taylor 
1579e39c5baSBill Taylor 	/*
1589e39c5baSBill Taylor 	 * The bottom bits of "offset" are undefined (number depends on
1599e39c5baSBill Taylor 	 * system PAGESIZE).  Shifting these off leaves us with a "key".
1609e39c5baSBill Taylor 	 * The "key" is actually a combination of both a real key value
1619e39c5baSBill Taylor 	 * (for the purpose of database lookup) and a "type" value.  We
1629e39c5baSBill Taylor 	 * extract this information before doing the database lookup.
1639e39c5baSBill Taylor 	 */
1649e39c5baSBill Taylor 	key  = off >> PAGESHIFT;
1659e39c5baSBill Taylor 	type = key & MLNX_UMAP_RSRC_TYPE_MASK;
1669e39c5baSBill Taylor 	key  = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
1679e39c5baSBill Taylor 	if (type == MLNX_UMAP_BLUEFLAMEPG_RSRC) {
1689e39c5baSBill Taylor 		if (state->hs_devlim.blu_flm == 0) {
1699e39c5baSBill Taylor 			return (EFAULT);
1709e39c5baSBill Taylor 		}
1719e39c5baSBill Taylor 		bf_offset = state->hs_bf_offset;
1729e39c5baSBill Taylor 		type = MLNX_UMAP_UARPG_RSRC;
1739e39c5baSBill Taylor 	}
1749e39c5baSBill Taylor 	status = hermon_umap_db_find(instance, key, type, &value, 0, NULL);
1759e39c5baSBill Taylor 	if (status == DDI_SUCCESS) {
1769e39c5baSBill Taylor 		rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
1779e39c5baSBill Taylor 
1789e39c5baSBill Taylor 		switch (type) {
1799e39c5baSBill Taylor 		case MLNX_UMAP_UARPG_RSRC:
1809e39c5baSBill Taylor 			/*
1819e39c5baSBill Taylor 			 * Double check that process who open()'d Hermon is
1829e39c5baSBill Taylor 			 * same process attempting to mmap() UAR page.
1839e39c5baSBill Taylor 			 */
1849e39c5baSBill Taylor 			if (key != ddi_get_pid()) {
1859e39c5baSBill Taylor 				return (EINVAL);
1869e39c5baSBill Taylor 			}
1879e39c5baSBill Taylor 
1889e39c5baSBill Taylor 			/* Map the UAR page out for userland access */
1899e39c5baSBill Taylor 			status = hermon_umap_uarpg(state, dhp, rsrcp, bf_offset,
1909e39c5baSBill Taylor 			    maplen, &err);
1919e39c5baSBill Taylor 			if (status != DDI_SUCCESS) {
1929e39c5baSBill Taylor 				return (err);
1939e39c5baSBill Taylor 			}
1949e39c5baSBill Taylor 			break;
1959e39c5baSBill Taylor 
1969e39c5baSBill Taylor 		case MLNX_UMAP_CQMEM_RSRC:
1979e39c5baSBill Taylor 			/* Map the CQ memory out for userland access */
1989e39c5baSBill Taylor 			status = hermon_umap_cqmem(state, dhp, rsrcp, off,
1999e39c5baSBill Taylor 			    maplen, &err);
2009e39c5baSBill Taylor 			if (status != DDI_SUCCESS) {
2019e39c5baSBill Taylor 				return (err);
2029e39c5baSBill Taylor 			}
2039e39c5baSBill Taylor 			break;
2049e39c5baSBill Taylor 
2059e39c5baSBill Taylor 		case MLNX_UMAP_QPMEM_RSRC:
2069e39c5baSBill Taylor 			/* Map the QP memory out for userland access */
2079e39c5baSBill Taylor 			status = hermon_umap_qpmem(state, dhp, rsrcp, off,
2089e39c5baSBill Taylor 			    maplen, &err);
2099e39c5baSBill Taylor 			if (status != DDI_SUCCESS) {
2109e39c5baSBill Taylor 				return (err);
2119e39c5baSBill Taylor 			}
2129e39c5baSBill Taylor 			break;
2139e39c5baSBill Taylor 
2149e39c5baSBill Taylor 		case MLNX_UMAP_SRQMEM_RSRC:
2159e39c5baSBill Taylor 			/* Map the SRQ memory out for userland access */
2169e39c5baSBill Taylor 			status = hermon_umap_srqmem(state, dhp, rsrcp, off,
2179e39c5baSBill Taylor 			    maplen, &err);
2189e39c5baSBill Taylor 			if (status != DDI_SUCCESS) {
2199e39c5baSBill Taylor 				return (err);
2209e39c5baSBill Taylor 			}
2219e39c5baSBill Taylor 			break;
2229e39c5baSBill Taylor 
2239e39c5baSBill Taylor 		case MLNX_UMAP_DBRMEM_RSRC:
2249e39c5baSBill Taylor 			/*
2259e39c5baSBill Taylor 			 * Map the doorbell record memory out for
2269e39c5baSBill Taylor 			 * userland access
2279e39c5baSBill Taylor 			 */
2289e39c5baSBill Taylor 			status = hermon_umap_dbrecmem(state, dhp, rsrcp, off,
2299e39c5baSBill Taylor 			    maplen, &err);
2309e39c5baSBill Taylor 			if (status != DDI_SUCCESS) {
2319e39c5baSBill Taylor 				return (err);
2329e39c5baSBill Taylor 			}
2339e39c5baSBill Taylor 			break;
2349e39c5baSBill Taylor 
2359e39c5baSBill Taylor 		default:
2369e39c5baSBill Taylor 			HERMON_WARNING(state, "unexpected rsrc type in devmap");
2379e39c5baSBill Taylor 			return (EINVAL);
2389e39c5baSBill Taylor 		}
2399e39c5baSBill Taylor 	} else {
2409e39c5baSBill Taylor 		return (EINVAL);
2419e39c5baSBill Taylor 	}
2429e39c5baSBill Taylor 
2439e39c5baSBill Taylor 	return (0);
2449e39c5baSBill Taylor }
2459e39c5baSBill Taylor 
2469e39c5baSBill Taylor 
2479e39c5baSBill Taylor /*
2489e39c5baSBill Taylor  * hermon_umap_uarpg()
2499e39c5baSBill Taylor  *    Context: Can be called from user context.
2509e39c5baSBill Taylor  */
2519e39c5baSBill Taylor static int
2529e39c5baSBill Taylor hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp,
2539e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err)
2549e39c5baSBill Taylor {
2559e39c5baSBill Taylor 	int			status;
2569e39c5baSBill Taylor 	uint_t			maxprot;
2579e39c5baSBill Taylor 	ddi_device_acc_attr_t	*accattrp = &state->hs_reg_accattr;
2589e39c5baSBill Taylor 	ddi_device_acc_attr_t	accattr;
2599e39c5baSBill Taylor 
2609e39c5baSBill Taylor 	if (offset != 0) {	/* Hermon Blueflame */
2619e39c5baSBill Taylor 		/* Try to use write coalescing data ordering */
2629e39c5baSBill Taylor 		accattr = *accattrp;
2639e39c5baSBill Taylor 		accattr.devacc_attr_dataorder = DDI_STORECACHING_OK_ACC;
2649e39c5baSBill Taylor 		accattrp = &accattr;
2659e39c5baSBill Taylor 	}
2669e39c5baSBill Taylor 
2679e39c5baSBill Taylor 	/* Map out the UAR page (doorbell page) */
2689e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
2699e39c5baSBill Taylor 	status = devmap_devmem_setup(dhp, state->hs_dip,
2709e39c5baSBill Taylor 	    &hermon_devmap_devmem_cbops, HERMON_UAR_BAR, (rsrcp->hr_indx <<
2719e39c5baSBill Taylor 	    PAGESHIFT) + offset, PAGESIZE, maxprot, DEVMAP_ALLOW_REMAP,
2729e39c5baSBill Taylor 	    accattrp);
2739e39c5baSBill Taylor 	if (status < 0) {
2749e39c5baSBill Taylor 		*err = status;
2759e39c5baSBill Taylor 		return (DDI_FAILURE);
2769e39c5baSBill Taylor 	}
2779e39c5baSBill Taylor 
2789e39c5baSBill Taylor 	*maplen = PAGESIZE;
2799e39c5baSBill Taylor 	return (DDI_SUCCESS);
2809e39c5baSBill Taylor }
2819e39c5baSBill Taylor 
2829e39c5baSBill Taylor 
2839e39c5baSBill Taylor /*
2849e39c5baSBill Taylor  * hermon_umap_cqmem()
2859e39c5baSBill Taylor  *    Context: Can be called from user context.
2869e39c5baSBill Taylor  */
2879e39c5baSBill Taylor /* ARGSUSED */
2889e39c5baSBill Taylor static int
2899e39c5baSBill Taylor hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp,
2909e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
2919e39c5baSBill Taylor {
2929e39c5baSBill Taylor 	hermon_cqhdl_t	cq;
2939e39c5baSBill Taylor 	size_t		size;
2949e39c5baSBill Taylor 	uint_t		maxprot;
2959e39c5baSBill Taylor 	int		status;
2969e39c5baSBill Taylor 
2979e39c5baSBill Taylor 	/* Extract the Hermon CQ handle pointer from the hermon_rsrc_t */
2989e39c5baSBill Taylor 	cq = (hermon_cqhdl_t)rsrcp->hr_addr;
2999e39c5baSBill Taylor 
3009e39c5baSBill Taylor 	/* Round-up the CQ size to system page size */
3019e39c5baSBill Taylor 	size = ptob(btopr(cq->cq_resize_hdl ?
3029e39c5baSBill Taylor 	    cq->cq_resize_hdl->cq_cqinfo.qa_size : cq->cq_cqinfo.qa_size));
3039e39c5baSBill Taylor 
3049e39c5baSBill Taylor 	/* Map out the CQ memory - use resize_hdl if non-NULL */
3059e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
3069e39c5baSBill Taylor 	status = devmap_umem_setup(dhp, state->hs_dip,
3079e39c5baSBill Taylor 	    &hermon_devmap_umem_cbops, cq->cq_resize_hdl ?
3089e39c5baSBill Taylor 	    cq->cq_resize_hdl->cq_cqinfo.qa_umemcookie :
3099e39c5baSBill Taylor 	    cq->cq_cqinfo.qa_umemcookie, 0, size,
3109e39c5baSBill Taylor 	    maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
3119e39c5baSBill Taylor 	if (status < 0) {
3129e39c5baSBill Taylor 		*err = status;
3139e39c5baSBill Taylor 		return (DDI_FAILURE);
3149e39c5baSBill Taylor 	}
3159e39c5baSBill Taylor 	*maplen = size;
3169e39c5baSBill Taylor 
3179e39c5baSBill Taylor 	return (DDI_SUCCESS);
3189e39c5baSBill Taylor }
3199e39c5baSBill Taylor 
3209e39c5baSBill Taylor 
3219e39c5baSBill Taylor /*
3229e39c5baSBill Taylor  * hermon_umap_qpmem()
3239e39c5baSBill Taylor  *    Context: Can be called from user context.
3249e39c5baSBill Taylor  */
3259e39c5baSBill Taylor /* ARGSUSED */
3269e39c5baSBill Taylor static int
3279e39c5baSBill Taylor hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp,
3289e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
3299e39c5baSBill Taylor {
3309e39c5baSBill Taylor 	hermon_qphdl_t	qp;
3319e39c5baSBill Taylor 	offset_t	offset;
3329e39c5baSBill Taylor 	size_t		size;
3339e39c5baSBill Taylor 	uint_t		maxprot;
3349e39c5baSBill Taylor 	int		status;
3359e39c5baSBill Taylor 
3369e39c5baSBill Taylor 	/* Extract the Hermon QP handle pointer from the hermon_rsrc_t */
3379e39c5baSBill Taylor 	qp = (hermon_qphdl_t)rsrcp->hr_addr;
3389e39c5baSBill Taylor 
3399e39c5baSBill Taylor 	/*
3409e39c5baSBill Taylor 	 * Calculate the offset of the first work queue (send or recv) into
3419e39c5baSBill Taylor 	 * the memory (ddi_umem_alloc()) allocated previously for the QP.
3429e39c5baSBill Taylor 	 */
3439e39c5baSBill Taylor 	offset = (offset_t)((uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
3449e39c5baSBill Taylor 	    (uintptr_t)qp->qp_wqinfo.qa_buf_real);
3459e39c5baSBill Taylor 
3469e39c5baSBill Taylor 	/* Round-up the QP work queue sizes to system page size */
3479e39c5baSBill Taylor 	size = ptob(btopr(qp->qp_wqinfo.qa_size));
3489e39c5baSBill Taylor 
3499e39c5baSBill Taylor 	/* Map out the QP memory */
3509e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
3519e39c5baSBill Taylor 	status = devmap_umem_setup(dhp, state->hs_dip,
3529e39c5baSBill Taylor 	    &hermon_devmap_umem_cbops, qp->qp_wqinfo.qa_umemcookie, offset,
3539e39c5baSBill Taylor 	    size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
3549e39c5baSBill Taylor 	if (status < 0) {
3559e39c5baSBill Taylor 		*err = status;
3569e39c5baSBill Taylor 		return (DDI_FAILURE);
3579e39c5baSBill Taylor 	}
3589e39c5baSBill Taylor 	*maplen = size;
3599e39c5baSBill Taylor 
3609e39c5baSBill Taylor 	return (DDI_SUCCESS);
3619e39c5baSBill Taylor }
3629e39c5baSBill Taylor 
3639e39c5baSBill Taylor 
3649e39c5baSBill Taylor /*
3659e39c5baSBill Taylor  * hermon_umap_srqmem()
3669e39c5baSBill Taylor  *    Context: Can be called from user context.
3679e39c5baSBill Taylor  */
3689e39c5baSBill Taylor /* ARGSUSED */
3699e39c5baSBill Taylor static int
3709e39c5baSBill Taylor hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp,
3719e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
3729e39c5baSBill Taylor {
3739e39c5baSBill Taylor 	hermon_srqhdl_t	srq;
3749e39c5baSBill Taylor 	offset_t	offset;
3759e39c5baSBill Taylor 	size_t		size;
3769e39c5baSBill Taylor 	uint_t		maxprot;
3779e39c5baSBill Taylor 	int		status;
3789e39c5baSBill Taylor 
3799e39c5baSBill Taylor 	/* Extract the Hermon SRQ handle pointer from the hermon_rsrc_t */
3809e39c5baSBill Taylor 	srq = (hermon_srqhdl_t)rsrcp->hr_addr;
3819e39c5baSBill Taylor 
3829e39c5baSBill Taylor 	/*
3839e39c5baSBill Taylor 	 * Calculate the offset of the first shared recv queue into the memory
3849e39c5baSBill Taylor 	 * (ddi_umem_alloc()) allocated previously for the SRQ.
3859e39c5baSBill Taylor 	 */
3869e39c5baSBill Taylor 	offset = (offset_t)((uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
3879e39c5baSBill Taylor 	    (uintptr_t)srq->srq_wqinfo.qa_buf_real);
3889e39c5baSBill Taylor 
3899e39c5baSBill Taylor 	/* Round-up the SRQ work queue sizes to system page size */
3909e39c5baSBill Taylor 	size = ptob(btopr(srq->srq_wqinfo.qa_size));
3919e39c5baSBill Taylor 
3929e39c5baSBill Taylor 	/* Map out the SRQ memory */
3939e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
3949e39c5baSBill Taylor 	status = devmap_umem_setup(dhp, state->hs_dip,
3959e39c5baSBill Taylor 	    &hermon_devmap_umem_cbops, srq->srq_wqinfo.qa_umemcookie, offset,
3969e39c5baSBill Taylor 	    size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
3979e39c5baSBill Taylor 	if (status < 0) {
3989e39c5baSBill Taylor 		*err = status;
3999e39c5baSBill Taylor 		return (DDI_FAILURE);
4009e39c5baSBill Taylor 	}
4019e39c5baSBill Taylor 	*maplen = size;
4029e39c5baSBill Taylor 
4039e39c5baSBill Taylor 	return (DDI_SUCCESS);
4049e39c5baSBill Taylor }
4059e39c5baSBill Taylor 
4069e39c5baSBill Taylor 
4079e39c5baSBill Taylor /*
4089e39c5baSBill Taylor  * hermon_devmap_dbrecmem()
4099e39c5baSBill Taylor  *    Context: Can be called from user context.
4109e39c5baSBill Taylor  */
4119e39c5baSBill Taylor /* ARGSUSED */
4129e39c5baSBill Taylor static int
4139e39c5baSBill Taylor hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp,
4149e39c5baSBill Taylor     hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err)
4159e39c5baSBill Taylor {
4169e39c5baSBill Taylor 	hermon_udbr_page_t *pagep;
4179e39c5baSBill Taylor 	offset_t	offset;
4189e39c5baSBill Taylor 	size_t		size;
4199e39c5baSBill Taylor 	uint_t		maxprot;
4209e39c5baSBill Taylor 	int		status;
4219e39c5baSBill Taylor 
4229e39c5baSBill Taylor 	/* We stored the udbr_page pointer, and not a hermon_rsrc_t */
4239e39c5baSBill Taylor 	pagep = (hermon_udbr_page_t *)rsrcp;
4249e39c5baSBill Taylor 
4259e39c5baSBill Taylor 	/*
4269e39c5baSBill Taylor 	 * Calculate the offset of the doorbell records into the memory
4279e39c5baSBill Taylor 	 * (ddi_umem_alloc()) allocated previously for them.
4289e39c5baSBill Taylor 	 */
4299e39c5baSBill Taylor 	offset = 0;
4309e39c5baSBill Taylor 
4319e39c5baSBill Taylor 	/* Round-up the doorbell page to system page size */
4329e39c5baSBill Taylor 	size = PAGESIZE;
4339e39c5baSBill Taylor 
4349e39c5baSBill Taylor 	/* Map out the Doorbell Record memory */
4359e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
4369e39c5baSBill Taylor 	status = devmap_umem_setup(dhp, state->hs_dip,
4379e39c5baSBill Taylor 	    &hermon_devmap_dbrecmem_cbops, pagep->upg_umemcookie, offset,
4389e39c5baSBill Taylor 	    size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL);
4399e39c5baSBill Taylor 	if (status < 0) {
4409e39c5baSBill Taylor 		*err = status;
4419e39c5baSBill Taylor 		return (DDI_FAILURE);
4429e39c5baSBill Taylor 	}
4439e39c5baSBill Taylor 	*maplen = size;
4449e39c5baSBill Taylor 
4459e39c5baSBill Taylor 	return (DDI_SUCCESS);
4469e39c5baSBill Taylor }
4479e39c5baSBill Taylor 
4489e39c5baSBill Taylor 
4499e39c5baSBill Taylor /*
4509e39c5baSBill Taylor  * hermon_devmap_umem_map()
4519e39c5baSBill Taylor  *    Context: Can be called from kernel context.
4529e39c5baSBill Taylor  */
4539e39c5baSBill Taylor /* ARGSUSED */
4549e39c5baSBill Taylor static int
4559e39c5baSBill Taylor hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
4569e39c5baSBill Taylor     offset_t off, size_t len, void **pvtp)
4579e39c5baSBill Taylor {
4589e39c5baSBill Taylor 	hermon_state_t		*state;
4599e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track;
4609e39c5baSBill Taylor 	hermon_cqhdl_t		cq;
4619e39c5baSBill Taylor 	hermon_qphdl_t		qp;
4629e39c5baSBill Taylor 	hermon_srqhdl_t		srq;
4639e39c5baSBill Taylor 	minor_t			instance;
4649e39c5baSBill Taylor 	uint64_t		key;
4659e39c5baSBill Taylor 	uint_t			type;
4669e39c5baSBill Taylor 
4679e39c5baSBill Taylor 	/* Get Hermon softstate structure from instance */
4689e39c5baSBill Taylor 	instance = HERMON_DEV_INSTANCE(dev);
4699e39c5baSBill Taylor 	state = ddi_get_soft_state(hermon_statep, instance);
4709e39c5baSBill Taylor 	if (state == NULL) {
4719e39c5baSBill Taylor 		return (ENXIO);
4729e39c5baSBill Taylor 	}
4739e39c5baSBill Taylor 
4749e39c5baSBill Taylor 	/*
4759e39c5baSBill Taylor 	 * The bottom bits of "offset" are undefined (number depends on
4769e39c5baSBill Taylor 	 * system PAGESIZE).  Shifting these off leaves us with a "key".
4779e39c5baSBill Taylor 	 * The "key" is actually a combination of both a real key value
4789e39c5baSBill Taylor 	 * (for the purpose of database lookup) and a "type" value.  Although
4799e39c5baSBill Taylor 	 * we are not going to do any database lookup per se, we do want
4809e39c5baSBill Taylor 	 * to extract the "key" and the "type" (to enable faster lookup of
4819e39c5baSBill Taylor 	 * the appropriate CQ or QP handle).
4829e39c5baSBill Taylor 	 */
4839e39c5baSBill Taylor 	key  = off >> PAGESHIFT;
4849e39c5baSBill Taylor 	type = key & MLNX_UMAP_RSRC_TYPE_MASK;
4859e39c5baSBill Taylor 	key  = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
4869e39c5baSBill Taylor 
4879e39c5baSBill Taylor 	/*
4889e39c5baSBill Taylor 	 * Allocate an entry to track the mapping and unmapping (specifically,
4899e39c5baSBill Taylor 	 * partial unmapping) of this resource.
4909e39c5baSBill Taylor 	 */
4919e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
4929e39c5baSBill Taylor 	    sizeof (hermon_devmap_track_t), KM_SLEEP);
4939e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
4949e39c5baSBill Taylor 	dvm_track->hdt_offset = off;
4959e39c5baSBill Taylor 	dvm_track->hdt_state  = state;
4969e39c5baSBill Taylor 	dvm_track->hdt_refcnt = 1;
4979e39c5baSBill Taylor 	mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
4989e39c5baSBill Taylor 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
4999e39c5baSBill Taylor 
5009e39c5baSBill Taylor 	/*
5019e39c5baSBill Taylor 	 * Depending of the type of resource that has been mapped out, we
5029e39c5baSBill Taylor 	 * need to update the QP or CQ handle to reflect that it has, in
5039e39c5baSBill Taylor 	 * fact, been mapped.  This allows the driver code which frees a QP
5049e39c5baSBill Taylor 	 * or a CQ to know whether it is appropriate to do a
5059e39c5baSBill Taylor 	 * devmap_devmem_remap() to invalidate the userland mapping for the
5069e39c5baSBill Taylor 	 * corresponding queue's memory.
5079e39c5baSBill Taylor 	 */
5089e39c5baSBill Taylor 	if (type == MLNX_UMAP_CQMEM_RSRC) {
5099e39c5baSBill Taylor 
5109e39c5baSBill Taylor 		/* Use "key" (CQ number) to do fast lookup of CQ handle */
5119e39c5baSBill Taylor 		cq = hermon_cqhdl_from_cqnum(state, key);
5129e39c5baSBill Taylor 
5139e39c5baSBill Taylor 		/*
5149e39c5baSBill Taylor 		 * Update the handle to the userland mapping.  Note:  If
5159e39c5baSBill Taylor 		 * the CQ already has a valid userland mapping, then stop
5169e39c5baSBill Taylor 		 * and return failure.
5179e39c5baSBill Taylor 		 */
5189e39c5baSBill Taylor 		mutex_enter(&cq->cq_lock);
5199e39c5baSBill Taylor 		if (cq->cq_umap_dhp == NULL) {
5209e39c5baSBill Taylor 			cq->cq_umap_dhp = dhp;
5219e39c5baSBill Taylor 			dvm_track->hdt_size = cq->cq_cqinfo.qa_size;
5229e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
5239e39c5baSBill Taylor 		} else if (cq->cq_resize_hdl &&
5249e39c5baSBill Taylor 		    (cq->cq_resize_hdl->cq_umap_dhp == NULL)) {
5259e39c5baSBill Taylor 			cq->cq_resize_hdl->cq_umap_dhp = dhp;
5269e39c5baSBill Taylor 			dvm_track->hdt_size =
5279e39c5baSBill Taylor 			    cq->cq_resize_hdl->cq_cqinfo.qa_size;
5289e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
5299e39c5baSBill Taylor 		} else {
5309e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
5319e39c5baSBill Taylor 			goto umem_map_fail;
5329e39c5baSBill Taylor 		}
5339e39c5baSBill Taylor 
5349e39c5baSBill Taylor 	} else if (type == MLNX_UMAP_QPMEM_RSRC) {
5359e39c5baSBill Taylor 
5369e39c5baSBill Taylor 		/* Use "key" (QP number) to do fast lookup of QP handle */
5379e39c5baSBill Taylor 		qp = hermon_qphdl_from_qpnum(state, key);
5389e39c5baSBill Taylor 
5399e39c5baSBill Taylor 		/*
5409e39c5baSBill Taylor 		 * Update the handle to the userland mapping.  Note:  If
5419e39c5baSBill Taylor 		 * the CQ already has a valid userland mapping, then stop
5429e39c5baSBill Taylor 		 * and return failure.
5439e39c5baSBill Taylor 		 */
5449e39c5baSBill Taylor 		mutex_enter(&qp->qp_lock);
5459e39c5baSBill Taylor 		if (qp->qp_umap_dhp == NULL) {
5469e39c5baSBill Taylor 			qp->qp_umap_dhp = dhp;
5479e39c5baSBill Taylor 			dvm_track->hdt_size = qp->qp_wqinfo.qa_size;
5489e39c5baSBill Taylor 			mutex_exit(&qp->qp_lock);
5499e39c5baSBill Taylor 		} else {
5509e39c5baSBill Taylor 			mutex_exit(&qp->qp_lock);
5519e39c5baSBill Taylor 			goto umem_map_fail;
5529e39c5baSBill Taylor 		}
5539e39c5baSBill Taylor 
5549e39c5baSBill Taylor 	} else if (type == MLNX_UMAP_SRQMEM_RSRC) {
5559e39c5baSBill Taylor 
5569e39c5baSBill Taylor 		/* Use "key" (SRQ number) to do fast lookup on SRQ handle */
5579e39c5baSBill Taylor 		srq = hermon_srqhdl_from_srqnum(state, key);
5589e39c5baSBill Taylor 
5599e39c5baSBill Taylor 		/*
5609e39c5baSBill Taylor 		 * Update the handle to the userland mapping.  Note:  If the
5619e39c5baSBill Taylor 		 * SRQ already has a valid userland mapping, then stop and
5629e39c5baSBill Taylor 		 * return failure.
5639e39c5baSBill Taylor 		 */
5649e39c5baSBill Taylor 		mutex_enter(&srq->srq_lock);
5659e39c5baSBill Taylor 		if (srq->srq_umap_dhp == NULL) {
5669e39c5baSBill Taylor 			srq->srq_umap_dhp = dhp;
5679e39c5baSBill Taylor 			dvm_track->hdt_size = srq->srq_wqinfo.qa_size;
5689e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
5699e39c5baSBill Taylor 		} else {
5709e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
5719e39c5baSBill Taylor 			goto umem_map_fail;
5729e39c5baSBill Taylor 		}
5739e39c5baSBill Taylor 	}
5749e39c5baSBill Taylor 
5759e39c5baSBill Taylor 	/*
5769e39c5baSBill Taylor 	 * Pass the private "Hermon devmap tracking structure" back.  This
5779e39c5baSBill Taylor 	 * pointer will be returned in subsequent "unmap" callbacks.
5789e39c5baSBill Taylor 	 */
5799e39c5baSBill Taylor 	*pvtp = dvm_track;
5809e39c5baSBill Taylor 
5819e39c5baSBill Taylor 	return (DDI_SUCCESS);
5829e39c5baSBill Taylor 
5839e39c5baSBill Taylor umem_map_fail:
5849e39c5baSBill Taylor 	mutex_destroy(&dvm_track->hdt_lock);
5859e39c5baSBill Taylor 	kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
5869e39c5baSBill Taylor 	return (DDI_FAILURE);
5879e39c5baSBill Taylor }
5889e39c5baSBill Taylor 
5899e39c5baSBill Taylor 
5909e39c5baSBill Taylor /*
5919e39c5baSBill Taylor  * hermon_devmap_umem_dup()
5929e39c5baSBill Taylor  *    Context: Can be called from kernel context.
5939e39c5baSBill Taylor  */
5949e39c5baSBill Taylor /* ARGSUSED */
5959e39c5baSBill Taylor static int
5969e39c5baSBill Taylor hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, devmap_cookie_t new_dhp,
5979e39c5baSBill Taylor     void **new_pvtp)
5989e39c5baSBill Taylor {
5999e39c5baSBill Taylor 	hermon_state_t		*state;
6009e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track, *new_dvm_track;
6019e39c5baSBill Taylor 	uint_t			maxprot;
6029e39c5baSBill Taylor 	int			status;
6039e39c5baSBill Taylor 
6049e39c5baSBill Taylor 	/*
6059e39c5baSBill Taylor 	 * Extract the Hermon softstate pointer from "Hermon devmap tracking
6069e39c5baSBill Taylor 	 * structure" (in "pvtp").
6079e39c5baSBill Taylor 	 */
6089e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)pvtp;
6099e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
6109e39c5baSBill Taylor 	state = dvm_track->hdt_state;
6119e39c5baSBill Taylor 
6129e39c5baSBill Taylor 	/*
6139e39c5baSBill Taylor 	 * Since this devmap_dup() entry point is generally called
6149e39c5baSBill Taylor 	 * when a process does fork(2), it is incumbent upon the driver
6159e39c5baSBill Taylor 	 * to insure that the child does not inherit a valid copy of
6169e39c5baSBill Taylor 	 * the parent's QP or CQ resource.  This is accomplished by using
6179e39c5baSBill Taylor 	 * devmap_devmem_remap() to invalidate the child's mapping to the
6189e39c5baSBill Taylor 	 * kernel memory.
6199e39c5baSBill Taylor 	 */
6209e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
6219e39c5baSBill Taylor 	status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
6229e39c5baSBill Taylor 	    dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
6239e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
6249e39c5baSBill Taylor 		HERMON_WARNING(state, "failed in hermon_devmap_umem_dup()");
6259e39c5baSBill Taylor 		return (status);
6269e39c5baSBill Taylor 	}
6279e39c5baSBill Taylor 
6289e39c5baSBill Taylor 	/*
6299e39c5baSBill Taylor 	 * Allocate a new entry to track the subsequent unmapping
6309e39c5baSBill Taylor 	 * (specifically, all partial unmappings) of the child's newly
6319e39c5baSBill Taylor 	 * invalidated resource.  Note: Setting the "hdt_size" field to
6329e39c5baSBill Taylor 	 * zero here is an indication to the devmap_unmap() entry point
6339e39c5baSBill Taylor 	 * that this mapping is invalid, and that its subsequent unmapping
6349e39c5baSBill Taylor 	 * should not affect any of the parent's CQ or QP resources.
6359e39c5baSBill Taylor 	 */
6369e39c5baSBill Taylor 	new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
6379e39c5baSBill Taylor 	    sizeof (hermon_devmap_track_t), KM_SLEEP);
6389e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track))
6399e39c5baSBill Taylor 	new_dvm_track->hdt_offset = 0;
6409e39c5baSBill Taylor 	new_dvm_track->hdt_state  = state;
6419e39c5baSBill Taylor 	new_dvm_track->hdt_refcnt = 1;
6429e39c5baSBill Taylor 	new_dvm_track->hdt_size	  = 0;
6439e39c5baSBill Taylor 	mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
6449e39c5baSBill Taylor 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
6459e39c5baSBill Taylor 	*new_pvtp = new_dvm_track;
6469e39c5baSBill Taylor 
6479e39c5baSBill Taylor 	return (DDI_SUCCESS);
6489e39c5baSBill Taylor }
6499e39c5baSBill Taylor 
6509e39c5baSBill Taylor 
6519e39c5baSBill Taylor /*
6529e39c5baSBill Taylor  * hermon_devmap_umem_unmap()
6539e39c5baSBill Taylor  *    Context: Can be called from kernel context.
6549e39c5baSBill Taylor  */
6559e39c5baSBill Taylor /* ARGSUSED */
6569e39c5baSBill Taylor static void
6579e39c5baSBill Taylor hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
6589e39c5baSBill Taylor     size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
6599e39c5baSBill Taylor     devmap_cookie_t new_dhp2, void **pvtp2)
6609e39c5baSBill Taylor {
6619e39c5baSBill Taylor 	hermon_state_t 		*state;
6629e39c5baSBill Taylor 	hermon_rsrc_t 		*rsrcp;
6639e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track;
6649e39c5baSBill Taylor 	hermon_cqhdl_t		cq;
6659e39c5baSBill Taylor 	hermon_qphdl_t		qp;
6669e39c5baSBill Taylor 	hermon_srqhdl_t		srq;
6679e39c5baSBill Taylor 	uint64_t		key, value;
6689e39c5baSBill Taylor 	uint_t			type;
6699e39c5baSBill Taylor 	uint_t			size;
6709e39c5baSBill Taylor 	int			status;
6719e39c5baSBill Taylor 
6729e39c5baSBill Taylor 	/*
6739e39c5baSBill Taylor 	 * Extract the Hermon softstate pointer from "Hermon devmap tracking
6749e39c5baSBill Taylor 	 * structure" (in "pvtp").
6759e39c5baSBill Taylor 	 */
6769e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)pvtp;
6779e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
6789e39c5baSBill Taylor 	state	  = dvm_track->hdt_state;
6799e39c5baSBill Taylor 
6809e39c5baSBill Taylor 	/*
6819e39c5baSBill Taylor 	 * Extract the "offset" from the "Hermon devmap tracking structure".
6829e39c5baSBill Taylor 	 * Note: The input argument "off" is ignored here because the
6839e39c5baSBill Taylor 	 * Hermon mapping interfaces define a very specific meaning to
6849e39c5baSBill Taylor 	 * each "logical offset".  Also extract the "key" and "type" encoded
6859e39c5baSBill Taylor 	 * in the logical offset.
6869e39c5baSBill Taylor 	 */
6879e39c5baSBill Taylor 	key  = dvm_track->hdt_offset >> PAGESHIFT;
6889e39c5baSBill Taylor 	type = key & MLNX_UMAP_RSRC_TYPE_MASK;
6899e39c5baSBill Taylor 	key  = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
6909e39c5baSBill Taylor 
6919e39c5baSBill Taylor 	/*
6929e39c5baSBill Taylor 	 * Extract the "size" of the mapping.  If this size is determined
6939e39c5baSBill Taylor 	 * to be zero, then it is an indication of a previously invalidated
6949e39c5baSBill Taylor 	 * mapping, and no CQ or QP resources should be affected.
6959e39c5baSBill Taylor 	 */
6969e39c5baSBill Taylor 	size = dvm_track->hdt_size;
6979e39c5baSBill Taylor 
6989e39c5baSBill Taylor 	/*
6999e39c5baSBill Taylor 	 * If only the "middle portion of a given mapping is being unmapped,
7009e39c5baSBill Taylor 	 * then we are effectively creating one new piece of mapped memory.
7019e39c5baSBill Taylor 	 * (Original region is divided into three pieces of which the middle
7029e39c5baSBill Taylor 	 * piece is being removed.  This leaves two pieces.  Since we started
7039e39c5baSBill Taylor 	 * with one piece and now have two pieces, we need to increment the
7049e39c5baSBill Taylor 	 * counter in the "Hermon devmap tracking structure".
7059e39c5baSBill Taylor 	 *
7069e39c5baSBill Taylor 	 * If, however, the whole mapped region is being unmapped, then we
7079e39c5baSBill Taylor 	 * have started with one region which we are completely removing.
7089e39c5baSBill Taylor 	 * In this case, we need to decrement the counter in the "Hermon
7099e39c5baSBill Taylor 	 * devmap tracking structure".
7109e39c5baSBill Taylor 	 *
7119e39c5baSBill Taylor 	 * In each of the remaining cases, we will have started with one
7129e39c5baSBill Taylor 	 * mapped region and ended with one (different) region.  So no counter
7139e39c5baSBill Taylor 	 * modification is necessary.
7149e39c5baSBill Taylor 	 */
7159e39c5baSBill Taylor 	mutex_enter(&dvm_track->hdt_lock);
7169e39c5baSBill Taylor 	if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) {
7179e39c5baSBill Taylor 		dvm_track->hdt_refcnt--;
7189e39c5baSBill Taylor 	} else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) {
7199e39c5baSBill Taylor 		dvm_track->hdt_refcnt++;
7209e39c5baSBill Taylor 	}
7219e39c5baSBill Taylor 	mutex_exit(&dvm_track->hdt_lock);
7229e39c5baSBill Taylor 
7239e39c5baSBill Taylor 	/*
7249e39c5baSBill Taylor 	 * For each of the cases where the region is being divided, then we
7259e39c5baSBill Taylor 	 * need to pass back the "Hermon devmap tracking structure".  This way
7269e39c5baSBill Taylor 	 * we get it back when each of the remaining pieces is subsequently
7279e39c5baSBill Taylor 	 * unmapped.
7289e39c5baSBill Taylor 	 */
7299e39c5baSBill Taylor 	if (new_dhp1 != NULL) {
7309e39c5baSBill Taylor 		*pvtp1 = pvtp;
7319e39c5baSBill Taylor 	}
7329e39c5baSBill Taylor 	if (new_dhp2 != NULL) {
7339e39c5baSBill Taylor 		*pvtp2 = pvtp;
7349e39c5baSBill Taylor 	}
7359e39c5baSBill Taylor 
7369e39c5baSBill Taylor 	/*
7379e39c5baSBill Taylor 	 * If the "Hermon devmap tracking structure" is no longer being
7389e39c5baSBill Taylor 	 * referenced, then free it up.  Otherwise, return.
7399e39c5baSBill Taylor 	 */
7409e39c5baSBill Taylor 	if (dvm_track->hdt_refcnt == 0) {
7419e39c5baSBill Taylor 		mutex_destroy(&dvm_track->hdt_lock);
7429e39c5baSBill Taylor 		kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
7439e39c5baSBill Taylor 
7449e39c5baSBill Taylor 		/*
7459e39c5baSBill Taylor 		 * If the mapping was invalid (see explanation above), then
7469e39c5baSBill Taylor 		 * no further processing is necessary.
7479e39c5baSBill Taylor 		 */
7489e39c5baSBill Taylor 		if (size == 0) {
7499e39c5baSBill Taylor 			return;
7509e39c5baSBill Taylor 		}
7519e39c5baSBill Taylor 	} else {
7529e39c5baSBill Taylor 		return;
7539e39c5baSBill Taylor 	}
7549e39c5baSBill Taylor 
7559e39c5baSBill Taylor 	/*
7569e39c5baSBill Taylor 	 * Now that we can guarantee that the user memory is fully unmapped,
7579e39c5baSBill Taylor 	 * we can use the "key" and "type" values to try to find the entry
7589e39c5baSBill Taylor 	 * in the "userland resources database".  If it's found, then it
7599e39c5baSBill Taylor 	 * indicates that the queue memory (CQ or QP) has not yet been freed.
7609e39c5baSBill Taylor 	 * In this case, we update the corresponding CQ or QP handle to
7619e39c5baSBill Taylor 	 * indicate that the "devmap_devmem_remap()" call will be unnecessary.
7629e39c5baSBill Taylor 	 * If it's _not_ found, then it indicates that the CQ or QP memory
7639e39c5baSBill Taylor 	 * was, in fact, freed before it was unmapped (thus requiring a
7649e39c5baSBill Taylor 	 * previous invalidation by remapping - which will already have
7659e39c5baSBill Taylor 	 * been done in the free routine).
7669e39c5baSBill Taylor 	 */
7679e39c5baSBill Taylor 	status = hermon_umap_db_find(state->hs_instance, key, type, &value,
7689e39c5baSBill Taylor 	    0, NULL);
7699e39c5baSBill Taylor 	if (status == DDI_SUCCESS) {
7709e39c5baSBill Taylor 		/*
7719e39c5baSBill Taylor 		 * Depending on the type of the mapped resource (CQ or QP),
7729e39c5baSBill Taylor 		 * update handle to indicate that no invalidation remapping
7739e39c5baSBill Taylor 		 * will be necessary.
7749e39c5baSBill Taylor 		 */
7759e39c5baSBill Taylor 		if (type == MLNX_UMAP_CQMEM_RSRC) {
7769e39c5baSBill Taylor 
7779e39c5baSBill Taylor 			/* Use "value" to convert to CQ handle */
7789e39c5baSBill Taylor 			rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
7799e39c5baSBill Taylor 			cq = (hermon_cqhdl_t)rsrcp->hr_addr;
7809e39c5baSBill Taylor 
7819e39c5baSBill Taylor 			/*
7829e39c5baSBill Taylor 			 * Invalidate the handle to the userland mapping.
7839e39c5baSBill Taylor 			 * Note: We must ensure that the mapping being
7849e39c5baSBill Taylor 			 * unmapped here is the current one for the CQ.  It
7859e39c5baSBill Taylor 			 * is possible that it might not be if this CQ has
7869e39c5baSBill Taylor 			 * been resized and the previous CQ memory has not
7879e39c5baSBill Taylor 			 * yet been unmapped.  But in that case, because of
7889e39c5baSBill Taylor 			 * the devmap_devmem_remap(), there is no longer any
7899e39c5baSBill Taylor 			 * association between the mapping and the real CQ
7909e39c5baSBill Taylor 			 * kernel memory.
7919e39c5baSBill Taylor 			 */
7929e39c5baSBill Taylor 			mutex_enter(&cq->cq_lock);
7939e39c5baSBill Taylor 			if (cq->cq_umap_dhp == dhp) {
7949e39c5baSBill Taylor 				cq->cq_umap_dhp = NULL;
7959e39c5baSBill Taylor 				if (cq->cq_resize_hdl) {
7969e39c5baSBill Taylor 					/* resize is DONE, switch queues */
7979e39c5baSBill Taylor 					hermon_cq_resize_helper(state, cq);
7989e39c5baSBill Taylor 				}
7999e39c5baSBill Taylor 			} else {
8009e39c5baSBill Taylor 				if (cq->cq_resize_hdl &&
8019e39c5baSBill Taylor 				    cq->cq_resize_hdl->cq_umap_dhp == dhp) {
8029e39c5baSBill Taylor 					/*
8039e39c5baSBill Taylor 					 * Unexpected case.  munmap of the
8049e39c5baSBill Taylor 					 * cq_resize buf, and not the
8059e39c5baSBill Taylor 					 * original buf.
8069e39c5baSBill Taylor 					 */
8079e39c5baSBill Taylor 					cq->cq_resize_hdl->cq_umap_dhp = NULL;
8089e39c5baSBill Taylor 				}
8099e39c5baSBill Taylor 			}
8109e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
8119e39c5baSBill Taylor 
8129e39c5baSBill Taylor 		} else if (type == MLNX_UMAP_QPMEM_RSRC) {
8139e39c5baSBill Taylor 
8149e39c5baSBill Taylor 			/* Use "value" to convert to QP handle */
8159e39c5baSBill Taylor 			rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
8169e39c5baSBill Taylor 			qp = (hermon_qphdl_t)rsrcp->hr_addr;
8179e39c5baSBill Taylor 
8189e39c5baSBill Taylor 			/*
8199e39c5baSBill Taylor 			 * Invalidate the handle to the userland mapping.
8209e39c5baSBill Taylor 			 * Note: we ensure that the mapping being unmapped
8219e39c5baSBill Taylor 			 * here is the current one for the QP.  This is
8229e39c5baSBill Taylor 			 * more of a sanity check here since, unlike CQs
8239e39c5baSBill Taylor 			 * (above) we do not support resize of QPs.
8249e39c5baSBill Taylor 			 */
8259e39c5baSBill Taylor 			mutex_enter(&qp->qp_lock);
8269e39c5baSBill Taylor 			if (qp->qp_umap_dhp == dhp) {
8279e39c5baSBill Taylor 				qp->qp_umap_dhp = NULL;
8289e39c5baSBill Taylor 			}
8299e39c5baSBill Taylor 			mutex_exit(&qp->qp_lock);
8309e39c5baSBill Taylor 
8319e39c5baSBill Taylor 		} else if (type == MLNX_UMAP_SRQMEM_RSRC) {
8329e39c5baSBill Taylor 
8339e39c5baSBill Taylor 			/* Use "value" to convert to SRQ handle */
8349e39c5baSBill Taylor 			rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
8359e39c5baSBill Taylor 			srq = (hermon_srqhdl_t)rsrcp->hr_addr;
8369e39c5baSBill Taylor 
8379e39c5baSBill Taylor 			/*
8389e39c5baSBill Taylor 			 * Invalidate the handle to the userland mapping.
8399e39c5baSBill Taylor 			 * Note: we ensure that the mapping being unmapped
8409e39c5baSBill Taylor 			 * here is the current one for the QP.  This is
8419e39c5baSBill Taylor 			 * more of a sanity check here since, unlike CQs
8429e39c5baSBill Taylor 			 * (above) we do not support resize of QPs.
8439e39c5baSBill Taylor 			 */
8449e39c5baSBill Taylor 			mutex_enter(&srq->srq_lock);
8459e39c5baSBill Taylor 			if (srq->srq_umap_dhp == dhp) {
8469e39c5baSBill Taylor 				srq->srq_umap_dhp = NULL;
8479e39c5baSBill Taylor 			}
8489e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
8499e39c5baSBill Taylor 		}
8509e39c5baSBill Taylor 	}
8519e39c5baSBill Taylor }
8529e39c5baSBill Taylor 
8539e39c5baSBill Taylor 
8549e39c5baSBill Taylor /*
8559e39c5baSBill Taylor  * hermon_devmap_devmem_map()
8569e39c5baSBill Taylor  *    Context: Can be called from kernel context.
8579e39c5baSBill Taylor  */
8589e39c5baSBill Taylor /* ARGSUSED */
8599e39c5baSBill Taylor static int
8609e39c5baSBill Taylor hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
8619e39c5baSBill Taylor     offset_t off, size_t len, void **pvtp)
8629e39c5baSBill Taylor {
8639e39c5baSBill Taylor 	hermon_state_t		*state;
8649e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track;
8659e39c5baSBill Taylor 	hermon_cqhdl_t		cq;
8669e39c5baSBill Taylor 	hermon_qphdl_t		qp;
8679e39c5baSBill Taylor 	hermon_srqhdl_t		srq;
8689e39c5baSBill Taylor 	minor_t			instance;
8699e39c5baSBill Taylor 	uint64_t		key;
8709e39c5baSBill Taylor 	uint_t			type;
8719e39c5baSBill Taylor 
8729e39c5baSBill Taylor 	/* Get Hermon softstate structure from instance */
8739e39c5baSBill Taylor 	instance = HERMON_DEV_INSTANCE(dev);
8749e39c5baSBill Taylor 	state = ddi_get_soft_state(hermon_statep, instance);
8759e39c5baSBill Taylor 	if (state == NULL) {
8769e39c5baSBill Taylor 		return (ENXIO);
8779e39c5baSBill Taylor 	}
8789e39c5baSBill Taylor 
8799e39c5baSBill Taylor 	/*
8809e39c5baSBill Taylor 	 * The bottom bits of "offset" are undefined (number depends on
8819e39c5baSBill Taylor 	 * system PAGESIZE).  Shifting these off leaves us with a "key".
8829e39c5baSBill Taylor 	 * The "key" is actually a combination of both a real key value
8839e39c5baSBill Taylor 	 * (for the purpose of database lookup) and a "type" value.  Although
8849e39c5baSBill Taylor 	 * we are not going to do any database lookup per se, we do want
8859e39c5baSBill Taylor 	 * to extract the "key" and the "type" (to enable faster lookup of
8869e39c5baSBill Taylor 	 * the appropriate CQ or QP handle).
8879e39c5baSBill Taylor 	 */
8889e39c5baSBill Taylor 	key  = off >> PAGESHIFT;
8899e39c5baSBill Taylor 	type = key & MLNX_UMAP_RSRC_TYPE_MASK;
8909e39c5baSBill Taylor 	key  = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
8919e39c5baSBill Taylor 
8929e39c5baSBill Taylor 	/*
8939e39c5baSBill Taylor 	 * Allocate an entry to track the mapping and unmapping (specifically,
8949e39c5baSBill Taylor 	 * partial unmapping) of this resource.
8959e39c5baSBill Taylor 	 */
8969e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
8979e39c5baSBill Taylor 	    sizeof (hermon_devmap_track_t), KM_SLEEP);
8989e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
8999e39c5baSBill Taylor 	dvm_track->hdt_offset = off;
9009e39c5baSBill Taylor 	dvm_track->hdt_state  = state;
9019e39c5baSBill Taylor 	dvm_track->hdt_refcnt = 1;
9029e39c5baSBill Taylor 	mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
9039e39c5baSBill Taylor 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
9049e39c5baSBill Taylor 
9059e39c5baSBill Taylor 	/*
9069e39c5baSBill Taylor 	 * Depending of the type of resource that has been mapped out, we
9079e39c5baSBill Taylor 	 * need to update the QP or CQ handle to reflect that it has, in
9089e39c5baSBill Taylor 	 * fact, been mapped.  This allows the driver code which frees a QP
9099e39c5baSBill Taylor 	 * or a CQ to know whether it is appropriate to do a
9109e39c5baSBill Taylor 	 * devmap_devmem_remap() to invalidate the userland mapping for the
9119e39c5baSBill Taylor 	 * corresponding queue's memory.
9129e39c5baSBill Taylor 	 */
9139e39c5baSBill Taylor 	if (type == MLNX_UMAP_CQMEM_RSRC) {
9149e39c5baSBill Taylor 
9159e39c5baSBill Taylor 		/* Use "key" (CQ number) to do fast lookup of CQ handle */
9169e39c5baSBill Taylor 		cq = hermon_cqhdl_from_cqnum(state, key);
9179e39c5baSBill Taylor 
9189e39c5baSBill Taylor 		/*
9199e39c5baSBill Taylor 		 * Update the handle to the userland mapping.  Note:  If
9209e39c5baSBill Taylor 		 * the CQ already has a valid userland mapping, then stop
9219e39c5baSBill Taylor 		 * and return failure.
9229e39c5baSBill Taylor 		 */
9239e39c5baSBill Taylor 		mutex_enter(&cq->cq_lock);
9249e39c5baSBill Taylor 		if (cq->cq_umap_dhp == NULL) {
9259e39c5baSBill Taylor 			cq->cq_umap_dhp = dhp;
9269e39c5baSBill Taylor 			dvm_track->hdt_size = cq->cq_cqinfo.qa_size;
9279e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
9289e39c5baSBill Taylor 		} else {
9299e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
9309e39c5baSBill Taylor 			goto umem_map_fail;
9319e39c5baSBill Taylor 		}
9329e39c5baSBill Taylor 
9339e39c5baSBill Taylor 	} else if (type == MLNX_UMAP_QPMEM_RSRC) {
9349e39c5baSBill Taylor 
9359e39c5baSBill Taylor 		/* Use "key" (QP number) to do fast lookup of QP handle */
9369e39c5baSBill Taylor 		qp = hermon_qphdl_from_qpnum(state, key);
9379e39c5baSBill Taylor 
9389e39c5baSBill Taylor 		/*
9399e39c5baSBill Taylor 		 * Update the handle to the userland mapping.  Note:  If
9409e39c5baSBill Taylor 		 * the CQ already has a valid userland mapping, then stop
9419e39c5baSBill Taylor 		 * and return failure.
9429e39c5baSBill Taylor 		 */
9439e39c5baSBill Taylor 		mutex_enter(&qp->qp_lock);
9449e39c5baSBill Taylor 		if (qp->qp_umap_dhp == NULL) {
9459e39c5baSBill Taylor 			qp->qp_umap_dhp = dhp;
9469e39c5baSBill Taylor 			dvm_track->hdt_size = qp->qp_wqinfo.qa_size;
9479e39c5baSBill Taylor 			mutex_exit(&qp->qp_lock);
9489e39c5baSBill Taylor 		} else {
9499e39c5baSBill Taylor 			mutex_exit(&qp->qp_lock);
9509e39c5baSBill Taylor 			goto umem_map_fail;
9519e39c5baSBill Taylor 		}
9529e39c5baSBill Taylor 
9539e39c5baSBill Taylor 	} else if (type == MLNX_UMAP_SRQMEM_RSRC) {
9549e39c5baSBill Taylor 
9559e39c5baSBill Taylor 		/* Use "key" (SRQ number) to do fast lookup on SRQ handle */
9569e39c5baSBill Taylor 		srq = hermon_srqhdl_from_srqnum(state, key);
9579e39c5baSBill Taylor 
9589e39c5baSBill Taylor 		/*
9599e39c5baSBill Taylor 		 * Update the handle to the userland mapping.  Note:  If the
9609e39c5baSBill Taylor 		 * SRQ already has a valid userland mapping, then stop and
9619e39c5baSBill Taylor 		 * return failure.
9629e39c5baSBill Taylor 		 */
9639e39c5baSBill Taylor 		mutex_enter(&srq->srq_lock);
9649e39c5baSBill Taylor 		if (srq->srq_umap_dhp == NULL) {
9659e39c5baSBill Taylor 			srq->srq_umap_dhp = dhp;
9669e39c5baSBill Taylor 			dvm_track->hdt_size = srq->srq_wqinfo.qa_size;
9679e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
9689e39c5baSBill Taylor 		} else {
9699e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
9709e39c5baSBill Taylor 			goto umem_map_fail;
9719e39c5baSBill Taylor 		}
9729e39c5baSBill Taylor 	}
9739e39c5baSBill Taylor 
9749e39c5baSBill Taylor 	/*
9759e39c5baSBill Taylor 	 * Pass the private "Hermon devmap tracking structure" back.  This
9769e39c5baSBill Taylor 	 * pointer will be returned in subsequent "unmap" callbacks.
9779e39c5baSBill Taylor 	 */
9789e39c5baSBill Taylor 	*pvtp = dvm_track;
9799e39c5baSBill Taylor 
9809e39c5baSBill Taylor 	return (DDI_SUCCESS);
9819e39c5baSBill Taylor 
9829e39c5baSBill Taylor umem_map_fail:
9839e39c5baSBill Taylor 	mutex_destroy(&dvm_track->hdt_lock);
9849e39c5baSBill Taylor 	kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
9859e39c5baSBill Taylor 	return (DDI_FAILURE);
9869e39c5baSBill Taylor }
9879e39c5baSBill Taylor 
9889e39c5baSBill Taylor 
9899e39c5baSBill Taylor /*
9909e39c5baSBill Taylor  * hermon_devmap_dbrecmem_dup()
9919e39c5baSBill Taylor  *    Context: Can be called from kernel context.
9929e39c5baSBill Taylor  */
9939e39c5baSBill Taylor /* ARGSUSED */
9949e39c5baSBill Taylor static int
9959e39c5baSBill Taylor hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp,
9969e39c5baSBill Taylor     devmap_cookie_t new_dhp, void **new_pvtp)
9979e39c5baSBill Taylor {
9989e39c5baSBill Taylor 	hermon_state_t		*state;
9999e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track, *new_dvm_track;
10009e39c5baSBill Taylor 	uint_t			maxprot;
10019e39c5baSBill Taylor 	int			status;
10029e39c5baSBill Taylor 
10039e39c5baSBill Taylor 	/*
10049e39c5baSBill Taylor 	 * Extract the Hermon softstate pointer from "Hermon devmap tracking
10059e39c5baSBill Taylor 	 * structure" (in "pvtp").
10069e39c5baSBill Taylor 	 */
10079e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)pvtp;
10089e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
10099e39c5baSBill Taylor 	state = dvm_track->hdt_state;
10109e39c5baSBill Taylor 
10119e39c5baSBill Taylor 	/*
10129e39c5baSBill Taylor 	 * Since this devmap_dup() entry point is generally called
10139e39c5baSBill Taylor 	 * when a process does fork(2), it is incumbent upon the driver
10149e39c5baSBill Taylor 	 * to insure that the child does not inherit a valid copy of
10159e39c5baSBill Taylor 	 * the parent's QP or CQ resource.  This is accomplished by using
10169e39c5baSBill Taylor 	 * devmap_devmem_remap() to invalidate the child's mapping to the
10179e39c5baSBill Taylor 	 * kernel memory.
10189e39c5baSBill Taylor 	 */
10199e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
10209e39c5baSBill Taylor 	status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
10219e39c5baSBill Taylor 	    dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
10229e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
10239e39c5baSBill Taylor 		HERMON_WARNING(state, "failed in hermon_devmap_dbrecmem_dup()");
10249e39c5baSBill Taylor 		return (status);
10259e39c5baSBill Taylor 	}
10269e39c5baSBill Taylor 
10279e39c5baSBill Taylor 	/*
10289e39c5baSBill Taylor 	 * Allocate a new entry to track the subsequent unmapping
10299e39c5baSBill Taylor 	 * (specifically, all partial unmappings) of the child's newly
10309e39c5baSBill Taylor 	 * invalidated resource.  Note: Setting the "hdt_size" field to
10319e39c5baSBill Taylor 	 * zero here is an indication to the devmap_unmap() entry point
10329e39c5baSBill Taylor 	 * that this mapping is invalid, and that its subsequent unmapping
10339e39c5baSBill Taylor 	 * should not affect any of the parent's CQ or QP resources.
10349e39c5baSBill Taylor 	 */
10359e39c5baSBill Taylor 	new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
10369e39c5baSBill Taylor 	    sizeof (hermon_devmap_track_t), KM_SLEEP);
10379e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track))
10389e39c5baSBill Taylor 	new_dvm_track->hdt_offset = 0;
10399e39c5baSBill Taylor 	new_dvm_track->hdt_state  = state;
10409e39c5baSBill Taylor 	new_dvm_track->hdt_refcnt = 1;
10419e39c5baSBill Taylor 	new_dvm_track->hdt_size	  = 0;
10429e39c5baSBill Taylor 	mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER,
10439e39c5baSBill Taylor 	    DDI_INTR_PRI(state->hs_intrmsi_pri));
10449e39c5baSBill Taylor 	*new_pvtp = new_dvm_track;
10459e39c5baSBill Taylor 
10469e39c5baSBill Taylor 	return (DDI_SUCCESS);
10479e39c5baSBill Taylor }
10489e39c5baSBill Taylor 
10499e39c5baSBill Taylor 
10509e39c5baSBill Taylor /*
10519e39c5baSBill Taylor  * hermon_devmap_dbrecmem_unmap()
10529e39c5baSBill Taylor  *    Context: Can be called from kernel context.
10539e39c5baSBill Taylor  */
10549e39c5baSBill Taylor /* ARGSUSED */
10559e39c5baSBill Taylor static void
10569e39c5baSBill Taylor hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
10579e39c5baSBill Taylor     size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
10589e39c5baSBill Taylor     devmap_cookie_t new_dhp2, void **pvtp2)
10599e39c5baSBill Taylor {
10609e39c5baSBill Taylor 	hermon_state_t 		*state;
10619e39c5baSBill Taylor 	hermon_rsrc_t 		*rsrcp;
10629e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track;
10639e39c5baSBill Taylor 	hermon_cqhdl_t		cq;
10649e39c5baSBill Taylor 	hermon_qphdl_t		qp;
10659e39c5baSBill Taylor 	hermon_srqhdl_t		srq;
10669e39c5baSBill Taylor 	uint64_t		key, value;
10679e39c5baSBill Taylor 	uint_t			type;
10689e39c5baSBill Taylor 	uint_t			size;
10699e39c5baSBill Taylor 	int			status;
10709e39c5baSBill Taylor 
10719e39c5baSBill Taylor 	/*
10729e39c5baSBill Taylor 	 * Extract the Hermon softstate pointer from "Hermon devmap tracking
10739e39c5baSBill Taylor 	 * structure" (in "pvtp").
10749e39c5baSBill Taylor 	 */
10759e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)pvtp;
10769e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
10779e39c5baSBill Taylor 	state	  = dvm_track->hdt_state;
10789e39c5baSBill Taylor 
10799e39c5baSBill Taylor 	/*
10809e39c5baSBill Taylor 	 * Extract the "offset" from the "Hermon devmap tracking structure".
10819e39c5baSBill Taylor 	 * Note: The input argument "off" is ignored here because the
10829e39c5baSBill Taylor 	 * Hermon mapping interfaces define a very specific meaning to
10839e39c5baSBill Taylor 	 * each "logical offset".  Also extract the "key" and "type" encoded
10849e39c5baSBill Taylor 	 * in the logical offset.
10859e39c5baSBill Taylor 	 */
10869e39c5baSBill Taylor 	key  = dvm_track->hdt_offset >> PAGESHIFT;
10879e39c5baSBill Taylor 	type = key & MLNX_UMAP_RSRC_TYPE_MASK;
10889e39c5baSBill Taylor 	key  = key >> MLNX_UMAP_RSRC_TYPE_SHIFT;
10899e39c5baSBill Taylor 
10909e39c5baSBill Taylor 	/*
10919e39c5baSBill Taylor 	 * Extract the "size" of the mapping.  If this size is determined
10929e39c5baSBill Taylor 	 * to be zero, then it is an indication of a previously invalidated
10939e39c5baSBill Taylor 	 * mapping, and no CQ or QP resources should be affected.
10949e39c5baSBill Taylor 	 */
10959e39c5baSBill Taylor 	size = dvm_track->hdt_size;
10969e39c5baSBill Taylor 
10979e39c5baSBill Taylor 	/*
10989e39c5baSBill Taylor 	 * If only the "middle portion of a given mapping is being unmapped,
10999e39c5baSBill Taylor 	 * then we are effectively creating one new piece of mapped memory.
11009e39c5baSBill Taylor 	 * (Original region is divided into three pieces of which the middle
11019e39c5baSBill Taylor 	 * piece is being removed.  This leaves two pieces.  Since we started
11029e39c5baSBill Taylor 	 * with one piece and now have two pieces, we need to increment the
11039e39c5baSBill Taylor 	 * counter in the "Hermon devmap tracking structure".
11049e39c5baSBill Taylor 	 *
11059e39c5baSBill Taylor 	 * If, however, the whole mapped region is being unmapped, then we
11069e39c5baSBill Taylor 	 * have started with one region which we are completely removing.
11079e39c5baSBill Taylor 	 * In this case, we need to decrement the counter in the "Hermon
11089e39c5baSBill Taylor 	 * devmap tracking structure".
11099e39c5baSBill Taylor 	 *
11109e39c5baSBill Taylor 	 * In each of the remaining cases, we will have started with one
11119e39c5baSBill Taylor 	 * mapped region and ended with one (different) region.  So no counter
11129e39c5baSBill Taylor 	 * modification is necessary.
11139e39c5baSBill Taylor 	 */
11149e39c5baSBill Taylor 	mutex_enter(&dvm_track->hdt_lock);
11159e39c5baSBill Taylor 	if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) {
11169e39c5baSBill Taylor 		dvm_track->hdt_refcnt--;
11179e39c5baSBill Taylor 	} else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) {
11189e39c5baSBill Taylor 		dvm_track->hdt_refcnt++;
11199e39c5baSBill Taylor 	}
11209e39c5baSBill Taylor 	mutex_exit(&dvm_track->hdt_lock);
11219e39c5baSBill Taylor 
11229e39c5baSBill Taylor 	/*
11239e39c5baSBill Taylor 	 * For each of the cases where the region is being divided, then we
11249e39c5baSBill Taylor 	 * need to pass back the "Hermon devmap tracking structure".  This way
11259e39c5baSBill Taylor 	 * we get it back when each of the remaining pieces is subsequently
11269e39c5baSBill Taylor 	 * unmapped.
11279e39c5baSBill Taylor 	 */
11289e39c5baSBill Taylor 	if (new_dhp1 != NULL) {
11299e39c5baSBill Taylor 		*pvtp1 = pvtp;
11309e39c5baSBill Taylor 	}
11319e39c5baSBill Taylor 	if (new_dhp2 != NULL) {
11329e39c5baSBill Taylor 		*pvtp2 = pvtp;
11339e39c5baSBill Taylor 	}
11349e39c5baSBill Taylor 
11359e39c5baSBill Taylor 	/*
11369e39c5baSBill Taylor 	 * If the "Hermon devmap tracking structure" is no longer being
11379e39c5baSBill Taylor 	 * referenced, then free it up.  Otherwise, return.
11389e39c5baSBill Taylor 	 */
11399e39c5baSBill Taylor 	if (dvm_track->hdt_refcnt == 0) {
11409e39c5baSBill Taylor 		mutex_destroy(&dvm_track->hdt_lock);
11419e39c5baSBill Taylor 		kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
11429e39c5baSBill Taylor 
11439e39c5baSBill Taylor 		/*
11449e39c5baSBill Taylor 		 * If the mapping was invalid (see explanation above), then
11459e39c5baSBill Taylor 		 * no further processing is necessary.
11469e39c5baSBill Taylor 		 */
11479e39c5baSBill Taylor 		if (size == 0) {
11489e39c5baSBill Taylor 			return;
11499e39c5baSBill Taylor 		}
11509e39c5baSBill Taylor 	} else {
11519e39c5baSBill Taylor 		return;
11529e39c5baSBill Taylor 	}
11539e39c5baSBill Taylor 
11549e39c5baSBill Taylor 	/*
11559e39c5baSBill Taylor 	 * Now that we can guarantee that the user memory is fully unmapped,
11569e39c5baSBill Taylor 	 * we can use the "key" and "type" values to try to find the entry
11579e39c5baSBill Taylor 	 * in the "userland resources database".  If it's found, then it
11589e39c5baSBill Taylor 	 * indicates that the queue memory (CQ or QP) has not yet been freed.
11599e39c5baSBill Taylor 	 * In this case, we update the corresponding CQ or QP handle to
11609e39c5baSBill Taylor 	 * indicate that the "devmap_devmem_remap()" call will be unnecessary.
11619e39c5baSBill Taylor 	 * If it's _not_ found, then it indicates that the CQ or QP memory
11629e39c5baSBill Taylor 	 * was, in fact, freed before it was unmapped (thus requiring a
11639e39c5baSBill Taylor 	 * previous invalidation by remapping - which will already have
11649e39c5baSBill Taylor 	 * been done in the free routine).
11659e39c5baSBill Taylor 	 */
11669e39c5baSBill Taylor 	status = hermon_umap_db_find(state->hs_instance, key, type, &value,
11679e39c5baSBill Taylor 	    0, NULL);
11689e39c5baSBill Taylor 	if (status == DDI_SUCCESS) {
11699e39c5baSBill Taylor 		/*
11709e39c5baSBill Taylor 		 * Depending on the type of the mapped resource (CQ or QP),
11719e39c5baSBill Taylor 		 * update handle to indicate that no invalidation remapping
11729e39c5baSBill Taylor 		 * will be necessary.
11739e39c5baSBill Taylor 		 */
11749e39c5baSBill Taylor 		if (type == MLNX_UMAP_CQMEM_RSRC) {
11759e39c5baSBill Taylor 
11769e39c5baSBill Taylor 			/* Use "value" to convert to CQ handle */
11779e39c5baSBill Taylor 			rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
11789e39c5baSBill Taylor 			cq = (hermon_cqhdl_t)rsrcp->hr_addr;
11799e39c5baSBill Taylor 
11809e39c5baSBill Taylor 			/*
11819e39c5baSBill Taylor 			 * Invalidate the handle to the userland mapping.
11829e39c5baSBill Taylor 			 * Note: We must ensure that the mapping being
11839e39c5baSBill Taylor 			 * unmapped here is the current one for the CQ.  It
11849e39c5baSBill Taylor 			 * is possible that it might not be if this CQ has
11859e39c5baSBill Taylor 			 * been resized and the previous CQ memory has not
11869e39c5baSBill Taylor 			 * yet been unmapped.  But in that case, because of
11879e39c5baSBill Taylor 			 * the devmap_devmem_remap(), there is no longer any
11889e39c5baSBill Taylor 			 * association between the mapping and the real CQ
11899e39c5baSBill Taylor 			 * kernel memory.
11909e39c5baSBill Taylor 			 */
11919e39c5baSBill Taylor 			mutex_enter(&cq->cq_lock);
11929e39c5baSBill Taylor 			if (cq->cq_umap_dhp == dhp) {
11939e39c5baSBill Taylor 				cq->cq_umap_dhp = NULL;
11949e39c5baSBill Taylor 			}
11959e39c5baSBill Taylor 			mutex_exit(&cq->cq_lock);
11969e39c5baSBill Taylor 
11979e39c5baSBill Taylor 		} else if (type == MLNX_UMAP_QPMEM_RSRC) {
11989e39c5baSBill Taylor 
11999e39c5baSBill Taylor 			/* Use "value" to convert to QP handle */
12009e39c5baSBill Taylor 			rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
12019e39c5baSBill Taylor 			qp = (hermon_qphdl_t)rsrcp->hr_addr;
12029e39c5baSBill Taylor 
12039e39c5baSBill Taylor 			/*
12049e39c5baSBill Taylor 			 * Invalidate the handle to the userland mapping.
12059e39c5baSBill Taylor 			 * Note: we ensure that the mapping being unmapped
12069e39c5baSBill Taylor 			 * here is the current one for the QP.  This is
12079e39c5baSBill Taylor 			 * more of a sanity check here since, unlike CQs
12089e39c5baSBill Taylor 			 * (above) we do not support resize of QPs.
12099e39c5baSBill Taylor 			 */
12109e39c5baSBill Taylor 			mutex_enter(&qp->qp_lock);
12119e39c5baSBill Taylor 			if (qp->qp_umap_dhp == dhp) {
12129e39c5baSBill Taylor 				qp->qp_umap_dhp = NULL;
12139e39c5baSBill Taylor 			}
12149e39c5baSBill Taylor 			mutex_exit(&qp->qp_lock);
12159e39c5baSBill Taylor 
12169e39c5baSBill Taylor 		} else if (type == MLNX_UMAP_SRQMEM_RSRC) {
12179e39c5baSBill Taylor 
12189e39c5baSBill Taylor 			/* Use "value" to convert to SRQ handle */
12199e39c5baSBill Taylor 			rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
12209e39c5baSBill Taylor 			srq = (hermon_srqhdl_t)rsrcp->hr_addr;
12219e39c5baSBill Taylor 
12229e39c5baSBill Taylor 			/*
12239e39c5baSBill Taylor 			 * Invalidate the handle to the userland mapping.
12249e39c5baSBill Taylor 			 * Note: we ensure that the mapping being unmapped
12259e39c5baSBill Taylor 			 * here is the current one for the QP.  This is
12269e39c5baSBill Taylor 			 * more of a sanity check here since, unlike CQs
12279e39c5baSBill Taylor 			 * (above) we do not support resize of QPs.
12289e39c5baSBill Taylor 			 */
12299e39c5baSBill Taylor 			mutex_enter(&srq->srq_lock);
12309e39c5baSBill Taylor 			if (srq->srq_umap_dhp == dhp) {
12319e39c5baSBill Taylor 				srq->srq_umap_dhp = NULL;
12329e39c5baSBill Taylor 			}
12339e39c5baSBill Taylor 			mutex_exit(&srq->srq_lock);
12349e39c5baSBill Taylor 		}
12359e39c5baSBill Taylor 	}
12369e39c5baSBill Taylor }
12379e39c5baSBill Taylor 
12389e39c5baSBill Taylor 
12399e39c5baSBill Taylor /*
12409e39c5baSBill Taylor  * hermon_devmap_devmem_map()
12419e39c5baSBill Taylor  *    Context: Can be called from kernel context.
12429e39c5baSBill Taylor  */
12439e39c5baSBill Taylor /* ARGSUSED */
12449e39c5baSBill Taylor static int
12459e39c5baSBill Taylor hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags,
12469e39c5baSBill Taylor     offset_t off, size_t len, void **pvtp)
12479e39c5baSBill Taylor {
12489e39c5baSBill Taylor 	hermon_state_t		*state;
12499e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track;
12509e39c5baSBill Taylor 	minor_t			instance;
12519e39c5baSBill Taylor 
12529e39c5baSBill Taylor 	/* Get Hermon softstate structure from instance */
12539e39c5baSBill Taylor 	instance = HERMON_DEV_INSTANCE(dev);
12549e39c5baSBill Taylor 	state = ddi_get_soft_state(hermon_statep, instance);
12559e39c5baSBill Taylor 	if (state == NULL) {
12569e39c5baSBill Taylor 		return (ENXIO);
12579e39c5baSBill Taylor 	}
12589e39c5baSBill Taylor 
12599e39c5baSBill Taylor 	/*
12609e39c5baSBill Taylor 	 * Allocate an entry to track the mapping and unmapping of this
12619e39c5baSBill Taylor 	 * resource.  Note:  We don't need to initialize the "refcnt" or
12629e39c5baSBill Taylor 	 * "offset" fields here, nor do we need to initialize the mutex
12639e39c5baSBill Taylor 	 * used with the "refcnt".  Since UAR pages are single pages, they
12649e39c5baSBill Taylor 	 * are not subject to "partial" unmappings.  This makes these other
12659e39c5baSBill Taylor 	 * fields unnecessary.
12669e39c5baSBill Taylor 	 */
12679e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)kmem_zalloc(
12689e39c5baSBill Taylor 	    sizeof (hermon_devmap_track_t), KM_SLEEP);
12699e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
12709e39c5baSBill Taylor 	dvm_track->hdt_state  = state;
12719e39c5baSBill Taylor 	dvm_track->hdt_size   = (uint_t)PAGESIZE;
12729e39c5baSBill Taylor 
12739e39c5baSBill Taylor 	/*
12749e39c5baSBill Taylor 	 * Pass the private "Hermon devmap tracking structure" back.  This
12759e39c5baSBill Taylor 	 * pointer will be returned in a subsequent "unmap" callback.
12769e39c5baSBill Taylor 	 */
12779e39c5baSBill Taylor 	*pvtp = dvm_track;
12789e39c5baSBill Taylor 
12799e39c5baSBill Taylor 	return (DDI_SUCCESS);
12809e39c5baSBill Taylor }
12819e39c5baSBill Taylor 
12829e39c5baSBill Taylor 
12839e39c5baSBill Taylor /*
12849e39c5baSBill Taylor  * hermon_devmap_devmem_dup()
12859e39c5baSBill Taylor  *    Context: Can be called from kernel context.
12869e39c5baSBill Taylor  */
12879e39c5baSBill Taylor /* ARGSUSED */
12889e39c5baSBill Taylor static int
12899e39c5baSBill Taylor hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp,
12909e39c5baSBill Taylor     devmap_cookie_t new_dhp, void **new_pvtp)
12919e39c5baSBill Taylor {
12929e39c5baSBill Taylor 	hermon_state_t		*state;
12939e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track;
12949e39c5baSBill Taylor 	uint_t			maxprot;
12959e39c5baSBill Taylor 	int			status;
12969e39c5baSBill Taylor 
12979e39c5baSBill Taylor 	/*
12989e39c5baSBill Taylor 	 * Extract the Hermon softstate pointer from "Hermon devmap tracking
12999e39c5baSBill Taylor 	 * structure" (in "pvtp").  Note: If the tracking structure is NULL
13009e39c5baSBill Taylor 	 * here, it means that the mapping corresponds to an invalid mapping.
13019e39c5baSBill Taylor 	 * In this case, it can be safely ignored ("new_pvtp" set to NULL).
13029e39c5baSBill Taylor 	 */
13039e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)pvtp;
13049e39c5baSBill Taylor 	if (dvm_track == NULL) {
13059e39c5baSBill Taylor 		*new_pvtp = NULL;
13069e39c5baSBill Taylor 		return (DDI_SUCCESS);
13079e39c5baSBill Taylor 	}
13089e39c5baSBill Taylor 
13099e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
13109e39c5baSBill Taylor 	state = dvm_track->hdt_state;
13119e39c5baSBill Taylor 
13129e39c5baSBill Taylor 	/*
13139e39c5baSBill Taylor 	 * Since this devmap_dup() entry point is generally called
13149e39c5baSBill Taylor 	 * when a process does fork(2), it is incumbent upon the driver
13159e39c5baSBill Taylor 	 * to insure that the child does not inherit a valid copy of
13169e39c5baSBill Taylor 	 * the parent's resource.  This is accomplished by using
13179e39c5baSBill Taylor 	 * devmap_devmem_remap() to invalidate the child's mapping to the
13189e39c5baSBill Taylor 	 * kernel memory.
13199e39c5baSBill Taylor 	 */
13209e39c5baSBill Taylor 	maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
13219e39c5baSBill Taylor 	status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0,
13229e39c5baSBill Taylor 	    dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL);
13239e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
13249e39c5baSBill Taylor 		HERMON_WARNING(state, "failed in hermon_devmap_devmem_dup()");
13259e39c5baSBill Taylor 		return (status);
13269e39c5baSBill Taylor 	}
13279e39c5baSBill Taylor 
13289e39c5baSBill Taylor 	/*
13299e39c5baSBill Taylor 	 * Since the region is invalid, there is no need for us to
13309e39c5baSBill Taylor 	 * allocate and continue to track an additional "Hermon devmap
13319e39c5baSBill Taylor 	 * tracking structure".  Instead we return NULL here, which is an
13329e39c5baSBill Taylor 	 * indication to the devmap_unmap() entry point that this entry
13339e39c5baSBill Taylor 	 * can be safely ignored.
13349e39c5baSBill Taylor 	 */
13359e39c5baSBill Taylor 	*new_pvtp = NULL;
13369e39c5baSBill Taylor 
13379e39c5baSBill Taylor 	return (DDI_SUCCESS);
13389e39c5baSBill Taylor }
13399e39c5baSBill Taylor 
13409e39c5baSBill Taylor 
13419e39c5baSBill Taylor /*
13429e39c5baSBill Taylor  * hermon_devmap_devmem_unmap()
13439e39c5baSBill Taylor  *    Context: Can be called from kernel context.
13449e39c5baSBill Taylor  */
13459e39c5baSBill Taylor /* ARGSUSED */
13469e39c5baSBill Taylor static void
13479e39c5baSBill Taylor hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off,
13489e39c5baSBill Taylor     size_t len, devmap_cookie_t new_dhp1, void **pvtp1,
13499e39c5baSBill Taylor     devmap_cookie_t new_dhp2, void **pvtp2)
13509e39c5baSBill Taylor {
13519e39c5baSBill Taylor 	hermon_devmap_track_t	*dvm_track;
13529e39c5baSBill Taylor 
13539e39c5baSBill Taylor 	/*
13549e39c5baSBill Taylor 	 * Free up the "Hermon devmap tracking structure" (in "pvtp").
13559e39c5baSBill Taylor 	 * There cannot be "partial" unmappings here because all UAR pages
13569e39c5baSBill Taylor 	 * are single pages.  Note: If the tracking structure is NULL here,
13579e39c5baSBill Taylor 	 * it means that the mapping corresponds to an invalid mapping.  In
13589e39c5baSBill Taylor 	 * this case, it can be safely ignored.
13599e39c5baSBill Taylor 	 */
13609e39c5baSBill Taylor 	dvm_track = (hermon_devmap_track_t *)pvtp;
13619e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track))
13629e39c5baSBill Taylor 	if (dvm_track == NULL) {
13639e39c5baSBill Taylor 		return;
13649e39c5baSBill Taylor 	}
13659e39c5baSBill Taylor 
13669e39c5baSBill Taylor 	kmem_free(dvm_track, sizeof (hermon_devmap_track_t));
13679e39c5baSBill Taylor }
13689e39c5baSBill Taylor 
13699e39c5baSBill Taylor 
13709e39c5baSBill Taylor /*
13719e39c5baSBill Taylor  * hermon_umap_ci_data_in()
13729e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
13739e39c5baSBill Taylor  */
13749e39c5baSBill Taylor /* ARGSUSED */
13759e39c5baSBill Taylor ibt_status_t
13769e39c5baSBill Taylor hermon_umap_ci_data_in(hermon_state_t *state, ibt_ci_data_flags_t flags,
13779e39c5baSBill Taylor     ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
13789e39c5baSBill Taylor {
13799e39c5baSBill Taylor 	int	status;
13809e39c5baSBill Taylor 
13819e39c5baSBill Taylor 	/*
13829e39c5baSBill Taylor 	 * Depending on the type of object about which additional information
13839e39c5baSBill Taylor 	 * is being provided (currently only MR is supported), we call the
13849e39c5baSBill Taylor 	 * appropriate resource-specific function.
13859e39c5baSBill Taylor 	 */
13869e39c5baSBill Taylor 	switch (object) {
13879e39c5baSBill Taylor 	case IBT_HDL_MR:
13889e39c5baSBill Taylor 		status = hermon_umap_mr_data_in((hermon_mrhdl_t)hdl,
13899e39c5baSBill Taylor 		    (ibt_mr_data_in_t *)data_p, data_sz);
13909e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
13919e39c5baSBill Taylor 			return (status);
13929e39c5baSBill Taylor 		}
13939e39c5baSBill Taylor 		break;
13949e39c5baSBill Taylor 
13959e39c5baSBill Taylor 	/*
13969e39c5baSBill Taylor 	 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
13979e39c5baSBill Taylor 	 * since the Hermon driver does not support these.
13989e39c5baSBill Taylor 	 */
13999e39c5baSBill Taylor 	case IBT_HDL_HCA:
14009e39c5baSBill Taylor 	case IBT_HDL_QP:
14019e39c5baSBill Taylor 	case IBT_HDL_CQ:
14029e39c5baSBill Taylor 	case IBT_HDL_PD:
14039e39c5baSBill Taylor 	case IBT_HDL_MW:
14049e39c5baSBill Taylor 	case IBT_HDL_AH:
14059e39c5baSBill Taylor 	case IBT_HDL_SCHED:
14069e39c5baSBill Taylor 	case IBT_HDL_EEC:
14079e39c5baSBill Taylor 	case IBT_HDL_RDD:
14089e39c5baSBill Taylor 	case IBT_HDL_SRQ:
14099e39c5baSBill Taylor 		return (IBT_NOT_SUPPORTED);
14109e39c5baSBill Taylor 
14119e39c5baSBill Taylor 	/*
14129e39c5baSBill Taylor 	 * Any other types are invalid.
14139e39c5baSBill Taylor 	 */
14149e39c5baSBill Taylor 	default:
14159e39c5baSBill Taylor 		return (IBT_INVALID_PARAM);
14169e39c5baSBill Taylor 	}
14179e39c5baSBill Taylor 
14189e39c5baSBill Taylor 	return (DDI_SUCCESS);
14199e39c5baSBill Taylor }
14209e39c5baSBill Taylor 
14219e39c5baSBill Taylor 
14229e39c5baSBill Taylor /*
14239e39c5baSBill Taylor  * hermon_umap_mr_data_in()
14249e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
14259e39c5baSBill Taylor  */
14269e39c5baSBill Taylor static ibt_status_t
14279e39c5baSBill Taylor hermon_umap_mr_data_in(hermon_mrhdl_t mr, ibt_mr_data_in_t *data,
14289e39c5baSBill Taylor     size_t data_sz)
14299e39c5baSBill Taylor {
14309e39c5baSBill Taylor 	if (data->mr_rev != IBT_MR_DATA_IN_IF_VERSION) {
14319e39c5baSBill Taylor 		return (IBT_NOT_SUPPORTED);
14329e39c5baSBill Taylor 	}
14339e39c5baSBill Taylor 
14349e39c5baSBill Taylor 	/* Check for valid MR handle pointer */
14359e39c5baSBill Taylor 	if (mr == NULL) {
14369e39c5baSBill Taylor 		return (IBT_MR_HDL_INVALID);
14379e39c5baSBill Taylor 	}
14389e39c5baSBill Taylor 
14399e39c5baSBill Taylor 	/* Check for valid MR input structure size */
14409e39c5baSBill Taylor 	if (data_sz < sizeof (ibt_mr_data_in_t)) {
14419e39c5baSBill Taylor 		return (IBT_INSUFF_RESOURCE);
14429e39c5baSBill Taylor 	}
14439e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
14449e39c5baSBill Taylor 
14459e39c5baSBill Taylor 	/*
14469e39c5baSBill Taylor 	 * Ensure that the MR corresponds to userland memory and that it is
14479e39c5baSBill Taylor 	 * a currently valid memory region as well.
14489e39c5baSBill Taylor 	 */
14499e39c5baSBill Taylor 	mutex_enter(&mr->mr_lock);
14509e39c5baSBill Taylor 	if ((mr->mr_is_umem == 0) || (mr->mr_umemcookie == NULL)) {
14519e39c5baSBill Taylor 		mutex_exit(&mr->mr_lock);
14529e39c5baSBill Taylor 		return (IBT_MR_HDL_INVALID);
14539e39c5baSBill Taylor 	}
14549e39c5baSBill Taylor 
14559e39c5baSBill Taylor 	/*
14569e39c5baSBill Taylor 	 * If it has passed all the above checks, then extract the callback
14579e39c5baSBill Taylor 	 * function and argument from the input structure.  Copy them into
14589e39c5baSBill Taylor 	 * the MR handle.  This function will be called only if the memory
14599e39c5baSBill Taylor 	 * corresponding to the MR handle gets a umem_lockmemory() callback.
14609e39c5baSBill Taylor 	 */
14619e39c5baSBill Taylor 	mr->mr_umem_cbfunc = data->mr_func;
14629e39c5baSBill Taylor 	mr->mr_umem_cbarg1 = data->mr_arg1;
14639e39c5baSBill Taylor 	mr->mr_umem_cbarg2 = data->mr_arg2;
14649e39c5baSBill Taylor 	mutex_exit(&mr->mr_lock);
14659e39c5baSBill Taylor 
14669e39c5baSBill Taylor 	return (DDI_SUCCESS);
14679e39c5baSBill Taylor }
14689e39c5baSBill Taylor 
14699e39c5baSBill Taylor 
14709e39c5baSBill Taylor /*
14719e39c5baSBill Taylor  * hermon_umap_ci_data_out()
14729e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
14739e39c5baSBill Taylor  */
14749e39c5baSBill Taylor /* ARGSUSED */
14759e39c5baSBill Taylor ibt_status_t
14769e39c5baSBill Taylor hermon_umap_ci_data_out(hermon_state_t *state, ibt_ci_data_flags_t flags,
14779e39c5baSBill Taylor     ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz)
14789e39c5baSBill Taylor {
14799e39c5baSBill Taylor 	int	status;
14809e39c5baSBill Taylor 
14819e39c5baSBill Taylor 	/*
14829e39c5baSBill Taylor 	 * Depending on the type of object about which additional information
14839e39c5baSBill Taylor 	 * is being requested (CQ or QP), we call the appropriate resource-
14849e39c5baSBill Taylor 	 * specific mapping function.
14859e39c5baSBill Taylor 	 */
14869e39c5baSBill Taylor 	switch (object) {
14879e39c5baSBill Taylor 	case IBT_HDL_CQ:
14889e39c5baSBill Taylor 		status = hermon_umap_cq_data_out((hermon_cqhdl_t)hdl,
14899e39c5baSBill Taylor 		    (mlnx_umap_cq_data_out_t *)data_p, data_sz);
14909e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
14919e39c5baSBill Taylor 			return (status);
14929e39c5baSBill Taylor 		}
14939e39c5baSBill Taylor 		break;
14949e39c5baSBill Taylor 
14959e39c5baSBill Taylor 	case IBT_HDL_QP:
14969e39c5baSBill Taylor 		status = hermon_umap_qp_data_out((hermon_qphdl_t)hdl,
14979e39c5baSBill Taylor 		    (mlnx_umap_qp_data_out_t *)data_p, data_sz);
14989e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
14999e39c5baSBill Taylor 			return (status);
15009e39c5baSBill Taylor 		}
15019e39c5baSBill Taylor 		break;
15029e39c5baSBill Taylor 
15039e39c5baSBill Taylor 	case IBT_HDL_SRQ:
15049e39c5baSBill Taylor 		status = hermon_umap_srq_data_out((hermon_srqhdl_t)hdl,
15059e39c5baSBill Taylor 		    (mlnx_umap_srq_data_out_t *)data_p, data_sz);
15069e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
15079e39c5baSBill Taylor 			return (status);
15089e39c5baSBill Taylor 		}
15099e39c5baSBill Taylor 		break;
15109e39c5baSBill Taylor 
15119e39c5baSBill Taylor 	case IBT_HDL_PD:
15129e39c5baSBill Taylor 		status = hermon_umap_pd_data_out((hermon_pdhdl_t)hdl,
15139e39c5baSBill Taylor 		    (mlnx_umap_pd_data_out_t *)data_p, data_sz);
15149e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
15159e39c5baSBill Taylor 			return (status);
15169e39c5baSBill Taylor 		}
15179e39c5baSBill Taylor 		break;
15189e39c5baSBill Taylor 
15199e39c5baSBill Taylor 	/*
15209e39c5baSBill Taylor 	 * For other possible valid IBT types, we return IBT_NOT_SUPPORTED,
15219e39c5baSBill Taylor 	 * since the Hermon driver does not support these.
15229e39c5baSBill Taylor 	 */
15239e39c5baSBill Taylor 	case IBT_HDL_HCA:
15249e39c5baSBill Taylor 	case IBT_HDL_MR:
15259e39c5baSBill Taylor 	case IBT_HDL_MW:
15269e39c5baSBill Taylor 	case IBT_HDL_AH:
15279e39c5baSBill Taylor 	case IBT_HDL_SCHED:
15289e39c5baSBill Taylor 	case IBT_HDL_EEC:
15299e39c5baSBill Taylor 	case IBT_HDL_RDD:
15309e39c5baSBill Taylor 		return (IBT_NOT_SUPPORTED);
15319e39c5baSBill Taylor 
15329e39c5baSBill Taylor 	/*
15339e39c5baSBill Taylor 	 * Any other types are invalid.
15349e39c5baSBill Taylor 	 */
15359e39c5baSBill Taylor 	default:
15369e39c5baSBill Taylor 		return (IBT_INVALID_PARAM);
15379e39c5baSBill Taylor 	}
15389e39c5baSBill Taylor 
15399e39c5baSBill Taylor 	return (DDI_SUCCESS);
15409e39c5baSBill Taylor }
15419e39c5baSBill Taylor 
15429e39c5baSBill Taylor 
15439e39c5baSBill Taylor /*
15449e39c5baSBill Taylor  * hermon_umap_cq_data_out()
15459e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
15469e39c5baSBill Taylor  */
15479e39c5baSBill Taylor static ibt_status_t
15489e39c5baSBill Taylor hermon_umap_cq_data_out(hermon_cqhdl_t cq, mlnx_umap_cq_data_out_t *data,
15499e39c5baSBill Taylor     size_t data_sz)
15509e39c5baSBill Taylor {
15519e39c5baSBill Taylor 	/* Check for valid CQ handle pointer */
15529e39c5baSBill Taylor 	if (cq == NULL) {
15539e39c5baSBill Taylor 		return (IBT_CQ_HDL_INVALID);
15549e39c5baSBill Taylor 	}
15559e39c5baSBill Taylor 
15569e39c5baSBill Taylor 	/* Check for valid CQ mapping structure size */
15579e39c5baSBill Taylor 	if (data_sz < sizeof (mlnx_umap_cq_data_out_t)) {
15589e39c5baSBill Taylor 		return (IBT_INSUFF_RESOURCE);
15599e39c5baSBill Taylor 	}
15609e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
15619e39c5baSBill Taylor 
15629e39c5baSBill Taylor 	/* deal with cq_alloc() verses cq_resize() */
15639e39c5baSBill Taylor 	if (cq->cq_resize_hdl) {
15649e39c5baSBill Taylor 		data->mcq_maplen = cq->cq_resize_hdl->cq_cqinfo.qa_size;
15659e39c5baSBill Taylor 		data->mcq_numcqe = cq->cq_resize_hdl->cq_bufsz;
15669e39c5baSBill Taylor 	} else {
15679e39c5baSBill Taylor 		data->mcq_maplen = cq->cq_cqinfo.qa_size;
15689e39c5baSBill Taylor 		data->mcq_numcqe = cq->cq_bufsz;
15699e39c5baSBill Taylor 	}
15709e39c5baSBill Taylor 
15719e39c5baSBill Taylor 	/*
15729e39c5baSBill Taylor 	 * If it has passed all the above checks, then fill in all the useful
15739e39c5baSBill Taylor 	 * mapping information (including the mapping offset that will be
15749e39c5baSBill Taylor 	 * passed back to the devmap() interface during a subsequent mmap()
15759e39c5baSBill Taylor 	 * call.
15769e39c5baSBill Taylor 	 *
15779e39c5baSBill Taylor 	 * The "offset" for CQ mmap()'s looks like this:
15789e39c5baSBill Taylor 	 * +----------------------------------------+--------+--------------+
15799e39c5baSBill Taylor 	 * |		   CQ Number		    |  0x33  | Reserved (0) |
15809e39c5baSBill Taylor 	 * +----------------------------------------+--------+--------------+
15819e39c5baSBill Taylor 	 *	   (64 - 8 - PAGESHIFT) bits	    8 bits	PAGESHIFT bits
15829e39c5baSBill Taylor 	 *
15839e39c5baSBill Taylor 	 * This returns information about the mapping offset, the length of
15849e39c5baSBill Taylor 	 * the CQ memory, the CQ number (for use in later CQ doorbells), the
15859e39c5baSBill Taylor 	 * number of CQEs the CQ memory can hold, and the size of each CQE.
15869e39c5baSBill Taylor 	 */
15879e39c5baSBill Taylor 	data->mcq_rev			= MLNX_UMAP_IF_VERSION;
15889e39c5baSBill Taylor 	data->mcq_mapoffset		= ((((uint64_t)cq->cq_cqnum <<
15899e39c5baSBill Taylor 	    MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_CQMEM_RSRC) << PAGESHIFT);
15909e39c5baSBill Taylor 	data->mcq_cqnum			= cq->cq_cqnum;
15919e39c5baSBill Taylor 	data->mcq_cqesz			= sizeof (hermon_hw_cqe_t);
15929e39c5baSBill Taylor 
15939e39c5baSBill Taylor 	/* doorbell record fields */
15949e39c5baSBill Taylor 	data->mcq_polldbr_mapoffset	= cq->cq_dbr_mapoffset;
15959e39c5baSBill Taylor 	data->mcq_polldbr_maplen	= PAGESIZE;
15969e39c5baSBill Taylor 	data->mcq_polldbr_offset	= (uintptr_t)cq->cq_arm_ci_vdbr &
15979e39c5baSBill Taylor 	    PAGEOFFSET;
15989e39c5baSBill Taylor 	data->mcq_armdbr_mapoffset	= cq->cq_dbr_mapoffset;
15999e39c5baSBill Taylor 	data->mcq_armdbr_maplen		= PAGESIZE;
16009e39c5baSBill Taylor 	data->mcq_armdbr_offset		= data->mcq_polldbr_offset + 4;
16019e39c5baSBill Taylor 
16029e39c5baSBill Taylor 	return (DDI_SUCCESS);
16039e39c5baSBill Taylor }
16049e39c5baSBill Taylor 
16059e39c5baSBill Taylor 
16069e39c5baSBill Taylor /*
16079e39c5baSBill Taylor  * hermon_umap_qp_data_out()
16089e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
16099e39c5baSBill Taylor  */
16109e39c5baSBill Taylor static ibt_status_t
16119e39c5baSBill Taylor hermon_umap_qp_data_out(hermon_qphdl_t qp, mlnx_umap_qp_data_out_t *data,
16129e39c5baSBill Taylor     size_t data_sz)
16139e39c5baSBill Taylor {
16149e39c5baSBill Taylor 	/* Check for valid QP handle pointer */
16159e39c5baSBill Taylor 	if (qp == NULL) {
16169e39c5baSBill Taylor 		return (IBT_QP_HDL_INVALID);
16179e39c5baSBill Taylor 	}
16189e39c5baSBill Taylor 
16199e39c5baSBill Taylor 	/* Check for valid QP mapping structure size */
16209e39c5baSBill Taylor 	if (data_sz < sizeof (mlnx_umap_qp_data_out_t)) {
16219e39c5baSBill Taylor 		return (IBT_INSUFF_RESOURCE);
16229e39c5baSBill Taylor 	}
16239e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
16249e39c5baSBill Taylor 
16259e39c5baSBill Taylor 	/*
16269e39c5baSBill Taylor 	 * If it has passed all the checks, then fill in all the useful
16279e39c5baSBill Taylor 	 * mapping information (including the mapping offset that will be
16289e39c5baSBill Taylor 	 * passed back to the devmap() interface during a subsequent mmap()
16299e39c5baSBill Taylor 	 * call.
16309e39c5baSBill Taylor 	 *
16319e39c5baSBill Taylor 	 * The "offset" for QP mmap()'s looks like this:
16329e39c5baSBill Taylor 	 * +----------------------------------------+--------+--------------+
16339e39c5baSBill Taylor 	 * |		   QP Number		    |  0x44  | Reserved (0) |
16349e39c5baSBill Taylor 	 * +----------------------------------------+--------+--------------+
16359e39c5baSBill Taylor 	 *	   (64 - 8 - PAGESHIFT) bits	    8 bits	PAGESHIFT bits
16369e39c5baSBill Taylor 	 *
16379e39c5baSBill Taylor 	 * This returns information about the mapping offset, the length of
16389e39c5baSBill Taylor 	 * the QP memory, and the QP number (for use in later send and recv
16399e39c5baSBill Taylor 	 * doorbells).  It also returns the following information for both
16409e39c5baSBill Taylor 	 * the receive work queue and the send work queue, respectively:  the
16419e39c5baSBill Taylor 	 * offset (from the base mapped address) of the start of the given
16429e39c5baSBill Taylor 	 * work queue, the 64-bit IB virtual address that corresponds to
16439e39c5baSBill Taylor 	 * the base mapped address (needed for posting WQEs though the
16449e39c5baSBill Taylor 	 * QP doorbells), the number of WQEs the given work queue can hold,
16459e39c5baSBill Taylor 	 * and the size of each WQE for the given work queue.
16469e39c5baSBill Taylor 	 */
16479e39c5baSBill Taylor 	data->mqp_rev		= MLNX_UMAP_IF_VERSION;
16489e39c5baSBill Taylor 	data->mqp_mapoffset	= ((((uint64_t)qp->qp_qpnum <<
16499e39c5baSBill Taylor 	    MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_QPMEM_RSRC) << PAGESHIFT);
16509e39c5baSBill Taylor 	data->mqp_maplen	= qp->qp_wqinfo.qa_size;
16519e39c5baSBill Taylor 	data->mqp_qpnum		= qp->qp_qpnum;
16529e39c5baSBill Taylor 
16539e39c5baSBill Taylor 	/*
16549e39c5baSBill Taylor 	 * If this QP is associated with a shared receive queue (SRQ),
16559e39c5baSBill Taylor 	 * then return invalid RecvQ parameters.  Otherwise, return
16569e39c5baSBill Taylor 	 * the proper parameter values.
16579e39c5baSBill Taylor 	 */
1658*17a2b317SBill Taylor 	if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) {
16599e39c5baSBill Taylor 		data->mqp_rq_off	= (uint32_t)qp->qp_wqinfo.qa_size;
16609e39c5baSBill Taylor 		data->mqp_rq_desc_addr	= (uint32_t)qp->qp_wqinfo.qa_size;
16619e39c5baSBill Taylor 		data->mqp_rq_numwqe	= 0;
16629e39c5baSBill Taylor 		data->mqp_rq_wqesz	= 0;
16639e39c5baSBill Taylor 		data->mqp_rdbr_mapoffset = 0;
16649e39c5baSBill Taylor 		data->mqp_rdbr_maplen	= 0;
16659e39c5baSBill Taylor 		data->mqp_rdbr_offset	= 0;
16669e39c5baSBill Taylor 	} else {
16679e39c5baSBill Taylor 		data->mqp_rq_off	= (uintptr_t)qp->qp_rq_buf -
16689e39c5baSBill Taylor 		    (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
16699e39c5baSBill Taylor 		data->mqp_rq_desc_addr	= (uint32_t)((uintptr_t)qp->qp_rq_buf -
16709e39c5baSBill Taylor 		    qp->qp_desc_off);
16719e39c5baSBill Taylor 		data->mqp_rq_numwqe	= qp->qp_rq_bufsz;
16729e39c5baSBill Taylor 		data->mqp_rq_wqesz	= (1 << qp->qp_rq_log_wqesz);
16739e39c5baSBill Taylor 
16749e39c5baSBill Taylor 		/* doorbell record fields */
16759e39c5baSBill Taylor 		data->mqp_rdbr_mapoffset = qp->qp_rdbr_mapoffset;
16769e39c5baSBill Taylor 		data->mqp_rdbr_maplen	= PAGESIZE;
16779e39c5baSBill Taylor 		data->mqp_rdbr_offset	= (uintptr_t)qp->qp_rq_vdbr &
16789e39c5baSBill Taylor 		    PAGEOFFSET;
16799e39c5baSBill Taylor 	}
16809e39c5baSBill Taylor 	data->mqp_sq_off		= (uintptr_t)qp->qp_sq_buf -
16819e39c5baSBill Taylor 	    (uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
16829e39c5baSBill Taylor 	data->mqp_sq_desc_addr	= (uint32_t)((uintptr_t)qp->qp_sq_buf -
16839e39c5baSBill Taylor 	    qp->qp_desc_off);
16849e39c5baSBill Taylor 	data->mqp_sq_numwqe	= qp->qp_sq_bufsz;
16859e39c5baSBill Taylor 	data->mqp_sq_wqesz	= (1 << qp->qp_sq_log_wqesz);
16869e39c5baSBill Taylor 	data->mqp_sq_headroomwqes = qp->qp_sq_hdrmwqes;
16879e39c5baSBill Taylor 
16889e39c5baSBill Taylor 	/* doorbell record fields */
16899e39c5baSBill Taylor 	data->mqp_sdbr_mapoffset = 0;
16909e39c5baSBill Taylor 	data->mqp_sdbr_maplen	= 0;
16919e39c5baSBill Taylor 	data->mqp_sdbr_offset	= 0;
16929e39c5baSBill Taylor 
16939e39c5baSBill Taylor 	return (DDI_SUCCESS);
16949e39c5baSBill Taylor }
16959e39c5baSBill Taylor 
16969e39c5baSBill Taylor 
16979e39c5baSBill Taylor /*
16989e39c5baSBill Taylor  * hermon_umap_srq_data_out()
16999e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
17009e39c5baSBill Taylor  */
17019e39c5baSBill Taylor static ibt_status_t
17029e39c5baSBill Taylor hermon_umap_srq_data_out(hermon_srqhdl_t srq, mlnx_umap_srq_data_out_t *data,
17039e39c5baSBill Taylor     size_t data_sz)
17049e39c5baSBill Taylor {
17059e39c5baSBill Taylor 	/* Check for valid SRQ handle pointer */
17069e39c5baSBill Taylor 	if (srq == NULL) {
17079e39c5baSBill Taylor 		return (IBT_SRQ_HDL_INVALID);
17089e39c5baSBill Taylor 	}
17099e39c5baSBill Taylor 
17109e39c5baSBill Taylor 	/* Check for valid SRQ mapping structure size */
17119e39c5baSBill Taylor 	if (data_sz < sizeof (mlnx_umap_srq_data_out_t)) {
17129e39c5baSBill Taylor 		return (IBT_INSUFF_RESOURCE);
17139e39c5baSBill Taylor 	}
17149e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
17159e39c5baSBill Taylor 
17169e39c5baSBill Taylor 	/*
17179e39c5baSBill Taylor 	 * If it has passed all the checks, then fill in all the useful
17189e39c5baSBill Taylor 	 * mapping information (including the mapping offset that will be
17199e39c5baSBill Taylor 	 * passed back to the devmap() interface during a subsequent mmap()
17209e39c5baSBill Taylor 	 * call.
17219e39c5baSBill Taylor 	 *
17229e39c5baSBill Taylor 	 * The "offset" for SRQ mmap()'s looks like this:
17239e39c5baSBill Taylor 	 * +----------------------------------------+--------+--------------+
17249e39c5baSBill Taylor 	 * |		   SRQ Number		    |  0x66  | Reserved (0) |
17259e39c5baSBill Taylor 	 * +----------------------------------------+--------+--------------+
17269e39c5baSBill Taylor 	 *	   (64 - 8 - PAGESHIFT) bits	    8 bits	PAGESHIFT bits
17279e39c5baSBill Taylor 	 *
17289e39c5baSBill Taylor 	 * This returns information about the mapping offset, the length of the
17299e39c5baSBill Taylor 	 * SRQ memory, and the SRQ number (for use in later send and recv
17309e39c5baSBill Taylor 	 * doorbells).  It also returns the following information for the
17319e39c5baSBill Taylor 	 * shared receive queue: the offset (from the base mapped address) of
17329e39c5baSBill Taylor 	 * the start of the given work queue, the 64-bit IB virtual address
17339e39c5baSBill Taylor 	 * that corresponds to the base mapped address (needed for posting WQEs
17349e39c5baSBill Taylor 	 * though the QP doorbells), the number of WQEs the given work queue
17359e39c5baSBill Taylor 	 * can hold, and the size of each WQE for the given work queue.
17369e39c5baSBill Taylor 	 */
17379e39c5baSBill Taylor 	data->msrq_rev		= MLNX_UMAP_IF_VERSION;
17389e39c5baSBill Taylor 	data->msrq_mapoffset	= ((((uint64_t)srq->srq_srqnum <<
17399e39c5baSBill Taylor 	    MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_SRQMEM_RSRC) << PAGESHIFT);
17409e39c5baSBill Taylor 	data->msrq_maplen	= srq->srq_wqinfo.qa_size;
17419e39c5baSBill Taylor 	data->msrq_srqnum	= srq->srq_srqnum;
17429e39c5baSBill Taylor 
17439e39c5baSBill Taylor 	data->msrq_desc_addr	= (uint32_t)((uintptr_t)srq->srq_wq_buf -
17449e39c5baSBill Taylor 	    srq->srq_desc_off);
17459e39c5baSBill Taylor 	data->msrq_numwqe	= srq->srq_wq_bufsz;
17469e39c5baSBill Taylor 	data->msrq_wqesz	= (1 << srq->srq_wq_log_wqesz);
17479e39c5baSBill Taylor 
17489e39c5baSBill Taylor 	/* doorbell record fields */
17499e39c5baSBill Taylor 	data->msrq_rdbr_mapoffset = srq->srq_rdbr_mapoffset;
17509e39c5baSBill Taylor 	data->msrq_rdbr_maplen	= PAGESIZE;
17519e39c5baSBill Taylor 	data->msrq_rdbr_offset	= (uintptr_t)srq->srq_wq_vdbr &
17529e39c5baSBill Taylor 	    PAGEOFFSET;
17539e39c5baSBill Taylor 
17549e39c5baSBill Taylor 	return (DDI_SUCCESS);
17559e39c5baSBill Taylor }
17569e39c5baSBill Taylor 
17579e39c5baSBill Taylor 
17589e39c5baSBill Taylor /*
17599e39c5baSBill Taylor  * hermon_umap_pd_data_out()
17609e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
17619e39c5baSBill Taylor  */
17629e39c5baSBill Taylor static ibt_status_t
17639e39c5baSBill Taylor hermon_umap_pd_data_out(hermon_pdhdl_t pd, mlnx_umap_pd_data_out_t *data,
17649e39c5baSBill Taylor     size_t data_sz)
17659e39c5baSBill Taylor {
17669e39c5baSBill Taylor 	/* Check for valid PD handle pointer */
17679e39c5baSBill Taylor 	if (pd == NULL) {
17689e39c5baSBill Taylor 		return (IBT_PD_HDL_INVALID);
17699e39c5baSBill Taylor 	}
17709e39c5baSBill Taylor 
17719e39c5baSBill Taylor 	/* Check for valid PD mapping structure size */
17729e39c5baSBill Taylor 	if (data_sz < sizeof (mlnx_umap_pd_data_out_t)) {
17739e39c5baSBill Taylor 		return (IBT_INSUFF_RESOURCE);
17749e39c5baSBill Taylor 	}
17759e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data))
17769e39c5baSBill Taylor 
17779e39c5baSBill Taylor 	/*
17789e39c5baSBill Taylor 	 * If it has passed all the checks, then fill the PD table index
17799e39c5baSBill Taylor 	 * (the PD table allocated index for the PD pd_pdnum).
17809e39c5baSBill Taylor 	 */
17819e39c5baSBill Taylor 	data->mpd_rev		= MLNX_UMAP_IF_VERSION;
17829e39c5baSBill Taylor 	data->mpd_pdnum		= pd->pd_pdnum;
17839e39c5baSBill Taylor 
17849e39c5baSBill Taylor 	return (DDI_SUCCESS);
17859e39c5baSBill Taylor }
17869e39c5baSBill Taylor 
17879e39c5baSBill Taylor 
17889e39c5baSBill Taylor /*
17899e39c5baSBill Taylor  * hermon_umap_db_init()
17909e39c5baSBill Taylor  *    Context: Only called from attach() path context
17919e39c5baSBill Taylor  */
17929e39c5baSBill Taylor void
17939e39c5baSBill Taylor hermon_umap_db_init(void)
17949e39c5baSBill Taylor {
17959e39c5baSBill Taylor 	/*
17969e39c5baSBill Taylor 	 * Initialize the lock used by the Hermon "userland resources database"
17979e39c5baSBill Taylor 	 * This is used to ensure atomic access to add, remove, and find
17989e39c5baSBill Taylor 	 * entries in the database.
17999e39c5baSBill Taylor 	 */
18009e39c5baSBill Taylor 	mutex_init(&hermon_userland_rsrc_db.hdl_umapdb_lock, NULL,
18019e39c5baSBill Taylor 	    MUTEX_DRIVER, NULL);
18029e39c5baSBill Taylor 
18039e39c5baSBill Taylor 	/*
18049e39c5baSBill Taylor 	 * Initialize the AVL tree used for the "userland resources
18059e39c5baSBill Taylor 	 * database".  Using an AVL tree here provides the ability to
18069e39c5baSBill Taylor 	 * scale the database size to large numbers of resources.  The
18079e39c5baSBill Taylor 	 * entries in the tree are "hermon_umap_db_entry_t" (see
18089e39c5baSBill Taylor 	 * hermon_umap.h).  The tree is searched with the help of the
18099e39c5baSBill Taylor 	 * hermon_umap_db_compare() routine.
18109e39c5baSBill Taylor 	 */
18119e39c5baSBill Taylor 	avl_create(&hermon_userland_rsrc_db.hdl_umapdb_avl,
18129e39c5baSBill Taylor 	    hermon_umap_db_compare, sizeof (hermon_umap_db_entry_t),
18139e39c5baSBill Taylor 	    offsetof(hermon_umap_db_entry_t, hdbe_avlnode));
18149e39c5baSBill Taylor }
18159e39c5baSBill Taylor 
18169e39c5baSBill Taylor 
18179e39c5baSBill Taylor /*
18189e39c5baSBill Taylor  * hermon_umap_db_fini()
18199e39c5baSBill Taylor  *    Context: Only called from attach() and/or detach() path contexts
18209e39c5baSBill Taylor  */
18219e39c5baSBill Taylor void
18229e39c5baSBill Taylor hermon_umap_db_fini(void)
18239e39c5baSBill Taylor {
18249e39c5baSBill Taylor 	/* Destroy the AVL tree for the "userland resources database" */
18259e39c5baSBill Taylor 	avl_destroy(&hermon_userland_rsrc_db.hdl_umapdb_avl);
18269e39c5baSBill Taylor 
18279e39c5baSBill Taylor 	/* Destroy the lock for the "userland resources database" */
18289e39c5baSBill Taylor 	mutex_destroy(&hermon_userland_rsrc_db.hdl_umapdb_lock);
18299e39c5baSBill Taylor }
18309e39c5baSBill Taylor 
18319e39c5baSBill Taylor 
18329e39c5baSBill Taylor /*
18339e39c5baSBill Taylor  * hermon_umap_db_alloc()
18349e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
18359e39c5baSBill Taylor  */
18369e39c5baSBill Taylor hermon_umap_db_entry_t *
18379e39c5baSBill Taylor hermon_umap_db_alloc(uint_t instance, uint64_t key, uint_t type, uint64_t value)
18389e39c5baSBill Taylor {
18399e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
18409e39c5baSBill Taylor 
18419e39c5baSBill Taylor 	/* Allocate an entry to add to the "userland resources database" */
18429e39c5baSBill Taylor 	umapdb = kmem_zalloc(sizeof (hermon_umap_db_entry_t), KM_NOSLEEP);
18439e39c5baSBill Taylor 	if (umapdb == NULL) {
18449e39c5baSBill Taylor 		return (NULL);
18459e39c5baSBill Taylor 	}
18469e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
18479e39c5baSBill Taylor 
18489e39c5baSBill Taylor 	/* Fill in the fields in the database entry */
18499e39c5baSBill Taylor 	umapdb->hdbe_common.hdb_instance  = instance;
18509e39c5baSBill Taylor 	umapdb->hdbe_common.hdb_type	  = type;
18519e39c5baSBill Taylor 	umapdb->hdbe_common.hdb_key	  = key;
18529e39c5baSBill Taylor 	umapdb->hdbe_common.hdb_value	  = value;
18539e39c5baSBill Taylor 
18549e39c5baSBill Taylor 	return (umapdb);
18559e39c5baSBill Taylor }
18569e39c5baSBill Taylor 
18579e39c5baSBill Taylor 
18589e39c5baSBill Taylor /*
18599e39c5baSBill Taylor  * hermon_umap_db_free()
18609e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
18619e39c5baSBill Taylor  */
18629e39c5baSBill Taylor void
18639e39c5baSBill Taylor hermon_umap_db_free(hermon_umap_db_entry_t *umapdb)
18649e39c5baSBill Taylor {
18659e39c5baSBill Taylor 	/* Free the database entry */
18669e39c5baSBill Taylor 	kmem_free(umapdb, sizeof (hermon_umap_db_entry_t));
18679e39c5baSBill Taylor }
18689e39c5baSBill Taylor 
18699e39c5baSBill Taylor 
18709e39c5baSBill Taylor /*
18719e39c5baSBill Taylor  * hermon_umap_db_add()
18729e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
18739e39c5baSBill Taylor  */
18749e39c5baSBill Taylor void
18759e39c5baSBill Taylor hermon_umap_db_add(hermon_umap_db_entry_t *umapdb)
18769e39c5baSBill Taylor {
18779e39c5baSBill Taylor 	mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
18789e39c5baSBill Taylor 	hermon_umap_db_add_nolock(umapdb);
18799e39c5baSBill Taylor 	mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
18809e39c5baSBill Taylor }
18819e39c5baSBill Taylor 
18829e39c5baSBill Taylor 
18839e39c5baSBill Taylor /*
18849e39c5baSBill Taylor  * hermon_umap_db_add_nolock()
18859e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
18869e39c5baSBill Taylor  */
18879e39c5baSBill Taylor void
18889e39c5baSBill Taylor hermon_umap_db_add_nolock(hermon_umap_db_entry_t *umapdb)
18899e39c5baSBill Taylor {
18909e39c5baSBill Taylor 	hermon_umap_db_query_t	query;
18919e39c5baSBill Taylor 	avl_index_t		where;
18929e39c5baSBill Taylor 
18939e39c5baSBill Taylor 	ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
18949e39c5baSBill Taylor 
18959e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
18969e39c5baSBill Taylor 
18979e39c5baSBill Taylor 	/*
18989e39c5baSBill Taylor 	 * Copy the common portion of the "to-be-added" database entry
18999e39c5baSBill Taylor 	 * into the "hermon_umap_db_query_t" structure.  We use this structure
19009e39c5baSBill Taylor 	 * (with no flags set) to find the appropriate location in the
19019e39c5baSBill Taylor 	 * "userland resources database" for the new entry to be added.
19029e39c5baSBill Taylor 	 *
19039e39c5baSBill Taylor 	 * Note: we expect that this entry should not be found in the
19049e39c5baSBill Taylor 	 * database (unless something bad has happened).
19059e39c5baSBill Taylor 	 */
19069e39c5baSBill Taylor 	query.hqdb_common = umapdb->hdbe_common;
19079e39c5baSBill Taylor 	query.hqdb_flags  = 0;
19089e39c5baSBill Taylor 	(void) avl_find(&hermon_userland_rsrc_db.hdl_umapdb_avl, &query,
19099e39c5baSBill Taylor 	    &where);
19109e39c5baSBill Taylor 
19119e39c5baSBill Taylor 	/*
19129e39c5baSBill Taylor 	 * Now, using the "where" field from the avl_find() operation
19139e39c5baSBill Taylor 	 * above, we will insert the new database entry ("umapdb").
19149e39c5baSBill Taylor 	 */
19159e39c5baSBill Taylor 	avl_insert(&hermon_userland_rsrc_db.hdl_umapdb_avl, umapdb,
19169e39c5baSBill Taylor 	    where);
19179e39c5baSBill Taylor }
19189e39c5baSBill Taylor 
19199e39c5baSBill Taylor 
19209e39c5baSBill Taylor /*
19219e39c5baSBill Taylor  * hermon_umap_db_find()
19229e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
19239e39c5baSBill Taylor  */
19249e39c5baSBill Taylor int
19259e39c5baSBill Taylor hermon_umap_db_find(uint_t instance, uint64_t key, uint_t type,
19269e39c5baSBill Taylor     uint64_t *value, uint_t flag, hermon_umap_db_entry_t	**umapdb)
19279e39c5baSBill Taylor {
19289e39c5baSBill Taylor 	int	status;
19299e39c5baSBill Taylor 
19309e39c5baSBill Taylor 	mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
19319e39c5baSBill Taylor 	status = hermon_umap_db_find_nolock(instance, key, type, value, flag,
19329e39c5baSBill Taylor 	    umapdb);
19339e39c5baSBill Taylor 	mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
19349e39c5baSBill Taylor 
19359e39c5baSBill Taylor 	return (status);
19369e39c5baSBill Taylor }
19379e39c5baSBill Taylor 
19389e39c5baSBill Taylor 
19399e39c5baSBill Taylor /*
19409e39c5baSBill Taylor  * hermon_umap_db_find_nolock()
19419e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
19429e39c5baSBill Taylor  */
19439e39c5baSBill Taylor int
19449e39c5baSBill Taylor hermon_umap_db_find_nolock(uint_t instance, uint64_t key, uint_t type,
19459e39c5baSBill Taylor     uint64_t *value, uint_t flags, hermon_umap_db_entry_t **umapdb)
19469e39c5baSBill Taylor {
19479e39c5baSBill Taylor 	hermon_umap_db_query_t	query;
19489e39c5baSBill Taylor 	hermon_umap_db_entry_t	*entry;
19499e39c5baSBill Taylor 	avl_index_t		where;
19509e39c5baSBill Taylor 
19519e39c5baSBill Taylor 	ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
19529e39c5baSBill Taylor 
19539e39c5baSBill Taylor 	/*
19549e39c5baSBill Taylor 	 * Fill in key, type, instance, and flags values of the
19559e39c5baSBill Taylor 	 * hermon_umap_db_query_t in preparation for the database
19569e39c5baSBill Taylor 	 * lookup.
19579e39c5baSBill Taylor 	 */
19589e39c5baSBill Taylor 	query.hqdb_flags		= flags;
19599e39c5baSBill Taylor 	query.hqdb_common.hdb_key	= key;
19609e39c5baSBill Taylor 	query.hqdb_common.hdb_type	= type;
19619e39c5baSBill Taylor 	query.hqdb_common.hdb_instance	= instance;
19629e39c5baSBill Taylor 
19639e39c5baSBill Taylor 	/*
19649e39c5baSBill Taylor 	 * Perform the database query.  If no entry is found, then
19659e39c5baSBill Taylor 	 * return failure, else continue.
19669e39c5baSBill Taylor 	 */
19679e39c5baSBill Taylor 	entry = (hermon_umap_db_entry_t *)avl_find(
19689e39c5baSBill Taylor 	    &hermon_userland_rsrc_db.hdl_umapdb_avl, &query, &where);
19699e39c5baSBill Taylor 	if (entry == NULL) {
19709e39c5baSBill Taylor 		return (DDI_FAILURE);
19719e39c5baSBill Taylor 	}
19729e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry))
19739e39c5baSBill Taylor 
19749e39c5baSBill Taylor 	/*
19759e39c5baSBill Taylor 	 * If the flags argument specifies that the entry should
19769e39c5baSBill Taylor 	 * be removed if found, then call avl_remove() to remove
19779e39c5baSBill Taylor 	 * the entry from the database.
19789e39c5baSBill Taylor 	 */
19799e39c5baSBill Taylor 	if (flags & HERMON_UMAP_DB_REMOVE) {
19809e39c5baSBill Taylor 
19819e39c5baSBill Taylor 		avl_remove(&hermon_userland_rsrc_db.hdl_umapdb_avl, entry);
19829e39c5baSBill Taylor 
19839e39c5baSBill Taylor 		/*
19849e39c5baSBill Taylor 		 * The database entry is returned with the expectation
19859e39c5baSBill Taylor 		 * that the caller will use hermon_umap_db_free() to
19869e39c5baSBill Taylor 		 * free the entry's memory.  ASSERT that this is non-NULL.
19879e39c5baSBill Taylor 		 * NULL pointer should never be passed for the
19889e39c5baSBill Taylor 		 * HERMON_UMAP_DB_REMOVE case.
19899e39c5baSBill Taylor 		 */
19909e39c5baSBill Taylor 		ASSERT(umapdb != NULL);
19919e39c5baSBill Taylor 	}
19929e39c5baSBill Taylor 
19939e39c5baSBill Taylor 	/*
19949e39c5baSBill Taylor 	 * If the caller would like visibility to the database entry
19959e39c5baSBill Taylor 	 * (indicated through the use of a non-NULL "umapdb" argument),
19969e39c5baSBill Taylor 	 * then fill it in.
19979e39c5baSBill Taylor 	 */
19989e39c5baSBill Taylor 	if (umapdb != NULL) {
19999e39c5baSBill Taylor 		*umapdb = entry;
20009e39c5baSBill Taylor 	}
20019e39c5baSBill Taylor 
20029e39c5baSBill Taylor 	/* Extract value field from database entry and return success */
20039e39c5baSBill Taylor 	*value = entry->hdbe_common.hdb_value;
20049e39c5baSBill Taylor 
20059e39c5baSBill Taylor 	return (DDI_SUCCESS);
20069e39c5baSBill Taylor }
20079e39c5baSBill Taylor 
20089e39c5baSBill Taylor 
20099e39c5baSBill Taylor /*
20109e39c5baSBill Taylor  * hermon_umap_umemlock_cb()
20119e39c5baSBill Taylor  *    Context: Can be called from callback context.
20129e39c5baSBill Taylor  */
20139e39c5baSBill Taylor void
20149e39c5baSBill Taylor hermon_umap_umemlock_cb(ddi_umem_cookie_t *umem_cookie)
20159e39c5baSBill Taylor {
20169e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
20179e39c5baSBill Taylor 	hermon_state_t		*state;
20189e39c5baSBill Taylor 	hermon_rsrc_t 		*rsrcp;
20199e39c5baSBill Taylor 	hermon_mrhdl_t		mr;
20209e39c5baSBill Taylor 	uint64_t		value;
20219e39c5baSBill Taylor 	uint_t			instance;
20229e39c5baSBill Taylor 	int			status;
20239e39c5baSBill Taylor 	void			(*mr_callback)(void *, void *);
20249e39c5baSBill Taylor 	void			*mr_cbarg1, *mr_cbarg2;
20259e39c5baSBill Taylor 
20269e39c5baSBill Taylor 	/*
20279e39c5baSBill Taylor 	 * If this was userland memory, then we need to remove its entry
20289e39c5baSBill Taylor 	 * from the "userland resources database".  Note:  We use the
20299e39c5baSBill Taylor 	 * HERMON_UMAP_DB_IGNORE_INSTANCE flag here because we don't know
20309e39c5baSBill Taylor 	 * which instance was used when the entry was added (but we want
20319e39c5baSBill Taylor 	 * to know after the entry is found using the other search criteria).
20329e39c5baSBill Taylor 	 */
20339e39c5baSBill Taylor 	status = hermon_umap_db_find(0, (uint64_t)(uintptr_t)umem_cookie,
20349e39c5baSBill Taylor 	    MLNX_UMAP_MRMEM_RSRC, &value, (HERMON_UMAP_DB_REMOVE |
20359e39c5baSBill Taylor 	    HERMON_UMAP_DB_IGNORE_INSTANCE), &umapdb);
20369e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb))
20379e39c5baSBill Taylor 	if (status == DDI_SUCCESS) {
20389e39c5baSBill Taylor 		instance = umapdb->hdbe_common.hdb_instance;
20399e39c5baSBill Taylor 		state = ddi_get_soft_state(hermon_statep, instance);
20409e39c5baSBill Taylor 		if (state == NULL) {
20419e39c5baSBill Taylor 			cmn_err(CE_WARN, "Unable to match Hermon instance\n");
20429e39c5baSBill Taylor 			return;
20439e39c5baSBill Taylor 		}
20449e39c5baSBill Taylor 
20459e39c5baSBill Taylor 		/* Free the database entry */
20469e39c5baSBill Taylor 		hermon_umap_db_free(umapdb);
20479e39c5baSBill Taylor 
20489e39c5baSBill Taylor 		/* Use "value" to convert to an MR handle */
20499e39c5baSBill Taylor 		rsrcp = (hermon_rsrc_t *)(uintptr_t)value;
20509e39c5baSBill Taylor 		mr = (hermon_mrhdl_t)rsrcp->hr_addr;
20519e39c5baSBill Taylor 
20529e39c5baSBill Taylor 		/*
20539e39c5baSBill Taylor 		 * If a callback has been provided, call it first.  This
20549e39c5baSBill Taylor 		 * callback is expected to do any cleanup necessary to
20559e39c5baSBill Taylor 		 * guarantee that the subsequent MR deregister (below)
20569e39c5baSBill Taylor 		 * will succeed.  Specifically, this means freeing up memory
20579e39c5baSBill Taylor 		 * windows which might have been associated with the MR.
20589e39c5baSBill Taylor 		 */
20599e39c5baSBill Taylor 		mutex_enter(&mr->mr_lock);
20609e39c5baSBill Taylor 		mr_callback = mr->mr_umem_cbfunc;
20619e39c5baSBill Taylor 		mr_cbarg1   = mr->mr_umem_cbarg1;
20629e39c5baSBill Taylor 		mr_cbarg2   = mr->mr_umem_cbarg2;
20639e39c5baSBill Taylor 		mutex_exit(&mr->mr_lock);
20649e39c5baSBill Taylor 		if (mr_callback != NULL) {
20659e39c5baSBill Taylor 			mr_callback(mr_cbarg1, mr_cbarg2);
20669e39c5baSBill Taylor 		}
20679e39c5baSBill Taylor 
20689e39c5baSBill Taylor 		/*
20699e39c5baSBill Taylor 		 * Then call hermon_mr_deregister() to release the resources
20709e39c5baSBill Taylor 		 * associated with the MR handle.  Note: Because this routine
20719e39c5baSBill Taylor 		 * will also check for whether the ddi_umem_cookie_t is in the
20729e39c5baSBill Taylor 		 * database, it will take responsibility for disabling the
20739e39c5baSBill Taylor 		 * memory region and calling ddi_umem_unlock().
20749e39c5baSBill Taylor 		 */
20759e39c5baSBill Taylor 		status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
20769e39c5baSBill Taylor 		    HERMON_SLEEP);
20779e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
20789e39c5baSBill Taylor 			HERMON_WARNING(state, "Unexpected failure in "
20799e39c5baSBill Taylor 			    "deregister from callback\n");
20809e39c5baSBill Taylor 		}
20819e39c5baSBill Taylor 	}
20829e39c5baSBill Taylor }
20839e39c5baSBill Taylor 
20849e39c5baSBill Taylor 
20859e39c5baSBill Taylor /*
20869e39c5baSBill Taylor  * hermon_umap_db_compare()
20879e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
20889e39c5baSBill Taylor  */
20899e39c5baSBill Taylor static int
20909e39c5baSBill Taylor hermon_umap_db_compare(const void *q, const void *e)
20919e39c5baSBill Taylor {
20929e39c5baSBill Taylor 	hermon_umap_db_common_t	*entry_common, *query_common;
20939e39c5baSBill Taylor 	uint_t			query_flags;
20949e39c5baSBill Taylor 
20959e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((hermon_umap_db_query_t *)q)))
20969e39c5baSBill Taylor 
20979e39c5baSBill Taylor 	entry_common = &((hermon_umap_db_entry_t *)e)->hdbe_common;
20989e39c5baSBill Taylor 	query_common = &((hermon_umap_db_query_t *)q)->hqdb_common;
20999e39c5baSBill Taylor 	query_flags  = ((hermon_umap_db_query_t *)q)->hqdb_flags;
21009e39c5baSBill Taylor 
21019e39c5baSBill Taylor 	/*
21029e39c5baSBill Taylor 	 * The first comparison is done on the "key" value in "query"
21039e39c5baSBill Taylor 	 * and "entry".  If they are not equal, then the appropriate
21049e39c5baSBill Taylor 	 * search direction is returned.  Else, we continue by
21059e39c5baSBill Taylor 	 * comparing "type".
21069e39c5baSBill Taylor 	 */
21079e39c5baSBill Taylor 	if (query_common->hdb_key < entry_common->hdb_key) {
21089e39c5baSBill Taylor 		return (-1);
21099e39c5baSBill Taylor 	} else if (query_common->hdb_key > entry_common->hdb_key) {
21109e39c5baSBill Taylor 		return (+1);
21119e39c5baSBill Taylor 	}
21129e39c5baSBill Taylor 
21139e39c5baSBill Taylor 	/*
21149e39c5baSBill Taylor 	 * If the search reaches this point, then "query" and "entry"
21159e39c5baSBill Taylor 	 * have equal key values.  So we continue be comparing their
21169e39c5baSBill Taylor 	 * "type" values.  Again, if they are not equal, then the
21179e39c5baSBill Taylor 	 * appropriate search direction is returned.  Else, we continue
21189e39c5baSBill Taylor 	 * by comparing "instance".
21199e39c5baSBill Taylor 	 */
21209e39c5baSBill Taylor 	if (query_common->hdb_type < entry_common->hdb_type) {
21219e39c5baSBill Taylor 		return (-1);
21229e39c5baSBill Taylor 	} else if (query_common->hdb_type > entry_common->hdb_type) {
21239e39c5baSBill Taylor 		return (+1);
21249e39c5baSBill Taylor 	}
21259e39c5baSBill Taylor 
21269e39c5baSBill Taylor 	/*
21279e39c5baSBill Taylor 	 * If the search reaches this point, then "query" and "entry"
21289e39c5baSBill Taylor 	 * have exactly the same key and type values.  Now we consult
21299e39c5baSBill Taylor 	 * the "flags" field in the query to determine whether the
21309e39c5baSBill Taylor 	 * "instance" is relevant to the search.  If the
21319e39c5baSBill Taylor 	 * HERMON_UMAP_DB_IGNORE_INSTANCE flags is set, then return
21329e39c5baSBill Taylor 	 * success (0) here.  Otherwise, continue the search by comparing
21339e39c5baSBill Taylor 	 * instance values and returning the appropriate search direction.
21349e39c5baSBill Taylor 	 */
21359e39c5baSBill Taylor 	if (query_flags & HERMON_UMAP_DB_IGNORE_INSTANCE) {
21369e39c5baSBill Taylor 		return (0);
21379e39c5baSBill Taylor 	}
21389e39c5baSBill Taylor 
21399e39c5baSBill Taylor 	/*
21409e39c5baSBill Taylor 	 * If the search has reached this point, then "query" and "entry"
21419e39c5baSBill Taylor 	 * can only be differentiated by their instance values.  If these
21429e39c5baSBill Taylor 	 * are not equal, then return the appropriate search direction.
21439e39c5baSBill Taylor 	 * Else, we return success (0).
21449e39c5baSBill Taylor 	 */
21459e39c5baSBill Taylor 	if (query_common->hdb_instance < entry_common->hdb_instance) {
21469e39c5baSBill Taylor 		return (-1);
21479e39c5baSBill Taylor 	} else if (query_common->hdb_instance > entry_common->hdb_instance) {
21489e39c5baSBill Taylor 		return (+1);
21499e39c5baSBill Taylor 	}
21509e39c5baSBill Taylor 
21519e39c5baSBill Taylor 	/* Everything matches... so return success */
21529e39c5baSBill Taylor 	return (0);
21539e39c5baSBill Taylor }
21549e39c5baSBill Taylor 
21559e39c5baSBill Taylor 
21569e39c5baSBill Taylor /*
21579e39c5baSBill Taylor  * hermon_umap_db_set_onclose_cb()
21589e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
21599e39c5baSBill Taylor  */
21609e39c5baSBill Taylor int
21619e39c5baSBill Taylor hermon_umap_db_set_onclose_cb(dev_t dev, uint64_t flag,
21629e39c5baSBill Taylor     int (*callback)(void *), void *arg)
21639e39c5baSBill Taylor {
21649e39c5baSBill Taylor 	hermon_umap_db_priv_t	*priv;
21659e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
21669e39c5baSBill Taylor 	minor_t			instance;
21679e39c5baSBill Taylor 	uint64_t		value;
21689e39c5baSBill Taylor 	int			status;
21699e39c5baSBill Taylor 
21709e39c5baSBill Taylor 	instance = HERMON_DEV_INSTANCE(dev);
21719e39c5baSBill Taylor 	if (instance == (minor_t)-1) {
21729e39c5baSBill Taylor 		return (DDI_FAILURE);
21739e39c5baSBill Taylor 	}
21749e39c5baSBill Taylor 
21759e39c5baSBill Taylor 	if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) {
21769e39c5baSBill Taylor 		return (DDI_FAILURE);
21779e39c5baSBill Taylor 	}
21789e39c5baSBill Taylor 
21799e39c5baSBill Taylor 	/*
21809e39c5baSBill Taylor 	 * Grab the lock for the "userland resources database" and find
21819e39c5baSBill Taylor 	 * the entry corresponding to this minor number.  Once it's found,
21829e39c5baSBill Taylor 	 * allocate (if necessary) and add an entry (in the "hdb_priv"
21839e39c5baSBill Taylor 	 * field) to indicate that further processing may be needed during
21849e39c5baSBill Taylor 	 * Hermon's close() handling.
21859e39c5baSBill Taylor 	 */
21869e39c5baSBill Taylor 	mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
21879e39c5baSBill Taylor 	status = hermon_umap_db_find_nolock(instance, dev,
21889e39c5baSBill Taylor 	    MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
21899e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
21909e39c5baSBill Taylor 		mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
21919e39c5baSBill Taylor 		return (DDI_FAILURE);
21929e39c5baSBill Taylor 	}
21939e39c5baSBill Taylor 
21949e39c5baSBill Taylor 	priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv;
21959e39c5baSBill Taylor 	if (priv == NULL) {
21969e39c5baSBill Taylor 		priv = (hermon_umap_db_priv_t *)kmem_zalloc(
21979e39c5baSBill Taylor 		    sizeof (hermon_umap_db_priv_t), KM_NOSLEEP);
21989e39c5baSBill Taylor 		if (priv == NULL) {
21999e39c5baSBill Taylor 			mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
22009e39c5baSBill Taylor 			return (DDI_FAILURE);
22019e39c5baSBill Taylor 		}
22029e39c5baSBill Taylor 	}
22039e39c5baSBill Taylor 
22049e39c5baSBill Taylor 	/*
22059e39c5baSBill Taylor 	 * Save away the callback and argument to be used during Hermon's
22069e39c5baSBill Taylor 	 * close() processing.
22079e39c5baSBill Taylor 	 */
22089e39c5baSBill Taylor 	priv->hdp_cb	= callback;
22099e39c5baSBill Taylor 	priv->hdp_arg	= arg;
22109e39c5baSBill Taylor 
22119e39c5baSBill Taylor 	umapdb->hdbe_common.hdb_priv = (void *)priv;
22129e39c5baSBill Taylor 	mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
22139e39c5baSBill Taylor 
22149e39c5baSBill Taylor 	return (DDI_SUCCESS);
22159e39c5baSBill Taylor }
22169e39c5baSBill Taylor 
22179e39c5baSBill Taylor 
22189e39c5baSBill Taylor /*
22199e39c5baSBill Taylor  * hermon_umap_db_clear_onclose_cb()
22209e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
22219e39c5baSBill Taylor  */
22229e39c5baSBill Taylor int
22239e39c5baSBill Taylor hermon_umap_db_clear_onclose_cb(dev_t dev, uint64_t flag)
22249e39c5baSBill Taylor {
22259e39c5baSBill Taylor 	hermon_umap_db_priv_t	*priv;
22269e39c5baSBill Taylor 	hermon_umap_db_entry_t	*umapdb;
22279e39c5baSBill Taylor 	minor_t			instance;
22289e39c5baSBill Taylor 	uint64_t		value;
22299e39c5baSBill Taylor 	int			status;
22309e39c5baSBill Taylor 
22319e39c5baSBill Taylor 	instance = HERMON_DEV_INSTANCE(dev);
22329e39c5baSBill Taylor 	if (instance == (minor_t)-1) {
22339e39c5baSBill Taylor 		return (DDI_FAILURE);
22349e39c5baSBill Taylor 	}
22359e39c5baSBill Taylor 
22369e39c5baSBill Taylor 	if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) {
22379e39c5baSBill Taylor 		return (DDI_FAILURE);
22389e39c5baSBill Taylor 	}
22399e39c5baSBill Taylor 
22409e39c5baSBill Taylor 	/*
22419e39c5baSBill Taylor 	 * Grab the lock for the "userland resources database" and find
22429e39c5baSBill Taylor 	 * the entry corresponding to this minor number.  Once it's found,
22439e39c5baSBill Taylor 	 * remove the entry (in the "hdb_priv" field) that indicated the
22449e39c5baSBill Taylor 	 * need for further processing during Hermon's close().  Free the
22459e39c5baSBill Taylor 	 * entry, if appropriate.
22469e39c5baSBill Taylor 	 */
22479e39c5baSBill Taylor 	mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
22489e39c5baSBill Taylor 	status = hermon_umap_db_find_nolock(instance, dev,
22499e39c5baSBill Taylor 	    MLNX_UMAP_PID_RSRC, &value, 0, &umapdb);
22509e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
22519e39c5baSBill Taylor 		mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
22529e39c5baSBill Taylor 		return (DDI_FAILURE);
22539e39c5baSBill Taylor 	}
22549e39c5baSBill Taylor 
22559e39c5baSBill Taylor 	priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv;
22569e39c5baSBill Taylor 	if (priv != NULL) {
22579e39c5baSBill Taylor 		kmem_free(priv, sizeof (hermon_umap_db_priv_t));
22589e39c5baSBill Taylor 		priv = NULL;
22599e39c5baSBill Taylor 	}
22609e39c5baSBill Taylor 
22619e39c5baSBill Taylor 	umapdb->hdbe_common.hdb_priv = (void *)priv;
22629e39c5baSBill Taylor 	mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
22639e39c5baSBill Taylor 	return (DDI_SUCCESS);
22649e39c5baSBill Taylor }
22659e39c5baSBill Taylor 
22669e39c5baSBill Taylor 
22679e39c5baSBill Taylor /*
22689e39c5baSBill Taylor  * hermon_umap_db_clear_onclose_cb()
22699e39c5baSBill Taylor  *    Context: Can be called from user or kernel context.
22709e39c5baSBill Taylor  */
22719e39c5baSBill Taylor int
22729e39c5baSBill Taylor hermon_umap_db_handle_onclose_cb(hermon_umap_db_priv_t *priv)
22739e39c5baSBill Taylor {
22749e39c5baSBill Taylor 	int	(*callback)(void *);
22759e39c5baSBill Taylor 
22769e39c5baSBill Taylor 	ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock));
22779e39c5baSBill Taylor 
22789e39c5baSBill Taylor 	/*
22799e39c5baSBill Taylor 	 * Call the callback.
22809e39c5baSBill Taylor 	 *    Note: Currently there is only one callback (in "hdp_cb"), but
22819e39c5baSBill Taylor 	 *    in the future there may be more, depending on what other types
22829e39c5baSBill Taylor 	 *    of interaction there are between userland processes and the
22839e39c5baSBill Taylor 	 *    driver.
22849e39c5baSBill Taylor 	 */
22859e39c5baSBill Taylor 	callback = priv->hdp_cb;
22869e39c5baSBill Taylor 	return (callback(priv->hdp_arg));
22879e39c5baSBill Taylor }
2288