19e39c5baSBill Taylor /* 29e39c5baSBill Taylor * CDDL HEADER START 39e39c5baSBill Taylor * 49e39c5baSBill Taylor * The contents of this file are subject to the terms of the 59e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 69e39c5baSBill Taylor * You may not use this file except in compliance with the License. 79e39c5baSBill Taylor * 89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 109e39c5baSBill Taylor * See the License for the specific language governing permissions 119e39c5baSBill Taylor * and limitations under the License. 129e39c5baSBill Taylor * 139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 189e39c5baSBill Taylor * 199e39c5baSBill Taylor * CDDL HEADER END 209e39c5baSBill Taylor */ 219e39c5baSBill Taylor 229e39c5baSBill Taylor /* 23*17a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 249e39c5baSBill Taylor */ 259e39c5baSBill Taylor 269e39c5baSBill Taylor /* 279e39c5baSBill Taylor * hermon_umap.c 289e39c5baSBill Taylor * Hermon Userland Mapping Routines 299e39c5baSBill Taylor * 309e39c5baSBill Taylor * Implements all the routines necessary for enabling direct userland 319e39c5baSBill Taylor * access to the Hermon hardware. This includes all routines necessary for 329e39c5baSBill Taylor * maintaining the "userland resources database" and all the support routines 339e39c5baSBill Taylor * for the devmap calls. 349e39c5baSBill Taylor */ 359e39c5baSBill Taylor 369e39c5baSBill Taylor #include <sys/types.h> 379e39c5baSBill Taylor #include <sys/conf.h> 389e39c5baSBill Taylor #include <sys/ddi.h> 399e39c5baSBill Taylor #include <sys/sunddi.h> 409e39c5baSBill Taylor #include <sys/modctl.h> 419e39c5baSBill Taylor #include <sys/file.h> 429e39c5baSBill Taylor #include <sys/avl.h> 439e39c5baSBill Taylor #include <sys/sysmacros.h> 449e39c5baSBill Taylor 459e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 469e39c5baSBill Taylor 479e39c5baSBill Taylor /* Hermon HCA state pointer (extern) */ 489e39c5baSBill Taylor extern void *hermon_statep; 499e39c5baSBill Taylor 509e39c5baSBill Taylor /* Hermon HCA Userland Resource Database (extern) */ 519e39c5baSBill Taylor extern hermon_umap_db_t hermon_userland_rsrc_db; 529e39c5baSBill Taylor 539e39c5baSBill Taylor static int hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp, 549e39c5baSBill Taylor hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err); 559e39c5baSBill Taylor static int hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp, 569e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err); 579e39c5baSBill Taylor static int hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp, 589e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err); 599e39c5baSBill Taylor static int hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp, 609e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err); 619e39c5baSBill Taylor static int hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp, 629e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err); 639e39c5baSBill Taylor static int hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags, 649e39c5baSBill Taylor offset_t off, size_t len, void **pvtp); 659e39c5baSBill Taylor static int hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, 669e39c5baSBill Taylor devmap_cookie_t new_dhp, void **new_pvtp); 679e39c5baSBill Taylor static void hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, 689e39c5baSBill Taylor offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1, 699e39c5baSBill Taylor devmap_cookie_t new_dhp2, void **pvtp2); 709e39c5baSBill Taylor static int hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev, 719e39c5baSBill Taylor uint_t flags, offset_t off, size_t len, void **pvtp); 729e39c5baSBill Taylor static int hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp, 739e39c5baSBill Taylor devmap_cookie_t new_dhp, void **new_pvtp); 749e39c5baSBill Taylor static void hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp, 759e39c5baSBill Taylor offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1, 769e39c5baSBill Taylor devmap_cookie_t new_dhp2, void **pvtp2); 779e39c5baSBill Taylor static int hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev, 789e39c5baSBill Taylor uint_t flags, offset_t off, size_t len, void **pvtp); 799e39c5baSBill Taylor static int hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp, 809e39c5baSBill Taylor devmap_cookie_t new_dhp, void **new_pvtp); 819e39c5baSBill Taylor static void hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, 829e39c5baSBill Taylor offset_t off, size_t len, devmap_cookie_t new_dhp1, void **pvtp1, 839e39c5baSBill Taylor devmap_cookie_t new_dhp2, void **pvtp2); 849e39c5baSBill Taylor static ibt_status_t hermon_umap_mr_data_in(hermon_mrhdl_t mr, 859e39c5baSBill Taylor ibt_mr_data_in_t *data, size_t data_sz); 869e39c5baSBill Taylor static ibt_status_t hermon_umap_cq_data_out(hermon_cqhdl_t cq, 879e39c5baSBill Taylor mlnx_umap_cq_data_out_t *data, size_t data_sz); 889e39c5baSBill Taylor static ibt_status_t hermon_umap_qp_data_out(hermon_qphdl_t qp, 899e39c5baSBill Taylor mlnx_umap_qp_data_out_t *data, size_t data_sz); 909e39c5baSBill Taylor static ibt_status_t hermon_umap_srq_data_out(hermon_srqhdl_t srq, 919e39c5baSBill Taylor mlnx_umap_srq_data_out_t *data, size_t data_sz); 929e39c5baSBill Taylor static ibt_status_t hermon_umap_pd_data_out(hermon_pdhdl_t pd, 939e39c5baSBill Taylor mlnx_umap_pd_data_out_t *data, size_t data_sz); 949e39c5baSBill Taylor static int hermon_umap_db_compare(const void *query, const void *entry); 959e39c5baSBill Taylor 969e39c5baSBill Taylor 979e39c5baSBill Taylor /* 989e39c5baSBill Taylor * These callbacks are passed to devmap_umem_setup() and devmap_devmem_setup(), 999e39c5baSBill Taylor * respectively. They are used to handle (among other things) partial 1009e39c5baSBill Taylor * unmappings and to provide a method for invalidating mappings inherited 1019e39c5baSBill Taylor * as a result of a fork(2) system call. 1029e39c5baSBill Taylor */ 1039e39c5baSBill Taylor static struct devmap_callback_ctl hermon_devmap_umem_cbops = { 1049e39c5baSBill Taylor DEVMAP_OPS_REV, 1059e39c5baSBill Taylor hermon_devmap_umem_map, 1069e39c5baSBill Taylor NULL, 1079e39c5baSBill Taylor hermon_devmap_umem_dup, 1089e39c5baSBill Taylor hermon_devmap_umem_unmap 1099e39c5baSBill Taylor }; 1109e39c5baSBill Taylor static struct devmap_callback_ctl hermon_devmap_devmem_cbops = { 1119e39c5baSBill Taylor DEVMAP_OPS_REV, 1129e39c5baSBill Taylor hermon_devmap_devmem_map, 1139e39c5baSBill Taylor NULL, 1149e39c5baSBill Taylor hermon_devmap_devmem_dup, 1159e39c5baSBill Taylor hermon_devmap_devmem_unmap 1169e39c5baSBill Taylor }; 1179e39c5baSBill Taylor static struct devmap_callback_ctl hermon_devmap_dbrecmem_cbops = { 1189e39c5baSBill Taylor DEVMAP_OPS_REV, 1199e39c5baSBill Taylor hermon_devmap_dbrecmem_map, 1209e39c5baSBill Taylor NULL, 1219e39c5baSBill Taylor hermon_devmap_dbrecmem_dup, 1229e39c5baSBill Taylor hermon_devmap_dbrecmem_unmap 1239e39c5baSBill Taylor }; 1249e39c5baSBill Taylor 1259e39c5baSBill Taylor /* 1269e39c5baSBill Taylor * hermon_devmap() 1279e39c5baSBill Taylor * Context: Can be called from user context. 1289e39c5baSBill Taylor */ 1299e39c5baSBill Taylor /* ARGSUSED */ 1309e39c5baSBill Taylor int 1319e39c5baSBill Taylor hermon_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len, 1329e39c5baSBill Taylor size_t *maplen, uint_t model) 1339e39c5baSBill Taylor { 1349e39c5baSBill Taylor hermon_state_t *state; 1359e39c5baSBill Taylor hermon_rsrc_t *rsrcp; 1369e39c5baSBill Taylor minor_t instance; 1379e39c5baSBill Taylor uint64_t key, value; 1389e39c5baSBill Taylor uint64_t bf_offset = 0; 1399e39c5baSBill Taylor uint_t type; 1409e39c5baSBill Taylor int err, status; 1419e39c5baSBill Taylor 1429e39c5baSBill Taylor /* Get Hermon softstate structure from instance */ 1439e39c5baSBill Taylor instance = HERMON_DEV_INSTANCE(dev); 1449e39c5baSBill Taylor state = ddi_get_soft_state(hermon_statep, instance); 1459e39c5baSBill Taylor if (state == NULL) { 1469e39c5baSBill Taylor return (ENXIO); 1479e39c5baSBill Taylor } 1489e39c5baSBill Taylor 1499e39c5baSBill Taylor /* 1509e39c5baSBill Taylor * Access to Hermon devmap interface is not allowed in 1519e39c5baSBill Taylor * "maintenance mode". 1529e39c5baSBill Taylor */ 1539e39c5baSBill Taylor if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) { 1549e39c5baSBill Taylor return (EFAULT); 1559e39c5baSBill Taylor } 1569e39c5baSBill Taylor 1579e39c5baSBill Taylor /* 1589e39c5baSBill Taylor * The bottom bits of "offset" are undefined (number depends on 1599e39c5baSBill Taylor * system PAGESIZE). Shifting these off leaves us with a "key". 1609e39c5baSBill Taylor * The "key" is actually a combination of both a real key value 1619e39c5baSBill Taylor * (for the purpose of database lookup) and a "type" value. We 1629e39c5baSBill Taylor * extract this information before doing the database lookup. 1639e39c5baSBill Taylor */ 1649e39c5baSBill Taylor key = off >> PAGESHIFT; 1659e39c5baSBill Taylor type = key & MLNX_UMAP_RSRC_TYPE_MASK; 1669e39c5baSBill Taylor key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT; 1679e39c5baSBill Taylor if (type == MLNX_UMAP_BLUEFLAMEPG_RSRC) { 1689e39c5baSBill Taylor if (state->hs_devlim.blu_flm == 0) { 1699e39c5baSBill Taylor return (EFAULT); 1709e39c5baSBill Taylor } 1719e39c5baSBill Taylor bf_offset = state->hs_bf_offset; 1729e39c5baSBill Taylor type = MLNX_UMAP_UARPG_RSRC; 1739e39c5baSBill Taylor } 1749e39c5baSBill Taylor status = hermon_umap_db_find(instance, key, type, &value, 0, NULL); 1759e39c5baSBill Taylor if (status == DDI_SUCCESS) { 1769e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 1779e39c5baSBill Taylor 1789e39c5baSBill Taylor switch (type) { 1799e39c5baSBill Taylor case MLNX_UMAP_UARPG_RSRC: 1809e39c5baSBill Taylor /* 1819e39c5baSBill Taylor * Double check that process who open()'d Hermon is 1829e39c5baSBill Taylor * same process attempting to mmap() UAR page. 1839e39c5baSBill Taylor */ 1849e39c5baSBill Taylor if (key != ddi_get_pid()) { 1859e39c5baSBill Taylor return (EINVAL); 1869e39c5baSBill Taylor } 1879e39c5baSBill Taylor 1889e39c5baSBill Taylor /* Map the UAR page out for userland access */ 1899e39c5baSBill Taylor status = hermon_umap_uarpg(state, dhp, rsrcp, bf_offset, 1909e39c5baSBill Taylor maplen, &err); 1919e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1929e39c5baSBill Taylor return (err); 1939e39c5baSBill Taylor } 1949e39c5baSBill Taylor break; 1959e39c5baSBill Taylor 1969e39c5baSBill Taylor case MLNX_UMAP_CQMEM_RSRC: 1979e39c5baSBill Taylor /* Map the CQ memory out for userland access */ 1989e39c5baSBill Taylor status = hermon_umap_cqmem(state, dhp, rsrcp, off, 1999e39c5baSBill Taylor maplen, &err); 2009e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2019e39c5baSBill Taylor return (err); 2029e39c5baSBill Taylor } 2039e39c5baSBill Taylor break; 2049e39c5baSBill Taylor 2059e39c5baSBill Taylor case MLNX_UMAP_QPMEM_RSRC: 2069e39c5baSBill Taylor /* Map the QP memory out for userland access */ 2079e39c5baSBill Taylor status = hermon_umap_qpmem(state, dhp, rsrcp, off, 2089e39c5baSBill Taylor maplen, &err); 2099e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2109e39c5baSBill Taylor return (err); 2119e39c5baSBill Taylor } 2129e39c5baSBill Taylor break; 2139e39c5baSBill Taylor 2149e39c5baSBill Taylor case MLNX_UMAP_SRQMEM_RSRC: 2159e39c5baSBill Taylor /* Map the SRQ memory out for userland access */ 2169e39c5baSBill Taylor status = hermon_umap_srqmem(state, dhp, rsrcp, off, 2179e39c5baSBill Taylor maplen, &err); 2189e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2199e39c5baSBill Taylor return (err); 2209e39c5baSBill Taylor } 2219e39c5baSBill Taylor break; 2229e39c5baSBill Taylor 2239e39c5baSBill Taylor case MLNX_UMAP_DBRMEM_RSRC: 2249e39c5baSBill Taylor /* 2259e39c5baSBill Taylor * Map the doorbell record memory out for 2269e39c5baSBill Taylor * userland access 2279e39c5baSBill Taylor */ 2289e39c5baSBill Taylor status = hermon_umap_dbrecmem(state, dhp, rsrcp, off, 2299e39c5baSBill Taylor maplen, &err); 2309e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2319e39c5baSBill Taylor return (err); 2329e39c5baSBill Taylor } 2339e39c5baSBill Taylor break; 2349e39c5baSBill Taylor 2359e39c5baSBill Taylor default: 2369e39c5baSBill Taylor HERMON_WARNING(state, "unexpected rsrc type in devmap"); 2379e39c5baSBill Taylor return (EINVAL); 2389e39c5baSBill Taylor } 2399e39c5baSBill Taylor } else { 2409e39c5baSBill Taylor return (EINVAL); 2419e39c5baSBill Taylor } 2429e39c5baSBill Taylor 2439e39c5baSBill Taylor return (0); 2449e39c5baSBill Taylor } 2459e39c5baSBill Taylor 2469e39c5baSBill Taylor 2479e39c5baSBill Taylor /* 2489e39c5baSBill Taylor * hermon_umap_uarpg() 2499e39c5baSBill Taylor * Context: Can be called from user context. 2509e39c5baSBill Taylor */ 2519e39c5baSBill Taylor static int 2529e39c5baSBill Taylor hermon_umap_uarpg(hermon_state_t *state, devmap_cookie_t dhp, 2539e39c5baSBill Taylor hermon_rsrc_t *rsrcp, uint64_t offset, size_t *maplen, int *err) 2549e39c5baSBill Taylor { 2559e39c5baSBill Taylor int status; 2569e39c5baSBill Taylor uint_t maxprot; 2579e39c5baSBill Taylor ddi_device_acc_attr_t *accattrp = &state->hs_reg_accattr; 2589e39c5baSBill Taylor ddi_device_acc_attr_t accattr; 2599e39c5baSBill Taylor 2609e39c5baSBill Taylor if (offset != 0) { /* Hermon Blueflame */ 2619e39c5baSBill Taylor /* Try to use write coalescing data ordering */ 2629e39c5baSBill Taylor accattr = *accattrp; 2639e39c5baSBill Taylor accattr.devacc_attr_dataorder = DDI_STORECACHING_OK_ACC; 2649e39c5baSBill Taylor accattrp = &accattr; 2659e39c5baSBill Taylor } 2669e39c5baSBill Taylor 2679e39c5baSBill Taylor /* Map out the UAR page (doorbell page) */ 2689e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 2699e39c5baSBill Taylor status = devmap_devmem_setup(dhp, state->hs_dip, 2709e39c5baSBill Taylor &hermon_devmap_devmem_cbops, HERMON_UAR_BAR, (rsrcp->hr_indx << 2719e39c5baSBill Taylor PAGESHIFT) + offset, PAGESIZE, maxprot, DEVMAP_ALLOW_REMAP, 2729e39c5baSBill Taylor accattrp); 2739e39c5baSBill Taylor if (status < 0) { 2749e39c5baSBill Taylor *err = status; 2759e39c5baSBill Taylor return (DDI_FAILURE); 2769e39c5baSBill Taylor } 2779e39c5baSBill Taylor 2789e39c5baSBill Taylor *maplen = PAGESIZE; 2799e39c5baSBill Taylor return (DDI_SUCCESS); 2809e39c5baSBill Taylor } 2819e39c5baSBill Taylor 2829e39c5baSBill Taylor 2839e39c5baSBill Taylor /* 2849e39c5baSBill Taylor * hermon_umap_cqmem() 2859e39c5baSBill Taylor * Context: Can be called from user context. 2869e39c5baSBill Taylor */ 2879e39c5baSBill Taylor /* ARGSUSED */ 2889e39c5baSBill Taylor static int 2899e39c5baSBill Taylor hermon_umap_cqmem(hermon_state_t *state, devmap_cookie_t dhp, 2909e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err) 2919e39c5baSBill Taylor { 2929e39c5baSBill Taylor hermon_cqhdl_t cq; 2939e39c5baSBill Taylor size_t size; 2949e39c5baSBill Taylor uint_t maxprot; 2959e39c5baSBill Taylor int status; 2969e39c5baSBill Taylor 2979e39c5baSBill Taylor /* Extract the Hermon CQ handle pointer from the hermon_rsrc_t */ 2989e39c5baSBill Taylor cq = (hermon_cqhdl_t)rsrcp->hr_addr; 2999e39c5baSBill Taylor 3009e39c5baSBill Taylor /* Round-up the CQ size to system page size */ 3019e39c5baSBill Taylor size = ptob(btopr(cq->cq_resize_hdl ? 3029e39c5baSBill Taylor cq->cq_resize_hdl->cq_cqinfo.qa_size : cq->cq_cqinfo.qa_size)); 3039e39c5baSBill Taylor 3049e39c5baSBill Taylor /* Map out the CQ memory - use resize_hdl if non-NULL */ 3059e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 3069e39c5baSBill Taylor status = devmap_umem_setup(dhp, state->hs_dip, 3079e39c5baSBill Taylor &hermon_devmap_umem_cbops, cq->cq_resize_hdl ? 3089e39c5baSBill Taylor cq->cq_resize_hdl->cq_cqinfo.qa_umemcookie : 3099e39c5baSBill Taylor cq->cq_cqinfo.qa_umemcookie, 0, size, 3109e39c5baSBill Taylor maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL); 3119e39c5baSBill Taylor if (status < 0) { 3129e39c5baSBill Taylor *err = status; 3139e39c5baSBill Taylor return (DDI_FAILURE); 3149e39c5baSBill Taylor } 3159e39c5baSBill Taylor *maplen = size; 3169e39c5baSBill Taylor 3179e39c5baSBill Taylor return (DDI_SUCCESS); 3189e39c5baSBill Taylor } 3199e39c5baSBill Taylor 3209e39c5baSBill Taylor 3219e39c5baSBill Taylor /* 3229e39c5baSBill Taylor * hermon_umap_qpmem() 3239e39c5baSBill Taylor * Context: Can be called from user context. 3249e39c5baSBill Taylor */ 3259e39c5baSBill Taylor /* ARGSUSED */ 3269e39c5baSBill Taylor static int 3279e39c5baSBill Taylor hermon_umap_qpmem(hermon_state_t *state, devmap_cookie_t dhp, 3289e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err) 3299e39c5baSBill Taylor { 3309e39c5baSBill Taylor hermon_qphdl_t qp; 3319e39c5baSBill Taylor offset_t offset; 3329e39c5baSBill Taylor size_t size; 3339e39c5baSBill Taylor uint_t maxprot; 3349e39c5baSBill Taylor int status; 3359e39c5baSBill Taylor 3369e39c5baSBill Taylor /* Extract the Hermon QP handle pointer from the hermon_rsrc_t */ 3379e39c5baSBill Taylor qp = (hermon_qphdl_t)rsrcp->hr_addr; 3389e39c5baSBill Taylor 3399e39c5baSBill Taylor /* 3409e39c5baSBill Taylor * Calculate the offset of the first work queue (send or recv) into 3419e39c5baSBill Taylor * the memory (ddi_umem_alloc()) allocated previously for the QP. 3429e39c5baSBill Taylor */ 3439e39c5baSBill Taylor offset = (offset_t)((uintptr_t)qp->qp_wqinfo.qa_buf_aligned - 3449e39c5baSBill Taylor (uintptr_t)qp->qp_wqinfo.qa_buf_real); 3459e39c5baSBill Taylor 3469e39c5baSBill Taylor /* Round-up the QP work queue sizes to system page size */ 3479e39c5baSBill Taylor size = ptob(btopr(qp->qp_wqinfo.qa_size)); 3489e39c5baSBill Taylor 3499e39c5baSBill Taylor /* Map out the QP memory */ 3509e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 3519e39c5baSBill Taylor status = devmap_umem_setup(dhp, state->hs_dip, 3529e39c5baSBill Taylor &hermon_devmap_umem_cbops, qp->qp_wqinfo.qa_umemcookie, offset, 3539e39c5baSBill Taylor size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL); 3549e39c5baSBill Taylor if (status < 0) { 3559e39c5baSBill Taylor *err = status; 3569e39c5baSBill Taylor return (DDI_FAILURE); 3579e39c5baSBill Taylor } 3589e39c5baSBill Taylor *maplen = size; 3599e39c5baSBill Taylor 3609e39c5baSBill Taylor return (DDI_SUCCESS); 3619e39c5baSBill Taylor } 3629e39c5baSBill Taylor 3639e39c5baSBill Taylor 3649e39c5baSBill Taylor /* 3659e39c5baSBill Taylor * hermon_umap_srqmem() 3669e39c5baSBill Taylor * Context: Can be called from user context. 3679e39c5baSBill Taylor */ 3689e39c5baSBill Taylor /* ARGSUSED */ 3699e39c5baSBill Taylor static int 3709e39c5baSBill Taylor hermon_umap_srqmem(hermon_state_t *state, devmap_cookie_t dhp, 3719e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err) 3729e39c5baSBill Taylor { 3739e39c5baSBill Taylor hermon_srqhdl_t srq; 3749e39c5baSBill Taylor offset_t offset; 3759e39c5baSBill Taylor size_t size; 3769e39c5baSBill Taylor uint_t maxprot; 3779e39c5baSBill Taylor int status; 3789e39c5baSBill Taylor 3799e39c5baSBill Taylor /* Extract the Hermon SRQ handle pointer from the hermon_rsrc_t */ 3809e39c5baSBill Taylor srq = (hermon_srqhdl_t)rsrcp->hr_addr; 3819e39c5baSBill Taylor 3829e39c5baSBill Taylor /* 3839e39c5baSBill Taylor * Calculate the offset of the first shared recv queue into the memory 3849e39c5baSBill Taylor * (ddi_umem_alloc()) allocated previously for the SRQ. 3859e39c5baSBill Taylor */ 3869e39c5baSBill Taylor offset = (offset_t)((uintptr_t)srq->srq_wqinfo.qa_buf_aligned - 3879e39c5baSBill Taylor (uintptr_t)srq->srq_wqinfo.qa_buf_real); 3889e39c5baSBill Taylor 3899e39c5baSBill Taylor /* Round-up the SRQ work queue sizes to system page size */ 3909e39c5baSBill Taylor size = ptob(btopr(srq->srq_wqinfo.qa_size)); 3919e39c5baSBill Taylor 3929e39c5baSBill Taylor /* Map out the SRQ memory */ 3939e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 3949e39c5baSBill Taylor status = devmap_umem_setup(dhp, state->hs_dip, 3959e39c5baSBill Taylor &hermon_devmap_umem_cbops, srq->srq_wqinfo.qa_umemcookie, offset, 3969e39c5baSBill Taylor size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL); 3979e39c5baSBill Taylor if (status < 0) { 3989e39c5baSBill Taylor *err = status; 3999e39c5baSBill Taylor return (DDI_FAILURE); 4009e39c5baSBill Taylor } 4019e39c5baSBill Taylor *maplen = size; 4029e39c5baSBill Taylor 4039e39c5baSBill Taylor return (DDI_SUCCESS); 4049e39c5baSBill Taylor } 4059e39c5baSBill Taylor 4069e39c5baSBill Taylor 4079e39c5baSBill Taylor /* 4089e39c5baSBill Taylor * hermon_devmap_dbrecmem() 4099e39c5baSBill Taylor * Context: Can be called from user context. 4109e39c5baSBill Taylor */ 4119e39c5baSBill Taylor /* ARGSUSED */ 4129e39c5baSBill Taylor static int 4139e39c5baSBill Taylor hermon_umap_dbrecmem(hermon_state_t *state, devmap_cookie_t dhp, 4149e39c5baSBill Taylor hermon_rsrc_t *rsrcp, offset_t off, size_t *maplen, int *err) 4159e39c5baSBill Taylor { 4169e39c5baSBill Taylor hermon_udbr_page_t *pagep; 4179e39c5baSBill Taylor offset_t offset; 4189e39c5baSBill Taylor size_t size; 4199e39c5baSBill Taylor uint_t maxprot; 4209e39c5baSBill Taylor int status; 4219e39c5baSBill Taylor 4229e39c5baSBill Taylor /* We stored the udbr_page pointer, and not a hermon_rsrc_t */ 4239e39c5baSBill Taylor pagep = (hermon_udbr_page_t *)rsrcp; 4249e39c5baSBill Taylor 4259e39c5baSBill Taylor /* 4269e39c5baSBill Taylor * Calculate the offset of the doorbell records into the memory 4279e39c5baSBill Taylor * (ddi_umem_alloc()) allocated previously for them. 4289e39c5baSBill Taylor */ 4299e39c5baSBill Taylor offset = 0; 4309e39c5baSBill Taylor 4319e39c5baSBill Taylor /* Round-up the doorbell page to system page size */ 4329e39c5baSBill Taylor size = PAGESIZE; 4339e39c5baSBill Taylor 4349e39c5baSBill Taylor /* Map out the Doorbell Record memory */ 4359e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 4369e39c5baSBill Taylor status = devmap_umem_setup(dhp, state->hs_dip, 4379e39c5baSBill Taylor &hermon_devmap_dbrecmem_cbops, pagep->upg_umemcookie, offset, 4389e39c5baSBill Taylor size, maxprot, (DEVMAP_ALLOW_REMAP | DEVMAP_DEFAULTS), NULL); 4399e39c5baSBill Taylor if (status < 0) { 4409e39c5baSBill Taylor *err = status; 4419e39c5baSBill Taylor return (DDI_FAILURE); 4429e39c5baSBill Taylor } 4439e39c5baSBill Taylor *maplen = size; 4449e39c5baSBill Taylor 4459e39c5baSBill Taylor return (DDI_SUCCESS); 4469e39c5baSBill Taylor } 4479e39c5baSBill Taylor 4489e39c5baSBill Taylor 4499e39c5baSBill Taylor /* 4509e39c5baSBill Taylor * hermon_devmap_umem_map() 4519e39c5baSBill Taylor * Context: Can be called from kernel context. 4529e39c5baSBill Taylor */ 4539e39c5baSBill Taylor /* ARGSUSED */ 4549e39c5baSBill Taylor static int 4559e39c5baSBill Taylor hermon_devmap_umem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags, 4569e39c5baSBill Taylor offset_t off, size_t len, void **pvtp) 4579e39c5baSBill Taylor { 4589e39c5baSBill Taylor hermon_state_t *state; 4599e39c5baSBill Taylor hermon_devmap_track_t *dvm_track; 4609e39c5baSBill Taylor hermon_cqhdl_t cq; 4619e39c5baSBill Taylor hermon_qphdl_t qp; 4629e39c5baSBill Taylor hermon_srqhdl_t srq; 4639e39c5baSBill Taylor minor_t instance; 4649e39c5baSBill Taylor uint64_t key; 4659e39c5baSBill Taylor uint_t type; 4669e39c5baSBill Taylor 4679e39c5baSBill Taylor /* Get Hermon softstate structure from instance */ 4689e39c5baSBill Taylor instance = HERMON_DEV_INSTANCE(dev); 4699e39c5baSBill Taylor state = ddi_get_soft_state(hermon_statep, instance); 4709e39c5baSBill Taylor if (state == NULL) { 4719e39c5baSBill Taylor return (ENXIO); 4729e39c5baSBill Taylor } 4739e39c5baSBill Taylor 4749e39c5baSBill Taylor /* 4759e39c5baSBill Taylor * The bottom bits of "offset" are undefined (number depends on 4769e39c5baSBill Taylor * system PAGESIZE). Shifting these off leaves us with a "key". 4779e39c5baSBill Taylor * The "key" is actually a combination of both a real key value 4789e39c5baSBill Taylor * (for the purpose of database lookup) and a "type" value. Although 4799e39c5baSBill Taylor * we are not going to do any database lookup per se, we do want 4809e39c5baSBill Taylor * to extract the "key" and the "type" (to enable faster lookup of 4819e39c5baSBill Taylor * the appropriate CQ or QP handle). 4829e39c5baSBill Taylor */ 4839e39c5baSBill Taylor key = off >> PAGESHIFT; 4849e39c5baSBill Taylor type = key & MLNX_UMAP_RSRC_TYPE_MASK; 4859e39c5baSBill Taylor key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT; 4869e39c5baSBill Taylor 4879e39c5baSBill Taylor /* 4889e39c5baSBill Taylor * Allocate an entry to track the mapping and unmapping (specifically, 4899e39c5baSBill Taylor * partial unmapping) of this resource. 4909e39c5baSBill Taylor */ 4919e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)kmem_zalloc( 4929e39c5baSBill Taylor sizeof (hermon_devmap_track_t), KM_SLEEP); 4939e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 4949e39c5baSBill Taylor dvm_track->hdt_offset = off; 4959e39c5baSBill Taylor dvm_track->hdt_state = state; 4969e39c5baSBill Taylor dvm_track->hdt_refcnt = 1; 4979e39c5baSBill Taylor mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER, 4989e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 4999e39c5baSBill Taylor 5009e39c5baSBill Taylor /* 5019e39c5baSBill Taylor * Depending of the type of resource that has been mapped out, we 5029e39c5baSBill Taylor * need to update the QP or CQ handle to reflect that it has, in 5039e39c5baSBill Taylor * fact, been mapped. This allows the driver code which frees a QP 5049e39c5baSBill Taylor * or a CQ to know whether it is appropriate to do a 5059e39c5baSBill Taylor * devmap_devmem_remap() to invalidate the userland mapping for the 5069e39c5baSBill Taylor * corresponding queue's memory. 5079e39c5baSBill Taylor */ 5089e39c5baSBill Taylor if (type == MLNX_UMAP_CQMEM_RSRC) { 5099e39c5baSBill Taylor 5109e39c5baSBill Taylor /* Use "key" (CQ number) to do fast lookup of CQ handle */ 5119e39c5baSBill Taylor cq = hermon_cqhdl_from_cqnum(state, key); 5129e39c5baSBill Taylor 5139e39c5baSBill Taylor /* 5149e39c5baSBill Taylor * Update the handle to the userland mapping. Note: If 5159e39c5baSBill Taylor * the CQ already has a valid userland mapping, then stop 5169e39c5baSBill Taylor * and return failure. 5179e39c5baSBill Taylor */ 5189e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 5199e39c5baSBill Taylor if (cq->cq_umap_dhp == NULL) { 5209e39c5baSBill Taylor cq->cq_umap_dhp = dhp; 5219e39c5baSBill Taylor dvm_track->hdt_size = cq->cq_cqinfo.qa_size; 5229e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 5239e39c5baSBill Taylor } else if (cq->cq_resize_hdl && 5249e39c5baSBill Taylor (cq->cq_resize_hdl->cq_umap_dhp == NULL)) { 5259e39c5baSBill Taylor cq->cq_resize_hdl->cq_umap_dhp = dhp; 5269e39c5baSBill Taylor dvm_track->hdt_size = 5279e39c5baSBill Taylor cq->cq_resize_hdl->cq_cqinfo.qa_size; 5289e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 5299e39c5baSBill Taylor } else { 5309e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 5319e39c5baSBill Taylor goto umem_map_fail; 5329e39c5baSBill Taylor } 5339e39c5baSBill Taylor 5349e39c5baSBill Taylor } else if (type == MLNX_UMAP_QPMEM_RSRC) { 5359e39c5baSBill Taylor 5369e39c5baSBill Taylor /* Use "key" (QP number) to do fast lookup of QP handle */ 5379e39c5baSBill Taylor qp = hermon_qphdl_from_qpnum(state, key); 5389e39c5baSBill Taylor 5399e39c5baSBill Taylor /* 5409e39c5baSBill Taylor * Update the handle to the userland mapping. Note: If 5419e39c5baSBill Taylor * the CQ already has a valid userland mapping, then stop 5429e39c5baSBill Taylor * and return failure. 5439e39c5baSBill Taylor */ 5449e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 5459e39c5baSBill Taylor if (qp->qp_umap_dhp == NULL) { 5469e39c5baSBill Taylor qp->qp_umap_dhp = dhp; 5479e39c5baSBill Taylor dvm_track->hdt_size = qp->qp_wqinfo.qa_size; 5489e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 5499e39c5baSBill Taylor } else { 5509e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 5519e39c5baSBill Taylor goto umem_map_fail; 5529e39c5baSBill Taylor } 5539e39c5baSBill Taylor 5549e39c5baSBill Taylor } else if (type == MLNX_UMAP_SRQMEM_RSRC) { 5559e39c5baSBill Taylor 5569e39c5baSBill Taylor /* Use "key" (SRQ number) to do fast lookup on SRQ handle */ 5579e39c5baSBill Taylor srq = hermon_srqhdl_from_srqnum(state, key); 5589e39c5baSBill Taylor 5599e39c5baSBill Taylor /* 5609e39c5baSBill Taylor * Update the handle to the userland mapping. Note: If the 5619e39c5baSBill Taylor * SRQ already has a valid userland mapping, then stop and 5629e39c5baSBill Taylor * return failure. 5639e39c5baSBill Taylor */ 5649e39c5baSBill Taylor mutex_enter(&srq->srq_lock); 5659e39c5baSBill Taylor if (srq->srq_umap_dhp == NULL) { 5669e39c5baSBill Taylor srq->srq_umap_dhp = dhp; 5679e39c5baSBill Taylor dvm_track->hdt_size = srq->srq_wqinfo.qa_size; 5689e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 5699e39c5baSBill Taylor } else { 5709e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 5719e39c5baSBill Taylor goto umem_map_fail; 5729e39c5baSBill Taylor } 5739e39c5baSBill Taylor } 5749e39c5baSBill Taylor 5759e39c5baSBill Taylor /* 5769e39c5baSBill Taylor * Pass the private "Hermon devmap tracking structure" back. This 5779e39c5baSBill Taylor * pointer will be returned in subsequent "unmap" callbacks. 5789e39c5baSBill Taylor */ 5799e39c5baSBill Taylor *pvtp = dvm_track; 5809e39c5baSBill Taylor 5819e39c5baSBill Taylor return (DDI_SUCCESS); 5829e39c5baSBill Taylor 5839e39c5baSBill Taylor umem_map_fail: 5849e39c5baSBill Taylor mutex_destroy(&dvm_track->hdt_lock); 5859e39c5baSBill Taylor kmem_free(dvm_track, sizeof (hermon_devmap_track_t)); 5869e39c5baSBill Taylor return (DDI_FAILURE); 5879e39c5baSBill Taylor } 5889e39c5baSBill Taylor 5899e39c5baSBill Taylor 5909e39c5baSBill Taylor /* 5919e39c5baSBill Taylor * hermon_devmap_umem_dup() 5929e39c5baSBill Taylor * Context: Can be called from kernel context. 5939e39c5baSBill Taylor */ 5949e39c5baSBill Taylor /* ARGSUSED */ 5959e39c5baSBill Taylor static int 5969e39c5baSBill Taylor hermon_devmap_umem_dup(devmap_cookie_t dhp, void *pvtp, devmap_cookie_t new_dhp, 5979e39c5baSBill Taylor void **new_pvtp) 5989e39c5baSBill Taylor { 5999e39c5baSBill Taylor hermon_state_t *state; 6009e39c5baSBill Taylor hermon_devmap_track_t *dvm_track, *new_dvm_track; 6019e39c5baSBill Taylor uint_t maxprot; 6029e39c5baSBill Taylor int status; 6039e39c5baSBill Taylor 6049e39c5baSBill Taylor /* 6059e39c5baSBill Taylor * Extract the Hermon softstate pointer from "Hermon devmap tracking 6069e39c5baSBill Taylor * structure" (in "pvtp"). 6079e39c5baSBill Taylor */ 6089e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)pvtp; 6099e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 6109e39c5baSBill Taylor state = dvm_track->hdt_state; 6119e39c5baSBill Taylor 6129e39c5baSBill Taylor /* 6139e39c5baSBill Taylor * Since this devmap_dup() entry point is generally called 6149e39c5baSBill Taylor * when a process does fork(2), it is incumbent upon the driver 6159e39c5baSBill Taylor * to insure that the child does not inherit a valid copy of 6169e39c5baSBill Taylor * the parent's QP or CQ resource. This is accomplished by using 6179e39c5baSBill Taylor * devmap_devmem_remap() to invalidate the child's mapping to the 6189e39c5baSBill Taylor * kernel memory. 6199e39c5baSBill Taylor */ 6209e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 6219e39c5baSBill Taylor status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0, 6229e39c5baSBill Taylor dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL); 6239e39c5baSBill Taylor if (status != DDI_SUCCESS) { 6249e39c5baSBill Taylor HERMON_WARNING(state, "failed in hermon_devmap_umem_dup()"); 6259e39c5baSBill Taylor return (status); 6269e39c5baSBill Taylor } 6279e39c5baSBill Taylor 6289e39c5baSBill Taylor /* 6299e39c5baSBill Taylor * Allocate a new entry to track the subsequent unmapping 6309e39c5baSBill Taylor * (specifically, all partial unmappings) of the child's newly 6319e39c5baSBill Taylor * invalidated resource. Note: Setting the "hdt_size" field to 6329e39c5baSBill Taylor * zero here is an indication to the devmap_unmap() entry point 6339e39c5baSBill Taylor * that this mapping is invalid, and that its subsequent unmapping 6349e39c5baSBill Taylor * should not affect any of the parent's CQ or QP resources. 6359e39c5baSBill Taylor */ 6369e39c5baSBill Taylor new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc( 6379e39c5baSBill Taylor sizeof (hermon_devmap_track_t), KM_SLEEP); 6389e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track)) 6399e39c5baSBill Taylor new_dvm_track->hdt_offset = 0; 6409e39c5baSBill Taylor new_dvm_track->hdt_state = state; 6419e39c5baSBill Taylor new_dvm_track->hdt_refcnt = 1; 6429e39c5baSBill Taylor new_dvm_track->hdt_size = 0; 6439e39c5baSBill Taylor mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER, 6449e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 6459e39c5baSBill Taylor *new_pvtp = new_dvm_track; 6469e39c5baSBill Taylor 6479e39c5baSBill Taylor return (DDI_SUCCESS); 6489e39c5baSBill Taylor } 6499e39c5baSBill Taylor 6509e39c5baSBill Taylor 6519e39c5baSBill Taylor /* 6529e39c5baSBill Taylor * hermon_devmap_umem_unmap() 6539e39c5baSBill Taylor * Context: Can be called from kernel context. 6549e39c5baSBill Taylor */ 6559e39c5baSBill Taylor /* ARGSUSED */ 6569e39c5baSBill Taylor static void 6579e39c5baSBill Taylor hermon_devmap_umem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off, 6589e39c5baSBill Taylor size_t len, devmap_cookie_t new_dhp1, void **pvtp1, 6599e39c5baSBill Taylor devmap_cookie_t new_dhp2, void **pvtp2) 6609e39c5baSBill Taylor { 6619e39c5baSBill Taylor hermon_state_t *state; 6629e39c5baSBill Taylor hermon_rsrc_t *rsrcp; 6639e39c5baSBill Taylor hermon_devmap_track_t *dvm_track; 6649e39c5baSBill Taylor hermon_cqhdl_t cq; 6659e39c5baSBill Taylor hermon_qphdl_t qp; 6669e39c5baSBill Taylor hermon_srqhdl_t srq; 6679e39c5baSBill Taylor uint64_t key, value; 6689e39c5baSBill Taylor uint_t type; 6699e39c5baSBill Taylor uint_t size; 6709e39c5baSBill Taylor int status; 6719e39c5baSBill Taylor 6729e39c5baSBill Taylor /* 6739e39c5baSBill Taylor * Extract the Hermon softstate pointer from "Hermon devmap tracking 6749e39c5baSBill Taylor * structure" (in "pvtp"). 6759e39c5baSBill Taylor */ 6769e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)pvtp; 6779e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 6789e39c5baSBill Taylor state = dvm_track->hdt_state; 6799e39c5baSBill Taylor 6809e39c5baSBill Taylor /* 6819e39c5baSBill Taylor * Extract the "offset" from the "Hermon devmap tracking structure". 6829e39c5baSBill Taylor * Note: The input argument "off" is ignored here because the 6839e39c5baSBill Taylor * Hermon mapping interfaces define a very specific meaning to 6849e39c5baSBill Taylor * each "logical offset". Also extract the "key" and "type" encoded 6859e39c5baSBill Taylor * in the logical offset. 6869e39c5baSBill Taylor */ 6879e39c5baSBill Taylor key = dvm_track->hdt_offset >> PAGESHIFT; 6889e39c5baSBill Taylor type = key & MLNX_UMAP_RSRC_TYPE_MASK; 6899e39c5baSBill Taylor key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT; 6909e39c5baSBill Taylor 6919e39c5baSBill Taylor /* 6929e39c5baSBill Taylor * Extract the "size" of the mapping. If this size is determined 6939e39c5baSBill Taylor * to be zero, then it is an indication of a previously invalidated 6949e39c5baSBill Taylor * mapping, and no CQ or QP resources should be affected. 6959e39c5baSBill Taylor */ 6969e39c5baSBill Taylor size = dvm_track->hdt_size; 6979e39c5baSBill Taylor 6989e39c5baSBill Taylor /* 6999e39c5baSBill Taylor * If only the "middle portion of a given mapping is being unmapped, 7009e39c5baSBill Taylor * then we are effectively creating one new piece of mapped memory. 7019e39c5baSBill Taylor * (Original region is divided into three pieces of which the middle 7029e39c5baSBill Taylor * piece is being removed. This leaves two pieces. Since we started 7039e39c5baSBill Taylor * with one piece and now have two pieces, we need to increment the 7049e39c5baSBill Taylor * counter in the "Hermon devmap tracking structure". 7059e39c5baSBill Taylor * 7069e39c5baSBill Taylor * If, however, the whole mapped region is being unmapped, then we 7079e39c5baSBill Taylor * have started with one region which we are completely removing. 7089e39c5baSBill Taylor * In this case, we need to decrement the counter in the "Hermon 7099e39c5baSBill Taylor * devmap tracking structure". 7109e39c5baSBill Taylor * 7119e39c5baSBill Taylor * In each of the remaining cases, we will have started with one 7129e39c5baSBill Taylor * mapped region and ended with one (different) region. So no counter 7139e39c5baSBill Taylor * modification is necessary. 7149e39c5baSBill Taylor */ 7159e39c5baSBill Taylor mutex_enter(&dvm_track->hdt_lock); 7169e39c5baSBill Taylor if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) { 7179e39c5baSBill Taylor dvm_track->hdt_refcnt--; 7189e39c5baSBill Taylor } else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) { 7199e39c5baSBill Taylor dvm_track->hdt_refcnt++; 7209e39c5baSBill Taylor } 7219e39c5baSBill Taylor mutex_exit(&dvm_track->hdt_lock); 7229e39c5baSBill Taylor 7239e39c5baSBill Taylor /* 7249e39c5baSBill Taylor * For each of the cases where the region is being divided, then we 7259e39c5baSBill Taylor * need to pass back the "Hermon devmap tracking structure". This way 7269e39c5baSBill Taylor * we get it back when each of the remaining pieces is subsequently 7279e39c5baSBill Taylor * unmapped. 7289e39c5baSBill Taylor */ 7299e39c5baSBill Taylor if (new_dhp1 != NULL) { 7309e39c5baSBill Taylor *pvtp1 = pvtp; 7319e39c5baSBill Taylor } 7329e39c5baSBill Taylor if (new_dhp2 != NULL) { 7339e39c5baSBill Taylor *pvtp2 = pvtp; 7349e39c5baSBill Taylor } 7359e39c5baSBill Taylor 7369e39c5baSBill Taylor /* 7379e39c5baSBill Taylor * If the "Hermon devmap tracking structure" is no longer being 7389e39c5baSBill Taylor * referenced, then free it up. Otherwise, return. 7399e39c5baSBill Taylor */ 7409e39c5baSBill Taylor if (dvm_track->hdt_refcnt == 0) { 7419e39c5baSBill Taylor mutex_destroy(&dvm_track->hdt_lock); 7429e39c5baSBill Taylor kmem_free(dvm_track, sizeof (hermon_devmap_track_t)); 7439e39c5baSBill Taylor 7449e39c5baSBill Taylor /* 7459e39c5baSBill Taylor * If the mapping was invalid (see explanation above), then 7469e39c5baSBill Taylor * no further processing is necessary. 7479e39c5baSBill Taylor */ 7489e39c5baSBill Taylor if (size == 0) { 7499e39c5baSBill Taylor return; 7509e39c5baSBill Taylor } 7519e39c5baSBill Taylor } else { 7529e39c5baSBill Taylor return; 7539e39c5baSBill Taylor } 7549e39c5baSBill Taylor 7559e39c5baSBill Taylor /* 7569e39c5baSBill Taylor * Now that we can guarantee that the user memory is fully unmapped, 7579e39c5baSBill Taylor * we can use the "key" and "type" values to try to find the entry 7589e39c5baSBill Taylor * in the "userland resources database". If it's found, then it 7599e39c5baSBill Taylor * indicates that the queue memory (CQ or QP) has not yet been freed. 7609e39c5baSBill Taylor * In this case, we update the corresponding CQ or QP handle to 7619e39c5baSBill Taylor * indicate that the "devmap_devmem_remap()" call will be unnecessary. 7629e39c5baSBill Taylor * If it's _not_ found, then it indicates that the CQ or QP memory 7639e39c5baSBill Taylor * was, in fact, freed before it was unmapped (thus requiring a 7649e39c5baSBill Taylor * previous invalidation by remapping - which will already have 7659e39c5baSBill Taylor * been done in the free routine). 7669e39c5baSBill Taylor */ 7679e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, key, type, &value, 7689e39c5baSBill Taylor 0, NULL); 7699e39c5baSBill Taylor if (status == DDI_SUCCESS) { 7709e39c5baSBill Taylor /* 7719e39c5baSBill Taylor * Depending on the type of the mapped resource (CQ or QP), 7729e39c5baSBill Taylor * update handle to indicate that no invalidation remapping 7739e39c5baSBill Taylor * will be necessary. 7749e39c5baSBill Taylor */ 7759e39c5baSBill Taylor if (type == MLNX_UMAP_CQMEM_RSRC) { 7769e39c5baSBill Taylor 7779e39c5baSBill Taylor /* Use "value" to convert to CQ handle */ 7789e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 7799e39c5baSBill Taylor cq = (hermon_cqhdl_t)rsrcp->hr_addr; 7809e39c5baSBill Taylor 7819e39c5baSBill Taylor /* 7829e39c5baSBill Taylor * Invalidate the handle to the userland mapping. 7839e39c5baSBill Taylor * Note: We must ensure that the mapping being 7849e39c5baSBill Taylor * unmapped here is the current one for the CQ. It 7859e39c5baSBill Taylor * is possible that it might not be if this CQ has 7869e39c5baSBill Taylor * been resized and the previous CQ memory has not 7879e39c5baSBill Taylor * yet been unmapped. But in that case, because of 7889e39c5baSBill Taylor * the devmap_devmem_remap(), there is no longer any 7899e39c5baSBill Taylor * association between the mapping and the real CQ 7909e39c5baSBill Taylor * kernel memory. 7919e39c5baSBill Taylor */ 7929e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 7939e39c5baSBill Taylor if (cq->cq_umap_dhp == dhp) { 7949e39c5baSBill Taylor cq->cq_umap_dhp = NULL; 7959e39c5baSBill Taylor if (cq->cq_resize_hdl) { 7969e39c5baSBill Taylor /* resize is DONE, switch queues */ 7979e39c5baSBill Taylor hermon_cq_resize_helper(state, cq); 7989e39c5baSBill Taylor } 7999e39c5baSBill Taylor } else { 8009e39c5baSBill Taylor if (cq->cq_resize_hdl && 8019e39c5baSBill Taylor cq->cq_resize_hdl->cq_umap_dhp == dhp) { 8029e39c5baSBill Taylor /* 8039e39c5baSBill Taylor * Unexpected case. munmap of the 8049e39c5baSBill Taylor * cq_resize buf, and not the 8059e39c5baSBill Taylor * original buf. 8069e39c5baSBill Taylor */ 8079e39c5baSBill Taylor cq->cq_resize_hdl->cq_umap_dhp = NULL; 8089e39c5baSBill Taylor } 8099e39c5baSBill Taylor } 8109e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 8119e39c5baSBill Taylor 8129e39c5baSBill Taylor } else if (type == MLNX_UMAP_QPMEM_RSRC) { 8139e39c5baSBill Taylor 8149e39c5baSBill Taylor /* Use "value" to convert to QP handle */ 8159e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 8169e39c5baSBill Taylor qp = (hermon_qphdl_t)rsrcp->hr_addr; 8179e39c5baSBill Taylor 8189e39c5baSBill Taylor /* 8199e39c5baSBill Taylor * Invalidate the handle to the userland mapping. 8209e39c5baSBill Taylor * Note: we ensure that the mapping being unmapped 8219e39c5baSBill Taylor * here is the current one for the QP. This is 8229e39c5baSBill Taylor * more of a sanity check here since, unlike CQs 8239e39c5baSBill Taylor * (above) we do not support resize of QPs. 8249e39c5baSBill Taylor */ 8259e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 8269e39c5baSBill Taylor if (qp->qp_umap_dhp == dhp) { 8279e39c5baSBill Taylor qp->qp_umap_dhp = NULL; 8289e39c5baSBill Taylor } 8299e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 8309e39c5baSBill Taylor 8319e39c5baSBill Taylor } else if (type == MLNX_UMAP_SRQMEM_RSRC) { 8329e39c5baSBill Taylor 8339e39c5baSBill Taylor /* Use "value" to convert to SRQ handle */ 8349e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 8359e39c5baSBill Taylor srq = (hermon_srqhdl_t)rsrcp->hr_addr; 8369e39c5baSBill Taylor 8379e39c5baSBill Taylor /* 8389e39c5baSBill Taylor * Invalidate the handle to the userland mapping. 8399e39c5baSBill Taylor * Note: we ensure that the mapping being unmapped 8409e39c5baSBill Taylor * here is the current one for the QP. This is 8419e39c5baSBill Taylor * more of a sanity check here since, unlike CQs 8429e39c5baSBill Taylor * (above) we do not support resize of QPs. 8439e39c5baSBill Taylor */ 8449e39c5baSBill Taylor mutex_enter(&srq->srq_lock); 8459e39c5baSBill Taylor if (srq->srq_umap_dhp == dhp) { 8469e39c5baSBill Taylor srq->srq_umap_dhp = NULL; 8479e39c5baSBill Taylor } 8489e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 8499e39c5baSBill Taylor } 8509e39c5baSBill Taylor } 8519e39c5baSBill Taylor } 8529e39c5baSBill Taylor 8539e39c5baSBill Taylor 8549e39c5baSBill Taylor /* 8559e39c5baSBill Taylor * hermon_devmap_devmem_map() 8569e39c5baSBill Taylor * Context: Can be called from kernel context. 8579e39c5baSBill Taylor */ 8589e39c5baSBill Taylor /* ARGSUSED */ 8599e39c5baSBill Taylor static int 8609e39c5baSBill Taylor hermon_devmap_dbrecmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags, 8619e39c5baSBill Taylor offset_t off, size_t len, void **pvtp) 8629e39c5baSBill Taylor { 8639e39c5baSBill Taylor hermon_state_t *state; 8649e39c5baSBill Taylor hermon_devmap_track_t *dvm_track; 8659e39c5baSBill Taylor hermon_cqhdl_t cq; 8669e39c5baSBill Taylor hermon_qphdl_t qp; 8679e39c5baSBill Taylor hermon_srqhdl_t srq; 8689e39c5baSBill Taylor minor_t instance; 8699e39c5baSBill Taylor uint64_t key; 8709e39c5baSBill Taylor uint_t type; 8719e39c5baSBill Taylor 8729e39c5baSBill Taylor /* Get Hermon softstate structure from instance */ 8739e39c5baSBill Taylor instance = HERMON_DEV_INSTANCE(dev); 8749e39c5baSBill Taylor state = ddi_get_soft_state(hermon_statep, instance); 8759e39c5baSBill Taylor if (state == NULL) { 8769e39c5baSBill Taylor return (ENXIO); 8779e39c5baSBill Taylor } 8789e39c5baSBill Taylor 8799e39c5baSBill Taylor /* 8809e39c5baSBill Taylor * The bottom bits of "offset" are undefined (number depends on 8819e39c5baSBill Taylor * system PAGESIZE). Shifting these off leaves us with a "key". 8829e39c5baSBill Taylor * The "key" is actually a combination of both a real key value 8839e39c5baSBill Taylor * (for the purpose of database lookup) and a "type" value. Although 8849e39c5baSBill Taylor * we are not going to do any database lookup per se, we do want 8859e39c5baSBill Taylor * to extract the "key" and the "type" (to enable faster lookup of 8869e39c5baSBill Taylor * the appropriate CQ or QP handle). 8879e39c5baSBill Taylor */ 8889e39c5baSBill Taylor key = off >> PAGESHIFT; 8899e39c5baSBill Taylor type = key & MLNX_UMAP_RSRC_TYPE_MASK; 8909e39c5baSBill Taylor key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT; 8919e39c5baSBill Taylor 8929e39c5baSBill Taylor /* 8939e39c5baSBill Taylor * Allocate an entry to track the mapping and unmapping (specifically, 8949e39c5baSBill Taylor * partial unmapping) of this resource. 8959e39c5baSBill Taylor */ 8969e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)kmem_zalloc( 8979e39c5baSBill Taylor sizeof (hermon_devmap_track_t), KM_SLEEP); 8989e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 8999e39c5baSBill Taylor dvm_track->hdt_offset = off; 9009e39c5baSBill Taylor dvm_track->hdt_state = state; 9019e39c5baSBill Taylor dvm_track->hdt_refcnt = 1; 9029e39c5baSBill Taylor mutex_init(&dvm_track->hdt_lock, NULL, MUTEX_DRIVER, 9039e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 9049e39c5baSBill Taylor 9059e39c5baSBill Taylor /* 9069e39c5baSBill Taylor * Depending of the type of resource that has been mapped out, we 9079e39c5baSBill Taylor * need to update the QP or CQ handle to reflect that it has, in 9089e39c5baSBill Taylor * fact, been mapped. This allows the driver code which frees a QP 9099e39c5baSBill Taylor * or a CQ to know whether it is appropriate to do a 9109e39c5baSBill Taylor * devmap_devmem_remap() to invalidate the userland mapping for the 9119e39c5baSBill Taylor * corresponding queue's memory. 9129e39c5baSBill Taylor */ 9139e39c5baSBill Taylor if (type == MLNX_UMAP_CQMEM_RSRC) { 9149e39c5baSBill Taylor 9159e39c5baSBill Taylor /* Use "key" (CQ number) to do fast lookup of CQ handle */ 9169e39c5baSBill Taylor cq = hermon_cqhdl_from_cqnum(state, key); 9179e39c5baSBill Taylor 9189e39c5baSBill Taylor /* 9199e39c5baSBill Taylor * Update the handle to the userland mapping. Note: If 9209e39c5baSBill Taylor * the CQ already has a valid userland mapping, then stop 9219e39c5baSBill Taylor * and return failure. 9229e39c5baSBill Taylor */ 9239e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 9249e39c5baSBill Taylor if (cq->cq_umap_dhp == NULL) { 9259e39c5baSBill Taylor cq->cq_umap_dhp = dhp; 9269e39c5baSBill Taylor dvm_track->hdt_size = cq->cq_cqinfo.qa_size; 9279e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 9289e39c5baSBill Taylor } else { 9299e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 9309e39c5baSBill Taylor goto umem_map_fail; 9319e39c5baSBill Taylor } 9329e39c5baSBill Taylor 9339e39c5baSBill Taylor } else if (type == MLNX_UMAP_QPMEM_RSRC) { 9349e39c5baSBill Taylor 9359e39c5baSBill Taylor /* Use "key" (QP number) to do fast lookup of QP handle */ 9369e39c5baSBill Taylor qp = hermon_qphdl_from_qpnum(state, key); 9379e39c5baSBill Taylor 9389e39c5baSBill Taylor /* 9399e39c5baSBill Taylor * Update the handle to the userland mapping. Note: If 9409e39c5baSBill Taylor * the CQ already has a valid userland mapping, then stop 9419e39c5baSBill Taylor * and return failure. 9429e39c5baSBill Taylor */ 9439e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 9449e39c5baSBill Taylor if (qp->qp_umap_dhp == NULL) { 9459e39c5baSBill Taylor qp->qp_umap_dhp = dhp; 9469e39c5baSBill Taylor dvm_track->hdt_size = qp->qp_wqinfo.qa_size; 9479e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 9489e39c5baSBill Taylor } else { 9499e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 9509e39c5baSBill Taylor goto umem_map_fail; 9519e39c5baSBill Taylor } 9529e39c5baSBill Taylor 9539e39c5baSBill Taylor } else if (type == MLNX_UMAP_SRQMEM_RSRC) { 9549e39c5baSBill Taylor 9559e39c5baSBill Taylor /* Use "key" (SRQ number) to do fast lookup on SRQ handle */ 9569e39c5baSBill Taylor srq = hermon_srqhdl_from_srqnum(state, key); 9579e39c5baSBill Taylor 9589e39c5baSBill Taylor /* 9599e39c5baSBill Taylor * Update the handle to the userland mapping. Note: If the 9609e39c5baSBill Taylor * SRQ already has a valid userland mapping, then stop and 9619e39c5baSBill Taylor * return failure. 9629e39c5baSBill Taylor */ 9639e39c5baSBill Taylor mutex_enter(&srq->srq_lock); 9649e39c5baSBill Taylor if (srq->srq_umap_dhp == NULL) { 9659e39c5baSBill Taylor srq->srq_umap_dhp = dhp; 9669e39c5baSBill Taylor dvm_track->hdt_size = srq->srq_wqinfo.qa_size; 9679e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 9689e39c5baSBill Taylor } else { 9699e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 9709e39c5baSBill Taylor goto umem_map_fail; 9719e39c5baSBill Taylor } 9729e39c5baSBill Taylor } 9739e39c5baSBill Taylor 9749e39c5baSBill Taylor /* 9759e39c5baSBill Taylor * Pass the private "Hermon devmap tracking structure" back. This 9769e39c5baSBill Taylor * pointer will be returned in subsequent "unmap" callbacks. 9779e39c5baSBill Taylor */ 9789e39c5baSBill Taylor *pvtp = dvm_track; 9799e39c5baSBill Taylor 9809e39c5baSBill Taylor return (DDI_SUCCESS); 9819e39c5baSBill Taylor 9829e39c5baSBill Taylor umem_map_fail: 9839e39c5baSBill Taylor mutex_destroy(&dvm_track->hdt_lock); 9849e39c5baSBill Taylor kmem_free(dvm_track, sizeof (hermon_devmap_track_t)); 9859e39c5baSBill Taylor return (DDI_FAILURE); 9869e39c5baSBill Taylor } 9879e39c5baSBill Taylor 9889e39c5baSBill Taylor 9899e39c5baSBill Taylor /* 9909e39c5baSBill Taylor * hermon_devmap_dbrecmem_dup() 9919e39c5baSBill Taylor * Context: Can be called from kernel context. 9929e39c5baSBill Taylor */ 9939e39c5baSBill Taylor /* ARGSUSED */ 9949e39c5baSBill Taylor static int 9959e39c5baSBill Taylor hermon_devmap_dbrecmem_dup(devmap_cookie_t dhp, void *pvtp, 9969e39c5baSBill Taylor devmap_cookie_t new_dhp, void **new_pvtp) 9979e39c5baSBill Taylor { 9989e39c5baSBill Taylor hermon_state_t *state; 9999e39c5baSBill Taylor hermon_devmap_track_t *dvm_track, *new_dvm_track; 10009e39c5baSBill Taylor uint_t maxprot; 10019e39c5baSBill Taylor int status; 10029e39c5baSBill Taylor 10039e39c5baSBill Taylor /* 10049e39c5baSBill Taylor * Extract the Hermon softstate pointer from "Hermon devmap tracking 10059e39c5baSBill Taylor * structure" (in "pvtp"). 10069e39c5baSBill Taylor */ 10079e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)pvtp; 10089e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 10099e39c5baSBill Taylor state = dvm_track->hdt_state; 10109e39c5baSBill Taylor 10119e39c5baSBill Taylor /* 10129e39c5baSBill Taylor * Since this devmap_dup() entry point is generally called 10139e39c5baSBill Taylor * when a process does fork(2), it is incumbent upon the driver 10149e39c5baSBill Taylor * to insure that the child does not inherit a valid copy of 10159e39c5baSBill Taylor * the parent's QP or CQ resource. This is accomplished by using 10169e39c5baSBill Taylor * devmap_devmem_remap() to invalidate the child's mapping to the 10179e39c5baSBill Taylor * kernel memory. 10189e39c5baSBill Taylor */ 10199e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 10209e39c5baSBill Taylor status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0, 10219e39c5baSBill Taylor dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL); 10229e39c5baSBill Taylor if (status != DDI_SUCCESS) { 10239e39c5baSBill Taylor HERMON_WARNING(state, "failed in hermon_devmap_dbrecmem_dup()"); 10249e39c5baSBill Taylor return (status); 10259e39c5baSBill Taylor } 10269e39c5baSBill Taylor 10279e39c5baSBill Taylor /* 10289e39c5baSBill Taylor * Allocate a new entry to track the subsequent unmapping 10299e39c5baSBill Taylor * (specifically, all partial unmappings) of the child's newly 10309e39c5baSBill Taylor * invalidated resource. Note: Setting the "hdt_size" field to 10319e39c5baSBill Taylor * zero here is an indication to the devmap_unmap() entry point 10329e39c5baSBill Taylor * that this mapping is invalid, and that its subsequent unmapping 10339e39c5baSBill Taylor * should not affect any of the parent's CQ or QP resources. 10349e39c5baSBill Taylor */ 10359e39c5baSBill Taylor new_dvm_track = (hermon_devmap_track_t *)kmem_zalloc( 10369e39c5baSBill Taylor sizeof (hermon_devmap_track_t), KM_SLEEP); 10379e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*new_dvm_track)) 10389e39c5baSBill Taylor new_dvm_track->hdt_offset = 0; 10399e39c5baSBill Taylor new_dvm_track->hdt_state = state; 10409e39c5baSBill Taylor new_dvm_track->hdt_refcnt = 1; 10419e39c5baSBill Taylor new_dvm_track->hdt_size = 0; 10429e39c5baSBill Taylor mutex_init(&new_dvm_track->hdt_lock, NULL, MUTEX_DRIVER, 10439e39c5baSBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 10449e39c5baSBill Taylor *new_pvtp = new_dvm_track; 10459e39c5baSBill Taylor 10469e39c5baSBill Taylor return (DDI_SUCCESS); 10479e39c5baSBill Taylor } 10489e39c5baSBill Taylor 10499e39c5baSBill Taylor 10509e39c5baSBill Taylor /* 10519e39c5baSBill Taylor * hermon_devmap_dbrecmem_unmap() 10529e39c5baSBill Taylor * Context: Can be called from kernel context. 10539e39c5baSBill Taylor */ 10549e39c5baSBill Taylor /* ARGSUSED */ 10559e39c5baSBill Taylor static void 10569e39c5baSBill Taylor hermon_devmap_dbrecmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off, 10579e39c5baSBill Taylor size_t len, devmap_cookie_t new_dhp1, void **pvtp1, 10589e39c5baSBill Taylor devmap_cookie_t new_dhp2, void **pvtp2) 10599e39c5baSBill Taylor { 10609e39c5baSBill Taylor hermon_state_t *state; 10619e39c5baSBill Taylor hermon_rsrc_t *rsrcp; 10629e39c5baSBill Taylor hermon_devmap_track_t *dvm_track; 10639e39c5baSBill Taylor hermon_cqhdl_t cq; 10649e39c5baSBill Taylor hermon_qphdl_t qp; 10659e39c5baSBill Taylor hermon_srqhdl_t srq; 10669e39c5baSBill Taylor uint64_t key, value; 10679e39c5baSBill Taylor uint_t type; 10689e39c5baSBill Taylor uint_t size; 10699e39c5baSBill Taylor int status; 10709e39c5baSBill Taylor 10719e39c5baSBill Taylor /* 10729e39c5baSBill Taylor * Extract the Hermon softstate pointer from "Hermon devmap tracking 10739e39c5baSBill Taylor * structure" (in "pvtp"). 10749e39c5baSBill Taylor */ 10759e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)pvtp; 10769e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 10779e39c5baSBill Taylor state = dvm_track->hdt_state; 10789e39c5baSBill Taylor 10799e39c5baSBill Taylor /* 10809e39c5baSBill Taylor * Extract the "offset" from the "Hermon devmap tracking structure". 10819e39c5baSBill Taylor * Note: The input argument "off" is ignored here because the 10829e39c5baSBill Taylor * Hermon mapping interfaces define a very specific meaning to 10839e39c5baSBill Taylor * each "logical offset". Also extract the "key" and "type" encoded 10849e39c5baSBill Taylor * in the logical offset. 10859e39c5baSBill Taylor */ 10869e39c5baSBill Taylor key = dvm_track->hdt_offset >> PAGESHIFT; 10879e39c5baSBill Taylor type = key & MLNX_UMAP_RSRC_TYPE_MASK; 10889e39c5baSBill Taylor key = key >> MLNX_UMAP_RSRC_TYPE_SHIFT; 10899e39c5baSBill Taylor 10909e39c5baSBill Taylor /* 10919e39c5baSBill Taylor * Extract the "size" of the mapping. If this size is determined 10929e39c5baSBill Taylor * to be zero, then it is an indication of a previously invalidated 10939e39c5baSBill Taylor * mapping, and no CQ or QP resources should be affected. 10949e39c5baSBill Taylor */ 10959e39c5baSBill Taylor size = dvm_track->hdt_size; 10969e39c5baSBill Taylor 10979e39c5baSBill Taylor /* 10989e39c5baSBill Taylor * If only the "middle portion of a given mapping is being unmapped, 10999e39c5baSBill Taylor * then we are effectively creating one new piece of mapped memory. 11009e39c5baSBill Taylor * (Original region is divided into three pieces of which the middle 11019e39c5baSBill Taylor * piece is being removed. This leaves two pieces. Since we started 11029e39c5baSBill Taylor * with one piece and now have two pieces, we need to increment the 11039e39c5baSBill Taylor * counter in the "Hermon devmap tracking structure". 11049e39c5baSBill Taylor * 11059e39c5baSBill Taylor * If, however, the whole mapped region is being unmapped, then we 11069e39c5baSBill Taylor * have started with one region which we are completely removing. 11079e39c5baSBill Taylor * In this case, we need to decrement the counter in the "Hermon 11089e39c5baSBill Taylor * devmap tracking structure". 11099e39c5baSBill Taylor * 11109e39c5baSBill Taylor * In each of the remaining cases, we will have started with one 11119e39c5baSBill Taylor * mapped region and ended with one (different) region. So no counter 11129e39c5baSBill Taylor * modification is necessary. 11139e39c5baSBill Taylor */ 11149e39c5baSBill Taylor mutex_enter(&dvm_track->hdt_lock); 11159e39c5baSBill Taylor if ((new_dhp1 == NULL) && (new_dhp2 == NULL)) { 11169e39c5baSBill Taylor dvm_track->hdt_refcnt--; 11179e39c5baSBill Taylor } else if ((new_dhp1 != NULL) && (new_dhp2 != NULL)) { 11189e39c5baSBill Taylor dvm_track->hdt_refcnt++; 11199e39c5baSBill Taylor } 11209e39c5baSBill Taylor mutex_exit(&dvm_track->hdt_lock); 11219e39c5baSBill Taylor 11229e39c5baSBill Taylor /* 11239e39c5baSBill Taylor * For each of the cases where the region is being divided, then we 11249e39c5baSBill Taylor * need to pass back the "Hermon devmap tracking structure". This way 11259e39c5baSBill Taylor * we get it back when each of the remaining pieces is subsequently 11269e39c5baSBill Taylor * unmapped. 11279e39c5baSBill Taylor */ 11289e39c5baSBill Taylor if (new_dhp1 != NULL) { 11299e39c5baSBill Taylor *pvtp1 = pvtp; 11309e39c5baSBill Taylor } 11319e39c5baSBill Taylor if (new_dhp2 != NULL) { 11329e39c5baSBill Taylor *pvtp2 = pvtp; 11339e39c5baSBill Taylor } 11349e39c5baSBill Taylor 11359e39c5baSBill Taylor /* 11369e39c5baSBill Taylor * If the "Hermon devmap tracking structure" is no longer being 11379e39c5baSBill Taylor * referenced, then free it up. Otherwise, return. 11389e39c5baSBill Taylor */ 11399e39c5baSBill Taylor if (dvm_track->hdt_refcnt == 0) { 11409e39c5baSBill Taylor mutex_destroy(&dvm_track->hdt_lock); 11419e39c5baSBill Taylor kmem_free(dvm_track, sizeof (hermon_devmap_track_t)); 11429e39c5baSBill Taylor 11439e39c5baSBill Taylor /* 11449e39c5baSBill Taylor * If the mapping was invalid (see explanation above), then 11459e39c5baSBill Taylor * no further processing is necessary. 11469e39c5baSBill Taylor */ 11479e39c5baSBill Taylor if (size == 0) { 11489e39c5baSBill Taylor return; 11499e39c5baSBill Taylor } 11509e39c5baSBill Taylor } else { 11519e39c5baSBill Taylor return; 11529e39c5baSBill Taylor } 11539e39c5baSBill Taylor 11549e39c5baSBill Taylor /* 11559e39c5baSBill Taylor * Now that we can guarantee that the user memory is fully unmapped, 11569e39c5baSBill Taylor * we can use the "key" and "type" values to try to find the entry 11579e39c5baSBill Taylor * in the "userland resources database". If it's found, then it 11589e39c5baSBill Taylor * indicates that the queue memory (CQ or QP) has not yet been freed. 11599e39c5baSBill Taylor * In this case, we update the corresponding CQ or QP handle to 11609e39c5baSBill Taylor * indicate that the "devmap_devmem_remap()" call will be unnecessary. 11619e39c5baSBill Taylor * If it's _not_ found, then it indicates that the CQ or QP memory 11629e39c5baSBill Taylor * was, in fact, freed before it was unmapped (thus requiring a 11639e39c5baSBill Taylor * previous invalidation by remapping - which will already have 11649e39c5baSBill Taylor * been done in the free routine). 11659e39c5baSBill Taylor */ 11669e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, key, type, &value, 11679e39c5baSBill Taylor 0, NULL); 11689e39c5baSBill Taylor if (status == DDI_SUCCESS) { 11699e39c5baSBill Taylor /* 11709e39c5baSBill Taylor * Depending on the type of the mapped resource (CQ or QP), 11719e39c5baSBill Taylor * update handle to indicate that no invalidation remapping 11729e39c5baSBill Taylor * will be necessary. 11739e39c5baSBill Taylor */ 11749e39c5baSBill Taylor if (type == MLNX_UMAP_CQMEM_RSRC) { 11759e39c5baSBill Taylor 11769e39c5baSBill Taylor /* Use "value" to convert to CQ handle */ 11779e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 11789e39c5baSBill Taylor cq = (hermon_cqhdl_t)rsrcp->hr_addr; 11799e39c5baSBill Taylor 11809e39c5baSBill Taylor /* 11819e39c5baSBill Taylor * Invalidate the handle to the userland mapping. 11829e39c5baSBill Taylor * Note: We must ensure that the mapping being 11839e39c5baSBill Taylor * unmapped here is the current one for the CQ. It 11849e39c5baSBill Taylor * is possible that it might not be if this CQ has 11859e39c5baSBill Taylor * been resized and the previous CQ memory has not 11869e39c5baSBill Taylor * yet been unmapped. But in that case, because of 11879e39c5baSBill Taylor * the devmap_devmem_remap(), there is no longer any 11889e39c5baSBill Taylor * association between the mapping and the real CQ 11899e39c5baSBill Taylor * kernel memory. 11909e39c5baSBill Taylor */ 11919e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 11929e39c5baSBill Taylor if (cq->cq_umap_dhp == dhp) { 11939e39c5baSBill Taylor cq->cq_umap_dhp = NULL; 11949e39c5baSBill Taylor } 11959e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 11969e39c5baSBill Taylor 11979e39c5baSBill Taylor } else if (type == MLNX_UMAP_QPMEM_RSRC) { 11989e39c5baSBill Taylor 11999e39c5baSBill Taylor /* Use "value" to convert to QP handle */ 12009e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 12019e39c5baSBill Taylor qp = (hermon_qphdl_t)rsrcp->hr_addr; 12029e39c5baSBill Taylor 12039e39c5baSBill Taylor /* 12049e39c5baSBill Taylor * Invalidate the handle to the userland mapping. 12059e39c5baSBill Taylor * Note: we ensure that the mapping being unmapped 12069e39c5baSBill Taylor * here is the current one for the QP. This is 12079e39c5baSBill Taylor * more of a sanity check here since, unlike CQs 12089e39c5baSBill Taylor * (above) we do not support resize of QPs. 12099e39c5baSBill Taylor */ 12109e39c5baSBill Taylor mutex_enter(&qp->qp_lock); 12119e39c5baSBill Taylor if (qp->qp_umap_dhp == dhp) { 12129e39c5baSBill Taylor qp->qp_umap_dhp = NULL; 12139e39c5baSBill Taylor } 12149e39c5baSBill Taylor mutex_exit(&qp->qp_lock); 12159e39c5baSBill Taylor 12169e39c5baSBill Taylor } else if (type == MLNX_UMAP_SRQMEM_RSRC) { 12179e39c5baSBill Taylor 12189e39c5baSBill Taylor /* Use "value" to convert to SRQ handle */ 12199e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 12209e39c5baSBill Taylor srq = (hermon_srqhdl_t)rsrcp->hr_addr; 12219e39c5baSBill Taylor 12229e39c5baSBill Taylor /* 12239e39c5baSBill Taylor * Invalidate the handle to the userland mapping. 12249e39c5baSBill Taylor * Note: we ensure that the mapping being unmapped 12259e39c5baSBill Taylor * here is the current one for the QP. This is 12269e39c5baSBill Taylor * more of a sanity check here since, unlike CQs 12279e39c5baSBill Taylor * (above) we do not support resize of QPs. 12289e39c5baSBill Taylor */ 12299e39c5baSBill Taylor mutex_enter(&srq->srq_lock); 12309e39c5baSBill Taylor if (srq->srq_umap_dhp == dhp) { 12319e39c5baSBill Taylor srq->srq_umap_dhp = NULL; 12329e39c5baSBill Taylor } 12339e39c5baSBill Taylor mutex_exit(&srq->srq_lock); 12349e39c5baSBill Taylor } 12359e39c5baSBill Taylor } 12369e39c5baSBill Taylor } 12379e39c5baSBill Taylor 12389e39c5baSBill Taylor 12399e39c5baSBill Taylor /* 12409e39c5baSBill Taylor * hermon_devmap_devmem_map() 12419e39c5baSBill Taylor * Context: Can be called from kernel context. 12429e39c5baSBill Taylor */ 12439e39c5baSBill Taylor /* ARGSUSED */ 12449e39c5baSBill Taylor static int 12459e39c5baSBill Taylor hermon_devmap_devmem_map(devmap_cookie_t dhp, dev_t dev, uint_t flags, 12469e39c5baSBill Taylor offset_t off, size_t len, void **pvtp) 12479e39c5baSBill Taylor { 12489e39c5baSBill Taylor hermon_state_t *state; 12499e39c5baSBill Taylor hermon_devmap_track_t *dvm_track; 12509e39c5baSBill Taylor minor_t instance; 12519e39c5baSBill Taylor 12529e39c5baSBill Taylor /* Get Hermon softstate structure from instance */ 12539e39c5baSBill Taylor instance = HERMON_DEV_INSTANCE(dev); 12549e39c5baSBill Taylor state = ddi_get_soft_state(hermon_statep, instance); 12559e39c5baSBill Taylor if (state == NULL) { 12569e39c5baSBill Taylor return (ENXIO); 12579e39c5baSBill Taylor } 12589e39c5baSBill Taylor 12599e39c5baSBill Taylor /* 12609e39c5baSBill Taylor * Allocate an entry to track the mapping and unmapping of this 12619e39c5baSBill Taylor * resource. Note: We don't need to initialize the "refcnt" or 12629e39c5baSBill Taylor * "offset" fields here, nor do we need to initialize the mutex 12639e39c5baSBill Taylor * used with the "refcnt". Since UAR pages are single pages, they 12649e39c5baSBill Taylor * are not subject to "partial" unmappings. This makes these other 12659e39c5baSBill Taylor * fields unnecessary. 12669e39c5baSBill Taylor */ 12679e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)kmem_zalloc( 12689e39c5baSBill Taylor sizeof (hermon_devmap_track_t), KM_SLEEP); 12699e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 12709e39c5baSBill Taylor dvm_track->hdt_state = state; 12719e39c5baSBill Taylor dvm_track->hdt_size = (uint_t)PAGESIZE; 12729e39c5baSBill Taylor 12739e39c5baSBill Taylor /* 12749e39c5baSBill Taylor * Pass the private "Hermon devmap tracking structure" back. This 12759e39c5baSBill Taylor * pointer will be returned in a subsequent "unmap" callback. 12769e39c5baSBill Taylor */ 12779e39c5baSBill Taylor *pvtp = dvm_track; 12789e39c5baSBill Taylor 12799e39c5baSBill Taylor return (DDI_SUCCESS); 12809e39c5baSBill Taylor } 12819e39c5baSBill Taylor 12829e39c5baSBill Taylor 12839e39c5baSBill Taylor /* 12849e39c5baSBill Taylor * hermon_devmap_devmem_dup() 12859e39c5baSBill Taylor * Context: Can be called from kernel context. 12869e39c5baSBill Taylor */ 12879e39c5baSBill Taylor /* ARGSUSED */ 12889e39c5baSBill Taylor static int 12899e39c5baSBill Taylor hermon_devmap_devmem_dup(devmap_cookie_t dhp, void *pvtp, 12909e39c5baSBill Taylor devmap_cookie_t new_dhp, void **new_pvtp) 12919e39c5baSBill Taylor { 12929e39c5baSBill Taylor hermon_state_t *state; 12939e39c5baSBill Taylor hermon_devmap_track_t *dvm_track; 12949e39c5baSBill Taylor uint_t maxprot; 12959e39c5baSBill Taylor int status; 12969e39c5baSBill Taylor 12979e39c5baSBill Taylor /* 12989e39c5baSBill Taylor * Extract the Hermon softstate pointer from "Hermon devmap tracking 12999e39c5baSBill Taylor * structure" (in "pvtp"). Note: If the tracking structure is NULL 13009e39c5baSBill Taylor * here, it means that the mapping corresponds to an invalid mapping. 13019e39c5baSBill Taylor * In this case, it can be safely ignored ("new_pvtp" set to NULL). 13029e39c5baSBill Taylor */ 13039e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)pvtp; 13049e39c5baSBill Taylor if (dvm_track == NULL) { 13059e39c5baSBill Taylor *new_pvtp = NULL; 13069e39c5baSBill Taylor return (DDI_SUCCESS); 13079e39c5baSBill Taylor } 13089e39c5baSBill Taylor 13099e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 13109e39c5baSBill Taylor state = dvm_track->hdt_state; 13119e39c5baSBill Taylor 13129e39c5baSBill Taylor /* 13139e39c5baSBill Taylor * Since this devmap_dup() entry point is generally called 13149e39c5baSBill Taylor * when a process does fork(2), it is incumbent upon the driver 13159e39c5baSBill Taylor * to insure that the child does not inherit a valid copy of 13169e39c5baSBill Taylor * the parent's resource. This is accomplished by using 13179e39c5baSBill Taylor * devmap_devmem_remap() to invalidate the child's mapping to the 13189e39c5baSBill Taylor * kernel memory. 13199e39c5baSBill Taylor */ 13209e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 13219e39c5baSBill Taylor status = devmap_devmem_remap(new_dhp, state->hs_dip, 0, 0, 13229e39c5baSBill Taylor dvm_track->hdt_size, maxprot, DEVMAP_MAPPING_INVALID, NULL); 13239e39c5baSBill Taylor if (status != DDI_SUCCESS) { 13249e39c5baSBill Taylor HERMON_WARNING(state, "failed in hermon_devmap_devmem_dup()"); 13259e39c5baSBill Taylor return (status); 13269e39c5baSBill Taylor } 13279e39c5baSBill Taylor 13289e39c5baSBill Taylor /* 13299e39c5baSBill Taylor * Since the region is invalid, there is no need for us to 13309e39c5baSBill Taylor * allocate and continue to track an additional "Hermon devmap 13319e39c5baSBill Taylor * tracking structure". Instead we return NULL here, which is an 13329e39c5baSBill Taylor * indication to the devmap_unmap() entry point that this entry 13339e39c5baSBill Taylor * can be safely ignored. 13349e39c5baSBill Taylor */ 13359e39c5baSBill Taylor *new_pvtp = NULL; 13369e39c5baSBill Taylor 13379e39c5baSBill Taylor return (DDI_SUCCESS); 13389e39c5baSBill Taylor } 13399e39c5baSBill Taylor 13409e39c5baSBill Taylor 13419e39c5baSBill Taylor /* 13429e39c5baSBill Taylor * hermon_devmap_devmem_unmap() 13439e39c5baSBill Taylor * Context: Can be called from kernel context. 13449e39c5baSBill Taylor */ 13459e39c5baSBill Taylor /* ARGSUSED */ 13469e39c5baSBill Taylor static void 13479e39c5baSBill Taylor hermon_devmap_devmem_unmap(devmap_cookie_t dhp, void *pvtp, offset_t off, 13489e39c5baSBill Taylor size_t len, devmap_cookie_t new_dhp1, void **pvtp1, 13499e39c5baSBill Taylor devmap_cookie_t new_dhp2, void **pvtp2) 13509e39c5baSBill Taylor { 13519e39c5baSBill Taylor hermon_devmap_track_t *dvm_track; 13529e39c5baSBill Taylor 13539e39c5baSBill Taylor /* 13549e39c5baSBill Taylor * Free up the "Hermon devmap tracking structure" (in "pvtp"). 13559e39c5baSBill Taylor * There cannot be "partial" unmappings here because all UAR pages 13569e39c5baSBill Taylor * are single pages. Note: If the tracking structure is NULL here, 13579e39c5baSBill Taylor * it means that the mapping corresponds to an invalid mapping. In 13589e39c5baSBill Taylor * this case, it can be safely ignored. 13599e39c5baSBill Taylor */ 13609e39c5baSBill Taylor dvm_track = (hermon_devmap_track_t *)pvtp; 13619e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dvm_track)) 13629e39c5baSBill Taylor if (dvm_track == NULL) { 13639e39c5baSBill Taylor return; 13649e39c5baSBill Taylor } 13659e39c5baSBill Taylor 13669e39c5baSBill Taylor kmem_free(dvm_track, sizeof (hermon_devmap_track_t)); 13679e39c5baSBill Taylor } 13689e39c5baSBill Taylor 13699e39c5baSBill Taylor 13709e39c5baSBill Taylor /* 13719e39c5baSBill Taylor * hermon_umap_ci_data_in() 13729e39c5baSBill Taylor * Context: Can be called from user or kernel context. 13739e39c5baSBill Taylor */ 13749e39c5baSBill Taylor /* ARGSUSED */ 13759e39c5baSBill Taylor ibt_status_t 13769e39c5baSBill Taylor hermon_umap_ci_data_in(hermon_state_t *state, ibt_ci_data_flags_t flags, 13779e39c5baSBill Taylor ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz) 13789e39c5baSBill Taylor { 13799e39c5baSBill Taylor int status; 13809e39c5baSBill Taylor 13819e39c5baSBill Taylor /* 13829e39c5baSBill Taylor * Depending on the type of object about which additional information 13839e39c5baSBill Taylor * is being provided (currently only MR is supported), we call the 13849e39c5baSBill Taylor * appropriate resource-specific function. 13859e39c5baSBill Taylor */ 13869e39c5baSBill Taylor switch (object) { 13879e39c5baSBill Taylor case IBT_HDL_MR: 13889e39c5baSBill Taylor status = hermon_umap_mr_data_in((hermon_mrhdl_t)hdl, 13899e39c5baSBill Taylor (ibt_mr_data_in_t *)data_p, data_sz); 13909e39c5baSBill Taylor if (status != DDI_SUCCESS) { 13919e39c5baSBill Taylor return (status); 13929e39c5baSBill Taylor } 13939e39c5baSBill Taylor break; 13949e39c5baSBill Taylor 13959e39c5baSBill Taylor /* 13969e39c5baSBill Taylor * For other possible valid IBT types, we return IBT_NOT_SUPPORTED, 13979e39c5baSBill Taylor * since the Hermon driver does not support these. 13989e39c5baSBill Taylor */ 13999e39c5baSBill Taylor case IBT_HDL_HCA: 14009e39c5baSBill Taylor case IBT_HDL_QP: 14019e39c5baSBill Taylor case IBT_HDL_CQ: 14029e39c5baSBill Taylor case IBT_HDL_PD: 14039e39c5baSBill Taylor case IBT_HDL_MW: 14049e39c5baSBill Taylor case IBT_HDL_AH: 14059e39c5baSBill Taylor case IBT_HDL_SCHED: 14069e39c5baSBill Taylor case IBT_HDL_EEC: 14079e39c5baSBill Taylor case IBT_HDL_RDD: 14089e39c5baSBill Taylor case IBT_HDL_SRQ: 14099e39c5baSBill Taylor return (IBT_NOT_SUPPORTED); 14109e39c5baSBill Taylor 14119e39c5baSBill Taylor /* 14129e39c5baSBill Taylor * Any other types are invalid. 14139e39c5baSBill Taylor */ 14149e39c5baSBill Taylor default: 14159e39c5baSBill Taylor return (IBT_INVALID_PARAM); 14169e39c5baSBill Taylor } 14179e39c5baSBill Taylor 14189e39c5baSBill Taylor return (DDI_SUCCESS); 14199e39c5baSBill Taylor } 14209e39c5baSBill Taylor 14219e39c5baSBill Taylor 14229e39c5baSBill Taylor /* 14239e39c5baSBill Taylor * hermon_umap_mr_data_in() 14249e39c5baSBill Taylor * Context: Can be called from user or kernel context. 14259e39c5baSBill Taylor */ 14269e39c5baSBill Taylor static ibt_status_t 14279e39c5baSBill Taylor hermon_umap_mr_data_in(hermon_mrhdl_t mr, ibt_mr_data_in_t *data, 14289e39c5baSBill Taylor size_t data_sz) 14299e39c5baSBill Taylor { 14309e39c5baSBill Taylor if (data->mr_rev != IBT_MR_DATA_IN_IF_VERSION) { 14319e39c5baSBill Taylor return (IBT_NOT_SUPPORTED); 14329e39c5baSBill Taylor } 14339e39c5baSBill Taylor 14349e39c5baSBill Taylor /* Check for valid MR handle pointer */ 14359e39c5baSBill Taylor if (mr == NULL) { 14369e39c5baSBill Taylor return (IBT_MR_HDL_INVALID); 14379e39c5baSBill Taylor } 14389e39c5baSBill Taylor 14399e39c5baSBill Taylor /* Check for valid MR input structure size */ 14409e39c5baSBill Taylor if (data_sz < sizeof (ibt_mr_data_in_t)) { 14419e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 14429e39c5baSBill Taylor } 14439e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data)) 14449e39c5baSBill Taylor 14459e39c5baSBill Taylor /* 14469e39c5baSBill Taylor * Ensure that the MR corresponds to userland memory and that it is 14479e39c5baSBill Taylor * a currently valid memory region as well. 14489e39c5baSBill Taylor */ 14499e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 14509e39c5baSBill Taylor if ((mr->mr_is_umem == 0) || (mr->mr_umemcookie == NULL)) { 14519e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 14529e39c5baSBill Taylor return (IBT_MR_HDL_INVALID); 14539e39c5baSBill Taylor } 14549e39c5baSBill Taylor 14559e39c5baSBill Taylor /* 14569e39c5baSBill Taylor * If it has passed all the above checks, then extract the callback 14579e39c5baSBill Taylor * function and argument from the input structure. Copy them into 14589e39c5baSBill Taylor * the MR handle. This function will be called only if the memory 14599e39c5baSBill Taylor * corresponding to the MR handle gets a umem_lockmemory() callback. 14609e39c5baSBill Taylor */ 14619e39c5baSBill Taylor mr->mr_umem_cbfunc = data->mr_func; 14629e39c5baSBill Taylor mr->mr_umem_cbarg1 = data->mr_arg1; 14639e39c5baSBill Taylor mr->mr_umem_cbarg2 = data->mr_arg2; 14649e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 14659e39c5baSBill Taylor 14669e39c5baSBill Taylor return (DDI_SUCCESS); 14679e39c5baSBill Taylor } 14689e39c5baSBill Taylor 14699e39c5baSBill Taylor 14709e39c5baSBill Taylor /* 14719e39c5baSBill Taylor * hermon_umap_ci_data_out() 14729e39c5baSBill Taylor * Context: Can be called from user or kernel context. 14739e39c5baSBill Taylor */ 14749e39c5baSBill Taylor /* ARGSUSED */ 14759e39c5baSBill Taylor ibt_status_t 14769e39c5baSBill Taylor hermon_umap_ci_data_out(hermon_state_t *state, ibt_ci_data_flags_t flags, 14779e39c5baSBill Taylor ibt_object_type_t object, void *hdl, void *data_p, size_t data_sz) 14789e39c5baSBill Taylor { 14799e39c5baSBill Taylor int status; 14809e39c5baSBill Taylor 14819e39c5baSBill Taylor /* 14829e39c5baSBill Taylor * Depending on the type of object about which additional information 14839e39c5baSBill Taylor * is being requested (CQ or QP), we call the appropriate resource- 14849e39c5baSBill Taylor * specific mapping function. 14859e39c5baSBill Taylor */ 14869e39c5baSBill Taylor switch (object) { 14879e39c5baSBill Taylor case IBT_HDL_CQ: 14889e39c5baSBill Taylor status = hermon_umap_cq_data_out((hermon_cqhdl_t)hdl, 14899e39c5baSBill Taylor (mlnx_umap_cq_data_out_t *)data_p, data_sz); 14909e39c5baSBill Taylor if (status != DDI_SUCCESS) { 14919e39c5baSBill Taylor return (status); 14929e39c5baSBill Taylor } 14939e39c5baSBill Taylor break; 14949e39c5baSBill Taylor 14959e39c5baSBill Taylor case IBT_HDL_QP: 14969e39c5baSBill Taylor status = hermon_umap_qp_data_out((hermon_qphdl_t)hdl, 14979e39c5baSBill Taylor (mlnx_umap_qp_data_out_t *)data_p, data_sz); 14989e39c5baSBill Taylor if (status != DDI_SUCCESS) { 14999e39c5baSBill Taylor return (status); 15009e39c5baSBill Taylor } 15019e39c5baSBill Taylor break; 15029e39c5baSBill Taylor 15039e39c5baSBill Taylor case IBT_HDL_SRQ: 15049e39c5baSBill Taylor status = hermon_umap_srq_data_out((hermon_srqhdl_t)hdl, 15059e39c5baSBill Taylor (mlnx_umap_srq_data_out_t *)data_p, data_sz); 15069e39c5baSBill Taylor if (status != DDI_SUCCESS) { 15079e39c5baSBill Taylor return (status); 15089e39c5baSBill Taylor } 15099e39c5baSBill Taylor break; 15109e39c5baSBill Taylor 15119e39c5baSBill Taylor case IBT_HDL_PD: 15129e39c5baSBill Taylor status = hermon_umap_pd_data_out((hermon_pdhdl_t)hdl, 15139e39c5baSBill Taylor (mlnx_umap_pd_data_out_t *)data_p, data_sz); 15149e39c5baSBill Taylor if (status != DDI_SUCCESS) { 15159e39c5baSBill Taylor return (status); 15169e39c5baSBill Taylor } 15179e39c5baSBill Taylor break; 15189e39c5baSBill Taylor 15199e39c5baSBill Taylor /* 15209e39c5baSBill Taylor * For other possible valid IBT types, we return IBT_NOT_SUPPORTED, 15219e39c5baSBill Taylor * since the Hermon driver does not support these. 15229e39c5baSBill Taylor */ 15239e39c5baSBill Taylor case IBT_HDL_HCA: 15249e39c5baSBill Taylor case IBT_HDL_MR: 15259e39c5baSBill Taylor case IBT_HDL_MW: 15269e39c5baSBill Taylor case IBT_HDL_AH: 15279e39c5baSBill Taylor case IBT_HDL_SCHED: 15289e39c5baSBill Taylor case IBT_HDL_EEC: 15299e39c5baSBill Taylor case IBT_HDL_RDD: 15309e39c5baSBill Taylor return (IBT_NOT_SUPPORTED); 15319e39c5baSBill Taylor 15329e39c5baSBill Taylor /* 15339e39c5baSBill Taylor * Any other types are invalid. 15349e39c5baSBill Taylor */ 15359e39c5baSBill Taylor default: 15369e39c5baSBill Taylor return (IBT_INVALID_PARAM); 15379e39c5baSBill Taylor } 15389e39c5baSBill Taylor 15399e39c5baSBill Taylor return (DDI_SUCCESS); 15409e39c5baSBill Taylor } 15419e39c5baSBill Taylor 15429e39c5baSBill Taylor 15439e39c5baSBill Taylor /* 15449e39c5baSBill Taylor * hermon_umap_cq_data_out() 15459e39c5baSBill Taylor * Context: Can be called from user or kernel context. 15469e39c5baSBill Taylor */ 15479e39c5baSBill Taylor static ibt_status_t 15489e39c5baSBill Taylor hermon_umap_cq_data_out(hermon_cqhdl_t cq, mlnx_umap_cq_data_out_t *data, 15499e39c5baSBill Taylor size_t data_sz) 15509e39c5baSBill Taylor { 15519e39c5baSBill Taylor /* Check for valid CQ handle pointer */ 15529e39c5baSBill Taylor if (cq == NULL) { 15539e39c5baSBill Taylor return (IBT_CQ_HDL_INVALID); 15549e39c5baSBill Taylor } 15559e39c5baSBill Taylor 15569e39c5baSBill Taylor /* Check for valid CQ mapping structure size */ 15579e39c5baSBill Taylor if (data_sz < sizeof (mlnx_umap_cq_data_out_t)) { 15589e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 15599e39c5baSBill Taylor } 15609e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data)) 15619e39c5baSBill Taylor 15629e39c5baSBill Taylor /* deal with cq_alloc() verses cq_resize() */ 15639e39c5baSBill Taylor if (cq->cq_resize_hdl) { 15649e39c5baSBill Taylor data->mcq_maplen = cq->cq_resize_hdl->cq_cqinfo.qa_size; 15659e39c5baSBill Taylor data->mcq_numcqe = cq->cq_resize_hdl->cq_bufsz; 15669e39c5baSBill Taylor } else { 15679e39c5baSBill Taylor data->mcq_maplen = cq->cq_cqinfo.qa_size; 15689e39c5baSBill Taylor data->mcq_numcqe = cq->cq_bufsz; 15699e39c5baSBill Taylor } 15709e39c5baSBill Taylor 15719e39c5baSBill Taylor /* 15729e39c5baSBill Taylor * If it has passed all the above checks, then fill in all the useful 15739e39c5baSBill Taylor * mapping information (including the mapping offset that will be 15749e39c5baSBill Taylor * passed back to the devmap() interface during a subsequent mmap() 15759e39c5baSBill Taylor * call. 15769e39c5baSBill Taylor * 15779e39c5baSBill Taylor * The "offset" for CQ mmap()'s looks like this: 15789e39c5baSBill Taylor * +----------------------------------------+--------+--------------+ 15799e39c5baSBill Taylor * | CQ Number | 0x33 | Reserved (0) | 15809e39c5baSBill Taylor * +----------------------------------------+--------+--------------+ 15819e39c5baSBill Taylor * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits 15829e39c5baSBill Taylor * 15839e39c5baSBill Taylor * This returns information about the mapping offset, the length of 15849e39c5baSBill Taylor * the CQ memory, the CQ number (for use in later CQ doorbells), the 15859e39c5baSBill Taylor * number of CQEs the CQ memory can hold, and the size of each CQE. 15869e39c5baSBill Taylor */ 15879e39c5baSBill Taylor data->mcq_rev = MLNX_UMAP_IF_VERSION; 15889e39c5baSBill Taylor data->mcq_mapoffset = ((((uint64_t)cq->cq_cqnum << 15899e39c5baSBill Taylor MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_CQMEM_RSRC) << PAGESHIFT); 15909e39c5baSBill Taylor data->mcq_cqnum = cq->cq_cqnum; 15919e39c5baSBill Taylor data->mcq_cqesz = sizeof (hermon_hw_cqe_t); 15929e39c5baSBill Taylor 15939e39c5baSBill Taylor /* doorbell record fields */ 15949e39c5baSBill Taylor data->mcq_polldbr_mapoffset = cq->cq_dbr_mapoffset; 15959e39c5baSBill Taylor data->mcq_polldbr_maplen = PAGESIZE; 15969e39c5baSBill Taylor data->mcq_polldbr_offset = (uintptr_t)cq->cq_arm_ci_vdbr & 15979e39c5baSBill Taylor PAGEOFFSET; 15989e39c5baSBill Taylor data->mcq_armdbr_mapoffset = cq->cq_dbr_mapoffset; 15999e39c5baSBill Taylor data->mcq_armdbr_maplen = PAGESIZE; 16009e39c5baSBill Taylor data->mcq_armdbr_offset = data->mcq_polldbr_offset + 4; 16019e39c5baSBill Taylor 16029e39c5baSBill Taylor return (DDI_SUCCESS); 16039e39c5baSBill Taylor } 16049e39c5baSBill Taylor 16059e39c5baSBill Taylor 16069e39c5baSBill Taylor /* 16079e39c5baSBill Taylor * hermon_umap_qp_data_out() 16089e39c5baSBill Taylor * Context: Can be called from user or kernel context. 16099e39c5baSBill Taylor */ 16109e39c5baSBill Taylor static ibt_status_t 16119e39c5baSBill Taylor hermon_umap_qp_data_out(hermon_qphdl_t qp, mlnx_umap_qp_data_out_t *data, 16129e39c5baSBill Taylor size_t data_sz) 16139e39c5baSBill Taylor { 16149e39c5baSBill Taylor /* Check for valid QP handle pointer */ 16159e39c5baSBill Taylor if (qp == NULL) { 16169e39c5baSBill Taylor return (IBT_QP_HDL_INVALID); 16179e39c5baSBill Taylor } 16189e39c5baSBill Taylor 16199e39c5baSBill Taylor /* Check for valid QP mapping structure size */ 16209e39c5baSBill Taylor if (data_sz < sizeof (mlnx_umap_qp_data_out_t)) { 16219e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 16229e39c5baSBill Taylor } 16239e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data)) 16249e39c5baSBill Taylor 16259e39c5baSBill Taylor /* 16269e39c5baSBill Taylor * If it has passed all the checks, then fill in all the useful 16279e39c5baSBill Taylor * mapping information (including the mapping offset that will be 16289e39c5baSBill Taylor * passed back to the devmap() interface during a subsequent mmap() 16299e39c5baSBill Taylor * call. 16309e39c5baSBill Taylor * 16319e39c5baSBill Taylor * The "offset" for QP mmap()'s looks like this: 16329e39c5baSBill Taylor * +----------------------------------------+--------+--------------+ 16339e39c5baSBill Taylor * | QP Number | 0x44 | Reserved (0) | 16349e39c5baSBill Taylor * +----------------------------------------+--------+--------------+ 16359e39c5baSBill Taylor * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits 16369e39c5baSBill Taylor * 16379e39c5baSBill Taylor * This returns information about the mapping offset, the length of 16389e39c5baSBill Taylor * the QP memory, and the QP number (for use in later send and recv 16399e39c5baSBill Taylor * doorbells). It also returns the following information for both 16409e39c5baSBill Taylor * the receive work queue and the send work queue, respectively: the 16419e39c5baSBill Taylor * offset (from the base mapped address) of the start of the given 16429e39c5baSBill Taylor * work queue, the 64-bit IB virtual address that corresponds to 16439e39c5baSBill Taylor * the base mapped address (needed for posting WQEs though the 16449e39c5baSBill Taylor * QP doorbells), the number of WQEs the given work queue can hold, 16459e39c5baSBill Taylor * and the size of each WQE for the given work queue. 16469e39c5baSBill Taylor */ 16479e39c5baSBill Taylor data->mqp_rev = MLNX_UMAP_IF_VERSION; 16489e39c5baSBill Taylor data->mqp_mapoffset = ((((uint64_t)qp->qp_qpnum << 16499e39c5baSBill Taylor MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_QPMEM_RSRC) << PAGESHIFT); 16509e39c5baSBill Taylor data->mqp_maplen = qp->qp_wqinfo.qa_size; 16519e39c5baSBill Taylor data->mqp_qpnum = qp->qp_qpnum; 16529e39c5baSBill Taylor 16539e39c5baSBill Taylor /* 16549e39c5baSBill Taylor * If this QP is associated with a shared receive queue (SRQ), 16559e39c5baSBill Taylor * then return invalid RecvQ parameters. Otherwise, return 16569e39c5baSBill Taylor * the proper parameter values. 16579e39c5baSBill Taylor */ 1658*17a2b317SBill Taylor if (qp->qp_alloc_flags & IBT_QP_USES_SRQ) { 16599e39c5baSBill Taylor data->mqp_rq_off = (uint32_t)qp->qp_wqinfo.qa_size; 16609e39c5baSBill Taylor data->mqp_rq_desc_addr = (uint32_t)qp->qp_wqinfo.qa_size; 16619e39c5baSBill Taylor data->mqp_rq_numwqe = 0; 16629e39c5baSBill Taylor data->mqp_rq_wqesz = 0; 16639e39c5baSBill Taylor data->mqp_rdbr_mapoffset = 0; 16649e39c5baSBill Taylor data->mqp_rdbr_maplen = 0; 16659e39c5baSBill Taylor data->mqp_rdbr_offset = 0; 16669e39c5baSBill Taylor } else { 16679e39c5baSBill Taylor data->mqp_rq_off = (uintptr_t)qp->qp_rq_buf - 16689e39c5baSBill Taylor (uintptr_t)qp->qp_wqinfo.qa_buf_aligned; 16699e39c5baSBill Taylor data->mqp_rq_desc_addr = (uint32_t)((uintptr_t)qp->qp_rq_buf - 16709e39c5baSBill Taylor qp->qp_desc_off); 16719e39c5baSBill Taylor data->mqp_rq_numwqe = qp->qp_rq_bufsz; 16729e39c5baSBill Taylor data->mqp_rq_wqesz = (1 << qp->qp_rq_log_wqesz); 16739e39c5baSBill Taylor 16749e39c5baSBill Taylor /* doorbell record fields */ 16759e39c5baSBill Taylor data->mqp_rdbr_mapoffset = qp->qp_rdbr_mapoffset; 16769e39c5baSBill Taylor data->mqp_rdbr_maplen = PAGESIZE; 16779e39c5baSBill Taylor data->mqp_rdbr_offset = (uintptr_t)qp->qp_rq_vdbr & 16789e39c5baSBill Taylor PAGEOFFSET; 16799e39c5baSBill Taylor } 16809e39c5baSBill Taylor data->mqp_sq_off = (uintptr_t)qp->qp_sq_buf - 16819e39c5baSBill Taylor (uintptr_t)qp->qp_wqinfo.qa_buf_aligned; 16829e39c5baSBill Taylor data->mqp_sq_desc_addr = (uint32_t)((uintptr_t)qp->qp_sq_buf - 16839e39c5baSBill Taylor qp->qp_desc_off); 16849e39c5baSBill Taylor data->mqp_sq_numwqe = qp->qp_sq_bufsz; 16859e39c5baSBill Taylor data->mqp_sq_wqesz = (1 << qp->qp_sq_log_wqesz); 16869e39c5baSBill Taylor data->mqp_sq_headroomwqes = qp->qp_sq_hdrmwqes; 16879e39c5baSBill Taylor 16889e39c5baSBill Taylor /* doorbell record fields */ 16899e39c5baSBill Taylor data->mqp_sdbr_mapoffset = 0; 16909e39c5baSBill Taylor data->mqp_sdbr_maplen = 0; 16919e39c5baSBill Taylor data->mqp_sdbr_offset = 0; 16929e39c5baSBill Taylor 16939e39c5baSBill Taylor return (DDI_SUCCESS); 16949e39c5baSBill Taylor } 16959e39c5baSBill Taylor 16969e39c5baSBill Taylor 16979e39c5baSBill Taylor /* 16989e39c5baSBill Taylor * hermon_umap_srq_data_out() 16999e39c5baSBill Taylor * Context: Can be called from user or kernel context. 17009e39c5baSBill Taylor */ 17019e39c5baSBill Taylor static ibt_status_t 17029e39c5baSBill Taylor hermon_umap_srq_data_out(hermon_srqhdl_t srq, mlnx_umap_srq_data_out_t *data, 17039e39c5baSBill Taylor size_t data_sz) 17049e39c5baSBill Taylor { 17059e39c5baSBill Taylor /* Check for valid SRQ handle pointer */ 17069e39c5baSBill Taylor if (srq == NULL) { 17079e39c5baSBill Taylor return (IBT_SRQ_HDL_INVALID); 17089e39c5baSBill Taylor } 17099e39c5baSBill Taylor 17109e39c5baSBill Taylor /* Check for valid SRQ mapping structure size */ 17119e39c5baSBill Taylor if (data_sz < sizeof (mlnx_umap_srq_data_out_t)) { 17129e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 17139e39c5baSBill Taylor } 17149e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data)) 17159e39c5baSBill Taylor 17169e39c5baSBill Taylor /* 17179e39c5baSBill Taylor * If it has passed all the checks, then fill in all the useful 17189e39c5baSBill Taylor * mapping information (including the mapping offset that will be 17199e39c5baSBill Taylor * passed back to the devmap() interface during a subsequent mmap() 17209e39c5baSBill Taylor * call. 17219e39c5baSBill Taylor * 17229e39c5baSBill Taylor * The "offset" for SRQ mmap()'s looks like this: 17239e39c5baSBill Taylor * +----------------------------------------+--------+--------------+ 17249e39c5baSBill Taylor * | SRQ Number | 0x66 | Reserved (0) | 17259e39c5baSBill Taylor * +----------------------------------------+--------+--------------+ 17269e39c5baSBill Taylor * (64 - 8 - PAGESHIFT) bits 8 bits PAGESHIFT bits 17279e39c5baSBill Taylor * 17289e39c5baSBill Taylor * This returns information about the mapping offset, the length of the 17299e39c5baSBill Taylor * SRQ memory, and the SRQ number (for use in later send and recv 17309e39c5baSBill Taylor * doorbells). It also returns the following information for the 17319e39c5baSBill Taylor * shared receive queue: the offset (from the base mapped address) of 17329e39c5baSBill Taylor * the start of the given work queue, the 64-bit IB virtual address 17339e39c5baSBill Taylor * that corresponds to the base mapped address (needed for posting WQEs 17349e39c5baSBill Taylor * though the QP doorbells), the number of WQEs the given work queue 17359e39c5baSBill Taylor * can hold, and the size of each WQE for the given work queue. 17369e39c5baSBill Taylor */ 17379e39c5baSBill Taylor data->msrq_rev = MLNX_UMAP_IF_VERSION; 17389e39c5baSBill Taylor data->msrq_mapoffset = ((((uint64_t)srq->srq_srqnum << 17399e39c5baSBill Taylor MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_SRQMEM_RSRC) << PAGESHIFT); 17409e39c5baSBill Taylor data->msrq_maplen = srq->srq_wqinfo.qa_size; 17419e39c5baSBill Taylor data->msrq_srqnum = srq->srq_srqnum; 17429e39c5baSBill Taylor 17439e39c5baSBill Taylor data->msrq_desc_addr = (uint32_t)((uintptr_t)srq->srq_wq_buf - 17449e39c5baSBill Taylor srq->srq_desc_off); 17459e39c5baSBill Taylor data->msrq_numwqe = srq->srq_wq_bufsz; 17469e39c5baSBill Taylor data->msrq_wqesz = (1 << srq->srq_wq_log_wqesz); 17479e39c5baSBill Taylor 17489e39c5baSBill Taylor /* doorbell record fields */ 17499e39c5baSBill Taylor data->msrq_rdbr_mapoffset = srq->srq_rdbr_mapoffset; 17509e39c5baSBill Taylor data->msrq_rdbr_maplen = PAGESIZE; 17519e39c5baSBill Taylor data->msrq_rdbr_offset = (uintptr_t)srq->srq_wq_vdbr & 17529e39c5baSBill Taylor PAGEOFFSET; 17539e39c5baSBill Taylor 17549e39c5baSBill Taylor return (DDI_SUCCESS); 17559e39c5baSBill Taylor } 17569e39c5baSBill Taylor 17579e39c5baSBill Taylor 17589e39c5baSBill Taylor /* 17599e39c5baSBill Taylor * hermon_umap_pd_data_out() 17609e39c5baSBill Taylor * Context: Can be called from user or kernel context. 17619e39c5baSBill Taylor */ 17629e39c5baSBill Taylor static ibt_status_t 17639e39c5baSBill Taylor hermon_umap_pd_data_out(hermon_pdhdl_t pd, mlnx_umap_pd_data_out_t *data, 17649e39c5baSBill Taylor size_t data_sz) 17659e39c5baSBill Taylor { 17669e39c5baSBill Taylor /* Check for valid PD handle pointer */ 17679e39c5baSBill Taylor if (pd == NULL) { 17689e39c5baSBill Taylor return (IBT_PD_HDL_INVALID); 17699e39c5baSBill Taylor } 17709e39c5baSBill Taylor 17719e39c5baSBill Taylor /* Check for valid PD mapping structure size */ 17729e39c5baSBill Taylor if (data_sz < sizeof (mlnx_umap_pd_data_out_t)) { 17739e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 17749e39c5baSBill Taylor } 17759e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*data)) 17769e39c5baSBill Taylor 17779e39c5baSBill Taylor /* 17789e39c5baSBill Taylor * If it has passed all the checks, then fill the PD table index 17799e39c5baSBill Taylor * (the PD table allocated index for the PD pd_pdnum). 17809e39c5baSBill Taylor */ 17819e39c5baSBill Taylor data->mpd_rev = MLNX_UMAP_IF_VERSION; 17829e39c5baSBill Taylor data->mpd_pdnum = pd->pd_pdnum; 17839e39c5baSBill Taylor 17849e39c5baSBill Taylor return (DDI_SUCCESS); 17859e39c5baSBill Taylor } 17869e39c5baSBill Taylor 17879e39c5baSBill Taylor 17889e39c5baSBill Taylor /* 17899e39c5baSBill Taylor * hermon_umap_db_init() 17909e39c5baSBill Taylor * Context: Only called from attach() path context 17919e39c5baSBill Taylor */ 17929e39c5baSBill Taylor void 17939e39c5baSBill Taylor hermon_umap_db_init(void) 17949e39c5baSBill Taylor { 17959e39c5baSBill Taylor /* 17969e39c5baSBill Taylor * Initialize the lock used by the Hermon "userland resources database" 17979e39c5baSBill Taylor * This is used to ensure atomic access to add, remove, and find 17989e39c5baSBill Taylor * entries in the database. 17999e39c5baSBill Taylor */ 18009e39c5baSBill Taylor mutex_init(&hermon_userland_rsrc_db.hdl_umapdb_lock, NULL, 18019e39c5baSBill Taylor MUTEX_DRIVER, NULL); 18029e39c5baSBill Taylor 18039e39c5baSBill Taylor /* 18049e39c5baSBill Taylor * Initialize the AVL tree used for the "userland resources 18059e39c5baSBill Taylor * database". Using an AVL tree here provides the ability to 18069e39c5baSBill Taylor * scale the database size to large numbers of resources. The 18079e39c5baSBill Taylor * entries in the tree are "hermon_umap_db_entry_t" (see 18089e39c5baSBill Taylor * hermon_umap.h). The tree is searched with the help of the 18099e39c5baSBill Taylor * hermon_umap_db_compare() routine. 18109e39c5baSBill Taylor */ 18119e39c5baSBill Taylor avl_create(&hermon_userland_rsrc_db.hdl_umapdb_avl, 18129e39c5baSBill Taylor hermon_umap_db_compare, sizeof (hermon_umap_db_entry_t), 18139e39c5baSBill Taylor offsetof(hermon_umap_db_entry_t, hdbe_avlnode)); 18149e39c5baSBill Taylor } 18159e39c5baSBill Taylor 18169e39c5baSBill Taylor 18179e39c5baSBill Taylor /* 18189e39c5baSBill Taylor * hermon_umap_db_fini() 18199e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts 18209e39c5baSBill Taylor */ 18219e39c5baSBill Taylor void 18229e39c5baSBill Taylor hermon_umap_db_fini(void) 18239e39c5baSBill Taylor { 18249e39c5baSBill Taylor /* Destroy the AVL tree for the "userland resources database" */ 18259e39c5baSBill Taylor avl_destroy(&hermon_userland_rsrc_db.hdl_umapdb_avl); 18269e39c5baSBill Taylor 18279e39c5baSBill Taylor /* Destroy the lock for the "userland resources database" */ 18289e39c5baSBill Taylor mutex_destroy(&hermon_userland_rsrc_db.hdl_umapdb_lock); 18299e39c5baSBill Taylor } 18309e39c5baSBill Taylor 18319e39c5baSBill Taylor 18329e39c5baSBill Taylor /* 18339e39c5baSBill Taylor * hermon_umap_db_alloc() 18349e39c5baSBill Taylor * Context: Can be called from user or kernel context. 18359e39c5baSBill Taylor */ 18369e39c5baSBill Taylor hermon_umap_db_entry_t * 18379e39c5baSBill Taylor hermon_umap_db_alloc(uint_t instance, uint64_t key, uint_t type, uint64_t value) 18389e39c5baSBill Taylor { 18399e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 18409e39c5baSBill Taylor 18419e39c5baSBill Taylor /* Allocate an entry to add to the "userland resources database" */ 18429e39c5baSBill Taylor umapdb = kmem_zalloc(sizeof (hermon_umap_db_entry_t), KM_NOSLEEP); 18439e39c5baSBill Taylor if (umapdb == NULL) { 18449e39c5baSBill Taylor return (NULL); 18459e39c5baSBill Taylor } 18469e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb)) 18479e39c5baSBill Taylor 18489e39c5baSBill Taylor /* Fill in the fields in the database entry */ 18499e39c5baSBill Taylor umapdb->hdbe_common.hdb_instance = instance; 18509e39c5baSBill Taylor umapdb->hdbe_common.hdb_type = type; 18519e39c5baSBill Taylor umapdb->hdbe_common.hdb_key = key; 18529e39c5baSBill Taylor umapdb->hdbe_common.hdb_value = value; 18539e39c5baSBill Taylor 18549e39c5baSBill Taylor return (umapdb); 18559e39c5baSBill Taylor } 18569e39c5baSBill Taylor 18579e39c5baSBill Taylor 18589e39c5baSBill Taylor /* 18599e39c5baSBill Taylor * hermon_umap_db_free() 18609e39c5baSBill Taylor * Context: Can be called from user or kernel context. 18619e39c5baSBill Taylor */ 18629e39c5baSBill Taylor void 18639e39c5baSBill Taylor hermon_umap_db_free(hermon_umap_db_entry_t *umapdb) 18649e39c5baSBill Taylor { 18659e39c5baSBill Taylor /* Free the database entry */ 18669e39c5baSBill Taylor kmem_free(umapdb, sizeof (hermon_umap_db_entry_t)); 18679e39c5baSBill Taylor } 18689e39c5baSBill Taylor 18699e39c5baSBill Taylor 18709e39c5baSBill Taylor /* 18719e39c5baSBill Taylor * hermon_umap_db_add() 18729e39c5baSBill Taylor * Context: Can be called from user or kernel context. 18739e39c5baSBill Taylor */ 18749e39c5baSBill Taylor void 18759e39c5baSBill Taylor hermon_umap_db_add(hermon_umap_db_entry_t *umapdb) 18769e39c5baSBill Taylor { 18779e39c5baSBill Taylor mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock); 18789e39c5baSBill Taylor hermon_umap_db_add_nolock(umapdb); 18799e39c5baSBill Taylor mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock); 18809e39c5baSBill Taylor } 18819e39c5baSBill Taylor 18829e39c5baSBill Taylor 18839e39c5baSBill Taylor /* 18849e39c5baSBill Taylor * hermon_umap_db_add_nolock() 18859e39c5baSBill Taylor * Context: Can be called from user or kernel context. 18869e39c5baSBill Taylor */ 18879e39c5baSBill Taylor void 18889e39c5baSBill Taylor hermon_umap_db_add_nolock(hermon_umap_db_entry_t *umapdb) 18899e39c5baSBill Taylor { 18909e39c5baSBill Taylor hermon_umap_db_query_t query; 18919e39c5baSBill Taylor avl_index_t where; 18929e39c5baSBill Taylor 18939e39c5baSBill Taylor ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock)); 18949e39c5baSBill Taylor 18959e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb)) 18969e39c5baSBill Taylor 18979e39c5baSBill Taylor /* 18989e39c5baSBill Taylor * Copy the common portion of the "to-be-added" database entry 18999e39c5baSBill Taylor * into the "hermon_umap_db_query_t" structure. We use this structure 19009e39c5baSBill Taylor * (with no flags set) to find the appropriate location in the 19019e39c5baSBill Taylor * "userland resources database" for the new entry to be added. 19029e39c5baSBill Taylor * 19039e39c5baSBill Taylor * Note: we expect that this entry should not be found in the 19049e39c5baSBill Taylor * database (unless something bad has happened). 19059e39c5baSBill Taylor */ 19069e39c5baSBill Taylor query.hqdb_common = umapdb->hdbe_common; 19079e39c5baSBill Taylor query.hqdb_flags = 0; 19089e39c5baSBill Taylor (void) avl_find(&hermon_userland_rsrc_db.hdl_umapdb_avl, &query, 19099e39c5baSBill Taylor &where); 19109e39c5baSBill Taylor 19119e39c5baSBill Taylor /* 19129e39c5baSBill Taylor * Now, using the "where" field from the avl_find() operation 19139e39c5baSBill Taylor * above, we will insert the new database entry ("umapdb"). 19149e39c5baSBill Taylor */ 19159e39c5baSBill Taylor avl_insert(&hermon_userland_rsrc_db.hdl_umapdb_avl, umapdb, 19169e39c5baSBill Taylor where); 19179e39c5baSBill Taylor } 19189e39c5baSBill Taylor 19199e39c5baSBill Taylor 19209e39c5baSBill Taylor /* 19219e39c5baSBill Taylor * hermon_umap_db_find() 19229e39c5baSBill Taylor * Context: Can be called from user or kernel context. 19239e39c5baSBill Taylor */ 19249e39c5baSBill Taylor int 19259e39c5baSBill Taylor hermon_umap_db_find(uint_t instance, uint64_t key, uint_t type, 19269e39c5baSBill Taylor uint64_t *value, uint_t flag, hermon_umap_db_entry_t **umapdb) 19279e39c5baSBill Taylor { 19289e39c5baSBill Taylor int status; 19299e39c5baSBill Taylor 19309e39c5baSBill Taylor mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock); 19319e39c5baSBill Taylor status = hermon_umap_db_find_nolock(instance, key, type, value, flag, 19329e39c5baSBill Taylor umapdb); 19339e39c5baSBill Taylor mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock); 19349e39c5baSBill Taylor 19359e39c5baSBill Taylor return (status); 19369e39c5baSBill Taylor } 19379e39c5baSBill Taylor 19389e39c5baSBill Taylor 19399e39c5baSBill Taylor /* 19409e39c5baSBill Taylor * hermon_umap_db_find_nolock() 19419e39c5baSBill Taylor * Context: Can be called from user or kernel context. 19429e39c5baSBill Taylor */ 19439e39c5baSBill Taylor int 19449e39c5baSBill Taylor hermon_umap_db_find_nolock(uint_t instance, uint64_t key, uint_t type, 19459e39c5baSBill Taylor uint64_t *value, uint_t flags, hermon_umap_db_entry_t **umapdb) 19469e39c5baSBill Taylor { 19479e39c5baSBill Taylor hermon_umap_db_query_t query; 19489e39c5baSBill Taylor hermon_umap_db_entry_t *entry; 19499e39c5baSBill Taylor avl_index_t where; 19509e39c5baSBill Taylor 19519e39c5baSBill Taylor ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock)); 19529e39c5baSBill Taylor 19539e39c5baSBill Taylor /* 19549e39c5baSBill Taylor * Fill in key, type, instance, and flags values of the 19559e39c5baSBill Taylor * hermon_umap_db_query_t in preparation for the database 19569e39c5baSBill Taylor * lookup. 19579e39c5baSBill Taylor */ 19589e39c5baSBill Taylor query.hqdb_flags = flags; 19599e39c5baSBill Taylor query.hqdb_common.hdb_key = key; 19609e39c5baSBill Taylor query.hqdb_common.hdb_type = type; 19619e39c5baSBill Taylor query.hqdb_common.hdb_instance = instance; 19629e39c5baSBill Taylor 19639e39c5baSBill Taylor /* 19649e39c5baSBill Taylor * Perform the database query. If no entry is found, then 19659e39c5baSBill Taylor * return failure, else continue. 19669e39c5baSBill Taylor */ 19679e39c5baSBill Taylor entry = (hermon_umap_db_entry_t *)avl_find( 19689e39c5baSBill Taylor &hermon_userland_rsrc_db.hdl_umapdb_avl, &query, &where); 19699e39c5baSBill Taylor if (entry == NULL) { 19709e39c5baSBill Taylor return (DDI_FAILURE); 19719e39c5baSBill Taylor } 19729e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*entry)) 19739e39c5baSBill Taylor 19749e39c5baSBill Taylor /* 19759e39c5baSBill Taylor * If the flags argument specifies that the entry should 19769e39c5baSBill Taylor * be removed if found, then call avl_remove() to remove 19779e39c5baSBill Taylor * the entry from the database. 19789e39c5baSBill Taylor */ 19799e39c5baSBill Taylor if (flags & HERMON_UMAP_DB_REMOVE) { 19809e39c5baSBill Taylor 19819e39c5baSBill Taylor avl_remove(&hermon_userland_rsrc_db.hdl_umapdb_avl, entry); 19829e39c5baSBill Taylor 19839e39c5baSBill Taylor /* 19849e39c5baSBill Taylor * The database entry is returned with the expectation 19859e39c5baSBill Taylor * that the caller will use hermon_umap_db_free() to 19869e39c5baSBill Taylor * free the entry's memory. ASSERT that this is non-NULL. 19879e39c5baSBill Taylor * NULL pointer should never be passed for the 19889e39c5baSBill Taylor * HERMON_UMAP_DB_REMOVE case. 19899e39c5baSBill Taylor */ 19909e39c5baSBill Taylor ASSERT(umapdb != NULL); 19919e39c5baSBill Taylor } 19929e39c5baSBill Taylor 19939e39c5baSBill Taylor /* 19949e39c5baSBill Taylor * If the caller would like visibility to the database entry 19959e39c5baSBill Taylor * (indicated through the use of a non-NULL "umapdb" argument), 19969e39c5baSBill Taylor * then fill it in. 19979e39c5baSBill Taylor */ 19989e39c5baSBill Taylor if (umapdb != NULL) { 19999e39c5baSBill Taylor *umapdb = entry; 20009e39c5baSBill Taylor } 20019e39c5baSBill Taylor 20029e39c5baSBill Taylor /* Extract value field from database entry and return success */ 20039e39c5baSBill Taylor *value = entry->hdbe_common.hdb_value; 20049e39c5baSBill Taylor 20059e39c5baSBill Taylor return (DDI_SUCCESS); 20069e39c5baSBill Taylor } 20079e39c5baSBill Taylor 20089e39c5baSBill Taylor 20099e39c5baSBill Taylor /* 20109e39c5baSBill Taylor * hermon_umap_umemlock_cb() 20119e39c5baSBill Taylor * Context: Can be called from callback context. 20129e39c5baSBill Taylor */ 20139e39c5baSBill Taylor void 20149e39c5baSBill Taylor hermon_umap_umemlock_cb(ddi_umem_cookie_t *umem_cookie) 20159e39c5baSBill Taylor { 20169e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 20179e39c5baSBill Taylor hermon_state_t *state; 20189e39c5baSBill Taylor hermon_rsrc_t *rsrcp; 20199e39c5baSBill Taylor hermon_mrhdl_t mr; 20209e39c5baSBill Taylor uint64_t value; 20219e39c5baSBill Taylor uint_t instance; 20229e39c5baSBill Taylor int status; 20239e39c5baSBill Taylor void (*mr_callback)(void *, void *); 20249e39c5baSBill Taylor void *mr_cbarg1, *mr_cbarg2; 20259e39c5baSBill Taylor 20269e39c5baSBill Taylor /* 20279e39c5baSBill Taylor * If this was userland memory, then we need to remove its entry 20289e39c5baSBill Taylor * from the "userland resources database". Note: We use the 20299e39c5baSBill Taylor * HERMON_UMAP_DB_IGNORE_INSTANCE flag here because we don't know 20309e39c5baSBill Taylor * which instance was used when the entry was added (but we want 20319e39c5baSBill Taylor * to know after the entry is found using the other search criteria). 20329e39c5baSBill Taylor */ 20339e39c5baSBill Taylor status = hermon_umap_db_find(0, (uint64_t)(uintptr_t)umem_cookie, 20349e39c5baSBill Taylor MLNX_UMAP_MRMEM_RSRC, &value, (HERMON_UMAP_DB_REMOVE | 20359e39c5baSBill Taylor HERMON_UMAP_DB_IGNORE_INSTANCE), &umapdb); 20369e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*umapdb)) 20379e39c5baSBill Taylor if (status == DDI_SUCCESS) { 20389e39c5baSBill Taylor instance = umapdb->hdbe_common.hdb_instance; 20399e39c5baSBill Taylor state = ddi_get_soft_state(hermon_statep, instance); 20409e39c5baSBill Taylor if (state == NULL) { 20419e39c5baSBill Taylor cmn_err(CE_WARN, "Unable to match Hermon instance\n"); 20429e39c5baSBill Taylor return; 20439e39c5baSBill Taylor } 20449e39c5baSBill Taylor 20459e39c5baSBill Taylor /* Free the database entry */ 20469e39c5baSBill Taylor hermon_umap_db_free(umapdb); 20479e39c5baSBill Taylor 20489e39c5baSBill Taylor /* Use "value" to convert to an MR handle */ 20499e39c5baSBill Taylor rsrcp = (hermon_rsrc_t *)(uintptr_t)value; 20509e39c5baSBill Taylor mr = (hermon_mrhdl_t)rsrcp->hr_addr; 20519e39c5baSBill Taylor 20529e39c5baSBill Taylor /* 20539e39c5baSBill Taylor * If a callback has been provided, call it first. This 20549e39c5baSBill Taylor * callback is expected to do any cleanup necessary to 20559e39c5baSBill Taylor * guarantee that the subsequent MR deregister (below) 20569e39c5baSBill Taylor * will succeed. Specifically, this means freeing up memory 20579e39c5baSBill Taylor * windows which might have been associated with the MR. 20589e39c5baSBill Taylor */ 20599e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 20609e39c5baSBill Taylor mr_callback = mr->mr_umem_cbfunc; 20619e39c5baSBill Taylor mr_cbarg1 = mr->mr_umem_cbarg1; 20629e39c5baSBill Taylor mr_cbarg2 = mr->mr_umem_cbarg2; 20639e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 20649e39c5baSBill Taylor if (mr_callback != NULL) { 20659e39c5baSBill Taylor mr_callback(mr_cbarg1, mr_cbarg2); 20669e39c5baSBill Taylor } 20679e39c5baSBill Taylor 20689e39c5baSBill Taylor /* 20699e39c5baSBill Taylor * Then call hermon_mr_deregister() to release the resources 20709e39c5baSBill Taylor * associated with the MR handle. Note: Because this routine 20719e39c5baSBill Taylor * will also check for whether the ddi_umem_cookie_t is in the 20729e39c5baSBill Taylor * database, it will take responsibility for disabling the 20739e39c5baSBill Taylor * memory region and calling ddi_umem_unlock(). 20749e39c5baSBill Taylor */ 20759e39c5baSBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL, 20769e39c5baSBill Taylor HERMON_SLEEP); 20779e39c5baSBill Taylor if (status != DDI_SUCCESS) { 20789e39c5baSBill Taylor HERMON_WARNING(state, "Unexpected failure in " 20799e39c5baSBill Taylor "deregister from callback\n"); 20809e39c5baSBill Taylor } 20819e39c5baSBill Taylor } 20829e39c5baSBill Taylor } 20839e39c5baSBill Taylor 20849e39c5baSBill Taylor 20859e39c5baSBill Taylor /* 20869e39c5baSBill Taylor * hermon_umap_db_compare() 20879e39c5baSBill Taylor * Context: Can be called from user or kernel context. 20889e39c5baSBill Taylor */ 20899e39c5baSBill Taylor static int 20909e39c5baSBill Taylor hermon_umap_db_compare(const void *q, const void *e) 20919e39c5baSBill Taylor { 20929e39c5baSBill Taylor hermon_umap_db_common_t *entry_common, *query_common; 20939e39c5baSBill Taylor uint_t query_flags; 20949e39c5baSBill Taylor 20959e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((hermon_umap_db_query_t *)q))) 20969e39c5baSBill Taylor 20979e39c5baSBill Taylor entry_common = &((hermon_umap_db_entry_t *)e)->hdbe_common; 20989e39c5baSBill Taylor query_common = &((hermon_umap_db_query_t *)q)->hqdb_common; 20999e39c5baSBill Taylor query_flags = ((hermon_umap_db_query_t *)q)->hqdb_flags; 21009e39c5baSBill Taylor 21019e39c5baSBill Taylor /* 21029e39c5baSBill Taylor * The first comparison is done on the "key" value in "query" 21039e39c5baSBill Taylor * and "entry". If they are not equal, then the appropriate 21049e39c5baSBill Taylor * search direction is returned. Else, we continue by 21059e39c5baSBill Taylor * comparing "type". 21069e39c5baSBill Taylor */ 21079e39c5baSBill Taylor if (query_common->hdb_key < entry_common->hdb_key) { 21089e39c5baSBill Taylor return (-1); 21099e39c5baSBill Taylor } else if (query_common->hdb_key > entry_common->hdb_key) { 21109e39c5baSBill Taylor return (+1); 21119e39c5baSBill Taylor } 21129e39c5baSBill Taylor 21139e39c5baSBill Taylor /* 21149e39c5baSBill Taylor * If the search reaches this point, then "query" and "entry" 21159e39c5baSBill Taylor * have equal key values. So we continue be comparing their 21169e39c5baSBill Taylor * "type" values. Again, if they are not equal, then the 21179e39c5baSBill Taylor * appropriate search direction is returned. Else, we continue 21189e39c5baSBill Taylor * by comparing "instance". 21199e39c5baSBill Taylor */ 21209e39c5baSBill Taylor if (query_common->hdb_type < entry_common->hdb_type) { 21219e39c5baSBill Taylor return (-1); 21229e39c5baSBill Taylor } else if (query_common->hdb_type > entry_common->hdb_type) { 21239e39c5baSBill Taylor return (+1); 21249e39c5baSBill Taylor } 21259e39c5baSBill Taylor 21269e39c5baSBill Taylor /* 21279e39c5baSBill Taylor * If the search reaches this point, then "query" and "entry" 21289e39c5baSBill Taylor * have exactly the same key and type values. Now we consult 21299e39c5baSBill Taylor * the "flags" field in the query to determine whether the 21309e39c5baSBill Taylor * "instance" is relevant to the search. If the 21319e39c5baSBill Taylor * HERMON_UMAP_DB_IGNORE_INSTANCE flags is set, then return 21329e39c5baSBill Taylor * success (0) here. Otherwise, continue the search by comparing 21339e39c5baSBill Taylor * instance values and returning the appropriate search direction. 21349e39c5baSBill Taylor */ 21359e39c5baSBill Taylor if (query_flags & HERMON_UMAP_DB_IGNORE_INSTANCE) { 21369e39c5baSBill Taylor return (0); 21379e39c5baSBill Taylor } 21389e39c5baSBill Taylor 21399e39c5baSBill Taylor /* 21409e39c5baSBill Taylor * If the search has reached this point, then "query" and "entry" 21419e39c5baSBill Taylor * can only be differentiated by their instance values. If these 21429e39c5baSBill Taylor * are not equal, then return the appropriate search direction. 21439e39c5baSBill Taylor * Else, we return success (0). 21449e39c5baSBill Taylor */ 21459e39c5baSBill Taylor if (query_common->hdb_instance < entry_common->hdb_instance) { 21469e39c5baSBill Taylor return (-1); 21479e39c5baSBill Taylor } else if (query_common->hdb_instance > entry_common->hdb_instance) { 21489e39c5baSBill Taylor return (+1); 21499e39c5baSBill Taylor } 21509e39c5baSBill Taylor 21519e39c5baSBill Taylor /* Everything matches... so return success */ 21529e39c5baSBill Taylor return (0); 21539e39c5baSBill Taylor } 21549e39c5baSBill Taylor 21559e39c5baSBill Taylor 21569e39c5baSBill Taylor /* 21579e39c5baSBill Taylor * hermon_umap_db_set_onclose_cb() 21589e39c5baSBill Taylor * Context: Can be called from user or kernel context. 21599e39c5baSBill Taylor */ 21609e39c5baSBill Taylor int 21619e39c5baSBill Taylor hermon_umap_db_set_onclose_cb(dev_t dev, uint64_t flag, 21629e39c5baSBill Taylor int (*callback)(void *), void *arg) 21639e39c5baSBill Taylor { 21649e39c5baSBill Taylor hermon_umap_db_priv_t *priv; 21659e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 21669e39c5baSBill Taylor minor_t instance; 21679e39c5baSBill Taylor uint64_t value; 21689e39c5baSBill Taylor int status; 21699e39c5baSBill Taylor 21709e39c5baSBill Taylor instance = HERMON_DEV_INSTANCE(dev); 21719e39c5baSBill Taylor if (instance == (minor_t)-1) { 21729e39c5baSBill Taylor return (DDI_FAILURE); 21739e39c5baSBill Taylor } 21749e39c5baSBill Taylor 21759e39c5baSBill Taylor if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) { 21769e39c5baSBill Taylor return (DDI_FAILURE); 21779e39c5baSBill Taylor } 21789e39c5baSBill Taylor 21799e39c5baSBill Taylor /* 21809e39c5baSBill Taylor * Grab the lock for the "userland resources database" and find 21819e39c5baSBill Taylor * the entry corresponding to this minor number. Once it's found, 21829e39c5baSBill Taylor * allocate (if necessary) and add an entry (in the "hdb_priv" 21839e39c5baSBill Taylor * field) to indicate that further processing may be needed during 21849e39c5baSBill Taylor * Hermon's close() handling. 21859e39c5baSBill Taylor */ 21869e39c5baSBill Taylor mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock); 21879e39c5baSBill Taylor status = hermon_umap_db_find_nolock(instance, dev, 21889e39c5baSBill Taylor MLNX_UMAP_PID_RSRC, &value, 0, &umapdb); 21899e39c5baSBill Taylor if (status != DDI_SUCCESS) { 21909e39c5baSBill Taylor mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock); 21919e39c5baSBill Taylor return (DDI_FAILURE); 21929e39c5baSBill Taylor } 21939e39c5baSBill Taylor 21949e39c5baSBill Taylor priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv; 21959e39c5baSBill Taylor if (priv == NULL) { 21969e39c5baSBill Taylor priv = (hermon_umap_db_priv_t *)kmem_zalloc( 21979e39c5baSBill Taylor sizeof (hermon_umap_db_priv_t), KM_NOSLEEP); 21989e39c5baSBill Taylor if (priv == NULL) { 21999e39c5baSBill Taylor mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock); 22009e39c5baSBill Taylor return (DDI_FAILURE); 22019e39c5baSBill Taylor } 22029e39c5baSBill Taylor } 22039e39c5baSBill Taylor 22049e39c5baSBill Taylor /* 22059e39c5baSBill Taylor * Save away the callback and argument to be used during Hermon's 22069e39c5baSBill Taylor * close() processing. 22079e39c5baSBill Taylor */ 22089e39c5baSBill Taylor priv->hdp_cb = callback; 22099e39c5baSBill Taylor priv->hdp_arg = arg; 22109e39c5baSBill Taylor 22119e39c5baSBill Taylor umapdb->hdbe_common.hdb_priv = (void *)priv; 22129e39c5baSBill Taylor mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock); 22139e39c5baSBill Taylor 22149e39c5baSBill Taylor return (DDI_SUCCESS); 22159e39c5baSBill Taylor } 22169e39c5baSBill Taylor 22179e39c5baSBill Taylor 22189e39c5baSBill Taylor /* 22199e39c5baSBill Taylor * hermon_umap_db_clear_onclose_cb() 22209e39c5baSBill Taylor * Context: Can be called from user or kernel context. 22219e39c5baSBill Taylor */ 22229e39c5baSBill Taylor int 22239e39c5baSBill Taylor hermon_umap_db_clear_onclose_cb(dev_t dev, uint64_t flag) 22249e39c5baSBill Taylor { 22259e39c5baSBill Taylor hermon_umap_db_priv_t *priv; 22269e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 22279e39c5baSBill Taylor minor_t instance; 22289e39c5baSBill Taylor uint64_t value; 22299e39c5baSBill Taylor int status; 22309e39c5baSBill Taylor 22319e39c5baSBill Taylor instance = HERMON_DEV_INSTANCE(dev); 22329e39c5baSBill Taylor if (instance == (minor_t)-1) { 22339e39c5baSBill Taylor return (DDI_FAILURE); 22349e39c5baSBill Taylor } 22359e39c5baSBill Taylor 22369e39c5baSBill Taylor if (flag != HERMON_ONCLOSE_FLASH_INPROGRESS) { 22379e39c5baSBill Taylor return (DDI_FAILURE); 22389e39c5baSBill Taylor } 22399e39c5baSBill Taylor 22409e39c5baSBill Taylor /* 22419e39c5baSBill Taylor * Grab the lock for the "userland resources database" and find 22429e39c5baSBill Taylor * the entry corresponding to this minor number. Once it's found, 22439e39c5baSBill Taylor * remove the entry (in the "hdb_priv" field) that indicated the 22449e39c5baSBill Taylor * need for further processing during Hermon's close(). Free the 22459e39c5baSBill Taylor * entry, if appropriate. 22469e39c5baSBill Taylor */ 22479e39c5baSBill Taylor mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock); 22489e39c5baSBill Taylor status = hermon_umap_db_find_nolock(instance, dev, 22499e39c5baSBill Taylor MLNX_UMAP_PID_RSRC, &value, 0, &umapdb); 22509e39c5baSBill Taylor if (status != DDI_SUCCESS) { 22519e39c5baSBill Taylor mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock); 22529e39c5baSBill Taylor return (DDI_FAILURE); 22539e39c5baSBill Taylor } 22549e39c5baSBill Taylor 22559e39c5baSBill Taylor priv = (hermon_umap_db_priv_t *)umapdb->hdbe_common.hdb_priv; 22569e39c5baSBill Taylor if (priv != NULL) { 22579e39c5baSBill Taylor kmem_free(priv, sizeof (hermon_umap_db_priv_t)); 22589e39c5baSBill Taylor priv = NULL; 22599e39c5baSBill Taylor } 22609e39c5baSBill Taylor 22619e39c5baSBill Taylor umapdb->hdbe_common.hdb_priv = (void *)priv; 22629e39c5baSBill Taylor mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock); 22639e39c5baSBill Taylor return (DDI_SUCCESS); 22649e39c5baSBill Taylor } 22659e39c5baSBill Taylor 22669e39c5baSBill Taylor 22679e39c5baSBill Taylor /* 22689e39c5baSBill Taylor * hermon_umap_db_clear_onclose_cb() 22699e39c5baSBill Taylor * Context: Can be called from user or kernel context. 22709e39c5baSBill Taylor */ 22719e39c5baSBill Taylor int 22729e39c5baSBill Taylor hermon_umap_db_handle_onclose_cb(hermon_umap_db_priv_t *priv) 22739e39c5baSBill Taylor { 22749e39c5baSBill Taylor int (*callback)(void *); 22759e39c5baSBill Taylor 22769e39c5baSBill Taylor ASSERT(MUTEX_HELD(&hermon_userland_rsrc_db.hdl_umapdb_lock)); 22779e39c5baSBill Taylor 22789e39c5baSBill Taylor /* 22799e39c5baSBill Taylor * Call the callback. 22809e39c5baSBill Taylor * Note: Currently there is only one callback (in "hdp_cb"), but 22819e39c5baSBill Taylor * in the future there may be more, depending on what other types 22829e39c5baSBill Taylor * of interaction there are between userland processes and the 22839e39c5baSBill Taylor * driver. 22849e39c5baSBill Taylor */ 22859e39c5baSBill Taylor callback = priv->hdp_cb; 22869e39c5baSBill Taylor return (callback(priv->hdp_arg)); 22879e39c5baSBill Taylor } 2288