19e39c5baSBill Taylor /* 29e39c5baSBill Taylor * CDDL HEADER START 39e39c5baSBill Taylor * 49e39c5baSBill Taylor * The contents of this file are subject to the terms of the 59e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 69e39c5baSBill Taylor * You may not use this file except in compliance with the License. 79e39c5baSBill Taylor * 89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 109e39c5baSBill Taylor * See the License for the specific language governing permissions 119e39c5baSBill Taylor * and limitations under the License. 129e39c5baSBill Taylor * 139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 189e39c5baSBill Taylor * 199e39c5baSBill Taylor * CDDL HEADER END 209e39c5baSBill Taylor */ 219e39c5baSBill Taylor 229e39c5baSBill Taylor /* 23949b58c7SBill Taylor * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 249e39c5baSBill Taylor * Use is subject to license terms. 259e39c5baSBill Taylor */ 269e39c5baSBill Taylor 279e39c5baSBill Taylor /* 289e39c5baSBill Taylor * hermon_mr.c 299e39c5baSBill Taylor * Hermon Memory Region/Window Routines 309e39c5baSBill Taylor * 319e39c5baSBill Taylor * Implements all the routines necessary to provide the requisite memory 329e39c5baSBill Taylor * registration verbs. These include operations like RegisterMemRegion(), 339e39c5baSBill Taylor * DeregisterMemRegion(), ReregisterMemRegion, RegisterSharedMemRegion, 349e39c5baSBill Taylor * etc., that affect Memory Regions. It also includes the verbs that 359e39c5baSBill Taylor * affect Memory Windows, including AllocMemWindow(), FreeMemWindow(), 369e39c5baSBill Taylor * and QueryMemWindow(). 379e39c5baSBill Taylor */ 389e39c5baSBill Taylor 399e39c5baSBill Taylor #include <sys/types.h> 409e39c5baSBill Taylor #include <sys/conf.h> 419e39c5baSBill Taylor #include <sys/ddi.h> 429e39c5baSBill Taylor #include <sys/sunddi.h> 439e39c5baSBill Taylor #include <sys/modctl.h> 449e39c5baSBill Taylor #include <sys/esunddi.h> 459e39c5baSBill Taylor 469e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 479e39c5baSBill Taylor 489e39c5baSBill Taylor extern uint32_t hermon_kernel_data_ro; 499e39c5baSBill Taylor extern uint32_t hermon_user_data_ro; 50*c7facc54SBill Taylor extern int hermon_rdma_debug; 519e39c5baSBill Taylor 529e39c5baSBill Taylor /* 539e39c5baSBill Taylor * Used by hermon_mr_keycalc() below to fill in the "unconstrained" portion 549e39c5baSBill Taylor * of Hermon memory keys (LKeys and RKeys) 559e39c5baSBill Taylor */ 569e39c5baSBill Taylor static uint_t hermon_memkey_cnt = 0x00; 57*c7facc54SBill Taylor #define HERMON_MEMKEY_SHIFT 24 58*c7facc54SBill Taylor 59*c7facc54SBill Taylor /* initial state of an MPT */ 60*c7facc54SBill Taylor #define HERMON_MPT_SW_OWNERSHIP 0xF /* memory regions */ 61*c7facc54SBill Taylor #define HERMON_MPT_FREE 0x3 /* allocate lkey */ 629e39c5baSBill Taylor 639e39c5baSBill Taylor static int hermon_mr_common_reg(hermon_state_t *state, hermon_pdhdl_t pd, 649e39c5baSBill Taylor hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op, 659e39c5baSBill Taylor hermon_mpt_rsrc_type_t mpt_type); 669e39c5baSBill Taylor static int hermon_mr_common_rereg(hermon_state_t *state, hermon_mrhdl_t mr, 679e39c5baSBill Taylor hermon_pdhdl_t pd, hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl_new, 689e39c5baSBill Taylor hermon_mr_options_t *op); 699e39c5baSBill Taylor static int hermon_mr_rereg_xlat_helper(hermon_state_t *state, hermon_mrhdl_t mr, 709e39c5baSBill Taylor hermon_bind_info_t *bind, hermon_mr_options_t *op, uint64_t *mtt_addr, 719e39c5baSBill Taylor uint_t sleep, uint_t *dereg_level); 729e39c5baSBill Taylor static uint64_t hermon_mr_nummtt_needed(hermon_state_t *state, 739e39c5baSBill Taylor hermon_bind_info_t *bind, uint_t *mtt_pgsize); 749e39c5baSBill Taylor static int hermon_mr_mem_bind(hermon_state_t *state, hermon_bind_info_t *bind, 759e39c5baSBill Taylor ddi_dma_handle_t dmahdl, uint_t sleep, uint_t is_buffer); 769e39c5baSBill Taylor static void hermon_mr_mem_unbind(hermon_state_t *state, 779e39c5baSBill Taylor hermon_bind_info_t *bind); 789e39c5baSBill Taylor static int hermon_mr_fast_mtt_write(hermon_state_t *state, hermon_rsrc_t *mtt, 799e39c5baSBill Taylor hermon_bind_info_t *bind, uint32_t mtt_pgsize_bits); 80*c7facc54SBill Taylor static int hermon_mr_fast_mtt_write_fmr(hermon_state_t *state, 81*c7facc54SBill Taylor hermon_rsrc_t *mtt, ibt_pmr_attr_t *mem_pattr, uint32_t mtt_pgsize_bits); 829e39c5baSBill Taylor static uint_t hermon_mtt_refcnt_inc(hermon_rsrc_t *rsrc); 839e39c5baSBill Taylor static uint_t hermon_mtt_refcnt_dec(hermon_rsrc_t *rsrc); 849e39c5baSBill Taylor 859e39c5baSBill Taylor 869e39c5baSBill Taylor /* 879e39c5baSBill Taylor * The Hermon umem_lockmemory() callback ops. When userland memory is 889e39c5baSBill Taylor * registered, these callback ops are specified. The hermon_umap_umemlock_cb() 899e39c5baSBill Taylor * callback will be called whenever the memory for the corresponding 909e39c5baSBill Taylor * ddi_umem_cookie_t is being freed. 919e39c5baSBill Taylor */ 929e39c5baSBill Taylor static struct umem_callback_ops hermon_umem_cbops = { 939e39c5baSBill Taylor UMEM_CALLBACK_VERSION, 949e39c5baSBill Taylor hermon_umap_umemlock_cb, 959e39c5baSBill Taylor }; 969e39c5baSBill Taylor 979e39c5baSBill Taylor 989e39c5baSBill Taylor 999e39c5baSBill Taylor /* 1009e39c5baSBill Taylor * hermon_mr_register() 1019e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1029e39c5baSBill Taylor */ 1039e39c5baSBill Taylor int 1049e39c5baSBill Taylor hermon_mr_register(hermon_state_t *state, hermon_pdhdl_t pd, 1059e39c5baSBill Taylor ibt_mr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op, 1069e39c5baSBill Taylor hermon_mpt_rsrc_type_t mpt_type) 1079e39c5baSBill Taylor { 1089e39c5baSBill Taylor hermon_bind_info_t bind; 1099e39c5baSBill Taylor int status; 1109e39c5baSBill Taylor 1119e39c5baSBill Taylor /* 1129e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority 1139e39c5baSBill Taylor * of the information that will be used to distinguish between an 1149e39c5baSBill Taylor * "addr" binding (as is the case here) and a "buf" binding (see 1159e39c5baSBill Taylor * below). The "bind" struct is later passed to hermon_mr_mem_bind() 1169e39c5baSBill Taylor * which does most of the "heavy lifting" for the Hermon memory 1179e39c5baSBill Taylor * registration routines. 1189e39c5baSBill Taylor */ 1199e39c5baSBill Taylor bind.bi_type = HERMON_BINDHDL_VADDR; 1209e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr; 1219e39c5baSBill Taylor bind.bi_len = mr_attr->mr_len; 1229e39c5baSBill Taylor bind.bi_as = mr_attr->mr_as; 1239e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags; 1249e39c5baSBill Taylor status = hermon_mr_common_reg(state, pd, &bind, mrhdl, op, 1259e39c5baSBill Taylor mpt_type); 1269e39c5baSBill Taylor return (status); 1279e39c5baSBill Taylor } 1289e39c5baSBill Taylor 1299e39c5baSBill Taylor 1309e39c5baSBill Taylor /* 1319e39c5baSBill Taylor * hermon_mr_register_buf() 1329e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1339e39c5baSBill Taylor */ 1349e39c5baSBill Taylor int 1359e39c5baSBill Taylor hermon_mr_register_buf(hermon_state_t *state, hermon_pdhdl_t pd, 1369e39c5baSBill Taylor ibt_smr_attr_t *mr_attr, struct buf *buf, hermon_mrhdl_t *mrhdl, 1379e39c5baSBill Taylor hermon_mr_options_t *op, hermon_mpt_rsrc_type_t mpt_type) 1389e39c5baSBill Taylor { 1399e39c5baSBill Taylor hermon_bind_info_t bind; 1409e39c5baSBill Taylor int status; 1419e39c5baSBill Taylor 1429e39c5baSBill Taylor /* 1439e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority 1449e39c5baSBill Taylor * of the information that will be used to distinguish between an 1459e39c5baSBill Taylor * "addr" binding (see above) and a "buf" binding (as is the case 1469e39c5baSBill Taylor * here). The "bind" struct is later passed to hermon_mr_mem_bind() 1479e39c5baSBill Taylor * which does most of the "heavy lifting" for the Hermon memory 1489e39c5baSBill Taylor * registration routines. Note: We have chosen to provide 1499e39c5baSBill Taylor * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is 1509e39c5baSBill Taylor * not set). It is not critical what value we choose here as it need 1519e39c5baSBill Taylor * only be unique for the given RKey (which will happen by default), 1529e39c5baSBill Taylor * so the choice here is somewhat arbitrary. 1539e39c5baSBill Taylor */ 1549e39c5baSBill Taylor bind.bi_type = HERMON_BINDHDL_BUF; 1559e39c5baSBill Taylor bind.bi_buf = buf; 1569e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_PHYS_IOVA) { 1579e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr; 1589e39c5baSBill Taylor } else { 1599e39c5baSBill Taylor bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr; 1609e39c5baSBill Taylor } 1619e39c5baSBill Taylor bind.bi_as = NULL; 1629e39c5baSBill Taylor bind.bi_len = (uint64_t)buf->b_bcount; 1639e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags; 1649e39c5baSBill Taylor status = hermon_mr_common_reg(state, pd, &bind, mrhdl, op, mpt_type); 1659e39c5baSBill Taylor return (status); 1669e39c5baSBill Taylor } 1679e39c5baSBill Taylor 1689e39c5baSBill Taylor 1699e39c5baSBill Taylor /* 1709e39c5baSBill Taylor * hermon_mr_register_shared() 1719e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 1729e39c5baSBill Taylor */ 1739e39c5baSBill Taylor int 1749e39c5baSBill Taylor hermon_mr_register_shared(hermon_state_t *state, hermon_mrhdl_t mrhdl, 1759e39c5baSBill Taylor hermon_pdhdl_t pd, ibt_smr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl_new) 1769e39c5baSBill Taylor { 1779e39c5baSBill Taylor hermon_rsrc_t *mpt, *mtt, *rsrc; 1789e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 1799e39c5baSBill Taylor hermon_hw_dmpt_t mpt_entry; 1809e39c5baSBill Taylor hermon_mrhdl_t mr; 1819e39c5baSBill Taylor hermon_bind_info_t *bind; 1829e39c5baSBill Taylor ddi_umem_cookie_t umem_cookie; 1839e39c5baSBill Taylor size_t umem_len; 1849e39c5baSBill Taylor caddr_t umem_addr; 1859e39c5baSBill Taylor uint64_t mtt_addr, pgsize_msk; 1869e39c5baSBill Taylor uint_t sleep, mr_is_umem; 1879e39c5baSBill Taylor int status, umem_flags; 1889e39c5baSBill Taylor 1899e39c5baSBill Taylor /* 1909e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 1919e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 1929e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 1939e39c5baSBill Taylor */ 1949e39c5baSBill Taylor sleep = (mr_attr->mr_flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP : 1959e39c5baSBill Taylor HERMON_SLEEP; 1969e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 1979e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 1989e39c5baSBill Taylor status = IBT_INVALID_PARAM; 1999e39c5baSBill Taylor goto mrshared_fail; 2009e39c5baSBill Taylor } 2019e39c5baSBill Taylor 2029e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */ 2039e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 2049e39c5baSBill Taylor 2059e39c5baSBill Taylor /* 2069e39c5baSBill Taylor * Allocate an MPT entry. This will be filled in with all the 2079e39c5baSBill Taylor * necessary parameters to define the shared memory region. 2089e39c5baSBill Taylor * Specifically, it will be made to reference the currently existing 2099e39c5baSBill Taylor * MTT entries and ownership of the MPT will be passed to the hardware 2109e39c5baSBill Taylor * in the last step below. If we fail here, we must undo the 2119e39c5baSBill Taylor * protection domain reference count. 2129e39c5baSBill Taylor */ 2139e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt); 2149e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2159e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 2169e39c5baSBill Taylor goto mrshared_fail1; 2179e39c5baSBill Taylor } 2189e39c5baSBill Taylor 2199e39c5baSBill Taylor /* 2209e39c5baSBill Taylor * Allocate the software structure for tracking the shared memory 2219e39c5baSBill Taylor * region (i.e. the Hermon Memory Region handle). If we fail here, we 2229e39c5baSBill Taylor * must undo the protection domain reference count and the previous 2239e39c5baSBill Taylor * resource allocation. 2249e39c5baSBill Taylor */ 2259e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc); 2269e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2279e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 2289e39c5baSBill Taylor goto mrshared_fail2; 2299e39c5baSBill Taylor } 2309e39c5baSBill Taylor mr = (hermon_mrhdl_t)rsrc->hr_addr; 2319e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) 2329e39c5baSBill Taylor 2339e39c5baSBill Taylor /* 2349e39c5baSBill Taylor * Setup and validate the memory region access flags. This means 2359e39c5baSBill Taylor * translating the IBTF's enable flags into the access flags that 2369e39c5baSBill Taylor * will be used in later operations. 2379e39c5baSBill Taylor */ 2389e39c5baSBill Taylor mr->mr_accflag = 0; 2399e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_WINDOW_BIND) 2409e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_WINDOW_BIND; 2419e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE) 2429e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_LOCAL_WRITE; 2439e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ) 2449e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_READ; 2459e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) 2469e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_WRITE; 2479e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) 2489e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC; 2499e39c5baSBill Taylor 2509e39c5baSBill Taylor /* 2519e39c5baSBill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed 2529e39c5baSBill Taylor * from a certain number of "constrained" bits (the least significant 2539e39c5baSBill Taylor * bits) and some number of "unconstrained" bits. The constrained 2549e39c5baSBill Taylor * bits must be set to the index of the entry in the MPT table, but 2559e39c5baSBill Taylor * the unconstrained bits can be set to any value we wish. Note: 2569e39c5baSBill Taylor * if no remote access is required, then the RKey value is not filled 2579e39c5baSBill Taylor * in. Otherwise both Rkey and LKey are given the same value. 2589e39c5baSBill Taylor */ 259*c7facc54SBill Taylor mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx); 2609e39c5baSBill Taylor 2619e39c5baSBill Taylor /* Grab the MR lock for the current memory region */ 2629e39c5baSBill Taylor mutex_enter(&mrhdl->mr_lock); 2639e39c5baSBill Taylor 2649e39c5baSBill Taylor /* 2659e39c5baSBill Taylor * Check here to see if the memory region has already been partially 2669e39c5baSBill Taylor * deregistered as a result of a hermon_umap_umemlock_cb() callback. 2679e39c5baSBill Taylor * If so, this is an error, return failure. 2689e39c5baSBill Taylor */ 2699e39c5baSBill Taylor if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) { 2709e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 2719e39c5baSBill Taylor status = IBT_MR_HDL_INVALID; 2729e39c5baSBill Taylor goto mrshared_fail3; 2739e39c5baSBill Taylor } 2749e39c5baSBill Taylor 2759e39c5baSBill Taylor /* 2769e39c5baSBill Taylor * Determine if the original memory was from userland and, if so, pin 2779e39c5baSBill Taylor * the pages (again) with umem_lockmemory(). This will guarantee a 2789e39c5baSBill Taylor * separate callback for each of this shared region's MR handles. 2799e39c5baSBill Taylor * If this is userland memory, then allocate an entry in the 2809e39c5baSBill Taylor * "userland resources database". This will later be added to 2819e39c5baSBill Taylor * the database (after all further memory registration operations are 2829e39c5baSBill Taylor * successful). If we fail here, we must undo all the above setup. 2839e39c5baSBill Taylor */ 2849e39c5baSBill Taylor mr_is_umem = mrhdl->mr_is_umem; 2859e39c5baSBill Taylor if (mr_is_umem) { 2869e39c5baSBill Taylor umem_len = ptob(btopr(mrhdl->mr_bindinfo.bi_len)); 2879e39c5baSBill Taylor umem_addr = (caddr_t)((uintptr_t)mrhdl->mr_bindinfo.bi_addr & 2889e39c5baSBill Taylor ~PAGEOFFSET); 2899e39c5baSBill Taylor umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ | 2909e39c5baSBill Taylor DDI_UMEMLOCK_LONGTERM); 2919e39c5baSBill Taylor status = umem_lockmemory(umem_addr, umem_len, umem_flags, 292d863b343SBill Taylor &umem_cookie, &hermon_umem_cbops, NULL); 2939e39c5baSBill Taylor if (status != 0) { 2949e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 2959e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 2969e39c5baSBill Taylor goto mrshared_fail3; 2979e39c5baSBill Taylor } 2989e39c5baSBill Taylor 2999e39c5baSBill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance, 3009e39c5baSBill Taylor (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC, 3019e39c5baSBill Taylor (uint64_t)(uintptr_t)rsrc); 3029e39c5baSBill Taylor if (umapdb == NULL) { 3039e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 3049e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 3059e39c5baSBill Taylor goto mrshared_fail4; 3069e39c5baSBill Taylor } 3079e39c5baSBill Taylor } 3089e39c5baSBill Taylor 3099e39c5baSBill Taylor /* 3109e39c5baSBill Taylor * Copy the MTT resource pointer (and additional parameters) from 3119e39c5baSBill Taylor * the original Hermon Memory Region handle. Note: this is normally 3129e39c5baSBill Taylor * where the hermon_mr_mem_bind() routine would be called, but because 3139e39c5baSBill Taylor * we already have bound and filled-in MTT entries it is simply a 3149e39c5baSBill Taylor * matter here of managing the MTT reference count and grabbing the 3159e39c5baSBill Taylor * address of the MTT table entries (for filling in the shared region's 3169e39c5baSBill Taylor * MPT entry). 3179e39c5baSBill Taylor */ 3189e39c5baSBill Taylor mr->mr_mttrsrcp = mrhdl->mr_mttrsrcp; 3199e39c5baSBill Taylor mr->mr_logmttpgsz = mrhdl->mr_logmttpgsz; 3209e39c5baSBill Taylor mr->mr_bindinfo = mrhdl->mr_bindinfo; 3219e39c5baSBill Taylor mr->mr_mttrefcntp = mrhdl->mr_mttrefcntp; 3229e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 3239e39c5baSBill Taylor bind = &mr->mr_bindinfo; 3249e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind)) 3259e39c5baSBill Taylor mtt = mr->mr_mttrsrcp; 3269e39c5baSBill Taylor 3279e39c5baSBill Taylor /* 3289e39c5baSBill Taylor * Increment the MTT reference count (to reflect the fact that 3299e39c5baSBill Taylor * the MTT is now shared) 3309e39c5baSBill Taylor */ 3319e39c5baSBill Taylor (void) hermon_mtt_refcnt_inc(mr->mr_mttrefcntp); 3329e39c5baSBill Taylor 3339e39c5baSBill Taylor /* 3349e39c5baSBill Taylor * Update the new "bind" virtual address. Do some extra work here 3359e39c5baSBill Taylor * to ensure proper alignment. That is, make sure that the page 3369e39c5baSBill Taylor * offset for the beginning of the old range is the same as the 3379e39c5baSBill Taylor * offset for this new mapping 3389e39c5baSBill Taylor */ 3399e39c5baSBill Taylor pgsize_msk = (((uint64_t)1 << mr->mr_logmttpgsz) - 1); 3409e39c5baSBill Taylor bind->bi_addr = ((mr_attr->mr_vaddr & ~pgsize_msk) | 3419e39c5baSBill Taylor (mr->mr_bindinfo.bi_addr & pgsize_msk)); 3429e39c5baSBill Taylor 3439e39c5baSBill Taylor /* 3449e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing 3459e39c5baSBill Taylor * ownership of the MPT entry to the Hermon hardware. We use all of 3469e39c5baSBill Taylor * the information collected/calculated above to fill in the 3479e39c5baSBill Taylor * requisite portions of the MPT. 3489e39c5baSBill Taylor */ 3499e39c5baSBill Taylor bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t)); 3509e39c5baSBill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0; 3519e39c5baSBill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0; 3529e39c5baSBill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0; 3539e39c5baSBill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0; 3549e39c5baSBill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0; 3559e39c5baSBill Taylor mpt_entry.lr = 1; 3569e39c5baSBill Taylor mpt_entry.reg_win = HERMON_MPT_IS_REGION; 3579e39c5baSBill Taylor mpt_entry.entity_sz = mr->mr_logmttpgsz; 3589e39c5baSBill Taylor mpt_entry.mem_key = mr->mr_lkey; 3599e39c5baSBill Taylor mpt_entry.pd = pd->pd_pdnum; 3609e39c5baSBill Taylor mpt_entry.start_addr = bind->bi_addr; 3619e39c5baSBill Taylor mpt_entry.reg_win_len = bind->bi_len; 3629e39c5baSBill Taylor mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT); 3639e39c5baSBill Taylor mpt_entry.mtt_addr_h = mtt_addr >> 32; 3649e39c5baSBill Taylor mpt_entry.mtt_addr_l = mtt_addr >> 3; 3659e39c5baSBill Taylor 3669e39c5baSBill Taylor /* 3679e39c5baSBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of 3689e39c5baSBill Taylor * the entry to the hardware. Note: in general, this operation 3699e39c5baSBill Taylor * shouldn't fail. But if it does, we have to undo everything we've 3709e39c5baSBill Taylor * done above before returning error. 3719e39c5baSBill Taylor */ 3729e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry, 3739e39c5baSBill Taylor sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep); 3749e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 3759e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n", 3769e39c5baSBill Taylor status); 3779e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 3789e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 3799e39c5baSBill Taylor } 3809e39c5baSBill Taylor status = ibc_get_ci_failure(0); 3819e39c5baSBill Taylor goto mrshared_fail5; 3829e39c5baSBill Taylor } 3839e39c5baSBill Taylor 3849e39c5baSBill Taylor /* 3859e39c5baSBill Taylor * Fill in the rest of the Hermon Memory Region handle. Having 3869e39c5baSBill Taylor * successfully transferred ownership of the MPT, we can update the 3879e39c5baSBill Taylor * following fields for use in further operations on the MR. 3889e39c5baSBill Taylor */ 3899e39c5baSBill Taylor mr->mr_mptrsrcp = mpt; 3909e39c5baSBill Taylor mr->mr_mttrsrcp = mtt; 3919e39c5baSBill Taylor mr->mr_mpt_type = HERMON_MPT_DMPT; 3929e39c5baSBill Taylor mr->mr_pdhdl = pd; 3939e39c5baSBill Taylor mr->mr_rsrcp = rsrc; 3949e39c5baSBill Taylor mr->mr_is_umem = mr_is_umem; 3959e39c5baSBill Taylor mr->mr_is_fmr = 0; 3969e39c5baSBill Taylor mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL; 3979e39c5baSBill Taylor mr->mr_umem_cbfunc = NULL; 3989e39c5baSBill Taylor mr->mr_umem_cbarg1 = NULL; 3999e39c5baSBill Taylor mr->mr_umem_cbarg2 = NULL; 4009e39c5baSBill Taylor mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey); 4019e39c5baSBill Taylor mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey); 4029e39c5baSBill Taylor 4039e39c5baSBill Taylor /* 4049e39c5baSBill Taylor * If this is userland memory, then we need to insert the previously 4059e39c5baSBill Taylor * allocated entry into the "userland resources database". This will 4069e39c5baSBill Taylor * allow for later coordination between the hermon_umap_umemlock_cb() 4079e39c5baSBill Taylor * callback and hermon_mr_deregister(). 4089e39c5baSBill Taylor */ 4099e39c5baSBill Taylor if (mr_is_umem) { 4109e39c5baSBill Taylor hermon_umap_db_add(umapdb); 4119e39c5baSBill Taylor } 4129e39c5baSBill Taylor 4139e39c5baSBill Taylor *mrhdl_new = mr; 4149e39c5baSBill Taylor 4159e39c5baSBill Taylor return (DDI_SUCCESS); 4169e39c5baSBill Taylor 4179e39c5baSBill Taylor /* 4189e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 4199e39c5baSBill Taylor */ 4209e39c5baSBill Taylor mrshared_fail5: 4219e39c5baSBill Taylor (void) hermon_mtt_refcnt_dec(mr->mr_mttrefcntp); 4229e39c5baSBill Taylor if (mr_is_umem) { 4239e39c5baSBill Taylor hermon_umap_db_free(umapdb); 4249e39c5baSBill Taylor } 4259e39c5baSBill Taylor mrshared_fail4: 4269e39c5baSBill Taylor if (mr_is_umem) { 4279e39c5baSBill Taylor ddi_umem_unlock(umem_cookie); 4289e39c5baSBill Taylor } 4299e39c5baSBill Taylor mrshared_fail3: 4309e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 4319e39c5baSBill Taylor mrshared_fail2: 4329e39c5baSBill Taylor hermon_rsrc_free(state, &mpt); 4339e39c5baSBill Taylor mrshared_fail1: 4349e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 4359e39c5baSBill Taylor mrshared_fail: 4369e39c5baSBill Taylor return (status); 4379e39c5baSBill Taylor } 4389e39c5baSBill Taylor 4399e39c5baSBill Taylor /* 4409e39c5baSBill Taylor * hermon_mr_alloc_fmr() 4419e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 4429e39c5baSBill Taylor */ 4439e39c5baSBill Taylor int 4449e39c5baSBill Taylor hermon_mr_alloc_fmr(hermon_state_t *state, hermon_pdhdl_t pd, 4459e39c5baSBill Taylor hermon_fmrhdl_t fmr_pool, hermon_mrhdl_t *mrhdl) 4469e39c5baSBill Taylor { 4479e39c5baSBill Taylor hermon_rsrc_t *mpt, *mtt, *rsrc; 448*c7facc54SBill Taylor hermon_hw_dmpt_t mpt_entry; 4499e39c5baSBill Taylor hermon_mrhdl_t mr; 4509e39c5baSBill Taylor hermon_bind_info_t bind; 4519e39c5baSBill Taylor uint64_t mtt_addr; 4529e39c5baSBill Taylor uint64_t nummtt; 4539e39c5baSBill Taylor uint_t sleep, mtt_pgsize_bits; 4549e39c5baSBill Taylor int status; 455*c7facc54SBill Taylor offset_t i; 456*c7facc54SBill Taylor hermon_icm_table_t *icm_table; 457*c7facc54SBill Taylor hermon_dma_info_t *dma_info; 458*c7facc54SBill Taylor uint32_t index1, index2, rindx; 4599e39c5baSBill Taylor 4609e39c5baSBill Taylor /* 4619e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 4629e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 4639e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 4649e39c5baSBill Taylor */ 4659e39c5baSBill Taylor sleep = (fmr_pool->fmr_flags & IBT_MR_SLEEP) ? HERMON_SLEEP : 4669e39c5baSBill Taylor HERMON_NOSLEEP; 4679e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 4689e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 4699e39c5baSBill Taylor return (IBT_INVALID_PARAM); 4709e39c5baSBill Taylor } 4719e39c5baSBill Taylor 4729e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */ 4739e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 4749e39c5baSBill Taylor 4759e39c5baSBill Taylor /* 4769e39c5baSBill Taylor * Allocate an MPT entry. This will be filled in with all the 4779e39c5baSBill Taylor * necessary parameters to define the FMR. Specifically, it will be 4789e39c5baSBill Taylor * made to reference the currently existing MTT entries and ownership 4799e39c5baSBill Taylor * of the MPT will be passed to the hardware in the last step below. 4809e39c5baSBill Taylor * If we fail here, we must undo the protection domain reference count. 4819e39c5baSBill Taylor */ 4829e39c5baSBill Taylor 4839e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt); 4849e39c5baSBill Taylor if (status != DDI_SUCCESS) { 4859e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 4869e39c5baSBill Taylor goto fmralloc_fail1; 4879e39c5baSBill Taylor } 4889e39c5baSBill Taylor 4899e39c5baSBill Taylor /* 4909e39c5baSBill Taylor * Allocate the software structure for tracking the fmr memory 4919e39c5baSBill Taylor * region (i.e. the Hermon Memory Region handle). If we fail here, we 4929e39c5baSBill Taylor * must undo the protection domain reference count and the previous 4939e39c5baSBill Taylor * resource allocation. 4949e39c5baSBill Taylor */ 4959e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc); 4969e39c5baSBill Taylor if (status != DDI_SUCCESS) { 4979e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 4989e39c5baSBill Taylor goto fmralloc_fail2; 4999e39c5baSBill Taylor } 5009e39c5baSBill Taylor mr = (hermon_mrhdl_t)rsrc->hr_addr; 5019e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) 5029e39c5baSBill Taylor 5039e39c5baSBill Taylor /* 5049e39c5baSBill Taylor * Setup and validate the memory region access flags. This means 5059e39c5baSBill Taylor * translating the IBTF's enable flags into the access flags that 5069e39c5baSBill Taylor * will be used in later operations. 5079e39c5baSBill Taylor */ 5089e39c5baSBill Taylor mr->mr_accflag = 0; 5099e39c5baSBill Taylor if (fmr_pool->fmr_flags & IBT_MR_ENABLE_LOCAL_WRITE) 5109e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_LOCAL_WRITE; 5119e39c5baSBill Taylor if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_READ) 5129e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_READ; 5139e39c5baSBill Taylor if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) 5149e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_WRITE; 5159e39c5baSBill Taylor if (fmr_pool->fmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) 5169e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC; 5179e39c5baSBill Taylor 5189e39c5baSBill Taylor /* 5199e39c5baSBill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed 5209e39c5baSBill Taylor * from a certain number of "constrained" bits (the least significant 5219e39c5baSBill Taylor * bits) and some number of "unconstrained" bits. The constrained 5229e39c5baSBill Taylor * bits must be set to the index of the entry in the MPT table, but 5239e39c5baSBill Taylor * the unconstrained bits can be set to any value we wish. Note: 5249e39c5baSBill Taylor * if no remote access is required, then the RKey value is not filled 5259e39c5baSBill Taylor * in. Otherwise both Rkey and LKey are given the same value. 5269e39c5baSBill Taylor */ 527*c7facc54SBill Taylor mr->mr_fmr_key = 1; /* ready for the next reload */ 528*c7facc54SBill Taylor mr->mr_rkey = mr->mr_lkey = mpt->hr_indx; 5299e39c5baSBill Taylor 5309e39c5baSBill Taylor /* 5319e39c5baSBill Taylor * Determine number of pages spanned. This routine uses the 5329e39c5baSBill Taylor * information in the "bind" struct to determine the required 5339e39c5baSBill Taylor * number of MTT entries needed (and returns the suggested page size - 5349e39c5baSBill Taylor * as a "power-of-2" - for each MTT entry). 5359e39c5baSBill Taylor */ 5369e39c5baSBill Taylor /* Assume address will be page aligned later */ 5379e39c5baSBill Taylor bind.bi_addr = 0; 5389e39c5baSBill Taylor /* Calculate size based on given max pages */ 5399e39c5baSBill Taylor bind.bi_len = fmr_pool->fmr_max_pages << PAGESHIFT; 5409e39c5baSBill Taylor nummtt = hermon_mr_nummtt_needed(state, &bind, &mtt_pgsize_bits); 5419e39c5baSBill Taylor 5429e39c5baSBill Taylor /* 5439e39c5baSBill Taylor * Allocate the MTT entries. Use the calculations performed above to 5449e39c5baSBill Taylor * allocate the required number of MTT entries. If we fail here, we 5459e39c5baSBill Taylor * must not only undo all the previous resource allocation (and PD 5469e39c5baSBill Taylor * reference count), but we must also unbind the memory. 5479e39c5baSBill Taylor */ 5489e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt, sleep, &mtt); 5499e39c5baSBill Taylor if (status != DDI_SUCCESS) { 5509e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 5519e39c5baSBill Taylor goto fmralloc_fail3; 5529e39c5baSBill Taylor } 5539e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits; 5549e39c5baSBill Taylor 5559e39c5baSBill Taylor /* 5569e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing 5579e39c5baSBill Taylor * ownership of the MPT entry to the Hermon hardware. We use all of 5589e39c5baSBill Taylor * the information collected/calculated above to fill in the 5599e39c5baSBill Taylor * requisite portions of the MPT. 5609e39c5baSBill Taylor */ 5619e39c5baSBill Taylor bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t)); 5629e39c5baSBill Taylor mpt_entry.en_bind = 0; 5639e39c5baSBill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0; 5649e39c5baSBill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0; 5659e39c5baSBill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0; 5669e39c5baSBill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0; 5679e39c5baSBill Taylor mpt_entry.lr = 1; 5689e39c5baSBill Taylor mpt_entry.reg_win = HERMON_MPT_IS_REGION; 5699e39c5baSBill Taylor mpt_entry.pd = pd->pd_pdnum; 5709e39c5baSBill Taylor 5719e39c5baSBill Taylor mpt_entry.entity_sz = mr->mr_logmttpgsz; 5729e39c5baSBill Taylor mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT); 573*c7facc54SBill Taylor mpt_entry.fast_reg_en = 1; 574*c7facc54SBill Taylor mpt_entry.mtt_size = (uint_t)nummtt; 5759e39c5baSBill Taylor mpt_entry.mtt_addr_h = mtt_addr >> 32; 5769e39c5baSBill Taylor mpt_entry.mtt_addr_l = mtt_addr >> 3; 5779e39c5baSBill Taylor mpt_entry.mem_key = mr->mr_lkey; 5789e39c5baSBill Taylor 5799e39c5baSBill Taylor /* 5809e39c5baSBill Taylor * FMR sets these to 0 for now. Later during actual fmr registration 5819e39c5baSBill Taylor * these values are filled in. 5829e39c5baSBill Taylor */ 5839e39c5baSBill Taylor mpt_entry.start_addr = 0; 5849e39c5baSBill Taylor mpt_entry.reg_win_len = 0; 5859e39c5baSBill Taylor 5869e39c5baSBill Taylor /* 5879e39c5baSBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of 5889e39c5baSBill Taylor * the entry to the hardware. Note: in general, this operation 5899e39c5baSBill Taylor * shouldn't fail. But if it does, we have to undo everything we've 5909e39c5baSBill Taylor * done above before returning error. 5919e39c5baSBill Taylor */ 5929e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry, 5939e39c5baSBill Taylor sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep); 5949e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 5959e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n", 5969e39c5baSBill Taylor status); 5979e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 5989e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 5999e39c5baSBill Taylor } 6009e39c5baSBill Taylor status = ibc_get_ci_failure(0); 6019e39c5baSBill Taylor goto fmralloc_fail4; 6029e39c5baSBill Taylor } 6039e39c5baSBill Taylor 6049e39c5baSBill Taylor /* 6059e39c5baSBill Taylor * Fill in the rest of the Hermon Memory Region handle. Having 6069e39c5baSBill Taylor * successfully transferred ownership of the MPT, we can update the 6079e39c5baSBill Taylor * following fields for use in further operations on the MR. Also, set 6089e39c5baSBill Taylor * that this is an FMR region. 6099e39c5baSBill Taylor */ 6109e39c5baSBill Taylor mr->mr_mptrsrcp = mpt; 6119e39c5baSBill Taylor mr->mr_mttrsrcp = mtt; 612*c7facc54SBill Taylor 6139e39c5baSBill Taylor mr->mr_mpt_type = HERMON_MPT_DMPT; 6149e39c5baSBill Taylor mr->mr_pdhdl = pd; 6159e39c5baSBill Taylor mr->mr_rsrcp = rsrc; 6169e39c5baSBill Taylor mr->mr_is_fmr = 1; 6179e39c5baSBill Taylor mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey); 6189e39c5baSBill Taylor mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey); 619*c7facc54SBill Taylor mr->mr_mttaddr = mtt_addr; 6209e39c5baSBill Taylor (void) memcpy(&mr->mr_bindinfo, &bind, sizeof (hermon_bind_info_t)); 6219e39c5baSBill Taylor 622*c7facc54SBill Taylor /* initialize hr_addr for use during register/deregister/invalidate */ 623*c7facc54SBill Taylor icm_table = &state->hs_icm[HERMON_DMPT]; 624*c7facc54SBill Taylor rindx = mpt->hr_indx; 625*c7facc54SBill Taylor hermon_index(index1, index2, rindx, icm_table, i); 626*c7facc54SBill Taylor dma_info = icm_table->icm_dma[index1] + index2; 627*c7facc54SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mpt)) 628*c7facc54SBill Taylor mpt->hr_addr = (void *)((uintptr_t)(dma_info->vaddr + i * mpt->hr_len)); 629*c7facc54SBill Taylor 6309e39c5baSBill Taylor *mrhdl = mr; 6319e39c5baSBill Taylor 6329e39c5baSBill Taylor return (DDI_SUCCESS); 6339e39c5baSBill Taylor 6349e39c5baSBill Taylor /* 6359e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 6369e39c5baSBill Taylor */ 6379e39c5baSBill Taylor fmralloc_fail4: 6389e39c5baSBill Taylor kmem_free(mtt, sizeof (hermon_rsrc_t) * nummtt); 6399e39c5baSBill Taylor fmralloc_fail3: 6409e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 6419e39c5baSBill Taylor fmralloc_fail2: 6429e39c5baSBill Taylor hermon_rsrc_free(state, &mpt); 6439e39c5baSBill Taylor fmralloc_fail1: 6449e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 6459e39c5baSBill Taylor fmralloc_fail: 6469e39c5baSBill Taylor return (status); 6479e39c5baSBill Taylor } 6489e39c5baSBill Taylor 649*c7facc54SBill Taylor 6509e39c5baSBill Taylor /* 6519e39c5baSBill Taylor * hermon_mr_register_physical_fmr() 6529e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 6539e39c5baSBill Taylor */ 6549e39c5baSBill Taylor /*ARGSUSED*/ 6559e39c5baSBill Taylor int 6569e39c5baSBill Taylor hermon_mr_register_physical_fmr(hermon_state_t *state, 6579e39c5baSBill Taylor ibt_pmr_attr_t *mem_pattr_p, hermon_mrhdl_t mr, ibt_pmr_desc_t *mem_desc_p) 6589e39c5baSBill Taylor { 6599e39c5baSBill Taylor hermon_rsrc_t *mpt; 6609e39c5baSBill Taylor uint64_t *mpt_table; 6619e39c5baSBill Taylor int status; 662*c7facc54SBill Taylor uint32_t key; 6639e39c5baSBill Taylor 6649e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 6659e39c5baSBill Taylor mpt = mr->mr_mptrsrcp; 6669e39c5baSBill Taylor mpt_table = (uint64_t *)mpt->hr_addr; 6679e39c5baSBill Taylor 6689e39c5baSBill Taylor /* Write MPT status to SW bit */ 669*c7facc54SBill Taylor *(uint8_t *)mpt_table = 0xF0; 670*c7facc54SBill Taylor 671*c7facc54SBill Taylor membar_producer(); 6729e39c5baSBill Taylor 6739e39c5baSBill Taylor /* 6749e39c5baSBill Taylor * Write the mapped addresses into the MTT entries. FMR needs to do 6759e39c5baSBill Taylor * this a little differently, so we call the fmr specific fast mtt 6769e39c5baSBill Taylor * write here. 6779e39c5baSBill Taylor */ 678*c7facc54SBill Taylor status = hermon_mr_fast_mtt_write_fmr(state, mr->mr_mttrsrcp, 679*c7facc54SBill Taylor mem_pattr_p, mr->mr_logmttpgsz); 6809e39c5baSBill Taylor if (status != DDI_SUCCESS) { 6819e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 6829e39c5baSBill Taylor status = ibc_get_ci_failure(0); 6839e39c5baSBill Taylor goto fmr_reg_fail1; 6849e39c5baSBill Taylor } 6859e39c5baSBill Taylor 6869e39c5baSBill Taylor /* 6879e39c5baSBill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed 6889e39c5baSBill Taylor * from a certain number of "constrained" bits (the least significant 6899e39c5baSBill Taylor * bits) and some number of "unconstrained" bits. The constrained 6909e39c5baSBill Taylor * bits must be set to the index of the entry in the MPT table, but 6919e39c5baSBill Taylor * the unconstrained bits can be set to any value we wish. Note: 6929e39c5baSBill Taylor * if no remote access is required, then the RKey value is not filled 6939e39c5baSBill Taylor * in. Otherwise both Rkey and LKey are given the same value. 6949e39c5baSBill Taylor */ 695*c7facc54SBill Taylor key = mpt->hr_indx | (mr->mr_fmr_key++ << HERMON_MEMKEY_SHIFT); 696*c7facc54SBill Taylor mr->mr_lkey = mr->mr_rkey = hermon_mr_key_swap(key); 6979e39c5baSBill Taylor 6989e39c5baSBill Taylor /* write mem key value */ 699*c7facc54SBill Taylor *(uint32_t *)&mpt_table[1] = htonl(key); 7009e39c5baSBill Taylor 7019e39c5baSBill Taylor /* write length value */ 702*c7facc54SBill Taylor mpt_table[3] = htonll(mem_pattr_p->pmr_len); 7039e39c5baSBill Taylor 7049e39c5baSBill Taylor /* write start addr value */ 705*c7facc54SBill Taylor mpt_table[2] = htonll(mem_pattr_p->pmr_iova); 7069e39c5baSBill Taylor 7079e39c5baSBill Taylor /* write lkey value */ 708*c7facc54SBill Taylor *(uint32_t *)&mpt_table[4] = htonl(key); 709*c7facc54SBill Taylor 710*c7facc54SBill Taylor membar_producer(); 7119e39c5baSBill Taylor 7129e39c5baSBill Taylor /* Write MPT status to HW bit */ 713*c7facc54SBill Taylor *(uint8_t *)mpt_table = 0x00; 7149e39c5baSBill Taylor 7159e39c5baSBill Taylor /* Fill in return parameters */ 7169e39c5baSBill Taylor mem_desc_p->pmd_lkey = mr->mr_lkey; 7179e39c5baSBill Taylor mem_desc_p->pmd_rkey = mr->mr_rkey; 7189e39c5baSBill Taylor mem_desc_p->pmd_iova = mem_pattr_p->pmr_iova; 7199e39c5baSBill Taylor mem_desc_p->pmd_phys_buf_list_sz = mem_pattr_p->pmr_len; 7209e39c5baSBill Taylor 7219e39c5baSBill Taylor /* Fill in MR bindinfo struct for later sync or query operations */ 7229e39c5baSBill Taylor mr->mr_bindinfo.bi_addr = mem_pattr_p->pmr_iova; 7239e39c5baSBill Taylor mr->mr_bindinfo.bi_flags = mem_pattr_p->pmr_flags & IBT_MR_NONCOHERENT; 7249e39c5baSBill Taylor 7259e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 7269e39c5baSBill Taylor 7279e39c5baSBill Taylor return (DDI_SUCCESS); 7289e39c5baSBill Taylor 7299e39c5baSBill Taylor fmr_reg_fail1: 7309e39c5baSBill Taylor /* 7319e39c5baSBill Taylor * Note, we fail here, and purposely leave the memory ownership in 7329e39c5baSBill Taylor * software. The memory tables may be corrupt, so we leave the region 7339e39c5baSBill Taylor * unregistered. 7349e39c5baSBill Taylor */ 735*c7facc54SBill Taylor return (status); 7369e39c5baSBill Taylor } 7379e39c5baSBill Taylor 7389e39c5baSBill Taylor 7399e39c5baSBill Taylor /* 7409e39c5baSBill Taylor * hermon_mr_deregister() 7419e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 7429e39c5baSBill Taylor */ 7439e39c5baSBill Taylor /* ARGSUSED */ 7449e39c5baSBill Taylor int 7459e39c5baSBill Taylor hermon_mr_deregister(hermon_state_t *state, hermon_mrhdl_t *mrhdl, uint_t level, 7469e39c5baSBill Taylor uint_t sleep) 7479e39c5baSBill Taylor { 7489e39c5baSBill Taylor hermon_rsrc_t *mpt, *mtt, *rsrc, *mtt_refcnt; 7499e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 7509e39c5baSBill Taylor hermon_pdhdl_t pd; 7519e39c5baSBill Taylor hermon_mrhdl_t mr; 7529e39c5baSBill Taylor hermon_bind_info_t *bind; 7539e39c5baSBill Taylor uint64_t value; 7549e39c5baSBill Taylor int status; 7559e39c5baSBill Taylor uint_t shared_mtt; 7569e39c5baSBill Taylor 7579e39c5baSBill Taylor /* 7589e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 7599e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 7609e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 7619e39c5baSBill Taylor */ 7629e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 7639e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 7649e39c5baSBill Taylor status = IBT_INVALID_PARAM; 7659e39c5baSBill Taylor return (status); 7669e39c5baSBill Taylor } 7679e39c5baSBill Taylor 7689e39c5baSBill Taylor /* 7699e39c5baSBill Taylor * Pull all the necessary information from the Hermon Memory Region 7709e39c5baSBill Taylor * handle. This is necessary here because the resource for the 7719e39c5baSBill Taylor * MR handle is going to be freed up as part of the this 7729e39c5baSBill Taylor * deregistration 7739e39c5baSBill Taylor */ 7749e39c5baSBill Taylor mr = *mrhdl; 7759e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 7769e39c5baSBill Taylor mpt = mr->mr_mptrsrcp; 7779e39c5baSBill Taylor mtt = mr->mr_mttrsrcp; 7789e39c5baSBill Taylor mtt_refcnt = mr->mr_mttrefcntp; 7799e39c5baSBill Taylor rsrc = mr->mr_rsrcp; 7809e39c5baSBill Taylor pd = mr->mr_pdhdl; 7819e39c5baSBill Taylor bind = &mr->mr_bindinfo; 7829e39c5baSBill Taylor 7839e39c5baSBill Taylor /* 7849e39c5baSBill Taylor * Check here if the memory region is really an FMR. If so, this is a 7859e39c5baSBill Taylor * bad thing and we shouldn't be here. Return failure. 7869e39c5baSBill Taylor */ 7879e39c5baSBill Taylor if (mr->mr_is_fmr) { 7889e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 7899e39c5baSBill Taylor return (IBT_INVALID_PARAM); 7909e39c5baSBill Taylor } 7919e39c5baSBill Taylor 7929e39c5baSBill Taylor /* 7939e39c5baSBill Taylor * Check here to see if the memory region has already been partially 7949e39c5baSBill Taylor * deregistered as a result of the hermon_umap_umemlock_cb() callback. 7959e39c5baSBill Taylor * If so, then jump to the end and free the remaining resources. 7969e39c5baSBill Taylor */ 7979e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) { 7989e39c5baSBill Taylor goto mrdereg_finish_cleanup; 7999e39c5baSBill Taylor } 800*c7facc54SBill Taylor if (hermon_rdma_debug & 0x4) 801*c7facc54SBill Taylor IBTF_DPRINTF_L2("mr", "dereg: mr %p key %x", 802*c7facc54SBill Taylor mr, mr->mr_rkey); 8039e39c5baSBill Taylor 8049e39c5baSBill Taylor /* 8059e39c5baSBill Taylor * We must drop the "mr_lock" here to ensure that both SLEEP and 8069e39c5baSBill Taylor * NOSLEEP calls into the firmware work as expected. Also, if two 8079e39c5baSBill Taylor * threads are attemping to access this MR (via de-register, 8089e39c5baSBill Taylor * re-register, or otherwise), then we allow the firmware to enforce 8099e39c5baSBill Taylor * the checking, that only one deregister is valid. 8109e39c5baSBill Taylor */ 8119e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 8129e39c5baSBill Taylor 8139e39c5baSBill Taylor /* 8149e39c5baSBill Taylor * Reclaim MPT entry from hardware (if necessary). Since the 8159e39c5baSBill Taylor * hermon_mr_deregister() routine is used in the memory region 8169e39c5baSBill Taylor * reregistration process as well, it is possible that we will 8179e39c5baSBill Taylor * not always wish to reclaim ownership of the MPT. Check the 8189e39c5baSBill Taylor * "level" arg and, if necessary, attempt to reclaim it. If 8199e39c5baSBill Taylor * the ownership transfer fails for any reason, we check to see 8209e39c5baSBill Taylor * what command status was returned from the hardware. The only 8219e39c5baSBill Taylor * "expected" error status is the one that indicates an attempt to 8229e39c5baSBill Taylor * deregister a memory region that has memory windows bound to it 8239e39c5baSBill Taylor */ 8249e39c5baSBill Taylor if (level >= HERMON_MR_DEREG_ALL) { 8259e39c5baSBill Taylor if (mr->mr_mpt_type >= HERMON_MPT_DMPT) { 8269e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT, 8279e39c5baSBill Taylor NULL, 0, mpt->hr_indx, sleep); 8289e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 8299e39c5baSBill Taylor if (status == HERMON_CMD_REG_BOUND) { 8309e39c5baSBill Taylor return (IBT_MR_IN_USE); 8319e39c5baSBill Taylor } else { 8329e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: HW2SW_MPT " 8339e39c5baSBill Taylor "command failed: %08x\n", status); 8349e39c5baSBill Taylor if (status == 8359e39c5baSBill Taylor HERMON_CMD_INVALID_STATUS) { 8369e39c5baSBill Taylor hermon_fm_ereport(state, 8379e39c5baSBill Taylor HCA_SYS_ERR, 8389e39c5baSBill Taylor DDI_SERVICE_LOST); 8399e39c5baSBill Taylor } 8409e39c5baSBill Taylor return (IBT_INVALID_PARAM); 8419e39c5baSBill Taylor } 8429e39c5baSBill Taylor } 8439e39c5baSBill Taylor } 8449e39c5baSBill Taylor } 8459e39c5baSBill Taylor 8469e39c5baSBill Taylor /* 8479e39c5baSBill Taylor * Re-grab the mr_lock here. Since further access to the protected 8489e39c5baSBill Taylor * 'mr' structure is needed, and we would have returned previously for 8499e39c5baSBill Taylor * the multiple deregistration case, we can safely grab the lock here. 8509e39c5baSBill Taylor */ 8519e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 8529e39c5baSBill Taylor 8539e39c5baSBill Taylor /* 8549e39c5baSBill Taylor * If the memory had come from userland, then we do a lookup in the 8559e39c5baSBill Taylor * "userland resources database". On success, we free the entry, call 8569e39c5baSBill Taylor * ddi_umem_unlock(), and continue the cleanup. On failure (which is 8579e39c5baSBill Taylor * an indication that the umem_lockmemory() callback has called 8589e39c5baSBill Taylor * hermon_mr_deregister()), we call ddi_umem_unlock() and invalidate 8599e39c5baSBill Taylor * the "mr_umemcookie" field in the MR handle (this will be used 8609e39c5baSBill Taylor * later to detect that only partial cleaup still remains to be done 8619e39c5baSBill Taylor * on the MR handle). 8629e39c5baSBill Taylor */ 8639e39c5baSBill Taylor if (mr->mr_is_umem) { 8649e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, 8659e39c5baSBill Taylor (uint64_t)(uintptr_t)mr->mr_umemcookie, 8669e39c5baSBill Taylor MLNX_UMAP_MRMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE, 8679e39c5baSBill Taylor &umapdb); 8689e39c5baSBill Taylor if (status == DDI_SUCCESS) { 8699e39c5baSBill Taylor hermon_umap_db_free(umapdb); 8709e39c5baSBill Taylor ddi_umem_unlock(mr->mr_umemcookie); 8719e39c5baSBill Taylor } else { 8729e39c5baSBill Taylor ddi_umem_unlock(mr->mr_umemcookie); 8739e39c5baSBill Taylor mr->mr_umemcookie = NULL; 8749e39c5baSBill Taylor } 8759e39c5baSBill Taylor } 8769e39c5baSBill Taylor 8779e39c5baSBill Taylor /* 8789e39c5baSBill Taylor * Decrement the MTT reference count. Since the MTT resource 8799e39c5baSBill Taylor * may be shared between multiple memory regions (as a result 8809e39c5baSBill Taylor * of a "RegisterSharedMR" verb) it is important that we not 8819e39c5baSBill Taylor * free up or unbind resources prematurely. If it's not shared (as 8829e39c5baSBill Taylor * indicated by the return status), then free the resource. 8839e39c5baSBill Taylor */ 8849e39c5baSBill Taylor shared_mtt = hermon_mtt_refcnt_dec(mtt_refcnt); 8859e39c5baSBill Taylor if (!shared_mtt) { 8869e39c5baSBill Taylor hermon_rsrc_free(state, &mtt_refcnt); 8879e39c5baSBill Taylor } 8889e39c5baSBill Taylor 8899e39c5baSBill Taylor /* 8909e39c5baSBill Taylor * Free up the MTT entries and unbind the memory. Here, as above, we 8919e39c5baSBill Taylor * attempt to free these resources only if it is appropriate to do so. 8929e39c5baSBill Taylor */ 8939e39c5baSBill Taylor if (!shared_mtt) { 8949e39c5baSBill Taylor if (level >= HERMON_MR_DEREG_NO_HW2SW_MPT) { 8959e39c5baSBill Taylor hermon_mr_mem_unbind(state, bind); 8969e39c5baSBill Taylor } 8979e39c5baSBill Taylor hermon_rsrc_free(state, &mtt); 8989e39c5baSBill Taylor } 8999e39c5baSBill Taylor 9009e39c5baSBill Taylor /* 9019e39c5baSBill Taylor * If the MR handle has been invalidated, then drop the 9029e39c5baSBill Taylor * lock and return success. Note: This only happens because 9039e39c5baSBill Taylor * the umem_lockmemory() callback has been triggered. The 9049e39c5baSBill Taylor * cleanup here is partial, and further cleanup (in a 9059e39c5baSBill Taylor * subsequent hermon_mr_deregister() call) will be necessary. 9069e39c5baSBill Taylor */ 9079e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) { 9089e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 9099e39c5baSBill Taylor return (DDI_SUCCESS); 9109e39c5baSBill Taylor } 9119e39c5baSBill Taylor 9129e39c5baSBill Taylor mrdereg_finish_cleanup: 9139e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 9149e39c5baSBill Taylor 9159e39c5baSBill Taylor /* Free the Hermon Memory Region handle */ 9169e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 9179e39c5baSBill Taylor 9189e39c5baSBill Taylor /* Free up the MPT entry resource */ 9199e39c5baSBill Taylor if (mpt != NULL) 9209e39c5baSBill Taylor hermon_rsrc_free(state, &mpt); 9219e39c5baSBill Taylor 9229e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */ 9239e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 9249e39c5baSBill Taylor 9259e39c5baSBill Taylor /* Set the mrhdl pointer to NULL and return success */ 9269e39c5baSBill Taylor *mrhdl = NULL; 9279e39c5baSBill Taylor 9289e39c5baSBill Taylor return (DDI_SUCCESS); 9299e39c5baSBill Taylor } 9309e39c5baSBill Taylor 9319e39c5baSBill Taylor /* 9329e39c5baSBill Taylor * hermon_mr_dealloc_fmr() 9339e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 9349e39c5baSBill Taylor */ 9359e39c5baSBill Taylor /* ARGSUSED */ 9369e39c5baSBill Taylor int 9379e39c5baSBill Taylor hermon_mr_dealloc_fmr(hermon_state_t *state, hermon_mrhdl_t *mrhdl) 9389e39c5baSBill Taylor { 9399e39c5baSBill Taylor hermon_rsrc_t *mpt, *mtt, *rsrc; 9409e39c5baSBill Taylor hermon_pdhdl_t pd; 9419e39c5baSBill Taylor hermon_mrhdl_t mr; 9429e39c5baSBill Taylor 9439e39c5baSBill Taylor /* 9449e39c5baSBill Taylor * Pull all the necessary information from the Hermon Memory Region 9459e39c5baSBill Taylor * handle. This is necessary here because the resource for the 9469e39c5baSBill Taylor * MR handle is going to be freed up as part of the this 9479e39c5baSBill Taylor * deregistration 9489e39c5baSBill Taylor */ 9499e39c5baSBill Taylor mr = *mrhdl; 9509e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 9519e39c5baSBill Taylor mpt = mr->mr_mptrsrcp; 9529e39c5baSBill Taylor mtt = mr->mr_mttrsrcp; 9539e39c5baSBill Taylor rsrc = mr->mr_rsrcp; 9549e39c5baSBill Taylor pd = mr->mr_pdhdl; 9559e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 9569e39c5baSBill Taylor 9579e39c5baSBill Taylor /* Free the MTT entries */ 9589e39c5baSBill Taylor hermon_rsrc_free(state, &mtt); 9599e39c5baSBill Taylor 9609e39c5baSBill Taylor /* Free the Hermon Memory Region handle */ 9619e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 9629e39c5baSBill Taylor 9639e39c5baSBill Taylor /* Free up the MPT entry resource */ 9649e39c5baSBill Taylor hermon_rsrc_free(state, &mpt); 9659e39c5baSBill Taylor 9669e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */ 9679e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 9689e39c5baSBill Taylor 9699e39c5baSBill Taylor /* Set the mrhdl pointer to NULL and return success */ 9709e39c5baSBill Taylor *mrhdl = NULL; 9719e39c5baSBill Taylor 9729e39c5baSBill Taylor return (DDI_SUCCESS); 9739e39c5baSBill Taylor } 9749e39c5baSBill Taylor 9759e39c5baSBill Taylor /* 9769e39c5baSBill Taylor * hermon_mr_invalidate_fmr() 9779e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 9789e39c5baSBill Taylor */ 9799e39c5baSBill Taylor /* ARGSUSED */ 9809e39c5baSBill Taylor int 9819e39c5baSBill Taylor hermon_mr_invalidate_fmr(hermon_state_t *state, hermon_mrhdl_t mr) 9829e39c5baSBill Taylor { 9839e39c5baSBill Taylor hermon_rsrc_t *mpt; 9849e39c5baSBill Taylor uint64_t *mpt_table; 9859e39c5baSBill Taylor 9869e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 9879e39c5baSBill Taylor mpt = mr->mr_mptrsrcp; 9889e39c5baSBill Taylor mpt_table = (uint64_t *)mpt->hr_addr; 9899e39c5baSBill Taylor 9909e39c5baSBill Taylor /* Write MPT status to SW bit */ 991*c7facc54SBill Taylor *(uint8_t *)&mpt_table[0] = 0xF0; 992*c7facc54SBill Taylor 993*c7facc54SBill Taylor membar_producer(); 9949e39c5baSBill Taylor 9959e39c5baSBill Taylor /* invalidate mem key value */ 996*c7facc54SBill Taylor *(uint32_t *)&mpt_table[1] = 0; 9979e39c5baSBill Taylor 9989e39c5baSBill Taylor /* invalidate lkey value */ 999*c7facc54SBill Taylor *(uint32_t *)&mpt_table[4] = 0; 1000*c7facc54SBill Taylor 1001*c7facc54SBill Taylor membar_producer(); 10029e39c5baSBill Taylor 10039e39c5baSBill Taylor /* Write MPT status to HW bit */ 1004*c7facc54SBill Taylor *(uint8_t *)&mpt_table[0] = 0x00; 10059e39c5baSBill Taylor 10069e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 10079e39c5baSBill Taylor 10089e39c5baSBill Taylor return (DDI_SUCCESS); 10099e39c5baSBill Taylor } 10109e39c5baSBill Taylor 10119e39c5baSBill Taylor /* 10129e39c5baSBill Taylor * hermon_mr_deregister_fmr() 10139e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 10149e39c5baSBill Taylor */ 10159e39c5baSBill Taylor /* ARGSUSED */ 10169e39c5baSBill Taylor int 10179e39c5baSBill Taylor hermon_mr_deregister_fmr(hermon_state_t *state, hermon_mrhdl_t mr) 10189e39c5baSBill Taylor { 10199e39c5baSBill Taylor hermon_rsrc_t *mpt; 10209e39c5baSBill Taylor uint64_t *mpt_table; 10219e39c5baSBill Taylor 10229e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 10239e39c5baSBill Taylor mpt = mr->mr_mptrsrcp; 10249e39c5baSBill Taylor mpt_table = (uint64_t *)mpt->hr_addr; 10259e39c5baSBill Taylor 10269e39c5baSBill Taylor /* Write MPT status to SW bit */ 1027*c7facc54SBill Taylor *(uint8_t *)&mpt_table[0] = 0xF0; 1028*c7facc54SBill Taylor 10299e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 10309e39c5baSBill Taylor 10319e39c5baSBill Taylor return (DDI_SUCCESS); 10329e39c5baSBill Taylor } 10339e39c5baSBill Taylor 10349e39c5baSBill Taylor 10359e39c5baSBill Taylor /* 10369e39c5baSBill Taylor * hermon_mr_query() 10379e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 10389e39c5baSBill Taylor */ 10399e39c5baSBill Taylor /* ARGSUSED */ 10409e39c5baSBill Taylor int 10419e39c5baSBill Taylor hermon_mr_query(hermon_state_t *state, hermon_mrhdl_t mr, 10429e39c5baSBill Taylor ibt_mr_query_attr_t *attr) 10439e39c5baSBill Taylor { 1044*c7facc54SBill Taylor int status; 1045*c7facc54SBill Taylor hermon_hw_dmpt_t mpt_entry; 1046*c7facc54SBill Taylor uint32_t lkey; 1047*c7facc54SBill Taylor 10489e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr)) 10499e39c5baSBill Taylor 10509e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 10519e39c5baSBill Taylor 10529e39c5baSBill Taylor /* 10539e39c5baSBill Taylor * Check here to see if the memory region has already been partially 10549e39c5baSBill Taylor * deregistered as a result of a hermon_umap_umemlock_cb() callback. 10559e39c5baSBill Taylor * If so, this is an error, return failure. 10569e39c5baSBill Taylor */ 10579e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) { 10589e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 10599e39c5baSBill Taylor return (IBT_MR_HDL_INVALID); 10609e39c5baSBill Taylor } 10619e39c5baSBill Taylor 1062*c7facc54SBill Taylor status = hermon_cmn_query_cmd_post(state, QUERY_MPT, 0, 1063*c7facc54SBill Taylor mr->mr_lkey >> 8, &mpt_entry, sizeof (hermon_hw_dmpt_t), 1064*c7facc54SBill Taylor HERMON_NOSLEEP); 1065*c7facc54SBill Taylor if (status != HERMON_CMD_SUCCESS) { 1066*c7facc54SBill Taylor cmn_err(CE_CONT, "Hermon: QUERY_MPT failed: status %x", status); 1067*c7facc54SBill Taylor mutex_exit(&mr->mr_lock); 1068*c7facc54SBill Taylor return (ibc_get_ci_failure(0)); 1069*c7facc54SBill Taylor } 1070*c7facc54SBill Taylor 1071*c7facc54SBill Taylor /* Update the mr sw struct from the hw struct. */ 1072*c7facc54SBill Taylor lkey = mpt_entry.mem_key; 1073*c7facc54SBill Taylor mr->mr_lkey = mr->mr_rkey = (lkey >> 8) | (lkey << 24); 1074*c7facc54SBill Taylor mr->mr_bindinfo.bi_addr = mpt_entry.start_addr; 1075*c7facc54SBill Taylor mr->mr_bindinfo.bi_len = mpt_entry.reg_win_len; 1076*c7facc54SBill Taylor mr->mr_accflag = (mr->mr_accflag & IBT_MR_RO_DISABLED) | 1077*c7facc54SBill Taylor (mpt_entry.lw ? IBT_MR_LOCAL_WRITE : 0) | 1078*c7facc54SBill Taylor (mpt_entry.rr ? IBT_MR_REMOTE_READ : 0) | 1079*c7facc54SBill Taylor (mpt_entry.rw ? IBT_MR_REMOTE_WRITE : 0) | 1080*c7facc54SBill Taylor (mpt_entry.atomic ? IBT_MR_REMOTE_ATOMIC : 0) | 1081*c7facc54SBill Taylor (mpt_entry.en_bind ? IBT_MR_WINDOW_BIND : 0); 1082*c7facc54SBill Taylor mr->mr_mttaddr = ((uint64_t)mpt_entry.mtt_addr_h << 32) | 1083*c7facc54SBill Taylor (mpt_entry.mtt_addr_l << 3); 1084*c7facc54SBill Taylor mr->mr_logmttpgsz = mpt_entry.entity_sz; 1085*c7facc54SBill Taylor 10869e39c5baSBill Taylor /* Fill in the queried attributes */ 1087*c7facc54SBill Taylor attr->mr_lkey_state = 1088*c7facc54SBill Taylor (mpt_entry.status == HERMON_MPT_FREE) ? IBT_KEY_FREE : 1089*c7facc54SBill Taylor (mpt_entry.status == HERMON_MPT_SW_OWNERSHIP) ? IBT_KEY_INVALID : 1090*c7facc54SBill Taylor IBT_KEY_VALID; 1091*c7facc54SBill Taylor attr->mr_phys_buf_list_sz = mpt_entry.mtt_size; 10929e39c5baSBill Taylor attr->mr_attr_flags = mr->mr_accflag; 1093*c7facc54SBill Taylor attr->mr_pd = (ibt_pd_hdl_t)mr->mr_pdhdl; 10949e39c5baSBill Taylor 10959e39c5baSBill Taylor /* Fill in the "local" attributes */ 10969e39c5baSBill Taylor attr->mr_lkey = (ibt_lkey_t)mr->mr_lkey; 10979e39c5baSBill Taylor attr->mr_lbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr; 10989e39c5baSBill Taylor attr->mr_lbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len; 10999e39c5baSBill Taylor 11009e39c5baSBill Taylor /* 11019e39c5baSBill Taylor * Fill in the "remote" attributes (if necessary). Note: the 11029e39c5baSBill Taylor * remote attributes are only valid if the memory region has one 11039e39c5baSBill Taylor * or more of the remote access flags set. 11049e39c5baSBill Taylor */ 11059e39c5baSBill Taylor if ((mr->mr_accflag & IBT_MR_REMOTE_READ) || 11069e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_WRITE) || 11079e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) { 11089e39c5baSBill Taylor attr->mr_rkey = (ibt_rkey_t)mr->mr_rkey; 11099e39c5baSBill Taylor attr->mr_rbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr; 11109e39c5baSBill Taylor attr->mr_rbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len; 11119e39c5baSBill Taylor } 11129e39c5baSBill Taylor 11139e39c5baSBill Taylor /* 11149e39c5baSBill Taylor * If region is mapped for streaming (i.e. noncoherent), then set sync 11159e39c5baSBill Taylor * is required 11169e39c5baSBill Taylor */ 11179e39c5baSBill Taylor attr->mr_sync_required = (mr->mr_bindinfo.bi_flags & 11189e39c5baSBill Taylor IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE; 11199e39c5baSBill Taylor 11209e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 11219e39c5baSBill Taylor return (DDI_SUCCESS); 11229e39c5baSBill Taylor } 11239e39c5baSBill Taylor 11249e39c5baSBill Taylor 11259e39c5baSBill Taylor /* 11269e39c5baSBill Taylor * hermon_mr_reregister() 11279e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 11289e39c5baSBill Taylor */ 11299e39c5baSBill Taylor int 11309e39c5baSBill Taylor hermon_mr_reregister(hermon_state_t *state, hermon_mrhdl_t mr, 11319e39c5baSBill Taylor hermon_pdhdl_t pd, ibt_mr_attr_t *mr_attr, hermon_mrhdl_t *mrhdl_new, 11329e39c5baSBill Taylor hermon_mr_options_t *op) 11339e39c5baSBill Taylor { 11349e39c5baSBill Taylor hermon_bind_info_t bind; 11359e39c5baSBill Taylor int status; 11369e39c5baSBill Taylor 11379e39c5baSBill Taylor /* 11389e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority 11399e39c5baSBill Taylor * of the information that will be used to distinguish between an 11409e39c5baSBill Taylor * "addr" binding (as is the case here) and a "buf" binding (see 11419e39c5baSBill Taylor * below). The "bind" struct is later passed to hermon_mr_mem_bind() 11429e39c5baSBill Taylor * which does most of the "heavy lifting" for the Hermon memory 11439e39c5baSBill Taylor * registration (and reregistration) routines. 11449e39c5baSBill Taylor */ 11459e39c5baSBill Taylor bind.bi_type = HERMON_BINDHDL_VADDR; 11469e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr; 11479e39c5baSBill Taylor bind.bi_len = mr_attr->mr_len; 11489e39c5baSBill Taylor bind.bi_as = mr_attr->mr_as; 11499e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags; 11509e39c5baSBill Taylor status = hermon_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op); 11519e39c5baSBill Taylor return (status); 11529e39c5baSBill Taylor } 11539e39c5baSBill Taylor 11549e39c5baSBill Taylor 11559e39c5baSBill Taylor /* 11569e39c5baSBill Taylor * hermon_mr_reregister_buf() 11579e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 11589e39c5baSBill Taylor */ 11599e39c5baSBill Taylor int 11609e39c5baSBill Taylor hermon_mr_reregister_buf(hermon_state_t *state, hermon_mrhdl_t mr, 11619e39c5baSBill Taylor hermon_pdhdl_t pd, ibt_smr_attr_t *mr_attr, struct buf *buf, 11629e39c5baSBill Taylor hermon_mrhdl_t *mrhdl_new, hermon_mr_options_t *op) 11639e39c5baSBill Taylor { 11649e39c5baSBill Taylor hermon_bind_info_t bind; 11659e39c5baSBill Taylor int status; 11669e39c5baSBill Taylor 11679e39c5baSBill Taylor /* 11689e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority 11699e39c5baSBill Taylor * of the information that will be used to distinguish between an 11709e39c5baSBill Taylor * "addr" binding (see above) and a "buf" binding (as is the case 11719e39c5baSBill Taylor * here). The "bind" struct is later passed to hermon_mr_mem_bind() 11729e39c5baSBill Taylor * which does most of the "heavy lifting" for the Hermon memory 11739e39c5baSBill Taylor * registration routines. Note: We have chosen to provide 11749e39c5baSBill Taylor * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is 11759e39c5baSBill Taylor * not set). It is not critical what value we choose here as it need 11769e39c5baSBill Taylor * only be unique for the given RKey (which will happen by default), 11779e39c5baSBill Taylor * so the choice here is somewhat arbitrary. 11789e39c5baSBill Taylor */ 11799e39c5baSBill Taylor bind.bi_type = HERMON_BINDHDL_BUF; 11809e39c5baSBill Taylor bind.bi_buf = buf; 11819e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_PHYS_IOVA) { 11829e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr; 11839e39c5baSBill Taylor } else { 11849e39c5baSBill Taylor bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr; 11859e39c5baSBill Taylor } 11869e39c5baSBill Taylor bind.bi_len = (uint64_t)buf->b_bcount; 11879e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags; 11889e39c5baSBill Taylor bind.bi_as = NULL; 11899e39c5baSBill Taylor status = hermon_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op); 11909e39c5baSBill Taylor return (status); 11919e39c5baSBill Taylor } 11929e39c5baSBill Taylor 11939e39c5baSBill Taylor 11949e39c5baSBill Taylor /* 11959e39c5baSBill Taylor * hermon_mr_sync() 11969e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 11979e39c5baSBill Taylor */ 11989e39c5baSBill Taylor /* ARGSUSED */ 11999e39c5baSBill Taylor int 12009e39c5baSBill Taylor hermon_mr_sync(hermon_state_t *state, ibt_mr_sync_t *mr_segs, size_t num_segs) 12019e39c5baSBill Taylor { 12029e39c5baSBill Taylor hermon_mrhdl_t mrhdl; 12039e39c5baSBill Taylor uint64_t seg_vaddr, seg_len, seg_end; 12049e39c5baSBill Taylor uint64_t mr_start, mr_end; 12059e39c5baSBill Taylor uint_t type; 12069e39c5baSBill Taylor int status, i; 12079e39c5baSBill Taylor 12089e39c5baSBill Taylor /* Process each of the ibt_mr_sync_t's */ 12099e39c5baSBill Taylor for (i = 0; i < num_segs; i++) { 12109e39c5baSBill Taylor mrhdl = (hermon_mrhdl_t)mr_segs[i].ms_handle; 12119e39c5baSBill Taylor 12129e39c5baSBill Taylor /* Check for valid memory region handle */ 12139e39c5baSBill Taylor if (mrhdl == NULL) { 12149e39c5baSBill Taylor status = IBT_MR_HDL_INVALID; 12159e39c5baSBill Taylor goto mrsync_fail; 12169e39c5baSBill Taylor } 12179e39c5baSBill Taylor 12189e39c5baSBill Taylor mutex_enter(&mrhdl->mr_lock); 12199e39c5baSBill Taylor 12209e39c5baSBill Taylor /* 12219e39c5baSBill Taylor * Check here to see if the memory region has already been 12229e39c5baSBill Taylor * partially deregistered as a result of a 12239e39c5baSBill Taylor * hermon_umap_umemlock_cb() callback. If so, this is an 12249e39c5baSBill Taylor * error, return failure. 12259e39c5baSBill Taylor */ 12269e39c5baSBill Taylor if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) { 12279e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 12289e39c5baSBill Taylor status = IBT_MR_HDL_INVALID; 12299e39c5baSBill Taylor goto mrsync_fail; 12309e39c5baSBill Taylor } 12319e39c5baSBill Taylor 12329e39c5baSBill Taylor /* Check for valid bounds on sync request */ 12339e39c5baSBill Taylor seg_vaddr = mr_segs[i].ms_vaddr; 12349e39c5baSBill Taylor seg_len = mr_segs[i].ms_len; 12359e39c5baSBill Taylor seg_end = seg_vaddr + seg_len - 1; 12369e39c5baSBill Taylor mr_start = mrhdl->mr_bindinfo.bi_addr; 12379e39c5baSBill Taylor mr_end = mr_start + mrhdl->mr_bindinfo.bi_len - 1; 12389e39c5baSBill Taylor if ((seg_vaddr < mr_start) || (seg_vaddr > mr_end)) { 12399e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 12409e39c5baSBill Taylor status = IBT_MR_VA_INVALID; 12419e39c5baSBill Taylor goto mrsync_fail; 12429e39c5baSBill Taylor } 12439e39c5baSBill Taylor if ((seg_end < mr_start) || (seg_end > mr_end)) { 12449e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 12459e39c5baSBill Taylor status = IBT_MR_LEN_INVALID; 12469e39c5baSBill Taylor goto mrsync_fail; 12479e39c5baSBill Taylor } 12489e39c5baSBill Taylor 12499e39c5baSBill Taylor /* Determine what type (i.e. direction) for sync */ 12509e39c5baSBill Taylor if (mr_segs[i].ms_flags & IBT_SYNC_READ) { 12519e39c5baSBill Taylor type = DDI_DMA_SYNC_FORDEV; 12529e39c5baSBill Taylor } else if (mr_segs[i].ms_flags & IBT_SYNC_WRITE) { 12539e39c5baSBill Taylor type = DDI_DMA_SYNC_FORCPU; 12549e39c5baSBill Taylor } else { 12559e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 12569e39c5baSBill Taylor status = IBT_INVALID_PARAM; 12579e39c5baSBill Taylor goto mrsync_fail; 12589e39c5baSBill Taylor } 12599e39c5baSBill Taylor 12609e39c5baSBill Taylor (void) ddi_dma_sync(mrhdl->mr_bindinfo.bi_dmahdl, 12619e39c5baSBill Taylor (off_t)(seg_vaddr - mr_start), (size_t)seg_len, type); 12629e39c5baSBill Taylor 12639e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock); 12649e39c5baSBill Taylor } 12659e39c5baSBill Taylor 12669e39c5baSBill Taylor return (DDI_SUCCESS); 12679e39c5baSBill Taylor 12689e39c5baSBill Taylor mrsync_fail: 12699e39c5baSBill Taylor return (status); 12709e39c5baSBill Taylor } 12719e39c5baSBill Taylor 12729e39c5baSBill Taylor 12739e39c5baSBill Taylor /* 12749e39c5baSBill Taylor * hermon_mw_alloc() 12759e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 12769e39c5baSBill Taylor */ 12779e39c5baSBill Taylor int 12789e39c5baSBill Taylor hermon_mw_alloc(hermon_state_t *state, hermon_pdhdl_t pd, ibt_mw_flags_t flags, 12799e39c5baSBill Taylor hermon_mwhdl_t *mwhdl) 12809e39c5baSBill Taylor { 12819e39c5baSBill Taylor hermon_rsrc_t *mpt, *rsrc; 12829e39c5baSBill Taylor hermon_hw_dmpt_t mpt_entry; 12839e39c5baSBill Taylor hermon_mwhdl_t mw; 12849e39c5baSBill Taylor uint_t sleep; 12859e39c5baSBill Taylor int status; 12869e39c5baSBill Taylor 12879e39c5baSBill Taylor if (state != NULL) /* XXX - bogus test that is always TRUE */ 12889e39c5baSBill Taylor return (IBT_INSUFF_RESOURCE); 12899e39c5baSBill Taylor 12909e39c5baSBill Taylor /* 12919e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 12929e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 12939e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 12949e39c5baSBill Taylor */ 12959e39c5baSBill Taylor sleep = (flags & IBT_MW_NOSLEEP) ? HERMON_NOSLEEP : HERMON_SLEEP; 12969e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 12979e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 12989e39c5baSBill Taylor status = IBT_INVALID_PARAM; 12999e39c5baSBill Taylor goto mwalloc_fail; 13009e39c5baSBill Taylor } 13019e39c5baSBill Taylor 13029e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */ 13039e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 13049e39c5baSBill Taylor 13059e39c5baSBill Taylor /* 13069e39c5baSBill Taylor * Allocate an MPT entry (for use as a memory window). Since the 13079e39c5baSBill Taylor * Hermon hardware uses the MPT entry for memory regions and for 13089e39c5baSBill Taylor * memory windows, we will fill in this MPT with all the necessary 13099e39c5baSBill Taylor * parameters for the memory window. And then (just as we do for 13109e39c5baSBill Taylor * memory regions) ownership will be passed to the hardware in the 13119e39c5baSBill Taylor * final step below. If we fail here, we must undo the protection 13129e39c5baSBill Taylor * domain reference count. 13139e39c5baSBill Taylor */ 13149e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt); 13159e39c5baSBill Taylor if (status != DDI_SUCCESS) { 13169e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 13179e39c5baSBill Taylor goto mwalloc_fail1; 13189e39c5baSBill Taylor } 13199e39c5baSBill Taylor 13209e39c5baSBill Taylor /* 13219e39c5baSBill Taylor * Allocate the software structure for tracking the memory window (i.e. 13229e39c5baSBill Taylor * the Hermon Memory Window handle). Note: This is actually the same 13239e39c5baSBill Taylor * software structure used for tracking memory regions, but since many 13249e39c5baSBill Taylor * of the same properties are needed, only a single structure is 13259e39c5baSBill Taylor * necessary. If we fail here, we must undo the protection domain 13269e39c5baSBill Taylor * reference count and the previous resource allocation. 13279e39c5baSBill Taylor */ 13289e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc); 13299e39c5baSBill Taylor if (status != DDI_SUCCESS) { 13309e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 13319e39c5baSBill Taylor goto mwalloc_fail2; 13329e39c5baSBill Taylor } 13339e39c5baSBill Taylor mw = (hermon_mwhdl_t)rsrc->hr_addr; 13349e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw)) 13359e39c5baSBill Taylor 13369e39c5baSBill Taylor /* 13379e39c5baSBill Taylor * Calculate an "unbound" RKey from MPT index. In much the same way 13389e39c5baSBill Taylor * as we do for memory regions (above), this key is constructed from 13399e39c5baSBill Taylor * a "constrained" (which depends on the MPT index) and an 13409e39c5baSBill Taylor * "unconstrained" portion (which may be arbitrarily chosen). 13419e39c5baSBill Taylor */ 13429e39c5baSBill Taylor mw->mr_rkey = hermon_mr_keycalc(mpt->hr_indx); 13439e39c5baSBill Taylor 13449e39c5baSBill Taylor /* 13459e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing 13469e39c5baSBill Taylor * ownership of the MPT entry to the Hermon hardware. We use all of 13479e39c5baSBill Taylor * the information collected/calculated above to fill in the 13489e39c5baSBill Taylor * requisite portions of the MPT. Note: fewer entries in the MPT 13499e39c5baSBill Taylor * entry are necessary to allocate a memory window. 13509e39c5baSBill Taylor */ 13519e39c5baSBill Taylor bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t)); 13529e39c5baSBill Taylor mpt_entry.reg_win = HERMON_MPT_IS_WINDOW; 13539e39c5baSBill Taylor mpt_entry.mem_key = mw->mr_rkey; 13549e39c5baSBill Taylor mpt_entry.pd = pd->pd_pdnum; 13559e39c5baSBill Taylor mpt_entry.lr = 1; 13569e39c5baSBill Taylor 13579e39c5baSBill Taylor /* 13589e39c5baSBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of 13599e39c5baSBill Taylor * the entry to the hardware. Note: in general, this operation 13609e39c5baSBill Taylor * shouldn't fail. But if it does, we have to undo everything we've 13619e39c5baSBill Taylor * done above before returning error. 13629e39c5baSBill Taylor */ 13639e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry, 13649e39c5baSBill Taylor sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep); 13659e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 13669e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n", 13679e39c5baSBill Taylor status); 13689e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 13699e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 13709e39c5baSBill Taylor } 13719e39c5baSBill Taylor status = ibc_get_ci_failure(0); 13729e39c5baSBill Taylor goto mwalloc_fail3; 13739e39c5baSBill Taylor } 13749e39c5baSBill Taylor 13759e39c5baSBill Taylor /* 13769e39c5baSBill Taylor * Fill in the rest of the Hermon Memory Window handle. Having 13779e39c5baSBill Taylor * successfully transferred ownership of the MPT, we can update the 13789e39c5baSBill Taylor * following fields for use in further operations on the MW. 13799e39c5baSBill Taylor */ 13809e39c5baSBill Taylor mw->mr_mptrsrcp = mpt; 13819e39c5baSBill Taylor mw->mr_pdhdl = pd; 13829e39c5baSBill Taylor mw->mr_rsrcp = rsrc; 13839e39c5baSBill Taylor mw->mr_rkey = hermon_mr_key_swap(mw->mr_rkey); 13849e39c5baSBill Taylor *mwhdl = mw; 13859e39c5baSBill Taylor 13869e39c5baSBill Taylor return (DDI_SUCCESS); 13879e39c5baSBill Taylor 13889e39c5baSBill Taylor mwalloc_fail3: 13899e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 13909e39c5baSBill Taylor mwalloc_fail2: 13919e39c5baSBill Taylor hermon_rsrc_free(state, &mpt); 13929e39c5baSBill Taylor mwalloc_fail1: 13939e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 13949e39c5baSBill Taylor mwalloc_fail: 13959e39c5baSBill Taylor return (status); 13969e39c5baSBill Taylor } 13979e39c5baSBill Taylor 13989e39c5baSBill Taylor 13999e39c5baSBill Taylor /* 14009e39c5baSBill Taylor * hermon_mw_free() 14019e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 14029e39c5baSBill Taylor */ 14039e39c5baSBill Taylor int 14049e39c5baSBill Taylor hermon_mw_free(hermon_state_t *state, hermon_mwhdl_t *mwhdl, uint_t sleep) 14059e39c5baSBill Taylor { 14069e39c5baSBill Taylor hermon_rsrc_t *mpt, *rsrc; 14079e39c5baSBill Taylor hermon_mwhdl_t mw; 14089e39c5baSBill Taylor int status; 14099e39c5baSBill Taylor hermon_pdhdl_t pd; 14109e39c5baSBill Taylor 14119e39c5baSBill Taylor /* 14129e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 14139e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 14149e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 14159e39c5baSBill Taylor */ 14169e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 14179e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 14189e39c5baSBill Taylor status = IBT_INVALID_PARAM; 14199e39c5baSBill Taylor return (status); 14209e39c5baSBill Taylor } 14219e39c5baSBill Taylor 14229e39c5baSBill Taylor /* 14239e39c5baSBill Taylor * Pull all the necessary information from the Hermon Memory Window 14249e39c5baSBill Taylor * handle. This is necessary here because the resource for the 14259e39c5baSBill Taylor * MW handle is going to be freed up as part of the this operation. 14269e39c5baSBill Taylor */ 14279e39c5baSBill Taylor mw = *mwhdl; 14289e39c5baSBill Taylor mutex_enter(&mw->mr_lock); 14299e39c5baSBill Taylor mpt = mw->mr_mptrsrcp; 14309e39c5baSBill Taylor rsrc = mw->mr_rsrcp; 14319e39c5baSBill Taylor pd = mw->mr_pdhdl; 14329e39c5baSBill Taylor mutex_exit(&mw->mr_lock); 14339e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw)) 14349e39c5baSBill Taylor 14359e39c5baSBill Taylor /* 14369e39c5baSBill Taylor * Reclaim the MPT entry from hardware. Note: in general, it is 14379e39c5baSBill Taylor * unexpected for this operation to return an error. 14389e39c5baSBill Taylor */ 14399e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT, NULL, 14409e39c5baSBill Taylor 0, mpt->hr_indx, sleep); 14419e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 14429e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: HW2SW_MPT command failed: %08x\n", 14439e39c5baSBill Taylor status); 14449e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 14459e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 14469e39c5baSBill Taylor } 14479e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 14489e39c5baSBill Taylor } 14499e39c5baSBill Taylor 14509e39c5baSBill Taylor /* Free the Hermon Memory Window handle */ 14519e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 14529e39c5baSBill Taylor 14539e39c5baSBill Taylor /* Free up the MPT entry resource */ 14549e39c5baSBill Taylor hermon_rsrc_free(state, &mpt); 14559e39c5baSBill Taylor 14569e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */ 14579e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 14589e39c5baSBill Taylor 14599e39c5baSBill Taylor /* Set the mwhdl pointer to NULL and return success */ 14609e39c5baSBill Taylor *mwhdl = NULL; 14619e39c5baSBill Taylor 14629e39c5baSBill Taylor return (DDI_SUCCESS); 14639e39c5baSBill Taylor } 14649e39c5baSBill Taylor 14659e39c5baSBill Taylor 14669e39c5baSBill Taylor /* 14679e39c5baSBill Taylor * hermon_mr_keycalc() 14689e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 14699e39c5baSBill Taylor * NOTE: Produces a key in the form of 14709e39c5baSBill Taylor * KKKKKKKK IIIIIIII IIIIIIII IIIIIIIII 14719e39c5baSBill Taylor * where K == the arbitrary bits and I == the index 14729e39c5baSBill Taylor */ 14739e39c5baSBill Taylor uint32_t 14749e39c5baSBill Taylor hermon_mr_keycalc(uint32_t indx) 14759e39c5baSBill Taylor { 14769e39c5baSBill Taylor uint32_t tmp_key, tmp_indx; 14779e39c5baSBill Taylor 14789e39c5baSBill Taylor /* 14799e39c5baSBill Taylor * Generate a simple key from counter. Note: We increment this 14809e39c5baSBill Taylor * static variable _intentionally_ without any kind of mutex around 14819e39c5baSBill Taylor * it. First, single-threading all operations through a single lock 14829e39c5baSBill Taylor * would be a bad idea (from a performance point-of-view). Second, 14839e39c5baSBill Taylor * the upper "unconstrained" bits don't really have to be unique 14849e39c5baSBill Taylor * because the lower bits are guaranteed to be (although we do make a 14859e39c5baSBill Taylor * best effort to ensure that they are). Third, the window for the 14869e39c5baSBill Taylor * race (where both threads read and update the counter at the same 14879e39c5baSBill Taylor * time) is incredibly small. 14889e39c5baSBill Taylor * And, lastly, we'd like to make this into a "random" key 14899e39c5baSBill Taylor */ 14909e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(hermon_memkey_cnt)) 14919e39c5baSBill Taylor tmp_key = (hermon_memkey_cnt++) << HERMON_MEMKEY_SHIFT; 14929e39c5baSBill Taylor tmp_indx = indx & 0xffffff; 14939e39c5baSBill Taylor return (tmp_key | tmp_indx); 14949e39c5baSBill Taylor } 14959e39c5baSBill Taylor 14969e39c5baSBill Taylor 14979e39c5baSBill Taylor /* 14989e39c5baSBill Taylor * hermon_mr_key_swap() 14999e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 15009e39c5baSBill Taylor * NOTE: Produces a key in the form of 15019e39c5baSBill Taylor * IIIIIIII IIIIIIII IIIIIIIII KKKKKKKK 15029e39c5baSBill Taylor * where K == the arbitrary bits and I == the index 15039e39c5baSBill Taylor */ 15049e39c5baSBill Taylor uint32_t 15059e39c5baSBill Taylor hermon_mr_key_swap(uint32_t indx) 15069e39c5baSBill Taylor { 15079e39c5baSBill Taylor /* 15089e39c5baSBill Taylor * The memory key format to pass down to the hardware is 15099e39c5baSBill Taylor * (key[7:0],index[23:0]), which defines the index to the 15109e39c5baSBill Taylor * hardware resource. When the driver passes this as a memory 15119e39c5baSBill Taylor * key, (i.e. to retrieve a resource) the format is 15129e39c5baSBill Taylor * (index[23:0],key[7:0]). 15139e39c5baSBill Taylor */ 15149e39c5baSBill Taylor return (((indx >> 24) & 0x000000ff) | ((indx << 8) & 0xffffff00)); 15159e39c5baSBill Taylor } 15169e39c5baSBill Taylor 15179e39c5baSBill Taylor /* 15189e39c5baSBill Taylor * hermon_mr_common_reg() 15199e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 15209e39c5baSBill Taylor */ 15219e39c5baSBill Taylor static int 15229e39c5baSBill Taylor hermon_mr_common_reg(hermon_state_t *state, hermon_pdhdl_t pd, 15239e39c5baSBill Taylor hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op, 15249e39c5baSBill Taylor hermon_mpt_rsrc_type_t mpt_type) 15259e39c5baSBill Taylor { 15269e39c5baSBill Taylor hermon_rsrc_t *mpt, *mtt, *rsrc, *mtt_refcnt; 15279e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 15289e39c5baSBill Taylor hermon_sw_refcnt_t *swrc_tmp; 15299e39c5baSBill Taylor hermon_hw_dmpt_t mpt_entry; 15309e39c5baSBill Taylor hermon_mrhdl_t mr; 15319e39c5baSBill Taylor ibt_mr_flags_t flags; 15329e39c5baSBill Taylor hermon_bind_info_t *bh; 15339e39c5baSBill Taylor ddi_dma_handle_t bind_dmahdl; 15349e39c5baSBill Taylor ddi_umem_cookie_t umem_cookie; 15359e39c5baSBill Taylor size_t umem_len; 15369e39c5baSBill Taylor caddr_t umem_addr; 15379e39c5baSBill Taylor uint64_t mtt_addr, max_sz; 15389e39c5baSBill Taylor uint_t sleep, mtt_pgsize_bits, bind_type, mr_is_umem; 15399e39c5baSBill Taylor int status, umem_flags, bind_override_addr; 15409e39c5baSBill Taylor 15419e39c5baSBill Taylor /* 15429e39c5baSBill Taylor * Check the "options" flag. Currently this flag tells the driver 15439e39c5baSBill Taylor * whether or not the region should be bound normally (i.e. with 15449e39c5baSBill Taylor * entries written into the PCI IOMMU), whether it should be 15459e39c5baSBill Taylor * registered to bypass the IOMMU, and whether or not the resulting 15469e39c5baSBill Taylor * address should be "zero-based" (to aid the alignment restrictions 15479e39c5baSBill Taylor * for QPs). 15489e39c5baSBill Taylor */ 15499e39c5baSBill Taylor if (op == NULL) { 15509e39c5baSBill Taylor bind_type = HERMON_BINDMEM_NORMAL; 15519e39c5baSBill Taylor bind_dmahdl = NULL; 15529e39c5baSBill Taylor bind_override_addr = 0; 15539e39c5baSBill Taylor } else { 15549e39c5baSBill Taylor bind_type = op->mro_bind_type; 15559e39c5baSBill Taylor bind_dmahdl = op->mro_bind_dmahdl; 15569e39c5baSBill Taylor bind_override_addr = op->mro_bind_override_addr; 15579e39c5baSBill Taylor } 15589e39c5baSBill Taylor 15599e39c5baSBill Taylor /* check what kind of mpt to use */ 15609e39c5baSBill Taylor 15619e39c5baSBill Taylor /* Extract the flags field from the hermon_bind_info_t */ 15629e39c5baSBill Taylor flags = bind->bi_flags; 15639e39c5baSBill Taylor 15649e39c5baSBill Taylor /* 15659e39c5baSBill Taylor * Check for invalid length. Check is the length is zero or if the 15669e39c5baSBill Taylor * length is larger than the maximum configured value. Return error 15679e39c5baSBill Taylor * if it is. 15689e39c5baSBill Taylor */ 15699e39c5baSBill Taylor max_sz = ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_mrw_sz); 15709e39c5baSBill Taylor if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) { 15719e39c5baSBill Taylor status = IBT_MR_LEN_INVALID; 15729e39c5baSBill Taylor goto mrcommon_fail; 15739e39c5baSBill Taylor } 15749e39c5baSBill Taylor 15759e39c5baSBill Taylor /* 15769e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 15779e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 15789e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 15799e39c5baSBill Taylor */ 15809e39c5baSBill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP: HERMON_SLEEP; 15819e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 15829e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 15839e39c5baSBill Taylor status = IBT_INVALID_PARAM; 15849e39c5baSBill Taylor goto mrcommon_fail; 15859e39c5baSBill Taylor } 15869e39c5baSBill Taylor 15879e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */ 15889e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 15899e39c5baSBill Taylor 15909e39c5baSBill Taylor /* 15919e39c5baSBill Taylor * Allocate an MPT entry. This will be filled in with all the 15929e39c5baSBill Taylor * necessary parameters to define the memory region. And then 15939e39c5baSBill Taylor * ownership will be passed to the hardware in the final step 15949e39c5baSBill Taylor * below. If we fail here, we must undo the protection domain 15959e39c5baSBill Taylor * reference count. 15969e39c5baSBill Taylor */ 15979e39c5baSBill Taylor if (mpt_type == HERMON_MPT_DMPT) { 15989e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_DMPT, 1, sleep, &mpt); 15999e39c5baSBill Taylor if (status != DDI_SUCCESS) { 16009e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 16019e39c5baSBill Taylor goto mrcommon_fail1; 16029e39c5baSBill Taylor } 16039e39c5baSBill Taylor } else { 16049e39c5baSBill Taylor mpt = NULL; 16059e39c5baSBill Taylor } 16069e39c5baSBill Taylor 16079e39c5baSBill Taylor /* 16089e39c5baSBill Taylor * Allocate the software structure for tracking the memory region (i.e. 16099e39c5baSBill Taylor * the Hermon Memory Region handle). If we fail here, we must undo 16109e39c5baSBill Taylor * the protection domain reference count and the previous resource 16119e39c5baSBill Taylor * allocation. 16129e39c5baSBill Taylor */ 16139e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_MRHDL, 1, sleep, &rsrc); 16149e39c5baSBill Taylor if (status != DDI_SUCCESS) { 16159e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 16169e39c5baSBill Taylor goto mrcommon_fail2; 16179e39c5baSBill Taylor } 16189e39c5baSBill Taylor mr = (hermon_mrhdl_t)rsrc->hr_addr; 16199e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) 16209e39c5baSBill Taylor 16219e39c5baSBill Taylor /* 16229e39c5baSBill Taylor * Setup and validate the memory region access flags. This means 16239e39c5baSBill Taylor * translating the IBTF's enable flags into the access flags that 16249e39c5baSBill Taylor * will be used in later operations. 16259e39c5baSBill Taylor */ 16269e39c5baSBill Taylor mr->mr_accflag = 0; 16279e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_WINDOW_BIND) 16289e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_WINDOW_BIND; 16299e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_LOCAL_WRITE) 16309e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_LOCAL_WRITE; 16319e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_READ) 16329e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_READ; 16339e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_WRITE) 16349e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_WRITE; 16359e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC) 16369e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC; 16379e39c5baSBill Taylor 16389e39c5baSBill Taylor /* 16399e39c5baSBill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed 16409e39c5baSBill Taylor * from a certain number of "constrained" bits (the least significant 16419e39c5baSBill Taylor * bits) and some number of "unconstrained" bits. The constrained 16429e39c5baSBill Taylor * bits must be set to the index of the entry in the MPT table, but 16439e39c5baSBill Taylor * the unconstrained bits can be set to any value we wish. Note: 16449e39c5baSBill Taylor * if no remote access is required, then the RKey value is not filled 16459e39c5baSBill Taylor * in. Otherwise both Rkey and LKey are given the same value. 16469e39c5baSBill Taylor */ 16479e39c5baSBill Taylor if (mpt) 1648*c7facc54SBill Taylor mr->mr_rkey = mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx); 16499e39c5baSBill Taylor 16509e39c5baSBill Taylor /* 16519e39c5baSBill Taylor * Determine if the memory is from userland and pin the pages 16529e39c5baSBill Taylor * with umem_lockmemory() if necessary. 16539e39c5baSBill Taylor * Then, if this is userland memory, allocate an entry in the 16549e39c5baSBill Taylor * "userland resources database". This will later be added to 16559e39c5baSBill Taylor * the database (after all further memory registration operations are 16569e39c5baSBill Taylor * successful). If we fail here, we must undo the reference counts 16579e39c5baSBill Taylor * and the previous resource allocations. 16589e39c5baSBill Taylor */ 16599e39c5baSBill Taylor mr_is_umem = (((bind->bi_as != NULL) && (bind->bi_as != &kas)) ? 1 : 0); 16609e39c5baSBill Taylor if (mr_is_umem) { 16619e39c5baSBill Taylor umem_len = ptob(btopr(bind->bi_len + 16629e39c5baSBill Taylor ((uintptr_t)bind->bi_addr & PAGEOFFSET))); 16639e39c5baSBill Taylor umem_addr = (caddr_t)((uintptr_t)bind->bi_addr & ~PAGEOFFSET); 16649e39c5baSBill Taylor umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ | 16659e39c5baSBill Taylor DDI_UMEMLOCK_LONGTERM); 16669e39c5baSBill Taylor status = umem_lockmemory(umem_addr, umem_len, umem_flags, 1667d863b343SBill Taylor &umem_cookie, &hermon_umem_cbops, NULL); 16689e39c5baSBill Taylor if (status != 0) { 16699e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 16709e39c5baSBill Taylor goto mrcommon_fail3; 16719e39c5baSBill Taylor } 16729e39c5baSBill Taylor 16739e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind)) 16749e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind->bi_buf)) 16759e39c5baSBill Taylor 16769e39c5baSBill Taylor bind->bi_buf = ddi_umem_iosetup(umem_cookie, 0, umem_len, 16779e39c5baSBill Taylor B_WRITE, 0, 0, NULL, DDI_UMEM_SLEEP); 16789e39c5baSBill Taylor if (bind->bi_buf == NULL) { 16799e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 16809e39c5baSBill Taylor goto mrcommon_fail3; 16819e39c5baSBill Taylor } 16829e39c5baSBill Taylor bind->bi_type = HERMON_BINDHDL_UBUF; 16839e39c5baSBill Taylor bind->bi_buf->b_flags |= B_READ; 16849e39c5baSBill Taylor 16859e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind->bi_buf)) 16869e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind)) 16879e39c5baSBill Taylor 16889e39c5baSBill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance, 16899e39c5baSBill Taylor (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC, 16909e39c5baSBill Taylor (uint64_t)(uintptr_t)rsrc); 16919e39c5baSBill Taylor if (umapdb == NULL) { 16929e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 16939e39c5baSBill Taylor goto mrcommon_fail4; 16949e39c5baSBill Taylor } 16959e39c5baSBill Taylor } 16969e39c5baSBill Taylor 16979e39c5baSBill Taylor /* 16989e39c5baSBill Taylor * Setup the bindinfo for the mtt bind call 16999e39c5baSBill Taylor */ 17009e39c5baSBill Taylor bh = &mr->mr_bindinfo; 17019e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bh)) 17029e39c5baSBill Taylor bcopy(bind, bh, sizeof (hermon_bind_info_t)); 17039e39c5baSBill Taylor bh->bi_bypass = bind_type; 17049e39c5baSBill Taylor status = hermon_mr_mtt_bind(state, bh, bind_dmahdl, &mtt, 17059e39c5baSBill Taylor &mtt_pgsize_bits, mpt != NULL); 17069e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1707949b58c7SBill Taylor /* 1708949b58c7SBill Taylor * When mtt_bind fails, freerbuf has already been done, 1709949b58c7SBill Taylor * so make sure not to call it again. 1710949b58c7SBill Taylor */ 1711949b58c7SBill Taylor bind->bi_type = bh->bi_type; 17129e39c5baSBill Taylor goto mrcommon_fail5; 17139e39c5baSBill Taylor } 17149e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits; 17159e39c5baSBill Taylor 17169e39c5baSBill Taylor /* 17179e39c5baSBill Taylor * Allocate MTT reference count (to track shared memory regions). 17189e39c5baSBill Taylor * This reference count resource may never be used on the given 17199e39c5baSBill Taylor * memory region, but if it is ever later registered as "shared" 17209e39c5baSBill Taylor * memory region then this resource will be necessary. If we fail 17219e39c5baSBill Taylor * here, we do pretty much the same as above to clean up. 17229e39c5baSBill Taylor */ 17239e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_REFCNT, 1, sleep, 17249e39c5baSBill Taylor &mtt_refcnt); 17259e39c5baSBill Taylor if (status != DDI_SUCCESS) { 17269e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 17279e39c5baSBill Taylor goto mrcommon_fail6; 17289e39c5baSBill Taylor } 17299e39c5baSBill Taylor mr->mr_mttrefcntp = mtt_refcnt; 17309e39c5baSBill Taylor swrc_tmp = (hermon_sw_refcnt_t *)mtt_refcnt->hr_addr; 17319e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_tmp)) 17329e39c5baSBill Taylor HERMON_MTT_REFCNT_INIT(swrc_tmp); 17339e39c5baSBill Taylor 17349e39c5baSBill Taylor mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT); 17359e39c5baSBill Taylor 17369e39c5baSBill Taylor /* 17379e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing 17389e39c5baSBill Taylor * ownership of the MPT entry to the Hermon hardware. We use all of 17399e39c5baSBill Taylor * the information collected/calculated above to fill in the 17409e39c5baSBill Taylor * requisite portions of the MPT. Do this ONLY for DMPTs. 17419e39c5baSBill Taylor */ 17429e39c5baSBill Taylor if (mpt == NULL) 17439e39c5baSBill Taylor goto no_passown; 17449e39c5baSBill Taylor 17459e39c5baSBill Taylor bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t)); 17469e39c5baSBill Taylor 17479e39c5baSBill Taylor mpt_entry.status = HERMON_MPT_SW_OWNERSHIP; 17489e39c5baSBill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0; 17499e39c5baSBill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0; 17509e39c5baSBill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0; 17519e39c5baSBill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0; 17529e39c5baSBill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0; 17539e39c5baSBill Taylor mpt_entry.lr = 1; 17549e39c5baSBill Taylor mpt_entry.phys_addr = 0; 17559e39c5baSBill Taylor mpt_entry.reg_win = HERMON_MPT_IS_REGION; 17569e39c5baSBill Taylor 17579e39c5baSBill Taylor mpt_entry.entity_sz = mr->mr_logmttpgsz; 17589e39c5baSBill Taylor mpt_entry.mem_key = mr->mr_lkey; 17599e39c5baSBill Taylor mpt_entry.pd = pd->pd_pdnum; 17609e39c5baSBill Taylor mpt_entry.rem_acc_en = 0; 17619e39c5baSBill Taylor mpt_entry.fast_reg_en = 0; 17629e39c5baSBill Taylor mpt_entry.en_inval = 0; 17639e39c5baSBill Taylor mpt_entry.lkey = 0; 17649e39c5baSBill Taylor mpt_entry.win_cnt = 0; 17659e39c5baSBill Taylor 17669e39c5baSBill Taylor if (bind_override_addr == 0) { 17679e39c5baSBill Taylor mpt_entry.start_addr = bh->bi_addr; 17689e39c5baSBill Taylor } else { 17699e39c5baSBill Taylor bh->bi_addr = bh->bi_addr & ((1 << mr->mr_logmttpgsz) - 1); 17709e39c5baSBill Taylor mpt_entry.start_addr = bh->bi_addr; 17719e39c5baSBill Taylor } 17729e39c5baSBill Taylor mpt_entry.reg_win_len = bh->bi_len; 17739e39c5baSBill Taylor 17749e39c5baSBill Taylor mpt_entry.mtt_addr_h = mtt_addr >> 32; /* only 8 more bits */ 17759e39c5baSBill Taylor mpt_entry.mtt_addr_l = mtt_addr >> 3; /* only 29 bits */ 17769e39c5baSBill Taylor 17779e39c5baSBill Taylor /* 17789e39c5baSBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of 17799e39c5baSBill Taylor * the entry to the hardware if needed. Note: in general, this 17809e39c5baSBill Taylor * operation shouldn't fail. But if it does, we have to undo 17819e39c5baSBill Taylor * everything we've done above before returning error. 17829e39c5baSBill Taylor * 17839e39c5baSBill Taylor * For Hermon, this routine (which is common to the contexts) will only 17849e39c5baSBill Taylor * set the ownership if needed - the process of passing the context 17859e39c5baSBill Taylor * itself to HW will take care of setting up the MPT (based on type 17869e39c5baSBill Taylor * and index). 17879e39c5baSBill Taylor */ 17889e39c5baSBill Taylor 17899e39c5baSBill Taylor mpt_entry.bnd_qp = 0; /* dMPT for a qp, check for window */ 17909e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry, 17919e39c5baSBill Taylor sizeof (hermon_hw_dmpt_t), mpt->hr_indx, sleep); 17929e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 17939e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n", 17949e39c5baSBill Taylor status); 17959e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 17969e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 17979e39c5baSBill Taylor } 17989e39c5baSBill Taylor status = ibc_get_ci_failure(0); 17999e39c5baSBill Taylor goto mrcommon_fail7; 18009e39c5baSBill Taylor } 1801*c7facc54SBill Taylor if (hermon_rdma_debug & 0x4) 1802*c7facc54SBill Taylor IBTF_DPRINTF_L2("mr", " reg: mr %p key %x", 1803*c7facc54SBill Taylor mr, hermon_mr_key_swap(mr->mr_rkey)); 18049e39c5baSBill Taylor no_passown: 18059e39c5baSBill Taylor 18069e39c5baSBill Taylor /* 18079e39c5baSBill Taylor * Fill in the rest of the Hermon Memory Region handle. Having 18089e39c5baSBill Taylor * successfully transferred ownership of the MPT, we can update the 18099e39c5baSBill Taylor * following fields for use in further operations on the MR. 18109e39c5baSBill Taylor */ 18119e39c5baSBill Taylor mr->mr_mttaddr = mtt_addr; 18129e39c5baSBill Taylor 18139e39c5baSBill Taylor mr->mr_log2_pgsz = (mr->mr_logmttpgsz - HERMON_PAGESHIFT); 18149e39c5baSBill Taylor mr->mr_mptrsrcp = mpt; 18159e39c5baSBill Taylor mr->mr_mttrsrcp = mtt; 18169e39c5baSBill Taylor mr->mr_pdhdl = pd; 18179e39c5baSBill Taylor mr->mr_rsrcp = rsrc; 18189e39c5baSBill Taylor mr->mr_is_umem = mr_is_umem; 18199e39c5baSBill Taylor mr->mr_is_fmr = 0; 18209e39c5baSBill Taylor mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL; 18219e39c5baSBill Taylor mr->mr_umem_cbfunc = NULL; 18229e39c5baSBill Taylor mr->mr_umem_cbarg1 = NULL; 18239e39c5baSBill Taylor mr->mr_umem_cbarg2 = NULL; 18249e39c5baSBill Taylor mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey); 18259e39c5baSBill Taylor mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey); 18269e39c5baSBill Taylor mr->mr_mpt_type = mpt_type; 18279e39c5baSBill Taylor 18289e39c5baSBill Taylor /* 18299e39c5baSBill Taylor * If this is userland memory, then we need to insert the previously 18309e39c5baSBill Taylor * allocated entry into the "userland resources database". This will 18319e39c5baSBill Taylor * allow for later coordination between the hermon_umap_umemlock_cb() 18329e39c5baSBill Taylor * callback and hermon_mr_deregister(). 18339e39c5baSBill Taylor */ 18349e39c5baSBill Taylor if (mr_is_umem) { 18359e39c5baSBill Taylor hermon_umap_db_add(umapdb); 18369e39c5baSBill Taylor } 18379e39c5baSBill Taylor 18389e39c5baSBill Taylor *mrhdl = mr; 18399e39c5baSBill Taylor 18409e39c5baSBill Taylor return (DDI_SUCCESS); 18419e39c5baSBill Taylor 18429e39c5baSBill Taylor /* 18439e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 18449e39c5baSBill Taylor */ 18459e39c5baSBill Taylor mrcommon_fail7: 18469e39c5baSBill Taylor hermon_rsrc_free(state, &mtt_refcnt); 18479e39c5baSBill Taylor mrcommon_fail6: 18489e39c5baSBill Taylor hermon_mr_mem_unbind(state, bh); 184936e3c695SPramod Gunjikar bind->bi_type = bh->bi_type; 18509e39c5baSBill Taylor mrcommon_fail5: 18519e39c5baSBill Taylor if (mr_is_umem) { 18529e39c5baSBill Taylor hermon_umap_db_free(umapdb); 18539e39c5baSBill Taylor } 18549e39c5baSBill Taylor mrcommon_fail4: 18559e39c5baSBill Taylor if (mr_is_umem) { 18569e39c5baSBill Taylor /* 18579e39c5baSBill Taylor * Free up the memory ddi_umem_iosetup() allocates 18589e39c5baSBill Taylor * internally. 18599e39c5baSBill Taylor */ 18609e39c5baSBill Taylor if (bind->bi_type == HERMON_BINDHDL_UBUF) { 18619e39c5baSBill Taylor freerbuf(bind->bi_buf); 18629e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind)) 18639e39c5baSBill Taylor bind->bi_type = HERMON_BINDHDL_NONE; 18649e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind)) 18659e39c5baSBill Taylor } 18669e39c5baSBill Taylor ddi_umem_unlock(umem_cookie); 18679e39c5baSBill Taylor } 18689e39c5baSBill Taylor mrcommon_fail3: 18699e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 18709e39c5baSBill Taylor mrcommon_fail2: 18719e39c5baSBill Taylor if (mpt != NULL) 18729e39c5baSBill Taylor hermon_rsrc_free(state, &mpt); 18739e39c5baSBill Taylor mrcommon_fail1: 18749e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 18759e39c5baSBill Taylor mrcommon_fail: 18769e39c5baSBill Taylor return (status); 18779e39c5baSBill Taylor } 18789e39c5baSBill Taylor 18799e39c5baSBill Taylor /* 18809e39c5baSBill Taylor * hermon_mr_mtt_bind() 18819e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 18829e39c5baSBill Taylor */ 18839e39c5baSBill Taylor int 18849e39c5baSBill Taylor hermon_mr_mtt_bind(hermon_state_t *state, hermon_bind_info_t *bind, 18859e39c5baSBill Taylor ddi_dma_handle_t bind_dmahdl, hermon_rsrc_t **mtt, uint_t *mtt_pgsize_bits, 18869e39c5baSBill Taylor uint_t is_buffer) 18879e39c5baSBill Taylor { 18889e39c5baSBill Taylor uint64_t nummtt; 18899e39c5baSBill Taylor uint_t sleep; 18909e39c5baSBill Taylor int status; 18919e39c5baSBill Taylor 18929e39c5baSBill Taylor /* 18939e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 18949e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 18959e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 18969e39c5baSBill Taylor */ 18979e39c5baSBill Taylor sleep = (bind->bi_flags & IBT_MR_NOSLEEP) ? 18989e39c5baSBill Taylor HERMON_NOSLEEP : HERMON_SLEEP; 18999e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 19009e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 19019e39c5baSBill Taylor status = IBT_INVALID_PARAM; 19029e39c5baSBill Taylor goto mrmttbind_fail; 19039e39c5baSBill Taylor } 19049e39c5baSBill Taylor 19059e39c5baSBill Taylor /* 19069e39c5baSBill Taylor * Bind the memory and determine the mapped addresses. This is 19079e39c5baSBill Taylor * the first of two routines that do all the "heavy lifting" for 19089e39c5baSBill Taylor * the Hermon memory registration routines. The hermon_mr_mem_bind() 19099e39c5baSBill Taylor * routine takes the "bind" struct with all its fields filled 19109e39c5baSBill Taylor * in and returns a list of DMA cookies (for the PCI mapped addresses 19119e39c5baSBill Taylor * corresponding to the specified address region) which are used by 19129e39c5baSBill Taylor * the hermon_mr_fast_mtt_write() routine below. If we fail here, we 19139e39c5baSBill Taylor * must undo all the previous resource allocation (and PD reference 19149e39c5baSBill Taylor * count). 19159e39c5baSBill Taylor */ 19169e39c5baSBill Taylor status = hermon_mr_mem_bind(state, bind, bind_dmahdl, sleep, is_buffer); 19179e39c5baSBill Taylor if (status != DDI_SUCCESS) { 19189e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 19199e39c5baSBill Taylor goto mrmttbind_fail; 19209e39c5baSBill Taylor } 19219e39c5baSBill Taylor 19229e39c5baSBill Taylor /* 19239e39c5baSBill Taylor * Determine number of pages spanned. This routine uses the 19249e39c5baSBill Taylor * information in the "bind" struct to determine the required 19259e39c5baSBill Taylor * number of MTT entries needed (and returns the suggested page size - 19269e39c5baSBill Taylor * as a "power-of-2" - for each MTT entry). 19279e39c5baSBill Taylor */ 19289e39c5baSBill Taylor nummtt = hermon_mr_nummtt_needed(state, bind, mtt_pgsize_bits); 19299e39c5baSBill Taylor 19309e39c5baSBill Taylor /* 19319e39c5baSBill Taylor * Allocate the MTT entries. Use the calculations performed above to 19329e39c5baSBill Taylor * allocate the required number of MTT entries. If we fail here, we 19339e39c5baSBill Taylor * must not only undo all the previous resource allocation (and PD 19349e39c5baSBill Taylor * reference count), but we must also unbind the memory. 19359e39c5baSBill Taylor */ 19369e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt, sleep, mtt); 19379e39c5baSBill Taylor if (status != DDI_SUCCESS) { 19389e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 19399e39c5baSBill Taylor goto mrmttbind_fail2; 19409e39c5baSBill Taylor } 19419e39c5baSBill Taylor 19429e39c5baSBill Taylor /* 19439e39c5baSBill Taylor * Write the mapped addresses into the MTT entries. This is part two 19449e39c5baSBill Taylor * of the "heavy lifting" routines that we talked about above. Note: 19459e39c5baSBill Taylor * we pass the suggested page size from the earlier operation here. 19469e39c5baSBill Taylor * And if we fail here, we again do pretty much the same huge clean up. 19479e39c5baSBill Taylor */ 19489e39c5baSBill Taylor status = hermon_mr_fast_mtt_write(state, *mtt, bind, *mtt_pgsize_bits); 19499e39c5baSBill Taylor if (status != DDI_SUCCESS) { 19509e39c5baSBill Taylor /* 19519e39c5baSBill Taylor * hermon_mr_fast_mtt_write() returns DDI_FAILURE 19529e39c5baSBill Taylor * only if it detects a HW error during DMA. 19539e39c5baSBill Taylor */ 19549e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 19559e39c5baSBill Taylor status = ibc_get_ci_failure(0); 19569e39c5baSBill Taylor goto mrmttbind_fail3; 19579e39c5baSBill Taylor } 19589e39c5baSBill Taylor return (DDI_SUCCESS); 19599e39c5baSBill Taylor 19609e39c5baSBill Taylor /* 19619e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 19629e39c5baSBill Taylor */ 19639e39c5baSBill Taylor mrmttbind_fail3: 19649e39c5baSBill Taylor hermon_rsrc_free(state, mtt); 19659e39c5baSBill Taylor mrmttbind_fail2: 19669e39c5baSBill Taylor hermon_mr_mem_unbind(state, bind); 19679e39c5baSBill Taylor mrmttbind_fail: 19689e39c5baSBill Taylor return (status); 19699e39c5baSBill Taylor } 19709e39c5baSBill Taylor 19719e39c5baSBill Taylor 19729e39c5baSBill Taylor /* 19739e39c5baSBill Taylor * hermon_mr_mtt_unbind() 19749e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 19759e39c5baSBill Taylor */ 19769e39c5baSBill Taylor int 19779e39c5baSBill Taylor hermon_mr_mtt_unbind(hermon_state_t *state, hermon_bind_info_t *bind, 19789e39c5baSBill Taylor hermon_rsrc_t *mtt) 19799e39c5baSBill Taylor { 19809e39c5baSBill Taylor /* 19819e39c5baSBill Taylor * Free up the MTT entries and unbind the memory. Here, as above, we 19829e39c5baSBill Taylor * attempt to free these resources only if it is appropriate to do so. 19839e39c5baSBill Taylor */ 19849e39c5baSBill Taylor hermon_mr_mem_unbind(state, bind); 19859e39c5baSBill Taylor hermon_rsrc_free(state, &mtt); 19869e39c5baSBill Taylor 19879e39c5baSBill Taylor return (DDI_SUCCESS); 19889e39c5baSBill Taylor } 19899e39c5baSBill Taylor 19909e39c5baSBill Taylor 19919e39c5baSBill Taylor /* 19929e39c5baSBill Taylor * hermon_mr_common_rereg() 19939e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 19949e39c5baSBill Taylor */ 19959e39c5baSBill Taylor static int 19969e39c5baSBill Taylor hermon_mr_common_rereg(hermon_state_t *state, hermon_mrhdl_t mr, 19979e39c5baSBill Taylor hermon_pdhdl_t pd, hermon_bind_info_t *bind, hermon_mrhdl_t *mrhdl_new, 19989e39c5baSBill Taylor hermon_mr_options_t *op) 19999e39c5baSBill Taylor { 20009e39c5baSBill Taylor hermon_rsrc_t *mpt; 20019e39c5baSBill Taylor ibt_mr_attr_flags_t acc_flags_to_use; 20029e39c5baSBill Taylor ibt_mr_flags_t flags; 20039e39c5baSBill Taylor hermon_pdhdl_t pd_to_use; 20049e39c5baSBill Taylor hermon_hw_dmpt_t mpt_entry; 20059e39c5baSBill Taylor uint64_t mtt_addr_to_use, vaddr_to_use, len_to_use; 20069e39c5baSBill Taylor uint_t sleep, dereg_level; 20079e39c5baSBill Taylor int status; 20089e39c5baSBill Taylor 20099e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind)) 20109e39c5baSBill Taylor 20119e39c5baSBill Taylor /* 20129e39c5baSBill Taylor * Check here to see if the memory region corresponds to a userland 20139e39c5baSBill Taylor * mapping. Reregistration of userland memory regions is not 20149e39c5baSBill Taylor * currently supported. Return failure. 20159e39c5baSBill Taylor */ 20169e39c5baSBill Taylor if (mr->mr_is_umem) { 20179e39c5baSBill Taylor status = IBT_MR_HDL_INVALID; 20189e39c5baSBill Taylor goto mrrereg_fail; 20199e39c5baSBill Taylor } 20209e39c5baSBill Taylor 20219e39c5baSBill Taylor mutex_enter(&mr->mr_lock); 20229e39c5baSBill Taylor 20239e39c5baSBill Taylor /* Pull MPT resource pointer from the Hermon Memory Region handle */ 20249e39c5baSBill Taylor mpt = mr->mr_mptrsrcp; 20259e39c5baSBill Taylor 20269e39c5baSBill Taylor /* Extract the flags field from the hermon_bind_info_t */ 20279e39c5baSBill Taylor flags = bind->bi_flags; 20289e39c5baSBill Taylor 20299e39c5baSBill Taylor /* 20309e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the 20319e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt 20329e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep). 20339e39c5baSBill Taylor */ 20349e39c5baSBill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? HERMON_NOSLEEP: HERMON_SLEEP; 20359e39c5baSBill Taylor if ((sleep == HERMON_SLEEP) && 20369e39c5baSBill Taylor (sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) { 20379e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 20389e39c5baSBill Taylor status = IBT_INVALID_PARAM; 20399e39c5baSBill Taylor goto mrrereg_fail; 20409e39c5baSBill Taylor } 20419e39c5baSBill Taylor 20429e39c5baSBill Taylor /* 20439e39c5baSBill Taylor * First step is to temporarily invalidate the MPT entry. This 20449e39c5baSBill Taylor * regains ownership from the hardware, and gives us the opportunity 20459e39c5baSBill Taylor * to modify the entry. Note: The HW2SW_MPT command returns the 20469e39c5baSBill Taylor * current MPT entry contents. These are saved away here because 20479e39c5baSBill Taylor * they will be reused in a later step below. If the region has 20489e39c5baSBill Taylor * bound memory windows that we fail returning an "in use" error code. 20499e39c5baSBill Taylor * Otherwise, this is an unexpected error and we deregister the 20509e39c5baSBill Taylor * memory region and return error. 20519e39c5baSBill Taylor * 20529e39c5baSBill Taylor * We use HERMON_CMD_NOSLEEP_SPIN here always because we must protect 20539e39c5baSBill Taylor * against holding the lock around this rereg call in all contexts. 20549e39c5baSBill Taylor */ 20559e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, HW2SW_MPT, &mpt_entry, 20569e39c5baSBill Taylor sizeof (hermon_hw_dmpt_t), mpt->hr_indx, HERMON_CMD_NOSLEEP_SPIN); 20579e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 20589e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 20599e39c5baSBill Taylor if (status == HERMON_CMD_REG_BOUND) { 20609e39c5baSBill Taylor return (IBT_MR_IN_USE); 20619e39c5baSBill Taylor } else { 20629e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: HW2SW_MPT command failed: " 20639e39c5baSBill Taylor "%08x\n", status); 20649e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 20659e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 20669e39c5baSBill Taylor HCA_ERR_SRV_LOST); 20679e39c5baSBill Taylor } 20689e39c5baSBill Taylor /* 20699e39c5baSBill Taylor * Call deregister and ensure that all current 20709e39c5baSBill Taylor * resources get freed up 20719e39c5baSBill Taylor */ 20729e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, 20739e39c5baSBill Taylor HERMON_MR_DEREG_ALL, sleep) != DDI_SUCCESS) { 20749e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister " 20759e39c5baSBill Taylor "memory region"); 20769e39c5baSBill Taylor } 20779e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 20789e39c5baSBill Taylor } 20799e39c5baSBill Taylor } 20809e39c5baSBill Taylor 20819e39c5baSBill Taylor /* 20829e39c5baSBill Taylor * If we're changing the protection domain, then validate the new one 20839e39c5baSBill Taylor */ 20849e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_PD) { 20859e39c5baSBill Taylor 20869e39c5baSBill Taylor /* Check for valid PD handle pointer */ 20879e39c5baSBill Taylor if (pd == NULL) { 20889e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 20899e39c5baSBill Taylor /* 20909e39c5baSBill Taylor * Call deregister and ensure that all current 20919e39c5baSBill Taylor * resources get properly freed up. Unnecessary 20929e39c5baSBill Taylor * here to attempt to regain software ownership 20939e39c5baSBill Taylor * of the MPT entry as that has already been 20949e39c5baSBill Taylor * done above. 20959e39c5baSBill Taylor */ 20969e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, 20979e39c5baSBill Taylor HERMON_MR_DEREG_NO_HW2SW_MPT, sleep) != 20989e39c5baSBill Taylor DDI_SUCCESS) { 20999e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister " 21009e39c5baSBill Taylor "memory region"); 21019e39c5baSBill Taylor } 21029e39c5baSBill Taylor status = IBT_PD_HDL_INVALID; 21039e39c5baSBill Taylor goto mrrereg_fail; 21049e39c5baSBill Taylor } 21059e39c5baSBill Taylor 21069e39c5baSBill Taylor /* Use the new PD handle in all operations below */ 21079e39c5baSBill Taylor pd_to_use = pd; 21089e39c5baSBill Taylor 21099e39c5baSBill Taylor } else { 21109e39c5baSBill Taylor /* Use the current PD handle in all operations below */ 21119e39c5baSBill Taylor pd_to_use = mr->mr_pdhdl; 21129e39c5baSBill Taylor } 21139e39c5baSBill Taylor 21149e39c5baSBill Taylor /* 21159e39c5baSBill Taylor * If we're changing access permissions, then validate the new ones 21169e39c5baSBill Taylor */ 21179e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_ACCESS) { 21189e39c5baSBill Taylor /* 21199e39c5baSBill Taylor * Validate the access flags. Both remote write and remote 21209e39c5baSBill Taylor * atomic require the local write flag to be set 21219e39c5baSBill Taylor */ 21229e39c5baSBill Taylor if (((flags & IBT_MR_ENABLE_REMOTE_WRITE) || 21239e39c5baSBill Taylor (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) && 21249e39c5baSBill Taylor !(flags & IBT_MR_ENABLE_LOCAL_WRITE)) { 21259e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21269e39c5baSBill Taylor /* 21279e39c5baSBill Taylor * Call deregister and ensure that all current 21289e39c5baSBill Taylor * resources get properly freed up. Unnecessary 21299e39c5baSBill Taylor * here to attempt to regain software ownership 21309e39c5baSBill Taylor * of the MPT entry as that has already been 21319e39c5baSBill Taylor * done above. 21329e39c5baSBill Taylor */ 21339e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, 21349e39c5baSBill Taylor HERMON_MR_DEREG_NO_HW2SW_MPT, sleep) != 21359e39c5baSBill Taylor DDI_SUCCESS) { 21369e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister " 21379e39c5baSBill Taylor "memory region"); 21389e39c5baSBill Taylor } 21399e39c5baSBill Taylor status = IBT_MR_ACCESS_REQ_INVALID; 21409e39c5baSBill Taylor goto mrrereg_fail; 21419e39c5baSBill Taylor } 21429e39c5baSBill Taylor 21439e39c5baSBill Taylor /* 21449e39c5baSBill Taylor * Setup and validate the memory region access flags. This 21459e39c5baSBill Taylor * means translating the IBTF's enable flags into the access 21469e39c5baSBill Taylor * flags that will be used in later operations. 21479e39c5baSBill Taylor */ 21489e39c5baSBill Taylor acc_flags_to_use = 0; 21499e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_WINDOW_BIND) 21509e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_WINDOW_BIND; 21519e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_LOCAL_WRITE) 21529e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_LOCAL_WRITE; 21539e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_READ) 21549e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_REMOTE_READ; 21559e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_WRITE) 21569e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_REMOTE_WRITE; 21579e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC) 21589e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_REMOTE_ATOMIC; 21599e39c5baSBill Taylor 21609e39c5baSBill Taylor } else { 21619e39c5baSBill Taylor acc_flags_to_use = mr->mr_accflag; 21629e39c5baSBill Taylor } 21639e39c5baSBill Taylor 21649e39c5baSBill Taylor /* 21659e39c5baSBill Taylor * If we're modifying the translation, then figure out whether 21669e39c5baSBill Taylor * we can reuse the current MTT resources. This means calling 21679e39c5baSBill Taylor * hermon_mr_rereg_xlat_helper() which does most of the heavy lifting 21689e39c5baSBill Taylor * for the reregistration. If the current memory region contains 21699e39c5baSBill Taylor * sufficient MTT entries for the new regions, then it will be 21709e39c5baSBill Taylor * reused and filled in. Otherwise, new entries will be allocated, 21719e39c5baSBill Taylor * the old ones will be freed, and the new entries will be filled 21729e39c5baSBill Taylor * in. Note: If we're not modifying the translation, then we 21739e39c5baSBill Taylor * should already have all the information we need to update the MPT. 21749e39c5baSBill Taylor * Also note: If hermon_mr_rereg_xlat_helper() fails, it will return 21759e39c5baSBill Taylor * a "dereg_level" which is the level of cleanup that needs to be 21769e39c5baSBill Taylor * passed to hermon_mr_deregister() to finish the cleanup. 21779e39c5baSBill Taylor */ 21789e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_TRANSLATION) { 21799e39c5baSBill Taylor status = hermon_mr_rereg_xlat_helper(state, mr, bind, op, 21809e39c5baSBill Taylor &mtt_addr_to_use, sleep, &dereg_level); 21819e39c5baSBill Taylor if (status != DDI_SUCCESS) { 21829e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 21839e39c5baSBill Taylor /* 21849e39c5baSBill Taylor * Call deregister and ensure that all resources get 21859e39c5baSBill Taylor * properly freed up. 21869e39c5baSBill Taylor */ 21879e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, dereg_level, 21889e39c5baSBill Taylor sleep) != DDI_SUCCESS) { 21899e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister " 21909e39c5baSBill Taylor "memory region"); 21919e39c5baSBill Taylor } 21929e39c5baSBill Taylor goto mrrereg_fail; 21939e39c5baSBill Taylor } 21949e39c5baSBill Taylor vaddr_to_use = mr->mr_bindinfo.bi_addr; 21959e39c5baSBill Taylor len_to_use = mr->mr_bindinfo.bi_len; 21969e39c5baSBill Taylor } else { 21979e39c5baSBill Taylor mtt_addr_to_use = mr->mr_mttaddr; 21989e39c5baSBill Taylor vaddr_to_use = mr->mr_bindinfo.bi_addr; 21999e39c5baSBill Taylor len_to_use = mr->mr_bindinfo.bi_len; 22009e39c5baSBill Taylor } 22019e39c5baSBill Taylor 22029e39c5baSBill Taylor /* 22039e39c5baSBill Taylor * Calculate new keys (Lkey, Rkey) from MPT index. Just like they were 22049e39c5baSBill Taylor * when the region was first registered, each key is formed from 22059e39c5baSBill Taylor * "constrained" bits and "unconstrained" bits. Note: If no remote 22069e39c5baSBill Taylor * access is required, then the RKey value is not filled in. Otherwise 22079e39c5baSBill Taylor * both Rkey and LKey are given the same value. 22089e39c5baSBill Taylor */ 22099e39c5baSBill Taylor mr->mr_lkey = hermon_mr_keycalc(mpt->hr_indx); 22109e39c5baSBill Taylor if ((acc_flags_to_use & IBT_MR_REMOTE_READ) || 22119e39c5baSBill Taylor (acc_flags_to_use & IBT_MR_REMOTE_WRITE) || 22129e39c5baSBill Taylor (acc_flags_to_use & IBT_MR_REMOTE_ATOMIC)) { 22139e39c5baSBill Taylor mr->mr_rkey = mr->mr_lkey; 22149e39c5baSBill Taylor } else 22159e39c5baSBill Taylor mr->mr_rkey = 0; 22169e39c5baSBill Taylor 22179e39c5baSBill Taylor /* 22189e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing 22199e39c5baSBill Taylor * ownership of the MPT entry to the Hermon hardware. We use all of 22209e39c5baSBill Taylor * the information collected/calculated above to fill in the 22219e39c5baSBill Taylor * requisite portions of the MPT. 22229e39c5baSBill Taylor */ 22239e39c5baSBill Taylor bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t)); 22249e39c5baSBill Taylor 22259e39c5baSBill Taylor mpt_entry.status = HERMON_MPT_SW_OWNERSHIP; 22269e39c5baSBill Taylor mpt_entry.en_bind = (acc_flags_to_use & IBT_MR_WINDOW_BIND) ? 1 : 0; 22279e39c5baSBill Taylor mpt_entry.atomic = (acc_flags_to_use & IBT_MR_REMOTE_ATOMIC) ? 1 : 0; 22289e39c5baSBill Taylor mpt_entry.rw = (acc_flags_to_use & IBT_MR_REMOTE_WRITE) ? 1 : 0; 22299e39c5baSBill Taylor mpt_entry.rr = (acc_flags_to_use & IBT_MR_REMOTE_READ) ? 1 : 0; 22309e39c5baSBill Taylor mpt_entry.lw = (acc_flags_to_use & IBT_MR_LOCAL_WRITE) ? 1 : 0; 22319e39c5baSBill Taylor mpt_entry.lr = 1; 22329e39c5baSBill Taylor mpt_entry.phys_addr = 0; 22339e39c5baSBill Taylor mpt_entry.reg_win = HERMON_MPT_IS_REGION; 22349e39c5baSBill Taylor 22359e39c5baSBill Taylor mpt_entry.entity_sz = mr->mr_logmttpgsz; 22369e39c5baSBill Taylor mpt_entry.mem_key = mr->mr_lkey; 22379e39c5baSBill Taylor mpt_entry.pd = pd_to_use->pd_pdnum; 22389e39c5baSBill Taylor 22399e39c5baSBill Taylor mpt_entry.start_addr = vaddr_to_use; 22409e39c5baSBill Taylor mpt_entry.reg_win_len = len_to_use; 22419e39c5baSBill Taylor mpt_entry.mtt_addr_h = mtt_addr_to_use >> 32; 22429e39c5baSBill Taylor mpt_entry.mtt_addr_l = mtt_addr_to_use >> 3; 22439e39c5baSBill Taylor 22449e39c5baSBill Taylor /* 22459e39c5baSBill Taylor * Write the updated MPT entry to hardware 22469e39c5baSBill Taylor * 22479e39c5baSBill Taylor * We use HERMON_CMD_NOSLEEP_SPIN here always because we must protect 22489e39c5baSBill Taylor * against holding the lock around this rereg call in all contexts. 22499e39c5baSBill Taylor */ 22509e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry, 22519e39c5baSBill Taylor sizeof (hermon_hw_dmpt_t), mpt->hr_indx, HERMON_CMD_NOSLEEP_SPIN); 22529e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 22539e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 22549e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_MPT command failed: %08x\n", 22559e39c5baSBill Taylor status); 22569e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 22579e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 22589e39c5baSBill Taylor } 22599e39c5baSBill Taylor /* 22609e39c5baSBill Taylor * Call deregister and ensure that all current resources get 22619e39c5baSBill Taylor * properly freed up. Unnecessary here to attempt to regain 22629e39c5baSBill Taylor * software ownership of the MPT entry as that has already 22639e39c5baSBill Taylor * been done above. 22649e39c5baSBill Taylor */ 22659e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, 22669e39c5baSBill Taylor HERMON_MR_DEREG_NO_HW2SW_MPT, sleep) != DDI_SUCCESS) { 22679e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister memory " 22689e39c5baSBill Taylor "region"); 22699e39c5baSBill Taylor } 22709e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 22719e39c5baSBill Taylor } 22729e39c5baSBill Taylor 22739e39c5baSBill Taylor /* 22749e39c5baSBill Taylor * If we're changing PD, then update their reference counts now. 22759e39c5baSBill Taylor * This means decrementing the reference count on the old PD and 22769e39c5baSBill Taylor * incrementing the reference count on the new PD. 22779e39c5baSBill Taylor */ 22789e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_PD) { 22799e39c5baSBill Taylor hermon_pd_refcnt_dec(mr->mr_pdhdl); 22809e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 22819e39c5baSBill Taylor } 22829e39c5baSBill Taylor 22839e39c5baSBill Taylor /* 22849e39c5baSBill Taylor * Update the contents of the Hermon Memory Region handle to reflect 22859e39c5baSBill Taylor * what has been changed. 22869e39c5baSBill Taylor */ 22879e39c5baSBill Taylor mr->mr_pdhdl = pd_to_use; 22889e39c5baSBill Taylor mr->mr_accflag = acc_flags_to_use; 22899e39c5baSBill Taylor mr->mr_is_umem = 0; 22909e39c5baSBill Taylor mr->mr_is_fmr = 0; 22919e39c5baSBill Taylor mr->mr_umemcookie = NULL; 22929e39c5baSBill Taylor mr->mr_lkey = hermon_mr_key_swap(mr->mr_lkey); 22939e39c5baSBill Taylor mr->mr_rkey = hermon_mr_key_swap(mr->mr_rkey); 22949e39c5baSBill Taylor 22959e39c5baSBill Taylor /* New MR handle is same as the old */ 22969e39c5baSBill Taylor *mrhdl_new = mr; 22979e39c5baSBill Taylor mutex_exit(&mr->mr_lock); 22989e39c5baSBill Taylor 22999e39c5baSBill Taylor return (DDI_SUCCESS); 23009e39c5baSBill Taylor 23019e39c5baSBill Taylor mrrereg_fail: 23029e39c5baSBill Taylor return (status); 23039e39c5baSBill Taylor } 23049e39c5baSBill Taylor 23059e39c5baSBill Taylor 23069e39c5baSBill Taylor /* 23079e39c5baSBill Taylor * hermon_mr_rereg_xlat_helper 23089e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 23099e39c5baSBill Taylor * Note: This routine expects the "mr_lock" to be held when it 23109e39c5baSBill Taylor * is called. Upon returning failure, this routine passes information 23119e39c5baSBill Taylor * about what "dereg_level" should be passed to hermon_mr_deregister(). 23129e39c5baSBill Taylor */ 23139e39c5baSBill Taylor static int 23149e39c5baSBill Taylor hermon_mr_rereg_xlat_helper(hermon_state_t *state, hermon_mrhdl_t mr, 23159e39c5baSBill Taylor hermon_bind_info_t *bind, hermon_mr_options_t *op, uint64_t *mtt_addr, 23169e39c5baSBill Taylor uint_t sleep, uint_t *dereg_level) 23179e39c5baSBill Taylor { 23189e39c5baSBill Taylor hermon_rsrc_t *mtt, *mtt_refcnt; 23199e39c5baSBill Taylor hermon_sw_refcnt_t *swrc_old, *swrc_new; 23209e39c5baSBill Taylor ddi_dma_handle_t dmahdl; 23219e39c5baSBill Taylor uint64_t nummtt_needed, nummtt_in_currrsrc, max_sz; 23229e39c5baSBill Taylor uint_t mtt_pgsize_bits, bind_type, reuse_dmahdl; 23239e39c5baSBill Taylor int status; 23249e39c5baSBill Taylor 23259e39c5baSBill Taylor ASSERT(MUTEX_HELD(&mr->mr_lock)); 23269e39c5baSBill Taylor 23279e39c5baSBill Taylor /* 23289e39c5baSBill Taylor * Check the "options" flag. Currently this flag tells the driver 23299e39c5baSBill Taylor * whether or not the region should be bound normally (i.e. with 23309e39c5baSBill Taylor * entries written into the PCI IOMMU) or whether it should be 23319e39c5baSBill Taylor * registered to bypass the IOMMU. 23329e39c5baSBill Taylor */ 23339e39c5baSBill Taylor if (op == NULL) { 23349e39c5baSBill Taylor bind_type = HERMON_BINDMEM_NORMAL; 23359e39c5baSBill Taylor } else { 23369e39c5baSBill Taylor bind_type = op->mro_bind_type; 23379e39c5baSBill Taylor } 23389e39c5baSBill Taylor 23399e39c5baSBill Taylor /* 23409e39c5baSBill Taylor * Check for invalid length. Check is the length is zero or if the 23419e39c5baSBill Taylor * length is larger than the maximum configured value. Return error 23429e39c5baSBill Taylor * if it is. 23439e39c5baSBill Taylor */ 23449e39c5baSBill Taylor max_sz = ((uint64_t)1 << state->hs_cfg_profile->cp_log_max_mrw_sz); 23459e39c5baSBill Taylor if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) { 23469e39c5baSBill Taylor /* 23479e39c5baSBill Taylor * Deregister will be called upon returning failure from this 23489e39c5baSBill Taylor * routine. This will ensure that all current resources get 23499e39c5baSBill Taylor * properly freed up. Unnecessary to attempt to regain 23509e39c5baSBill Taylor * software ownership of the MPT entry as that has already 23519e39c5baSBill Taylor * been done above (in hermon_mr_reregister()) 23529e39c5baSBill Taylor */ 23539e39c5baSBill Taylor *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT; 23549e39c5baSBill Taylor 23559e39c5baSBill Taylor status = IBT_MR_LEN_INVALID; 23569e39c5baSBill Taylor goto mrrereghelp_fail; 23579e39c5baSBill Taylor } 23589e39c5baSBill Taylor 23599e39c5baSBill Taylor /* 23609e39c5baSBill Taylor * Determine the number of pages necessary for new region and the 23619e39c5baSBill Taylor * number of pages supported by the current MTT resources 23629e39c5baSBill Taylor */ 23639e39c5baSBill Taylor nummtt_needed = hermon_mr_nummtt_needed(state, bind, &mtt_pgsize_bits); 23649e39c5baSBill Taylor nummtt_in_currrsrc = mr->mr_mttrsrcp->hr_len >> HERMON_MTT_SIZE_SHIFT; 23659e39c5baSBill Taylor 23669e39c5baSBill Taylor /* 23679e39c5baSBill Taylor * Depending on whether we have enough pages or not, the next step is 23689e39c5baSBill Taylor * to fill in a set of MTT entries that reflect the new mapping. In 23699e39c5baSBill Taylor * the first case below, we already have enough entries. This means 23709e39c5baSBill Taylor * we need to unbind the memory from the previous mapping, bind the 23719e39c5baSBill Taylor * memory for the new mapping, write the new MTT entries, and update 23729e39c5baSBill Taylor * the mr to reflect the changes. 23739e39c5baSBill Taylor * In the second case below, we do not have enough entries in the 23749e39c5baSBill Taylor * current mapping. So, in this case, we need not only to unbind the 23759e39c5baSBill Taylor * current mapping, but we need to free up the MTT resources associated 23769e39c5baSBill Taylor * with that mapping. After we've successfully done that, we continue 23779e39c5baSBill Taylor * by binding the new memory, allocating new MTT entries, writing the 23789e39c5baSBill Taylor * new MTT entries, and updating the mr to reflect the changes. 23799e39c5baSBill Taylor */ 23809e39c5baSBill Taylor 23819e39c5baSBill Taylor /* 23829e39c5baSBill Taylor * If this region is being shared (i.e. MTT refcount != 1), then we 23839e39c5baSBill Taylor * can't reuse the current MTT resources regardless of their size. 23849e39c5baSBill Taylor * Instead we'll need to alloc new ones (below) just as if there 23859e39c5baSBill Taylor * hadn't been enough room in the current entries. 23869e39c5baSBill Taylor */ 23879e39c5baSBill Taylor swrc_old = (hermon_sw_refcnt_t *)mr->mr_mttrefcntp->hr_addr; 23889e39c5baSBill Taylor if (HERMON_MTT_IS_NOT_SHARED(swrc_old) && 23899e39c5baSBill Taylor (nummtt_needed <= nummtt_in_currrsrc)) { 23909e39c5baSBill Taylor 23919e39c5baSBill Taylor /* 23929e39c5baSBill Taylor * Unbind the old mapping for this memory region, but retain 23939e39c5baSBill Taylor * the ddi_dma_handle_t (if possible) for reuse in the bind 23949e39c5baSBill Taylor * operation below. Note: If original memory region was 23959e39c5baSBill Taylor * bound for IOMMU bypass and the new region can not use 23969e39c5baSBill Taylor * bypass, then a new DMA handle will be necessary. 23979e39c5baSBill Taylor */ 23989e39c5baSBill Taylor if (HERMON_MR_REUSE_DMAHDL(mr, bind->bi_flags)) { 23999e39c5baSBill Taylor mr->mr_bindinfo.bi_free_dmahdl = 0; 24009e39c5baSBill Taylor hermon_mr_mem_unbind(state, &mr->mr_bindinfo); 24019e39c5baSBill Taylor dmahdl = mr->mr_bindinfo.bi_dmahdl; 24029e39c5baSBill Taylor reuse_dmahdl = 1; 24039e39c5baSBill Taylor } else { 24049e39c5baSBill Taylor hermon_mr_mem_unbind(state, &mr->mr_bindinfo); 24059e39c5baSBill Taylor dmahdl = NULL; 24069e39c5baSBill Taylor reuse_dmahdl = 0; 24079e39c5baSBill Taylor } 24089e39c5baSBill Taylor 24099e39c5baSBill Taylor /* 24109e39c5baSBill Taylor * Bind the new memory and determine the mapped addresses. 24119e39c5baSBill Taylor * As described, this routine and hermon_mr_fast_mtt_write() 24129e39c5baSBill Taylor * do the majority of the work for the memory registration 24139e39c5baSBill Taylor * operations. Note: When we successfully finish the binding, 24149e39c5baSBill Taylor * we will set the "bi_free_dmahdl" flag to indicate that 24159e39c5baSBill Taylor * even though we may have reused the ddi_dma_handle_t we do 24169e39c5baSBill Taylor * wish it to be freed up at some later time. Note also that 24179e39c5baSBill Taylor * if we fail, we may need to cleanup the ddi_dma_handle_t. 24189e39c5baSBill Taylor */ 24199e39c5baSBill Taylor bind->bi_bypass = bind_type; 24209e39c5baSBill Taylor status = hermon_mr_mem_bind(state, bind, dmahdl, sleep, 1); 24219e39c5baSBill Taylor if (status != DDI_SUCCESS) { 24229e39c5baSBill Taylor if (reuse_dmahdl) { 24239e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl); 24249e39c5baSBill Taylor } 24259e39c5baSBill Taylor 24269e39c5baSBill Taylor /* 24279e39c5baSBill Taylor * Deregister will be called upon returning failure 24289e39c5baSBill Taylor * from this routine. This will ensure that all 24299e39c5baSBill Taylor * current resources get properly freed up. 24309e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership 24319e39c5baSBill Taylor * of the MPT entry as that has already been done 24329e39c5baSBill Taylor * above (in hermon_mr_reregister()). Also unnecessary 24339e39c5baSBill Taylor * to attempt to unbind the memory. 24349e39c5baSBill Taylor */ 24359e39c5baSBill Taylor *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND; 24369e39c5baSBill Taylor 24379e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 24389e39c5baSBill Taylor goto mrrereghelp_fail; 24399e39c5baSBill Taylor } 24409e39c5baSBill Taylor if (reuse_dmahdl) { 24419e39c5baSBill Taylor bind->bi_free_dmahdl = 1; 24429e39c5baSBill Taylor } 24439e39c5baSBill Taylor 24449e39c5baSBill Taylor /* 24459e39c5baSBill Taylor * Using the new mapping, but reusing the current MTT 24469e39c5baSBill Taylor * resources, write the updated entries to MTT 24479e39c5baSBill Taylor */ 24489e39c5baSBill Taylor mtt = mr->mr_mttrsrcp; 24499e39c5baSBill Taylor status = hermon_mr_fast_mtt_write(state, mtt, bind, 24509e39c5baSBill Taylor mtt_pgsize_bits); 24519e39c5baSBill Taylor if (status != DDI_SUCCESS) { 24529e39c5baSBill Taylor /* 24539e39c5baSBill Taylor * Deregister will be called upon returning failure 24549e39c5baSBill Taylor * from this routine. This will ensure that all 24559e39c5baSBill Taylor * current resources get properly freed up. 24569e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership 24579e39c5baSBill Taylor * of the MPT entry as that has already been done 24589e39c5baSBill Taylor * above (in hermon_mr_reregister()). Also unnecessary 24599e39c5baSBill Taylor * to attempt to unbind the memory. 24609e39c5baSBill Taylor * 24619e39c5baSBill Taylor * But we do need to unbind the newly bound memory 24629e39c5baSBill Taylor * before returning. 24639e39c5baSBill Taylor */ 24649e39c5baSBill Taylor hermon_mr_mem_unbind(state, bind); 24659e39c5baSBill Taylor *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND; 24669e39c5baSBill Taylor 24679e39c5baSBill Taylor /* 24689e39c5baSBill Taylor * hermon_mr_fast_mtt_write() returns DDI_FAILURE 24699e39c5baSBill Taylor * only if it detects a HW error during DMA. 24709e39c5baSBill Taylor */ 24719e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 24729e39c5baSBill Taylor status = ibc_get_ci_failure(0); 24739e39c5baSBill Taylor goto mrrereghelp_fail; 24749e39c5baSBill Taylor } 24759e39c5baSBill Taylor 24769e39c5baSBill Taylor /* Put the updated information into the Mem Region handle */ 24779e39c5baSBill Taylor mr->mr_bindinfo = *bind; 24789e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits; 24799e39c5baSBill Taylor 24809e39c5baSBill Taylor } else { 24819e39c5baSBill Taylor /* 24829e39c5baSBill Taylor * Check if the memory region MTT is shared by any other MRs. 24839e39c5baSBill Taylor * Since the resource may be shared between multiple memory 24849e39c5baSBill Taylor * regions (as a result of a "RegisterSharedMR()" verb) it is 24859e39c5baSBill Taylor * important that we not unbind any resources prematurely. 24869e39c5baSBill Taylor */ 24879e39c5baSBill Taylor if (!HERMON_MTT_IS_SHARED(swrc_old)) { 24889e39c5baSBill Taylor /* 24899e39c5baSBill Taylor * Unbind the old mapping for this memory region, but 24909e39c5baSBill Taylor * retain the ddi_dma_handle_t for reuse in the bind 24919e39c5baSBill Taylor * operation below. Note: This can only be done here 24929e39c5baSBill Taylor * because the region being reregistered is not 24939e39c5baSBill Taylor * currently shared. Also if original memory region 24949e39c5baSBill Taylor * was bound for IOMMU bypass and the new region can 24959e39c5baSBill Taylor * not use bypass, then a new DMA handle will be 24969e39c5baSBill Taylor * necessary. 24979e39c5baSBill Taylor */ 24989e39c5baSBill Taylor if (HERMON_MR_REUSE_DMAHDL(mr, bind->bi_flags)) { 24999e39c5baSBill Taylor mr->mr_bindinfo.bi_free_dmahdl = 0; 25009e39c5baSBill Taylor hermon_mr_mem_unbind(state, &mr->mr_bindinfo); 25019e39c5baSBill Taylor dmahdl = mr->mr_bindinfo.bi_dmahdl; 25029e39c5baSBill Taylor reuse_dmahdl = 1; 25039e39c5baSBill Taylor } else { 25049e39c5baSBill Taylor hermon_mr_mem_unbind(state, &mr->mr_bindinfo); 25059e39c5baSBill Taylor dmahdl = NULL; 25069e39c5baSBill Taylor reuse_dmahdl = 0; 25079e39c5baSBill Taylor } 25089e39c5baSBill Taylor } else { 25099e39c5baSBill Taylor dmahdl = NULL; 25109e39c5baSBill Taylor reuse_dmahdl = 0; 25119e39c5baSBill Taylor } 25129e39c5baSBill Taylor 25139e39c5baSBill Taylor /* 25149e39c5baSBill Taylor * Bind the new memory and determine the mapped addresses. 25159e39c5baSBill Taylor * As described, this routine and hermon_mr_fast_mtt_write() 25169e39c5baSBill Taylor * do the majority of the work for the memory registration 25179e39c5baSBill Taylor * operations. Note: When we successfully finish the binding, 25189e39c5baSBill Taylor * we will set the "bi_free_dmahdl" flag to indicate that 25199e39c5baSBill Taylor * even though we may have reused the ddi_dma_handle_t we do 25209e39c5baSBill Taylor * wish it to be freed up at some later time. Note also that 25219e39c5baSBill Taylor * if we fail, we may need to cleanup the ddi_dma_handle_t. 25229e39c5baSBill Taylor */ 25239e39c5baSBill Taylor bind->bi_bypass = bind_type; 25249e39c5baSBill Taylor status = hermon_mr_mem_bind(state, bind, dmahdl, sleep, 1); 25259e39c5baSBill Taylor if (status != DDI_SUCCESS) { 25269e39c5baSBill Taylor if (reuse_dmahdl) { 25279e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl); 25289e39c5baSBill Taylor } 25299e39c5baSBill Taylor 25309e39c5baSBill Taylor /* 25319e39c5baSBill Taylor * Deregister will be called upon returning failure 25329e39c5baSBill Taylor * from this routine. This will ensure that all 25339e39c5baSBill Taylor * current resources get properly freed up. 25349e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership 25359e39c5baSBill Taylor * of the MPT entry as that has already been done 25369e39c5baSBill Taylor * above (in hermon_mr_reregister()). Also unnecessary 25379e39c5baSBill Taylor * to attempt to unbind the memory. 25389e39c5baSBill Taylor */ 25399e39c5baSBill Taylor *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND; 25409e39c5baSBill Taylor 25419e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 25429e39c5baSBill Taylor goto mrrereghelp_fail; 25439e39c5baSBill Taylor } 25449e39c5baSBill Taylor if (reuse_dmahdl) { 25459e39c5baSBill Taylor bind->bi_free_dmahdl = 1; 25469e39c5baSBill Taylor } 25479e39c5baSBill Taylor 25489e39c5baSBill Taylor /* 25499e39c5baSBill Taylor * Allocate the new MTT entries resource 25509e39c5baSBill Taylor */ 25519e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_MTT, nummtt_needed, 25529e39c5baSBill Taylor sleep, &mtt); 25539e39c5baSBill Taylor if (status != DDI_SUCCESS) { 25549e39c5baSBill Taylor /* 25559e39c5baSBill Taylor * Deregister will be called upon returning failure 25569e39c5baSBill Taylor * from this routine. This will ensure that all 25579e39c5baSBill Taylor * current resources get properly freed up. 25589e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership 25599e39c5baSBill Taylor * of the MPT entry as that has already been done 25609e39c5baSBill Taylor * above (in hermon_mr_reregister()). Also unnecessary 25619e39c5baSBill Taylor * to attempt to unbind the memory. 25629e39c5baSBill Taylor * 25639e39c5baSBill Taylor * But we do need to unbind the newly bound memory 25649e39c5baSBill Taylor * before returning. 25659e39c5baSBill Taylor */ 25669e39c5baSBill Taylor hermon_mr_mem_unbind(state, bind); 25679e39c5baSBill Taylor *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND; 25689e39c5baSBill Taylor 25699e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 25709e39c5baSBill Taylor goto mrrereghelp_fail; 25719e39c5baSBill Taylor } 25729e39c5baSBill Taylor 25739e39c5baSBill Taylor /* 25749e39c5baSBill Taylor * Allocate MTT reference count (to track shared memory 25759e39c5baSBill Taylor * regions). As mentioned elsewhere above, this reference 25769e39c5baSBill Taylor * count resource may never be used on the given memory region, 25779e39c5baSBill Taylor * but if it is ever later registered as a "shared" memory 25789e39c5baSBill Taylor * region then this resource will be necessary. Note: This 25799e39c5baSBill Taylor * is only necessary here if the existing memory region is 25809e39c5baSBill Taylor * already being shared (because otherwise we already have 25819e39c5baSBill Taylor * a useable reference count resource). 25829e39c5baSBill Taylor */ 25839e39c5baSBill Taylor if (HERMON_MTT_IS_SHARED(swrc_old)) { 25849e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_REFCNT, 1, 25859e39c5baSBill Taylor sleep, &mtt_refcnt); 25869e39c5baSBill Taylor if (status != DDI_SUCCESS) { 25879e39c5baSBill Taylor /* 25889e39c5baSBill Taylor * Deregister will be called upon returning 25899e39c5baSBill Taylor * failure from this routine. This will ensure 25909e39c5baSBill Taylor * that all current resources get properly 25919e39c5baSBill Taylor * freed up. Unnecessary to attempt to regain 25929e39c5baSBill Taylor * software ownership of the MPT entry as that 25939e39c5baSBill Taylor * has already been done above (in 25949e39c5baSBill Taylor * hermon_mr_reregister()). Also unnecessary 25959e39c5baSBill Taylor * to attempt to unbind the memory. 25969e39c5baSBill Taylor * 25979e39c5baSBill Taylor * But we need to unbind the newly bound 25989e39c5baSBill Taylor * memory and free up the newly allocated MTT 25999e39c5baSBill Taylor * entries before returning. 26009e39c5baSBill Taylor */ 26019e39c5baSBill Taylor hermon_mr_mem_unbind(state, bind); 26029e39c5baSBill Taylor hermon_rsrc_free(state, &mtt); 26039e39c5baSBill Taylor *dereg_level = 26049e39c5baSBill Taylor HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND; 26059e39c5baSBill Taylor 26069e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 26079e39c5baSBill Taylor goto mrrereghelp_fail; 26089e39c5baSBill Taylor } 26099e39c5baSBill Taylor swrc_new = (hermon_sw_refcnt_t *)mtt_refcnt->hr_addr; 26109e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_new)) 26119e39c5baSBill Taylor HERMON_MTT_REFCNT_INIT(swrc_new); 26129e39c5baSBill Taylor } else { 26139e39c5baSBill Taylor mtt_refcnt = mr->mr_mttrefcntp; 26149e39c5baSBill Taylor } 26159e39c5baSBill Taylor 26169e39c5baSBill Taylor /* 26179e39c5baSBill Taylor * Using the new mapping and the new MTT resources, write the 26189e39c5baSBill Taylor * updated entries to MTT 26199e39c5baSBill Taylor */ 26209e39c5baSBill Taylor status = hermon_mr_fast_mtt_write(state, mtt, bind, 26219e39c5baSBill Taylor mtt_pgsize_bits); 26229e39c5baSBill Taylor if (status != DDI_SUCCESS) { 26239e39c5baSBill Taylor /* 26249e39c5baSBill Taylor * Deregister will be called upon returning failure 26259e39c5baSBill Taylor * from this routine. This will ensure that all 26269e39c5baSBill Taylor * current resources get properly freed up. 26279e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership 26289e39c5baSBill Taylor * of the MPT entry as that has already been done 26299e39c5baSBill Taylor * above (in hermon_mr_reregister()). Also unnecessary 26309e39c5baSBill Taylor * to attempt to unbind the memory. 26319e39c5baSBill Taylor * 26329e39c5baSBill Taylor * But we need to unbind the newly bound memory, 26339e39c5baSBill Taylor * free up the newly allocated MTT entries, and 26349e39c5baSBill Taylor * (possibly) free the new MTT reference count 26359e39c5baSBill Taylor * resource before returning. 26369e39c5baSBill Taylor */ 26379e39c5baSBill Taylor if (HERMON_MTT_IS_SHARED(swrc_old)) { 26389e39c5baSBill Taylor hermon_rsrc_free(state, &mtt_refcnt); 26399e39c5baSBill Taylor } 26409e39c5baSBill Taylor hermon_mr_mem_unbind(state, bind); 26419e39c5baSBill Taylor hermon_rsrc_free(state, &mtt); 26429e39c5baSBill Taylor *dereg_level = HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND; 26439e39c5baSBill Taylor 26449e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 26459e39c5baSBill Taylor goto mrrereghelp_fail; 26469e39c5baSBill Taylor } 26479e39c5baSBill Taylor 26489e39c5baSBill Taylor /* 26499e39c5baSBill Taylor * Check if the memory region MTT is shared by any other MRs. 26509e39c5baSBill Taylor * Since the resource may be shared between multiple memory 26519e39c5baSBill Taylor * regions (as a result of a "RegisterSharedMR()" verb) it is 26529e39c5baSBill Taylor * important that we not free up any resources prematurely. 26539e39c5baSBill Taylor */ 26549e39c5baSBill Taylor if (HERMON_MTT_IS_SHARED(swrc_old)) { 26559e39c5baSBill Taylor /* Decrement MTT reference count for "old" region */ 26569e39c5baSBill Taylor (void) hermon_mtt_refcnt_dec(mr->mr_mttrefcntp); 26579e39c5baSBill Taylor } else { 26589e39c5baSBill Taylor /* Free up the old MTT entries resource */ 26599e39c5baSBill Taylor hermon_rsrc_free(state, &mr->mr_mttrsrcp); 26609e39c5baSBill Taylor } 26619e39c5baSBill Taylor 26629e39c5baSBill Taylor /* Put the updated information into the mrhdl */ 26639e39c5baSBill Taylor mr->mr_bindinfo = *bind; 26649e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits; 26659e39c5baSBill Taylor mr->mr_mttrsrcp = mtt; 26669e39c5baSBill Taylor mr->mr_mttrefcntp = mtt_refcnt; 26679e39c5baSBill Taylor } 26689e39c5baSBill Taylor 26699e39c5baSBill Taylor /* 26709e39c5baSBill Taylor * Calculate and return the updated MTT address (in the DDR address 26719e39c5baSBill Taylor * space). This will be used by the caller (hermon_mr_reregister) in 26729e39c5baSBill Taylor * the updated MPT entry 26739e39c5baSBill Taylor */ 26749e39c5baSBill Taylor *mtt_addr = mtt->hr_indx << HERMON_MTT_SIZE_SHIFT; 26759e39c5baSBill Taylor 26769e39c5baSBill Taylor return (DDI_SUCCESS); 26779e39c5baSBill Taylor 26789e39c5baSBill Taylor mrrereghelp_fail: 26799e39c5baSBill Taylor return (status); 26809e39c5baSBill Taylor } 26819e39c5baSBill Taylor 26829e39c5baSBill Taylor 26839e39c5baSBill Taylor /* 26849e39c5baSBill Taylor * hermon_mr_nummtt_needed() 26859e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 26869e39c5baSBill Taylor */ 26879e39c5baSBill Taylor /* ARGSUSED */ 26889e39c5baSBill Taylor static uint64_t 26899e39c5baSBill Taylor hermon_mr_nummtt_needed(hermon_state_t *state, hermon_bind_info_t *bind, 26909e39c5baSBill Taylor uint_t *mtt_pgsize_bits) 26919e39c5baSBill Taylor { 26929e39c5baSBill Taylor uint64_t pg_offset_mask; 26939e39c5baSBill Taylor uint64_t pg_offset, tmp_length; 26949e39c5baSBill Taylor 26959e39c5baSBill Taylor /* 26969e39c5baSBill Taylor * For now we specify the page size as 8Kb (the default page size for 26979e39c5baSBill Taylor * the sun4u architecture), or 4Kb for x86. Figure out optimal page 26989e39c5baSBill Taylor * size by examining the dmacookies 26999e39c5baSBill Taylor */ 27009e39c5baSBill Taylor *mtt_pgsize_bits = PAGESHIFT; 27019e39c5baSBill Taylor 27029e39c5baSBill Taylor pg_offset_mask = ((uint64_t)1 << *mtt_pgsize_bits) - 1; 27039e39c5baSBill Taylor pg_offset = bind->bi_addr & pg_offset_mask; 27049e39c5baSBill Taylor tmp_length = pg_offset + (bind->bi_len - 1); 27059e39c5baSBill Taylor return ((tmp_length >> *mtt_pgsize_bits) + 1); 27069e39c5baSBill Taylor } 27079e39c5baSBill Taylor 27089e39c5baSBill Taylor 27099e39c5baSBill Taylor /* 27109e39c5baSBill Taylor * hermon_mr_mem_bind() 27119e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 27129e39c5baSBill Taylor */ 27139e39c5baSBill Taylor static int 27149e39c5baSBill Taylor hermon_mr_mem_bind(hermon_state_t *state, hermon_bind_info_t *bind, 27159e39c5baSBill Taylor ddi_dma_handle_t dmahdl, uint_t sleep, uint_t is_buffer) 27169e39c5baSBill Taylor { 27179e39c5baSBill Taylor ddi_dma_attr_t dma_attr; 27189e39c5baSBill Taylor int (*callback)(caddr_t); 27199e39c5baSBill Taylor int status; 27209e39c5baSBill Taylor 27219e39c5baSBill Taylor /* bi_type must be set to a meaningful value to get a bind handle */ 27229e39c5baSBill Taylor ASSERT(bind->bi_type == HERMON_BINDHDL_VADDR || 27239e39c5baSBill Taylor bind->bi_type == HERMON_BINDHDL_BUF || 27249e39c5baSBill Taylor bind->bi_type == HERMON_BINDHDL_UBUF); 27259e39c5baSBill Taylor 27269e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind)) 27279e39c5baSBill Taylor 27289e39c5baSBill Taylor /* Set the callback flag appropriately */ 27299e39c5baSBill Taylor callback = (sleep == HERMON_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT; 27309e39c5baSBill Taylor 27319e39c5baSBill Taylor /* 27329e39c5baSBill Taylor * Initialize many of the default DMA attributes. Then, if we're 27339e39c5baSBill Taylor * bypassing the IOMMU, set the DDI_DMA_FORCE_PHYSICAL flag. 27349e39c5baSBill Taylor */ 27359e39c5baSBill Taylor if (dmahdl == NULL) { 27369e39c5baSBill Taylor hermon_dma_attr_init(state, &dma_attr); 27379e39c5baSBill Taylor #ifdef __sparc 27389e39c5baSBill Taylor if (bind->bi_bypass == HERMON_BINDMEM_BYPASS) { 27399e39c5baSBill Taylor dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL; 27409e39c5baSBill Taylor } 27419e39c5baSBill Taylor #endif 27429e39c5baSBill Taylor 27439e39c5baSBill Taylor /* set RO if needed - tunable set and 'is_buffer' is non-0 */ 27449e39c5baSBill Taylor if (is_buffer) { 27459e39c5baSBill Taylor if (! (bind->bi_flags & IBT_MR_DISABLE_RO)) { 27469e39c5baSBill Taylor if ((bind->bi_type != HERMON_BINDHDL_UBUF) && 27479e39c5baSBill Taylor (hermon_kernel_data_ro == 27489e39c5baSBill Taylor HERMON_RO_ENABLED)) { 27499e39c5baSBill Taylor dma_attr.dma_attr_flags |= 27509e39c5baSBill Taylor DDI_DMA_RELAXED_ORDERING; 27519e39c5baSBill Taylor } 27529e39c5baSBill Taylor if (((bind->bi_type == HERMON_BINDHDL_UBUF) && 27539e39c5baSBill Taylor (hermon_user_data_ro == 27549e39c5baSBill Taylor HERMON_RO_ENABLED))) { 27559e39c5baSBill Taylor dma_attr.dma_attr_flags |= 27569e39c5baSBill Taylor DDI_DMA_RELAXED_ORDERING; 27579e39c5baSBill Taylor } 27589e39c5baSBill Taylor } 27599e39c5baSBill Taylor } 27609e39c5baSBill Taylor 27619e39c5baSBill Taylor /* Allocate a DMA handle for the binding */ 27629e39c5baSBill Taylor status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, 27639e39c5baSBill Taylor callback, NULL, &bind->bi_dmahdl); 27649e39c5baSBill Taylor if (status != DDI_SUCCESS) { 27659e39c5baSBill Taylor return (status); 27669e39c5baSBill Taylor } 27679e39c5baSBill Taylor bind->bi_free_dmahdl = 1; 27689e39c5baSBill Taylor 27699e39c5baSBill Taylor } else { 27709e39c5baSBill Taylor bind->bi_dmahdl = dmahdl; 27719e39c5baSBill Taylor bind->bi_free_dmahdl = 0; 27729e39c5baSBill Taylor } 27739e39c5baSBill Taylor 27749e39c5baSBill Taylor 27759e39c5baSBill Taylor /* 27769e39c5baSBill Taylor * Bind the memory to get the PCI mapped addresses. The decision 27779e39c5baSBill Taylor * to call ddi_dma_addr_bind_handle() or ddi_dma_buf_bind_handle() 27789e39c5baSBill Taylor * is determined by the "bi_type" flag. Note: if the bind operation 27799e39c5baSBill Taylor * fails then we have to free up the DMA handle and return error. 27809e39c5baSBill Taylor */ 27819e39c5baSBill Taylor if (bind->bi_type == HERMON_BINDHDL_VADDR) { 27829e39c5baSBill Taylor status = ddi_dma_addr_bind_handle(bind->bi_dmahdl, NULL, 27839e39c5baSBill Taylor (caddr_t)(uintptr_t)bind->bi_addr, bind->bi_len, 27849e39c5baSBill Taylor (DDI_DMA_RDWR | DDI_DMA_CONSISTENT), callback, NULL, 27859e39c5baSBill Taylor &bind->bi_dmacookie, &bind->bi_cookiecnt); 27869e39c5baSBill Taylor 27879e39c5baSBill Taylor } else { /* HERMON_BINDHDL_BUF or HERMON_BINDHDL_UBUF */ 27889e39c5baSBill Taylor 27899e39c5baSBill Taylor status = ddi_dma_buf_bind_handle(bind->bi_dmahdl, 27909e39c5baSBill Taylor bind->bi_buf, (DDI_DMA_RDWR | DDI_DMA_CONSISTENT), callback, 27919e39c5baSBill Taylor NULL, &bind->bi_dmacookie, &bind->bi_cookiecnt); 27929e39c5baSBill Taylor } 27939e39c5baSBill Taylor if (status != DDI_DMA_MAPPED) { 27949e39c5baSBill Taylor if (bind->bi_free_dmahdl != 0) { 27959e39c5baSBill Taylor ddi_dma_free_handle(&bind->bi_dmahdl); 27969e39c5baSBill Taylor } 27979e39c5baSBill Taylor return (status); 27989e39c5baSBill Taylor } 27999e39c5baSBill Taylor 28009e39c5baSBill Taylor return (DDI_SUCCESS); 28019e39c5baSBill Taylor } 28029e39c5baSBill Taylor 28039e39c5baSBill Taylor 28049e39c5baSBill Taylor /* 28059e39c5baSBill Taylor * hermon_mr_mem_unbind() 28069e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 28079e39c5baSBill Taylor */ 28089e39c5baSBill Taylor static void 28099e39c5baSBill Taylor hermon_mr_mem_unbind(hermon_state_t *state, hermon_bind_info_t *bind) 28109e39c5baSBill Taylor { 28119e39c5baSBill Taylor int status; 28129e39c5baSBill Taylor 28139e39c5baSBill Taylor /* 28149e39c5baSBill Taylor * In case of HERMON_BINDHDL_UBUF, the memory bi_buf points to 28159e39c5baSBill Taylor * is actually allocated by ddi_umem_iosetup() internally, then 28169e39c5baSBill Taylor * it's required to free it here. Reset bi_type to HERMON_BINDHDL_NONE 28179e39c5baSBill Taylor * not to free it again later. 28189e39c5baSBill Taylor */ 28199e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind)) 28209e39c5baSBill Taylor if (bind->bi_type == HERMON_BINDHDL_UBUF) { 28219e39c5baSBill Taylor freerbuf(bind->bi_buf); 28229e39c5baSBill Taylor bind->bi_type = HERMON_BINDHDL_NONE; 28239e39c5baSBill Taylor } 28249e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind)) 28259e39c5baSBill Taylor 28269e39c5baSBill Taylor /* 28279e39c5baSBill Taylor * Unbind the DMA memory for the region 28289e39c5baSBill Taylor * 28299e39c5baSBill Taylor * Note: The only way ddi_dma_unbind_handle() currently 28309e39c5baSBill Taylor * can return an error is if the handle passed in is invalid. 28319e39c5baSBill Taylor * Since this should never happen, we choose to return void 28329e39c5baSBill Taylor * from this function! If this does return an error, however, 28339e39c5baSBill Taylor * then we print a warning message to the console. 28349e39c5baSBill Taylor */ 28359e39c5baSBill Taylor status = ddi_dma_unbind_handle(bind->bi_dmahdl); 28369e39c5baSBill Taylor if (status != DDI_SUCCESS) { 28379e39c5baSBill Taylor HERMON_WARNING(state, "failed to unbind DMA mapping"); 28389e39c5baSBill Taylor return; 28399e39c5baSBill Taylor } 28409e39c5baSBill Taylor 28419e39c5baSBill Taylor /* Free up the DMA handle */ 28429e39c5baSBill Taylor if (bind->bi_free_dmahdl != 0) { 28439e39c5baSBill Taylor ddi_dma_free_handle(&bind->bi_dmahdl); 28449e39c5baSBill Taylor } 28459e39c5baSBill Taylor } 28469e39c5baSBill Taylor 28479e39c5baSBill Taylor 28489e39c5baSBill Taylor /* 28499e39c5baSBill Taylor * hermon_mr_fast_mtt_write() 28509e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 28519e39c5baSBill Taylor */ 28529e39c5baSBill Taylor static int 28539e39c5baSBill Taylor hermon_mr_fast_mtt_write(hermon_state_t *state, hermon_rsrc_t *mtt, 28549e39c5baSBill Taylor hermon_bind_info_t *bind, uint32_t mtt_pgsize_bits) 28559e39c5baSBill Taylor { 28569e39c5baSBill Taylor hermon_icm_table_t *icm_table; 28579e39c5baSBill Taylor hermon_dma_info_t *dma_info; 28589e39c5baSBill Taylor uint32_t index1, index2, rindx; 28599e39c5baSBill Taylor ddi_dma_cookie_t dmacookie; 28609e39c5baSBill Taylor uint_t cookie_cnt; 28619e39c5baSBill Taylor uint64_t *mtt_table; 28629e39c5baSBill Taylor uint64_t mtt_entry; 28639e39c5baSBill Taylor uint64_t addr, endaddr; 28649e39c5baSBill Taylor uint64_t pagesize; 28659e39c5baSBill Taylor offset_t i, start; 28669e39c5baSBill Taylor uint_t per_span; 28679e39c5baSBill Taylor int sync_needed; 28689e39c5baSBill Taylor 28699e39c5baSBill Taylor /* 28709e39c5baSBill Taylor * XXX According to the PRM, we are to use the WRITE_MTT 28719e39c5baSBill Taylor * command to write out MTTs. Tavor does not do this, 28729e39c5baSBill Taylor * instead taking advantage of direct access to the MTTs, 28739e39c5baSBill Taylor * and knowledge that Mellanox FMR relies on our ability 28749e39c5baSBill Taylor * to write directly to the MTTs without any further 28759e39c5baSBill Taylor * notification to the firmware. Likewise, we will choose 28769e39c5baSBill Taylor * to not use the WRITE_MTT command, but to simply write 28779e39c5baSBill Taylor * out the MTTs. 28789e39c5baSBill Taylor */ 28799e39c5baSBill Taylor 28809e39c5baSBill Taylor /* Calculate page size from the suggested value passed in */ 28819e39c5baSBill Taylor pagesize = ((uint64_t)1 << mtt_pgsize_bits); 28829e39c5baSBill Taylor 28839e39c5baSBill Taylor /* Walk the "cookie list" and fill in the MTT table entries */ 28849e39c5baSBill Taylor dmacookie = bind->bi_dmacookie; 28859e39c5baSBill Taylor cookie_cnt = bind->bi_cookiecnt; 28869e39c5baSBill Taylor 28879e39c5baSBill Taylor icm_table = &state->hs_icm[HERMON_MTT]; 28889e39c5baSBill Taylor rindx = mtt->hr_indx; 28899e39c5baSBill Taylor hermon_index(index1, index2, rindx, icm_table, i); 28909e39c5baSBill Taylor start = i; 28919e39c5baSBill Taylor 28929e39c5baSBill Taylor per_span = icm_table->span; 28939e39c5baSBill Taylor dma_info = icm_table->icm_dma[index1] + index2; 28949e39c5baSBill Taylor mtt_table = (uint64_t *)(uintptr_t)dma_info->vaddr; 28959e39c5baSBill Taylor 28969e39c5baSBill Taylor sync_needed = 0; 28979e39c5baSBill Taylor while (cookie_cnt-- > 0) { 28989e39c5baSBill Taylor addr = dmacookie.dmac_laddress; 28999e39c5baSBill Taylor endaddr = addr + (dmacookie.dmac_size - 1); 29009e39c5baSBill Taylor addr = addr & ~((uint64_t)pagesize - 1); 29019e39c5baSBill Taylor 29029e39c5baSBill Taylor while (addr <= endaddr) { 29039e39c5baSBill Taylor 29049e39c5baSBill Taylor /* 29059e39c5baSBill Taylor * Fill in the mapped addresses (calculated above) and 29069e39c5baSBill Taylor * set HERMON_MTT_ENTRY_PRESENT flag for each MTT entry. 29079e39c5baSBill Taylor */ 29089e39c5baSBill Taylor mtt_entry = addr | HERMON_MTT_ENTRY_PRESENT; 29099e39c5baSBill Taylor mtt_table[i] = htonll(mtt_entry); 29109e39c5baSBill Taylor i++; 29119e39c5baSBill Taylor rindx++; 29129e39c5baSBill Taylor 29139e39c5baSBill Taylor if (i == per_span) { 29149e39c5baSBill Taylor 29159e39c5baSBill Taylor (void) ddi_dma_sync(dma_info->dma_hdl, 29169e39c5baSBill Taylor start * sizeof (hermon_hw_mtt_t), 29179e39c5baSBill Taylor (i - start) * sizeof (hermon_hw_mtt_t), 29189e39c5baSBill Taylor DDI_DMA_SYNC_FORDEV); 29199e39c5baSBill Taylor 29204bbb904cSBill Taylor if ((addr + pagesize > endaddr) && 29214bbb904cSBill Taylor (cookie_cnt == 0)) 29224bbb904cSBill Taylor return (DDI_SUCCESS); 29234bbb904cSBill Taylor 29249e39c5baSBill Taylor hermon_index(index1, index2, rindx, icm_table, 29259e39c5baSBill Taylor i); 29269e39c5baSBill Taylor start = i * sizeof (hermon_hw_mtt_t); 29279e39c5baSBill Taylor dma_info = icm_table->icm_dma[index1] + index2; 29289e39c5baSBill Taylor mtt_table = 29299e39c5baSBill Taylor (uint64_t *)(uintptr_t)dma_info->vaddr; 29309e39c5baSBill Taylor 29319e39c5baSBill Taylor sync_needed = 0; 29329e39c5baSBill Taylor } else { 29339e39c5baSBill Taylor sync_needed = 1; 29349e39c5baSBill Taylor } 29359e39c5baSBill Taylor 29369e39c5baSBill Taylor addr += pagesize; 29379e39c5baSBill Taylor if (addr == 0) { 29389e39c5baSBill Taylor static int do_once = 1; 29399e39c5baSBill Taylor _NOTE(SCHEME_PROTECTS_DATA("safe sharing", 29409e39c5baSBill Taylor do_once)) 29419e39c5baSBill Taylor if (do_once) { 29429e39c5baSBill Taylor do_once = 0; 29439e39c5baSBill Taylor cmn_err(CE_NOTE, "probable error in " 29449e39c5baSBill Taylor "dma_cookie address from caller\n"); 29459e39c5baSBill Taylor } 29469e39c5baSBill Taylor break; 29479e39c5baSBill Taylor } 29489e39c5baSBill Taylor } 29499e39c5baSBill Taylor 29509e39c5baSBill Taylor /* 29519e39c5baSBill Taylor * When we've reached the end of the current DMA cookie, 29529e39c5baSBill Taylor * jump to the next cookie (if there are more) 29539e39c5baSBill Taylor */ 29549e39c5baSBill Taylor if (cookie_cnt != 0) { 29559e39c5baSBill Taylor ddi_dma_nextcookie(bind->bi_dmahdl, &dmacookie); 29569e39c5baSBill Taylor } 29579e39c5baSBill Taylor } 29589e39c5baSBill Taylor 29599e39c5baSBill Taylor /* done all the cookies, now sync the memory for the device */ 29609e39c5baSBill Taylor if (sync_needed) 29619e39c5baSBill Taylor (void) ddi_dma_sync(dma_info->dma_hdl, 29629e39c5baSBill Taylor start * sizeof (hermon_hw_mtt_t), 29639e39c5baSBill Taylor (i - start) * sizeof (hermon_hw_mtt_t), 29649e39c5baSBill Taylor DDI_DMA_SYNC_FORDEV); 29659e39c5baSBill Taylor 29669e39c5baSBill Taylor return (DDI_SUCCESS); 29679e39c5baSBill Taylor } 29689e39c5baSBill Taylor 29699e39c5baSBill Taylor /* 29709e39c5baSBill Taylor * hermon_mr_fast_mtt_write_fmr() 29719e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 29729e39c5baSBill Taylor */ 2973*c7facc54SBill Taylor /* ARGSUSED */ 29749e39c5baSBill Taylor static int 2975*c7facc54SBill Taylor hermon_mr_fast_mtt_write_fmr(hermon_state_t *state, hermon_rsrc_t *mtt, 2976*c7facc54SBill Taylor ibt_pmr_attr_t *mem_pattr, uint32_t mtt_pgsize_bits) 29779e39c5baSBill Taylor { 2978*c7facc54SBill Taylor hermon_icm_table_t *icm_table; 2979*c7facc54SBill Taylor hermon_dma_info_t *dma_info; 2980*c7facc54SBill Taylor uint32_t index1, index2, rindx; 29819e39c5baSBill Taylor uint64_t *mtt_table; 2982*c7facc54SBill Taylor offset_t i, j; 2983*c7facc54SBill Taylor uint_t per_span; 29849e39c5baSBill Taylor 2985*c7facc54SBill Taylor icm_table = &state->hs_icm[HERMON_MTT]; 2986*c7facc54SBill Taylor rindx = mtt->hr_indx; 2987*c7facc54SBill Taylor hermon_index(index1, index2, rindx, icm_table, i); 2988*c7facc54SBill Taylor per_span = icm_table->span; 2989*c7facc54SBill Taylor dma_info = icm_table->icm_dma[index1] + index2; 2990*c7facc54SBill Taylor mtt_table = (uint64_t *)(uintptr_t)dma_info->vaddr; 29919e39c5baSBill Taylor 29929e39c5baSBill Taylor /* 2993*c7facc54SBill Taylor * Fill in the MTT table entries 29949e39c5baSBill Taylor */ 2995*c7facc54SBill Taylor for (j = 0; j < mem_pattr->pmr_num_buf; j++) { 2996*c7facc54SBill Taylor mtt_table[i] = mem_pattr->pmr_addr_list[j].p_laddr; 2997*c7facc54SBill Taylor i++; 2998*c7facc54SBill Taylor rindx++; 2999*c7facc54SBill Taylor if (i == per_span) { 3000*c7facc54SBill Taylor hermon_index(index1, index2, rindx, icm_table, i); 3001*c7facc54SBill Taylor dma_info = icm_table->icm_dma[index1] + index2; 3002*c7facc54SBill Taylor mtt_table = (uint64_t *)(uintptr_t)dma_info->vaddr; 30039e39c5baSBill Taylor } 30049e39c5baSBill Taylor } 30059e39c5baSBill Taylor 30069e39c5baSBill Taylor return (DDI_SUCCESS); 30079e39c5baSBill Taylor } 30089e39c5baSBill Taylor 30099e39c5baSBill Taylor 30109e39c5baSBill Taylor /* 30119e39c5baSBill Taylor * hermon_mtt_refcnt_inc() 30129e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 30139e39c5baSBill Taylor */ 30149e39c5baSBill Taylor static uint_t 30159e39c5baSBill Taylor hermon_mtt_refcnt_inc(hermon_rsrc_t *rsrc) 30169e39c5baSBill Taylor { 30179e39c5baSBill Taylor hermon_sw_refcnt_t *rc; 30189e39c5baSBill Taylor 30199e39c5baSBill Taylor rc = (hermon_sw_refcnt_t *)rsrc->hr_addr; 30209e39c5baSBill Taylor return (atomic_inc_uint_nv(&rc->swrc_refcnt)); 30219e39c5baSBill Taylor } 30229e39c5baSBill Taylor 30239e39c5baSBill Taylor 30249e39c5baSBill Taylor /* 30259e39c5baSBill Taylor * hermon_mtt_refcnt_dec() 30269e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 30279e39c5baSBill Taylor */ 30289e39c5baSBill Taylor static uint_t 30299e39c5baSBill Taylor hermon_mtt_refcnt_dec(hermon_rsrc_t *rsrc) 30309e39c5baSBill Taylor { 30319e39c5baSBill Taylor hermon_sw_refcnt_t *rc; 30329e39c5baSBill Taylor 30339e39c5baSBill Taylor rc = (hermon_sw_refcnt_t *)rsrc->hr_addr; 30349e39c5baSBill Taylor return (atomic_dec_uint_nv(&rc->swrc_refcnt)); 30359e39c5baSBill Taylor } 3036