Lines Matching refs:mr
56 static int tavor_mr_common_rereg(tavor_state_t *state, tavor_mrhdl_t mr,
59 static int tavor_mr_rereg_xlat_helper(tavor_state_t *state, tavor_mrhdl_t mr,
167 tavor_mrhdl_t mr; in tavor_mr_register_shared() local
214 mr = (tavor_mrhdl_t)rsrc->tr_addr; in tavor_mr_register_shared()
215 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) in tavor_mr_register_shared()
222 mr->mr_accflag = 0; in tavor_mr_register_shared()
224 mr->mr_accflag |= IBT_MR_WINDOW_BIND; in tavor_mr_register_shared()
226 mr->mr_accflag |= IBT_MR_LOCAL_WRITE; in tavor_mr_register_shared()
228 mr->mr_accflag |= IBT_MR_REMOTE_READ; in tavor_mr_register_shared()
230 mr->mr_accflag |= IBT_MR_REMOTE_WRITE; in tavor_mr_register_shared()
232 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC; in tavor_mr_register_shared()
243 tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey); in tavor_mr_register_shared()
244 if ((mr->mr_accflag & IBT_MR_REMOTE_READ) || in tavor_mr_register_shared()
245 (mr->mr_accflag & IBT_MR_REMOTE_WRITE) || in tavor_mr_register_shared()
246 (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) { in tavor_mr_register_shared()
247 mr->mr_rkey = mr->mr_lkey; in tavor_mr_register_shared()
305 mr->mr_mttrsrcp = mrhdl->mr_mttrsrcp; in tavor_mr_register_shared()
306 mr->mr_logmttpgsz = mrhdl->mr_logmttpgsz; in tavor_mr_register_shared()
307 mr->mr_bindinfo = mrhdl->mr_bindinfo; in tavor_mr_register_shared()
308 mr->mr_mttrefcntp = mrhdl->mr_mttrefcntp; in tavor_mr_register_shared()
310 bind = &mr->mr_bindinfo; in tavor_mr_register_shared()
312 mtt = mr->mr_mttrsrcp; in tavor_mr_register_shared()
318 (void) tavor_mtt_refcnt_inc(mr->mr_mttrefcntp); in tavor_mr_register_shared()
326 pgsize_msk = (((uint64_t)1 << mr->mr_logmttpgsz) - 1); in tavor_mr_register_shared()
328 (mr->mr_bindinfo.bi_addr & pgsize_msk)); in tavor_mr_register_shared()
345 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0; in tavor_mr_register_shared()
346 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0; in tavor_mr_register_shared()
347 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0; in tavor_mr_register_shared()
348 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0; in tavor_mr_register_shared()
349 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0; in tavor_mr_register_shared()
352 mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC; in tavor_mr_register_shared()
353 mpt_entry.mem_key = mr->mr_lkey; in tavor_mr_register_shared()
381 mr->mr_mptrsrcp = mpt; in tavor_mr_register_shared()
382 mr->mr_mttrsrcp = mtt; in tavor_mr_register_shared()
383 mr->mr_pdhdl = pd; in tavor_mr_register_shared()
384 mr->mr_rsrcp = rsrc; in tavor_mr_register_shared()
385 mr->mr_is_umem = mr_is_umem; in tavor_mr_register_shared()
386 mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL; in tavor_mr_register_shared()
387 mr->mr_umem_cbfunc = NULL; in tavor_mr_register_shared()
388 mr->mr_umem_cbarg1 = NULL; in tavor_mr_register_shared()
389 mr->mr_umem_cbarg2 = NULL; in tavor_mr_register_shared()
401 *mrhdl_new = mr; in tavor_mr_register_shared()
409 (void) tavor_mtt_refcnt_dec(mr->mr_mttrefcntp); in tavor_mr_register_shared()
440 tavor_mrhdl_t mr; in tavor_mr_deregister() local
461 mr = *mrhdl; in tavor_mr_deregister()
462 mutex_enter(&mr->mr_lock); in tavor_mr_deregister()
463 mpt = mr->mr_mptrsrcp; in tavor_mr_deregister()
464 mtt = mr->mr_mttrsrcp; in tavor_mr_deregister()
465 mtt_refcnt = mr->mr_mttrefcntp; in tavor_mr_deregister()
466 rsrc = mr->mr_rsrcp; in tavor_mr_deregister()
467 pd = mr->mr_pdhdl; in tavor_mr_deregister()
468 bind = &mr->mr_bindinfo; in tavor_mr_deregister()
475 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) { in tavor_mr_deregister()
486 mutex_exit(&mr->mr_lock); in tavor_mr_deregister()
518 mutex_enter(&mr->mr_lock); in tavor_mr_deregister()
530 if (mr->mr_is_umem) { in tavor_mr_deregister()
532 (uint64_t)(uintptr_t)mr->mr_umemcookie, in tavor_mr_deregister()
537 ddi_umem_unlock(mr->mr_umemcookie); in tavor_mr_deregister()
539 ddi_umem_unlock(mr->mr_umemcookie); in tavor_mr_deregister()
540 mr->mr_umemcookie = NULL; in tavor_mr_deregister()
578 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) { in tavor_mr_deregister()
579 mutex_exit(&mr->mr_lock); in tavor_mr_deregister()
584 mutex_exit(&mr->mr_lock); in tavor_mr_deregister()
608 tavor_mr_query(tavor_state_t *state, tavor_mrhdl_t mr, in tavor_mr_query() argument
613 mutex_enter(&mr->mr_lock); in tavor_mr_query()
620 if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) { in tavor_mr_query()
621 mutex_exit(&mr->mr_lock); in tavor_mr_query()
626 attr->mr_attr_flags = mr->mr_accflag; in tavor_mr_query()
627 attr->mr_pd = (ibt_pd_hdl_t)mr->mr_pdhdl; in tavor_mr_query()
630 attr->mr_lkey = (ibt_lkey_t)mr->mr_lkey; in tavor_mr_query()
631 attr->mr_lbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr; in tavor_mr_query()
632 attr->mr_lbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len; in tavor_mr_query()
639 if ((mr->mr_accflag & IBT_MR_REMOTE_READ) || in tavor_mr_query()
640 (mr->mr_accflag & IBT_MR_REMOTE_WRITE) || in tavor_mr_query()
641 (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) { in tavor_mr_query()
642 attr->mr_rkey = (ibt_rkey_t)mr->mr_rkey; in tavor_mr_query()
643 attr->mr_rbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr; in tavor_mr_query()
644 attr->mr_rbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len; in tavor_mr_query()
651 attr->mr_sync_required = (mr->mr_bindinfo.bi_flags & in tavor_mr_query()
654 mutex_exit(&mr->mr_lock); in tavor_mr_query()
664 tavor_mr_reregister(tavor_state_t *state, tavor_mrhdl_t mr, in tavor_mr_reregister() argument
684 status = tavor_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op); in tavor_mr_reregister()
695 tavor_mr_reregister_buf(tavor_state_t *state, tavor_mrhdl_t mr, in tavor_mr_reregister_buf() argument
724 status = tavor_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op); in tavor_mr_reregister_buf()
1021 tavor_mrhdl_t mr; in tavor_mr_common_reg() local
1106 mr = (tavor_mrhdl_t)rsrc->tr_addr; in tavor_mr_common_reg()
1107 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) in tavor_mr_common_reg()
1114 mr->mr_accflag = 0; in tavor_mr_common_reg()
1116 mr->mr_accflag |= IBT_MR_WINDOW_BIND; in tavor_mr_common_reg()
1118 mr->mr_accflag |= IBT_MR_LOCAL_WRITE; in tavor_mr_common_reg()
1120 mr->mr_accflag |= IBT_MR_REMOTE_READ; in tavor_mr_common_reg()
1122 mr->mr_accflag |= IBT_MR_REMOTE_WRITE; in tavor_mr_common_reg()
1124 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC; in tavor_mr_common_reg()
1135 tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey); in tavor_mr_common_reg()
1136 if ((mr->mr_accflag & IBT_MR_REMOTE_READ) || in tavor_mr_common_reg()
1137 (mr->mr_accflag & IBT_MR_REMOTE_WRITE) || in tavor_mr_common_reg()
1138 (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) { in tavor_mr_common_reg()
1139 mr->mr_rkey = mr->mr_lkey; in tavor_mr_common_reg()
1189 bh = &mr->mr_bindinfo; in tavor_mr_common_reg()
1203 mr->mr_logmttpgsz = mtt_pgsize_bits; in tavor_mr_common_reg()
1217 mr->mr_mttrefcntp = mtt_refcnt; in tavor_mr_common_reg()
1230 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0; in tavor_mr_common_reg()
1231 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0; in tavor_mr_common_reg()
1232 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0; in tavor_mr_common_reg()
1233 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0; in tavor_mr_common_reg()
1234 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0; in tavor_mr_common_reg()
1237 mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC; in tavor_mr_common_reg()
1238 mpt_entry.mem_key = mr->mr_lkey; in tavor_mr_common_reg()
1243 bh->bi_addr = bh->bi_addr & ((1 << mr->mr_logmttpgsz) - 1); in tavor_mr_common_reg()
1271 mr->mr_mptrsrcp = mpt; in tavor_mr_common_reg()
1272 mr->mr_mttrsrcp = mtt; in tavor_mr_common_reg()
1273 mr->mr_pdhdl = pd; in tavor_mr_common_reg()
1274 mr->mr_rsrcp = rsrc; in tavor_mr_common_reg()
1275 mr->mr_is_umem = mr_is_umem; in tavor_mr_common_reg()
1276 mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL; in tavor_mr_common_reg()
1277 mr->mr_umem_cbfunc = NULL; in tavor_mr_common_reg()
1278 mr->mr_umem_cbarg1 = NULL; in tavor_mr_common_reg()
1279 mr->mr_umem_cbarg2 = NULL; in tavor_mr_common_reg()
1291 *mrhdl = mr; in tavor_mr_common_reg()
1338 tavor_mrhdl_t mr; in tavor_dma_mr_register() local
1385 mr = (tavor_mrhdl_t)rsrc->tr_addr; in tavor_dma_mr_register()
1386 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) in tavor_dma_mr_register()
1387 bzero(mr, sizeof (*mr)); in tavor_dma_mr_register()
1394 mr->mr_accflag = 0; in tavor_dma_mr_register()
1396 mr->mr_accflag |= IBT_MR_WINDOW_BIND; in tavor_dma_mr_register()
1398 mr->mr_accflag |= IBT_MR_LOCAL_WRITE; in tavor_dma_mr_register()
1400 mr->mr_accflag |= IBT_MR_REMOTE_READ; in tavor_dma_mr_register()
1402 mr->mr_accflag |= IBT_MR_REMOTE_WRITE; in tavor_dma_mr_register()
1404 mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC; in tavor_dma_mr_register()
1415 tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey); in tavor_dma_mr_register()
1416 if ((mr->mr_accflag & IBT_MR_REMOTE_READ) || in tavor_dma_mr_register()
1417 (mr->mr_accflag & IBT_MR_REMOTE_WRITE) || in tavor_dma_mr_register()
1418 (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) { in tavor_dma_mr_register()
1419 mr->mr_rkey = mr->mr_lkey; in tavor_dma_mr_register()
1431 mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0; in tavor_dma_mr_register()
1432 mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0; in tavor_dma_mr_register()
1433 mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0; in tavor_dma_mr_register()
1434 mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0; in tavor_dma_mr_register()
1435 mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0; in tavor_dma_mr_register()
1440 mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC; in tavor_dma_mr_register()
1441 mpt_entry.mem_key = mr->mr_lkey; in tavor_dma_mr_register()
1477 mr->mr_mptrsrcp = mpt; in tavor_dma_mr_register()
1478 mr->mr_mttrsrcp = NULL; in tavor_dma_mr_register()
1479 mr->mr_pdhdl = pd; in tavor_dma_mr_register()
1480 mr->mr_rsrcp = rsrc; in tavor_dma_mr_register()
1481 mr->mr_is_umem = 0; in tavor_dma_mr_register()
1482 mr->mr_umemcookie = NULL; in tavor_dma_mr_register()
1483 mr->mr_umem_cbfunc = NULL; in tavor_dma_mr_register()
1484 mr->mr_umem_cbarg1 = NULL; in tavor_dma_mr_register()
1485 mr->mr_umem_cbarg2 = NULL; in tavor_dma_mr_register()
1487 *mrhdl = mr; in tavor_dma_mr_register()
1614 tavor_mr_common_rereg(tavor_state_t *state, tavor_mrhdl_t mr, in tavor_mr_common_rereg() argument
1634 if (mr->mr_is_umem) { in tavor_mr_common_rereg()
1638 mutex_enter(&mr->mr_lock); in tavor_mr_common_rereg()
1641 mpt = mr->mr_mptrsrcp; in tavor_mr_common_rereg()
1654 mutex_exit(&mr->mr_lock); in tavor_mr_common_rereg()
1674 mutex_exit(&mr->mr_lock); in tavor_mr_common_rereg()
1685 if (tavor_mr_deregister(state, &mr, in tavor_mr_common_rereg()
1701 mutex_exit(&mr->mr_lock); in tavor_mr_common_rereg()
1709 if (tavor_mr_deregister(state, &mr, in tavor_mr_common_rereg()
1723 pd_to_use = mr->mr_pdhdl; in tavor_mr_common_rereg()
1737 mutex_exit(&mr->mr_lock); in tavor_mr_common_rereg()
1745 if (tavor_mr_deregister(state, &mr, in tavor_mr_common_rereg()
1772 acc_flags_to_use = mr->mr_accflag; in tavor_mr_common_rereg()
1790 status = tavor_mr_rereg_xlat_helper(state, mr, bind, op, in tavor_mr_common_rereg()
1793 mutex_exit(&mr->mr_lock); in tavor_mr_common_rereg()
1798 if (tavor_mr_deregister(state, &mr, dereg_level, in tavor_mr_common_rereg()
1806 vaddr_to_use = mr->mr_bindinfo.bi_addr; in tavor_mr_common_rereg()
1807 len_to_use = mr->mr_bindinfo.bi_len; in tavor_mr_common_rereg()
1811 vaddr_to_use = mr->mr_bindinfo.bi_addr; in tavor_mr_common_rereg()
1812 len_to_use = mr->mr_bindinfo.bi_len; in tavor_mr_common_rereg()
1822 tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey); in tavor_mr_common_rereg()
1826 mr->mr_rkey = mr->mr_lkey; in tavor_mr_common_rereg()
1839 mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC; in tavor_mr_common_rereg()
1840 mpt_entry.mem_key = mr->mr_lkey; in tavor_mr_common_rereg()
1856 mutex_exit(&mr->mr_lock); in tavor_mr_common_rereg()
1865 if (tavor_mr_deregister(state, &mr, in tavor_mr_common_rereg()
1879 tavor_pd_refcnt_dec(mr->mr_pdhdl); in tavor_mr_common_rereg()
1887 mr->mr_pdhdl = pd_to_use; in tavor_mr_common_rereg()
1888 mr->mr_accflag = acc_flags_to_use; in tavor_mr_common_rereg()
1889 mr->mr_is_umem = 0; in tavor_mr_common_rereg()
1890 mr->mr_umemcookie = NULL; in tavor_mr_common_rereg()
1893 *mrhdl_new = mr; in tavor_mr_common_rereg()
1894 mutex_exit(&mr->mr_lock); in tavor_mr_common_rereg()
1911 tavor_mr_rereg_xlat_helper(tavor_state_t *state, tavor_mrhdl_t mr, in tavor_mr_rereg_xlat_helper() argument
1924 ASSERT(MUTEX_HELD(&mr->mr_lock)); in tavor_mr_rereg_xlat_helper()
1962 nummtt_in_currrsrc = mr->mr_mttrsrcp->tr_len >> TAVOR_MTT_SIZE_SHIFT; in tavor_mr_rereg_xlat_helper()
1985 swrc_old = (tavor_sw_refcnt_t *)mr->mr_mttrefcntp->tr_addr; in tavor_mr_rereg_xlat_helper()
1996 if (TAVOR_MR_REUSE_DMAHDL(mr, bind->bi_flags)) { in tavor_mr_rereg_xlat_helper()
1997 mr->mr_bindinfo.bi_free_dmahdl = 0; in tavor_mr_rereg_xlat_helper()
1998 tavor_mr_mem_unbind(state, &mr->mr_bindinfo); in tavor_mr_rereg_xlat_helper()
1999 dmahdl = mr->mr_bindinfo.bi_dmahdl; in tavor_mr_rereg_xlat_helper()
2002 tavor_mr_mem_unbind(state, &mr->mr_bindinfo); in tavor_mr_rereg_xlat_helper()
2045 mtt = mr->mr_mttrsrcp; in tavor_mr_rereg_xlat_helper()
2067 mr->mr_bindinfo = *bind; in tavor_mr_rereg_xlat_helper()
2068 mr->mr_logmttpgsz = mtt_pgsize_bits; in tavor_mr_rereg_xlat_helper()
2088 if (TAVOR_MR_REUSE_DMAHDL(mr, bind->bi_flags)) { in tavor_mr_rereg_xlat_helper()
2089 mr->mr_bindinfo.bi_free_dmahdl = 0; in tavor_mr_rereg_xlat_helper()
2090 tavor_mr_mem_unbind(state, &mr->mr_bindinfo); in tavor_mr_rereg_xlat_helper()
2091 dmahdl = mr->mr_bindinfo.bi_dmahdl; in tavor_mr_rereg_xlat_helper()
2094 tavor_mr_mem_unbind(state, &mr->mr_bindinfo); in tavor_mr_rereg_xlat_helper()
2200 mtt_refcnt = mr->mr_mttrefcntp; in tavor_mr_rereg_xlat_helper()
2241 (void) tavor_mtt_refcnt_dec(mr->mr_mttrefcntp); in tavor_mr_rereg_xlat_helper()
2244 tavor_rsrc_free(state, &mr->mr_mttrsrcp); in tavor_mr_rereg_xlat_helper()
2248 mr->mr_bindinfo = *bind; in tavor_mr_rereg_xlat_helper()
2249 mr->mr_logmttpgsz = mtt_pgsize_bits; in tavor_mr_rereg_xlat_helper()
2250 mr->mr_mttrsrcp = mtt; in tavor_mr_rereg_xlat_helper()
2251 mr->mr_mttrefcntp = mtt_refcnt; in tavor_mr_rereg_xlat_helper()