xref: /illumos-gate/usr/src/uts/common/rpc/rdma_subr.c (revision 25c95769d03db2f91662020bcbc62b7aa59a7642)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
50a701b1eSRobert Gordon  * Common Development and Distribution License (the "License").
60a701b1eSRobert Gordon  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2251f34d4bSRajkumar Sivaprakasam  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
260a701b1eSRobert Gordon /*
2751f34d4bSRajkumar Sivaprakasam  * Copyright (c) 2008, The Ohio State University. All rights reserved.
280a701b1eSRobert Gordon  *
290a701b1eSRobert Gordon  * Portions of this source code is developed by the team members of
300a701b1eSRobert Gordon  * The Ohio State University's Network-Based Computing Laboratory (NBCL),
310a701b1eSRobert Gordon  * headed by Professor Dhabaleswar K. (DK) Panda.
320a701b1eSRobert Gordon  *
330a701b1eSRobert Gordon  * Acknowledgements to contributions from developors:
340a701b1eSRobert Gordon  *   Ranjit Noronha: noronha@cse.ohio-state.edu
350a701b1eSRobert Gordon  *   Lei Chai      : chail@cse.ohio-state.edu
360a701b1eSRobert Gordon  *   Weikuan Yu    : yuw@cse.ohio-state.edu
370a701b1eSRobert Gordon  *
380a701b1eSRobert Gordon  */
397c478bd9Sstevel@tonic-gate 
407c478bd9Sstevel@tonic-gate #include <sys/systm.h>
417c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
427c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
430a701b1eSRobert Gordon #include <sys/sdt.h>
447c478bd9Sstevel@tonic-gate #include <rpc/rpc_rdma.h>
457c478bd9Sstevel@tonic-gate 
467c478bd9Sstevel@tonic-gate #include <sys/ib/ibtl/ibti.h>
477c478bd9Sstevel@tonic-gate 
487c478bd9Sstevel@tonic-gate uint_t rdma_minchunk = RDMA_MINCHUNK;
497c478bd9Sstevel@tonic-gate 
507c478bd9Sstevel@tonic-gate /*
517c478bd9Sstevel@tonic-gate  * Globals
527c478bd9Sstevel@tonic-gate  */
537c478bd9Sstevel@tonic-gate int rdma_modloaded = 0;		/* flag to load RDMA plugin modules */
547c478bd9Sstevel@tonic-gate int rdma_dev_available = 0;	/* if any RDMA device is loaded */
557c478bd9Sstevel@tonic-gate kmutex_t rdma_modload_lock;	/* protects rdma_modloaded flag */
5651f34d4bSRajkumar Sivaprakasam 
5751f34d4bSRajkumar Sivaprakasam rdma_svc_wait_t rdma_wait;
5851f34d4bSRajkumar Sivaprakasam 
597c478bd9Sstevel@tonic-gate rdma_registry_t	*rdma_mod_head = NULL;	/* head for RDMA modules */
607c478bd9Sstevel@tonic-gate krwlock_t	rdma_lock;		/* protects rdma_mod_head list */
617c478bd9Sstevel@tonic-gate ldi_ident_t rpcmod_li = NULL;	/* identifies us with ldi_ framework */
627c478bd9Sstevel@tonic-gate 
630a701b1eSRobert Gordon kmem_cache_t *clist_cache = NULL;
640a701b1eSRobert Gordon 
657c478bd9Sstevel@tonic-gate /*
667c478bd9Sstevel@tonic-gate  * Statics
677c478bd9Sstevel@tonic-gate  */
6851f34d4bSRajkumar Sivaprakasam ldi_handle_t rpcib_handle = NULL;
697c478bd9Sstevel@tonic-gate 
707c478bd9Sstevel@tonic-gate /*
717c478bd9Sstevel@tonic-gate  * Externs
727c478bd9Sstevel@tonic-gate  */
737c478bd9Sstevel@tonic-gate extern	kstat_named_t	*rdmarcstat_ptr;
747c478bd9Sstevel@tonic-gate extern	uint_t		rdmarcstat_ndata;
757c478bd9Sstevel@tonic-gate extern	kstat_named_t	*rdmarsstat_ptr;
767c478bd9Sstevel@tonic-gate extern	uint_t		rdmarsstat_ndata;
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate void rdma_kstat_init();
797c478bd9Sstevel@tonic-gate 
807c478bd9Sstevel@tonic-gate /*
817c478bd9Sstevel@tonic-gate  * RDMATF module registration routine.
827c478bd9Sstevel@tonic-gate  * This routine is expected to be called by the init routine in
837c478bd9Sstevel@tonic-gate  * the plugin modules.
847c478bd9Sstevel@tonic-gate  */
857c478bd9Sstevel@tonic-gate rdma_stat
867c478bd9Sstevel@tonic-gate rdma_register_mod(rdma_mod_t *mod)
877c478bd9Sstevel@tonic-gate {
887c478bd9Sstevel@tonic-gate 	rdma_registry_t **mp, *m;
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate 	if (mod->rdma_version != RDMATF_VERS) {
917c478bd9Sstevel@tonic-gate 		return (RDMA_BADVERS);
927c478bd9Sstevel@tonic-gate 	}
937c478bd9Sstevel@tonic-gate 
947c478bd9Sstevel@tonic-gate 	rw_enter(&rdma_lock, RW_WRITER);
957c478bd9Sstevel@tonic-gate 	/*
967c478bd9Sstevel@tonic-gate 	 * Ensure not already registered
977c478bd9Sstevel@tonic-gate 	 */
987c478bd9Sstevel@tonic-gate 	mp = &rdma_mod_head;
997c478bd9Sstevel@tonic-gate 	while (*mp != NULL) {
1007c478bd9Sstevel@tonic-gate 		if (strncmp((*mp)->r_mod->rdma_api, mod->rdma_api,
1017c478bd9Sstevel@tonic-gate 		    KNC_STRSIZE) == 0) {
10251f34d4bSRajkumar Sivaprakasam 			if ((*mp)->r_mod_state == RDMA_MOD_INACTIVE) {
10351f34d4bSRajkumar Sivaprakasam 				(*mp)->r_mod_state = RDMA_MOD_ACTIVE;
10451f34d4bSRajkumar Sivaprakasam 				(*mp)->r_mod->rdma_ops = mod->rdma_ops;
10551f34d4bSRajkumar Sivaprakasam 				(*mp)->r_mod->rdma_count = mod->rdma_count;
10651f34d4bSRajkumar Sivaprakasam 				goto announce_hca;
10751f34d4bSRajkumar Sivaprakasam 			}
1087c478bd9Sstevel@tonic-gate 			rw_exit(&rdma_lock);
1097c478bd9Sstevel@tonic-gate 			return (RDMA_REG_EXIST);
1107c478bd9Sstevel@tonic-gate 		}
1117c478bd9Sstevel@tonic-gate 		mp = &((*mp)->r_next);
1127c478bd9Sstevel@tonic-gate 	}
1137c478bd9Sstevel@tonic-gate 
1147c478bd9Sstevel@tonic-gate 	/*
1157c478bd9Sstevel@tonic-gate 	 * New one, create and add to registry
1167c478bd9Sstevel@tonic-gate 	 */
1177c478bd9Sstevel@tonic-gate 	m = kmem_alloc(sizeof (rdma_registry_t), KM_SLEEP);
1187c478bd9Sstevel@tonic-gate 	m->r_mod = kmem_alloc(sizeof (rdma_mod_t), KM_SLEEP);
1197c478bd9Sstevel@tonic-gate 	*m->r_mod = *mod;
1207c478bd9Sstevel@tonic-gate 	m->r_next = NULL;
1217c478bd9Sstevel@tonic-gate 	m->r_mod->rdma_api = kmem_zalloc(KNC_STRSIZE, KM_SLEEP);
1227c478bd9Sstevel@tonic-gate 	(void) strncpy(m->r_mod->rdma_api, mod->rdma_api, KNC_STRSIZE);
1237c478bd9Sstevel@tonic-gate 	m->r_mod->rdma_api[KNC_STRSIZE - 1] = '\0';
12451f34d4bSRajkumar Sivaprakasam 	m->r_mod_state = RDMA_MOD_ACTIVE;
1257c478bd9Sstevel@tonic-gate 	*mp = m;
12651f34d4bSRajkumar Sivaprakasam 
12751f34d4bSRajkumar Sivaprakasam announce_hca:
1287c478bd9Sstevel@tonic-gate 	rw_exit(&rdma_lock);
12951f34d4bSRajkumar Sivaprakasam 	/*
13051f34d4bSRajkumar Sivaprakasam 	 * Start the nfs service on the rdma xprts.
13151f34d4bSRajkumar Sivaprakasam 	 * (this notification mechanism will need to change when we support
13251f34d4bSRajkumar Sivaprakasam 	 * multiple hcas and have support for multiple rdma plugins).
13351f34d4bSRajkumar Sivaprakasam 	 */
13451f34d4bSRajkumar Sivaprakasam 	mutex_enter(&rdma_wait.svc_lock);
13551f34d4bSRajkumar Sivaprakasam 	rdma_wait.svc_stat = RDMA_HCA_ATTACH;
13651f34d4bSRajkumar Sivaprakasam 	cv_signal(&rdma_wait.svc_cv);
13751f34d4bSRajkumar Sivaprakasam 	mutex_exit(&rdma_wait.svc_lock);
1387c478bd9Sstevel@tonic-gate 
1397c478bd9Sstevel@tonic-gate 	return (RDMA_SUCCESS);
1407c478bd9Sstevel@tonic-gate }
1417c478bd9Sstevel@tonic-gate 
1427c478bd9Sstevel@tonic-gate /*
1437c478bd9Sstevel@tonic-gate  * RDMATF module unregistration routine.
1447c478bd9Sstevel@tonic-gate  * This routine is expected to be called by the fini routine in
1457c478bd9Sstevel@tonic-gate  * the plugin modules.
1467c478bd9Sstevel@tonic-gate  */
1477c478bd9Sstevel@tonic-gate rdma_stat
1487c478bd9Sstevel@tonic-gate rdma_unregister_mod(rdma_mod_t *mod)
1497c478bd9Sstevel@tonic-gate {
1507c478bd9Sstevel@tonic-gate 	rdma_registry_t **m, *mmod = NULL;
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate 	rw_enter(&rdma_lock, RW_WRITER);
1537c478bd9Sstevel@tonic-gate 
1547c478bd9Sstevel@tonic-gate 	m = &rdma_mod_head;
1557c478bd9Sstevel@tonic-gate 	while (*m != NULL) {
1567c478bd9Sstevel@tonic-gate 		if (strncmp((*m)->r_mod->rdma_api, mod->rdma_api,
1577c478bd9Sstevel@tonic-gate 		    KNC_STRSIZE) != 0) {
1587c478bd9Sstevel@tonic-gate 			m = &((*m)->r_next);
1597c478bd9Sstevel@tonic-gate 			continue;
1607c478bd9Sstevel@tonic-gate 		}
1617c478bd9Sstevel@tonic-gate 		/*
1627c478bd9Sstevel@tonic-gate 		 * Check if any device attached, if so return error
1637c478bd9Sstevel@tonic-gate 		 */
16451f34d4bSRajkumar Sivaprakasam 		if (mod->rdma_count != 0) {
1657c478bd9Sstevel@tonic-gate 			rw_exit(&rdma_lock);
1667c478bd9Sstevel@tonic-gate 			return (RDMA_FAILED);
1677c478bd9Sstevel@tonic-gate 		}
1687c478bd9Sstevel@tonic-gate 		/*
16951f34d4bSRajkumar Sivaprakasam 		 * Found entry. Mark it inactive.
1707c478bd9Sstevel@tonic-gate 		 */
1717c478bd9Sstevel@tonic-gate 		mmod = *m;
17251f34d4bSRajkumar Sivaprakasam 		mmod->r_mod->rdma_count = 0;
17351f34d4bSRajkumar Sivaprakasam 		mmod->r_mod_state = RDMA_MOD_INACTIVE;
17451f34d4bSRajkumar Sivaprakasam 		break;
1757c478bd9Sstevel@tonic-gate 	}
1767c478bd9Sstevel@tonic-gate 
17751f34d4bSRajkumar Sivaprakasam 	rdma_modloaded = 0;
17851f34d4bSRajkumar Sivaprakasam 	rdma_dev_available = 0;
17951f34d4bSRajkumar Sivaprakasam 	rw_exit(&rdma_lock);
18051f34d4bSRajkumar Sivaprakasam 
18151f34d4bSRajkumar Sivaprakasam 	/*
18251f34d4bSRajkumar Sivaprakasam 	 * Stop the nfs service running on the rdma xprts.
18351f34d4bSRajkumar Sivaprakasam 	 * (this notification mechanism will need to change when we support
18451f34d4bSRajkumar Sivaprakasam 	 * multiple hcas and have support for multiple rdma plugins).
18551f34d4bSRajkumar Sivaprakasam 	 */
18651f34d4bSRajkumar Sivaprakasam 	mutex_enter(&rdma_wait.svc_lock);
18751f34d4bSRajkumar Sivaprakasam 	rdma_wait.svc_stat = RDMA_HCA_DETACH;
18851f34d4bSRajkumar Sivaprakasam 	cv_signal(&rdma_wait.svc_cv);
18951f34d4bSRajkumar Sivaprakasam 	mutex_exit(&rdma_wait.svc_lock);
19051f34d4bSRajkumar Sivaprakasam 
1917c478bd9Sstevel@tonic-gate 	/*
1927c478bd9Sstevel@tonic-gate 	 * Not found.
1937c478bd9Sstevel@tonic-gate 	 */
19451f34d4bSRajkumar Sivaprakasam 	return (RDMA_SUCCESS);
1957c478bd9Sstevel@tonic-gate }
1967c478bd9Sstevel@tonic-gate 
1970a701b1eSRobert Gordon struct clist *
1980a701b1eSRobert Gordon clist_alloc(void)
1990a701b1eSRobert Gordon {
2000a701b1eSRobert Gordon 	struct clist *clp;
2010a701b1eSRobert Gordon 
2020a701b1eSRobert Gordon 	clp = kmem_cache_alloc(clist_cache, KM_SLEEP);
2030a701b1eSRobert Gordon 
2040a701b1eSRobert Gordon 	bzero(clp, sizeof (*clp));
2050a701b1eSRobert Gordon 
2060a701b1eSRobert Gordon 	return (clp);
2070a701b1eSRobert Gordon }
2080a701b1eSRobert Gordon 
209f837ee4aSSiddheshwar Mahesh uint32_t
210f837ee4aSSiddheshwar Mahesh clist_len(struct clist *cl)
211f837ee4aSSiddheshwar Mahesh {
212f837ee4aSSiddheshwar Mahesh 	uint32_t len = 0;
213f837ee4aSSiddheshwar Mahesh 	while (cl) {
214f837ee4aSSiddheshwar Mahesh 		len += cl->c_len;
215f837ee4aSSiddheshwar Mahesh 		cl = cl->c_next;
216f837ee4aSSiddheshwar Mahesh 	}
217f837ee4aSSiddheshwar Mahesh 	return (len);
218f837ee4aSSiddheshwar Mahesh }
219f837ee4aSSiddheshwar Mahesh 
220f837ee4aSSiddheshwar Mahesh void
221f837ee4aSSiddheshwar Mahesh clist_zero_len(struct clist *cl)
222f837ee4aSSiddheshwar Mahesh {
223f837ee4aSSiddheshwar Mahesh 	while (cl != NULL) {
224f837ee4aSSiddheshwar Mahesh 		if (cl->c_dmemhandle.mrc_rmr == 0)
225f837ee4aSSiddheshwar Mahesh 			break;
226f837ee4aSSiddheshwar Mahesh 		cl->c_len = 0;
227f837ee4aSSiddheshwar Mahesh 		cl = cl->c_next;
228f837ee4aSSiddheshwar Mahesh 	}
229f837ee4aSSiddheshwar Mahesh }
230f837ee4aSSiddheshwar Mahesh 
2317c478bd9Sstevel@tonic-gate /*
2327c478bd9Sstevel@tonic-gate  * Creates a new chunk list entry, and
2337c478bd9Sstevel@tonic-gate  * adds it to the end of a chunk list.
2347c478bd9Sstevel@tonic-gate  */
2357c478bd9Sstevel@tonic-gate void
2367c478bd9Sstevel@tonic-gate clist_add(struct clist **clp, uint32_t xdroff, int len,
237*25c95769SToomas Soome     struct mrc *shandle, caddr_t saddr,
238*25c95769SToomas Soome     struct mrc *dhandle, caddr_t daddr)
2397c478bd9Sstevel@tonic-gate {
2407c478bd9Sstevel@tonic-gate 	struct clist *cl;
2417c478bd9Sstevel@tonic-gate 
2427c478bd9Sstevel@tonic-gate 	/* Find the end of the list */
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 	while (*clp != NULL)
2457c478bd9Sstevel@tonic-gate 		clp = &((*clp)->c_next);
2467c478bd9Sstevel@tonic-gate 
2470a701b1eSRobert Gordon 	cl = clist_alloc();
2487c478bd9Sstevel@tonic-gate 	cl->c_xdroff = xdroff;
2497c478bd9Sstevel@tonic-gate 	cl->c_len = len;
2500a701b1eSRobert Gordon 	cl->w.c_saddr = (uint64_t)(uintptr_t)saddr;
2517c478bd9Sstevel@tonic-gate 	if (shandle)
2527c478bd9Sstevel@tonic-gate 		cl->c_smemhandle = *shandle;
2530a701b1eSRobert Gordon 	cl->u.c_daddr = (uint64_t)(uintptr_t)daddr;
2547c478bd9Sstevel@tonic-gate 	if (dhandle)
2557c478bd9Sstevel@tonic-gate 		cl->c_dmemhandle = *dhandle;
2567c478bd9Sstevel@tonic-gate 	cl->c_next = NULL;
2577c478bd9Sstevel@tonic-gate 
2587c478bd9Sstevel@tonic-gate 	*clp = cl;
2597c478bd9Sstevel@tonic-gate }
2607c478bd9Sstevel@tonic-gate 
2610a701b1eSRobert Gordon rdma_stat
2620a701b1eSRobert Gordon clist_register(CONN *conn, struct clist *cl, clist_dstsrc dstsrc)
2637c478bd9Sstevel@tonic-gate {
2647c478bd9Sstevel@tonic-gate 	struct clist *c;
2657c478bd9Sstevel@tonic-gate 	int status;
2667c478bd9Sstevel@tonic-gate 
2677c478bd9Sstevel@tonic-gate 	for (c = cl; c; c = c->c_next) {
2680a701b1eSRobert Gordon 		if (c->c_len <= 0)
2690a701b1eSRobert Gordon 			continue;
270f837ee4aSSiddheshwar Mahesh 
271f837ee4aSSiddheshwar Mahesh 		c->c_regtype = dstsrc;
272f837ee4aSSiddheshwar Mahesh 
2730a701b1eSRobert Gordon 		switch (dstsrc) {
2740a701b1eSRobert Gordon 		case CLIST_REG_SOURCE:
2757c478bd9Sstevel@tonic-gate 			status = RDMA_REGMEMSYNC(conn,
276f837ee4aSSiddheshwar Mahesh 			    (caddr_t)(struct as *)c->c_adspc,
2770a701b1eSRobert Gordon 			    (caddr_t)(uintptr_t)c->w.c_saddr3, c->c_len,
2780a701b1eSRobert Gordon 			    &c->c_smemhandle, (void **)&c->c_ssynchandle,
2790a701b1eSRobert Gordon 			    (void *)c->rb_longbuf.rb_private);
2800a701b1eSRobert Gordon 			break;
2810a701b1eSRobert Gordon 		case CLIST_REG_DST:
2827c478bd9Sstevel@tonic-gate 			status = RDMA_REGMEMSYNC(conn,
283f837ee4aSSiddheshwar Mahesh 			    (caddr_t)(struct as *)c->c_adspc,
2840a701b1eSRobert Gordon 			    (caddr_t)(uintptr_t)c->u.c_daddr3, c->c_len,
2850a701b1eSRobert Gordon 			    &c->c_dmemhandle, (void **)&c->c_dsynchandle,
2860a701b1eSRobert Gordon 			    (void *)c->rb_longbuf.rb_private);
2870a701b1eSRobert Gordon 			break;
2880a701b1eSRobert Gordon 		default:
2890a701b1eSRobert Gordon 			return (RDMA_INVAL);
2907c478bd9Sstevel@tonic-gate 		}
2917c478bd9Sstevel@tonic-gate 		if (status != RDMA_SUCCESS) {
292f837ee4aSSiddheshwar Mahesh 			(void) clist_deregister(conn, cl);
2937c478bd9Sstevel@tonic-gate 			return (status);
2947c478bd9Sstevel@tonic-gate 		}
2957c478bd9Sstevel@tonic-gate 	}
2967c478bd9Sstevel@tonic-gate 
2977c478bd9Sstevel@tonic-gate 	return (RDMA_SUCCESS);
2987c478bd9Sstevel@tonic-gate }
2997c478bd9Sstevel@tonic-gate 
3000a701b1eSRobert Gordon rdma_stat
301f837ee4aSSiddheshwar Mahesh clist_deregister(CONN *conn, struct clist *cl)
3027c478bd9Sstevel@tonic-gate {
3037c478bd9Sstevel@tonic-gate 	struct clist *c;
3047c478bd9Sstevel@tonic-gate 
3057c478bd9Sstevel@tonic-gate 	for (c = cl; c; c = c->c_next) {
306f837ee4aSSiddheshwar Mahesh 		switch (c->c_regtype) {
3070a701b1eSRobert Gordon 		case CLIST_REG_SOURCE:
3087c478bd9Sstevel@tonic-gate 			if (c->c_smemhandle.mrc_rmr != 0) {
3097c478bd9Sstevel@tonic-gate 				(void) RDMA_DEREGMEMSYNC(conn,
3100a701b1eSRobert Gordon 				    (caddr_t)(uintptr_t)c->w.c_saddr3,
3117c478bd9Sstevel@tonic-gate 				    c->c_smemhandle,
3120a701b1eSRobert Gordon 				    (void *)(uintptr_t)c->c_ssynchandle,
3130a701b1eSRobert Gordon 				    (void *)c->rb_longbuf.rb_private);
3147c478bd9Sstevel@tonic-gate 				c->c_smemhandle.mrc_rmr = 0;
315*25c95769SToomas Soome 				c->c_ssynchandle = 0;
3167c478bd9Sstevel@tonic-gate 			}
3170a701b1eSRobert Gordon 			break;
3180a701b1eSRobert Gordon 		case CLIST_REG_DST:
3197c478bd9Sstevel@tonic-gate 			if (c->c_dmemhandle.mrc_rmr != 0) {
3207c478bd9Sstevel@tonic-gate 				(void) RDMA_DEREGMEMSYNC(conn,
3210a701b1eSRobert Gordon 				    (caddr_t)(uintptr_t)c->u.c_daddr3,
3227c478bd9Sstevel@tonic-gate 				    c->c_dmemhandle,
3230a701b1eSRobert Gordon 				    (void *)(uintptr_t)c->c_dsynchandle,
3240a701b1eSRobert Gordon 				    (void *)c->rb_longbuf.rb_private);
3257c478bd9Sstevel@tonic-gate 				c->c_dmemhandle.mrc_rmr = 0;
326*25c95769SToomas Soome 				c->c_dsynchandle = 0;
3277c478bd9Sstevel@tonic-gate 			}
3280a701b1eSRobert Gordon 			break;
3290a701b1eSRobert Gordon 		default:
330f837ee4aSSiddheshwar Mahesh 			/* clist unregistered. continue */
331f837ee4aSSiddheshwar Mahesh 			break;
3327c478bd9Sstevel@tonic-gate 		}
3337c478bd9Sstevel@tonic-gate 	}
3347c478bd9Sstevel@tonic-gate 
3357c478bd9Sstevel@tonic-gate 	return (RDMA_SUCCESS);
3367c478bd9Sstevel@tonic-gate }
3377c478bd9Sstevel@tonic-gate 
3380a701b1eSRobert Gordon rdma_stat
3390a701b1eSRobert Gordon clist_syncmem(CONN *conn, struct clist *cl, clist_dstsrc dstsrc)
3400a701b1eSRobert Gordon {
3410a701b1eSRobert Gordon 	struct clist *c;
3420a701b1eSRobert Gordon 	rdma_stat status;
3430a701b1eSRobert Gordon 
3440a701b1eSRobert Gordon 	c = cl;
3450a701b1eSRobert Gordon 	switch (dstsrc) {
3460a701b1eSRobert Gordon 	case CLIST_REG_SOURCE:
3470a701b1eSRobert Gordon 		while (c != NULL) {
3480a701b1eSRobert Gordon 			if (c->c_ssynchandle) {
3490a701b1eSRobert Gordon 				status = RDMA_SYNCMEM(conn,
3500a701b1eSRobert Gordon 				    (void *)(uintptr_t)c->c_ssynchandle,
3510a701b1eSRobert Gordon 				    (caddr_t)(uintptr_t)c->w.c_saddr3,
3520a701b1eSRobert Gordon 				    c->c_len, 0);
3530a701b1eSRobert Gordon 				if (status != RDMA_SUCCESS)
3540a701b1eSRobert Gordon 					return (status);
3550a701b1eSRobert Gordon 			}
3560a701b1eSRobert Gordon 			c = c->c_next;
3570a701b1eSRobert Gordon 		}
3580a701b1eSRobert Gordon 		break;
3590a701b1eSRobert Gordon 	case CLIST_REG_DST:
3600a701b1eSRobert Gordon 		while (c != NULL) {
3610a701b1eSRobert Gordon 			if (c->c_ssynchandle) {
3620a701b1eSRobert Gordon 				status = RDMA_SYNCMEM(conn,
3630a701b1eSRobert Gordon 				    (void *)(uintptr_t)c->c_dsynchandle,
3640a701b1eSRobert Gordon 				    (caddr_t)(uintptr_t)c->u.c_daddr3,
3650a701b1eSRobert Gordon 				    c->c_len, 1);
3660a701b1eSRobert Gordon 				if (status != RDMA_SUCCESS)
3670a701b1eSRobert Gordon 					return (status);
3680a701b1eSRobert Gordon 			}
3690a701b1eSRobert Gordon 			c = c->c_next;
3700a701b1eSRobert Gordon 		}
3710a701b1eSRobert Gordon 		break;
3720a701b1eSRobert Gordon 	default:
3730a701b1eSRobert Gordon 		return (RDMA_INVAL);
3740a701b1eSRobert Gordon 	}
3750a701b1eSRobert Gordon 
3760a701b1eSRobert Gordon 	return (RDMA_SUCCESS);
3770a701b1eSRobert Gordon }
3780a701b1eSRobert Gordon 
3797c478bd9Sstevel@tonic-gate /*
3807c478bd9Sstevel@tonic-gate  * Frees up entries in chunk list
3817c478bd9Sstevel@tonic-gate  */
3827c478bd9Sstevel@tonic-gate void
3837c478bd9Sstevel@tonic-gate clist_free(struct clist *cl)
3847c478bd9Sstevel@tonic-gate {
3857c478bd9Sstevel@tonic-gate 	struct clist *c = cl;
3867c478bd9Sstevel@tonic-gate 
3877c478bd9Sstevel@tonic-gate 	while (c != NULL) {
3887c478bd9Sstevel@tonic-gate 		cl = cl->c_next;
3890a701b1eSRobert Gordon 		kmem_cache_free(clist_cache, c);
3907c478bd9Sstevel@tonic-gate 		c = cl;
3917c478bd9Sstevel@tonic-gate 	}
3927c478bd9Sstevel@tonic-gate }
3937c478bd9Sstevel@tonic-gate 
3947c478bd9Sstevel@tonic-gate rdma_stat
3957c478bd9Sstevel@tonic-gate rdma_clnt_postrecv(CONN *conn, uint32_t xid)
3967c478bd9Sstevel@tonic-gate {
3977c478bd9Sstevel@tonic-gate 	struct clist *cl = NULL;
3987c478bd9Sstevel@tonic-gate 	rdma_stat retval;
3990a701b1eSRobert Gordon 	rdma_buf_t rbuf = {0};
4007c478bd9Sstevel@tonic-gate 
4017c478bd9Sstevel@tonic-gate 	rbuf.type = RECV_BUFFER;
4027c478bd9Sstevel@tonic-gate 	if (RDMA_BUF_ALLOC(conn, &rbuf)) {
4030a701b1eSRobert Gordon 		return (RDMA_NORESOURCE);
4047c478bd9Sstevel@tonic-gate 	}
4050a701b1eSRobert Gordon 
4060a701b1eSRobert Gordon 	clist_add(&cl, 0, rbuf.len, &rbuf.handle, rbuf.addr,
4070a701b1eSRobert Gordon 	    NULL, NULL);
4080a701b1eSRobert Gordon 	retval = RDMA_CLNT_RECVBUF(conn, cl, xid);
4090a701b1eSRobert Gordon 	clist_free(cl);
4100a701b1eSRobert Gordon 
4117c478bd9Sstevel@tonic-gate 	return (retval);
4127c478bd9Sstevel@tonic-gate }
4137c478bd9Sstevel@tonic-gate 
4140a701b1eSRobert Gordon rdma_stat
4150a701b1eSRobert Gordon rdma_clnt_postrecv_remove(CONN *conn, uint32_t xid)
4160a701b1eSRobert Gordon {
4170a701b1eSRobert Gordon 	return (RDMA_CLNT_RECVBUF_REMOVE(conn, xid));
4180a701b1eSRobert Gordon }
4190a701b1eSRobert Gordon 
4207c478bd9Sstevel@tonic-gate rdma_stat
4217c478bd9Sstevel@tonic-gate rdma_svc_postrecv(CONN *conn)
4227c478bd9Sstevel@tonic-gate {
4237c478bd9Sstevel@tonic-gate 	struct clist *cl = NULL;
4247c478bd9Sstevel@tonic-gate 	rdma_stat retval;
4250a701b1eSRobert Gordon 	rdma_buf_t rbuf = {0};
4267c478bd9Sstevel@tonic-gate 
4277c478bd9Sstevel@tonic-gate 	rbuf.type = RECV_BUFFER;
4287c478bd9Sstevel@tonic-gate 	if (RDMA_BUF_ALLOC(conn, &rbuf)) {
4297c478bd9Sstevel@tonic-gate 		retval = RDMA_NORESOURCE;
4307c478bd9Sstevel@tonic-gate 	} else {
4317c478bd9Sstevel@tonic-gate 		clist_add(&cl, 0, rbuf.len, &rbuf.handle, rbuf.addr,
4320a701b1eSRobert Gordon 		    NULL, NULL);
4337c478bd9Sstevel@tonic-gate 		retval = RDMA_SVC_RECVBUF(conn, cl);
4347c478bd9Sstevel@tonic-gate 		clist_free(cl);
4357c478bd9Sstevel@tonic-gate 	}
4367c478bd9Sstevel@tonic-gate 	return (retval);
4377c478bd9Sstevel@tonic-gate }
4387c478bd9Sstevel@tonic-gate 
4397c478bd9Sstevel@tonic-gate rdma_stat
4400a701b1eSRobert Gordon rdma_buf_alloc(CONN *conn, rdma_buf_t *rbuf)
4417c478bd9Sstevel@tonic-gate {
4420a701b1eSRobert Gordon 	return (RDMA_BUF_ALLOC(conn, rbuf));
4437c478bd9Sstevel@tonic-gate }
4447c478bd9Sstevel@tonic-gate 
4457c478bd9Sstevel@tonic-gate void
4467c478bd9Sstevel@tonic-gate rdma_buf_free(CONN *conn, rdma_buf_t *rbuf)
4477c478bd9Sstevel@tonic-gate {
4487c478bd9Sstevel@tonic-gate 	if (!rbuf || rbuf->addr == NULL) {
4497c478bd9Sstevel@tonic-gate 		return;
4507c478bd9Sstevel@tonic-gate 	}
4510a701b1eSRobert Gordon 	RDMA_BUF_FREE(conn, rbuf);
4520a701b1eSRobert Gordon 	bzero(rbuf, sizeof (rdma_buf_t));
4537c478bd9Sstevel@tonic-gate }
4547c478bd9Sstevel@tonic-gate 
4557c478bd9Sstevel@tonic-gate /*
4567c478bd9Sstevel@tonic-gate  * Caller is holding rdma_modload_lock mutex
4577c478bd9Sstevel@tonic-gate  */
4587c478bd9Sstevel@tonic-gate int
4597c478bd9Sstevel@tonic-gate rdma_modload()
4607c478bd9Sstevel@tonic-gate {
4617c478bd9Sstevel@tonic-gate 	int status;
4627c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&rdma_modload_lock));
4637c478bd9Sstevel@tonic-gate 	/*
4647c478bd9Sstevel@tonic-gate 	 * Load all available RDMA plugins which right now is only IB plugin.
4657c478bd9Sstevel@tonic-gate 	 * If no IB hardware is present, then quit right away.
4667c478bd9Sstevel@tonic-gate 	 * ENODEV -- For no device on the system
4677c478bd9Sstevel@tonic-gate 	 * EPROTONOSUPPORT -- For module not avilable either due to failure to
4687c478bd9Sstevel@tonic-gate 	 * load or some other reason.
4697c478bd9Sstevel@tonic-gate 	 */
4707c478bd9Sstevel@tonic-gate 	rdma_modloaded = 1;
4717c478bd9Sstevel@tonic-gate 	if (ibt_hw_is_present() == 0) {
4727c478bd9Sstevel@tonic-gate 		rdma_dev_available = 0;
4737c478bd9Sstevel@tonic-gate 		return (ENODEV);
4747c478bd9Sstevel@tonic-gate 	}
4757c478bd9Sstevel@tonic-gate 
4767c478bd9Sstevel@tonic-gate 	rdma_dev_available = 1;
4777c478bd9Sstevel@tonic-gate 	if (rpcmod_li == NULL)
4787c478bd9Sstevel@tonic-gate 		return (EPROTONOSUPPORT);
4797c478bd9Sstevel@tonic-gate 
4807c478bd9Sstevel@tonic-gate 	status = ldi_open_by_name("/devices/ib/rpcib@0:rpcib",
4817c478bd9Sstevel@tonic-gate 	    FREAD | FWRITE, kcred,
4827c478bd9Sstevel@tonic-gate 	    &rpcib_handle, rpcmod_li);
48351f34d4bSRajkumar Sivaprakasam 
4847c478bd9Sstevel@tonic-gate 	if (status != 0)
4857c478bd9Sstevel@tonic-gate 		return (EPROTONOSUPPORT);
4867c478bd9Sstevel@tonic-gate 
4870a701b1eSRobert Gordon 
48851f34d4bSRajkumar Sivaprakasam 	/*
48951f34d4bSRajkumar Sivaprakasam 	 * We will need to reload the plugin module after it was unregistered
49051f34d4bSRajkumar Sivaprakasam 	 * but the resources below need to allocated only the first time.
49151f34d4bSRajkumar Sivaprakasam 	 */
49251f34d4bSRajkumar Sivaprakasam 	if (!clist_cache) {
49351f34d4bSRajkumar Sivaprakasam 		clist_cache = kmem_cache_create("rdma_clist",
49451f34d4bSRajkumar Sivaprakasam 		    sizeof (struct clist), _POINTER_ALIGNMENT, NULL,
49551f34d4bSRajkumar Sivaprakasam 		    NULL, NULL, NULL, 0, 0);
49651f34d4bSRajkumar Sivaprakasam 		rdma_kstat_init();
49751f34d4bSRajkumar Sivaprakasam 	}
49851f34d4bSRajkumar Sivaprakasam 
49951f34d4bSRajkumar Sivaprakasam 	(void) ldi_close(rpcib_handle, FREAD|FWRITE, kcred);
5000a701b1eSRobert Gordon 
5017c478bd9Sstevel@tonic-gate 	return (0);
5027c478bd9Sstevel@tonic-gate }
5037c478bd9Sstevel@tonic-gate 
5047c478bd9Sstevel@tonic-gate void
5057c478bd9Sstevel@tonic-gate rdma_kstat_init(void)
5067c478bd9Sstevel@tonic-gate {
5077c478bd9Sstevel@tonic-gate 	kstat_t *ksp;
5087c478bd9Sstevel@tonic-gate 
5097c478bd9Sstevel@tonic-gate 	/*
5107c478bd9Sstevel@tonic-gate 	 * The RDMA framework doesn't know how to deal with Zones, and is
5117c478bd9Sstevel@tonic-gate 	 * only available in the global zone.
5127c478bd9Sstevel@tonic-gate 	 */
5137c478bd9Sstevel@tonic-gate 	ASSERT(INGLOBALZONE(curproc));
5147c478bd9Sstevel@tonic-gate 	ksp = kstat_create_zone("unix", 0, "rpc_rdma_client", "rpc",
5157c478bd9Sstevel@tonic-gate 	    KSTAT_TYPE_NAMED, rdmarcstat_ndata,
5167c478bd9Sstevel@tonic-gate 	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE, GLOBAL_ZONEID);
5177c478bd9Sstevel@tonic-gate 	if (ksp) {
5187c478bd9Sstevel@tonic-gate 		ksp->ks_data = (void *) rdmarcstat_ptr;
5197c478bd9Sstevel@tonic-gate 		kstat_install(ksp);
5207c478bd9Sstevel@tonic-gate 	}
5217c478bd9Sstevel@tonic-gate 
5227c478bd9Sstevel@tonic-gate 	ksp = kstat_create_zone("unix", 0, "rpc_rdma_server", "rpc",
5237c478bd9Sstevel@tonic-gate 	    KSTAT_TYPE_NAMED, rdmarsstat_ndata,
5247c478bd9Sstevel@tonic-gate 	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE, GLOBAL_ZONEID);
5257c478bd9Sstevel@tonic-gate 	if (ksp) {
5267c478bd9Sstevel@tonic-gate 		ksp->ks_data = (void *) rdmarsstat_ptr;
5277c478bd9Sstevel@tonic-gate 		kstat_install(ksp);
5287c478bd9Sstevel@tonic-gate 	}
5297c478bd9Sstevel@tonic-gate }
53051f34d4bSRajkumar Sivaprakasam 
53151f34d4bSRajkumar Sivaprakasam rdma_stat
53251f34d4bSRajkumar Sivaprakasam rdma_kwait(void)
53351f34d4bSRajkumar Sivaprakasam {
53451f34d4bSRajkumar Sivaprakasam 	int ret;
53551f34d4bSRajkumar Sivaprakasam 	rdma_stat stat;
53651f34d4bSRajkumar Sivaprakasam 
53751f34d4bSRajkumar Sivaprakasam 	mutex_enter(&rdma_wait.svc_lock);
53851f34d4bSRajkumar Sivaprakasam 
53951f34d4bSRajkumar Sivaprakasam 	ret = cv_wait_sig(&rdma_wait.svc_cv, &rdma_wait.svc_lock);
54051f34d4bSRajkumar Sivaprakasam 
54151f34d4bSRajkumar Sivaprakasam 	/*
54251f34d4bSRajkumar Sivaprakasam 	 * If signalled by a hca attach/detach, pass the right
54351f34d4bSRajkumar Sivaprakasam 	 * stat back.
54451f34d4bSRajkumar Sivaprakasam 	 */
54551f34d4bSRajkumar Sivaprakasam 
54651f34d4bSRajkumar Sivaprakasam 	if (ret)
54751f34d4bSRajkumar Sivaprakasam 		stat =  rdma_wait.svc_stat;
54851f34d4bSRajkumar Sivaprakasam 	else
54951f34d4bSRajkumar Sivaprakasam 		stat = RDMA_INTR;
55051f34d4bSRajkumar Sivaprakasam 
55151f34d4bSRajkumar Sivaprakasam 	mutex_exit(&rdma_wait.svc_lock);
55251f34d4bSRajkumar Sivaprakasam 
55351f34d4bSRajkumar Sivaprakasam 	return (stat);
55451f34d4bSRajkumar Sivaprakasam }
555