17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
50a701b1Robert Gordon * Common Development and Distribution License (the "License").
60a701b1Robert Gordon * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
2251f34d4Rajkumar Sivaprakasam * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
247c478bdstevel@tonic-gate */
257c478bdstevel@tonic-gate
260a701b1Robert Gordon/*
2751f34d4Rajkumar Sivaprakasam * Copyright (c) 2008, The Ohio State University. All rights reserved.
280a701b1Robert Gordon *
290a701b1Robert Gordon * Portions of this source code is developed by the team members of
300a701b1Robert Gordon * The Ohio State University's Network-Based Computing Laboratory (NBCL),
310a701b1Robert Gordon * headed by Professor Dhabaleswar K. (DK) Panda.
320a701b1Robert Gordon *
330a701b1Robert Gordon * Acknowledgements to contributions from developors:
340a701b1Robert Gordon *   Ranjit Noronha: noronha@cse.ohio-state.edu
350a701b1Robert Gordon *   Lei Chai      : chail@cse.ohio-state.edu
360a701b1Robert Gordon *   Weikuan Yu    : yuw@cse.ohio-state.edu
370a701b1Robert Gordon *
380a701b1Robert Gordon */
397c478bdstevel@tonic-gate
407c478bdstevel@tonic-gate#include <sys/systm.h>
417c478bdstevel@tonic-gate#include <sys/kstat.h>
427c478bdstevel@tonic-gate#include <sys/modctl.h>
430a701b1Robert Gordon#include <sys/sdt.h>
447c478bdstevel@tonic-gate#include <rpc/rpc_rdma.h>
457c478bdstevel@tonic-gate
467c478bdstevel@tonic-gate#include <sys/ib/ibtl/ibti.h>
477c478bdstevel@tonic-gate
487c478bdstevel@tonic-gateuint_t rdma_minchunk = RDMA_MINCHUNK;
497c478bdstevel@tonic-gate
507c478bdstevel@tonic-gate/*
517c478bdstevel@tonic-gate * Globals
527c478bdstevel@tonic-gate */
537c478bdstevel@tonic-gateint rdma_modloaded = 0;		/* flag to load RDMA plugin modules */
547c478bdstevel@tonic-gateint rdma_dev_available = 0;	/* if any RDMA device is loaded */
557c478bdstevel@tonic-gatekmutex_t rdma_modload_lock;	/* protects rdma_modloaded flag */
5651f34d4Rajkumar Sivaprakasam
5751f34d4Rajkumar Sivaprakasamrdma_svc_wait_t rdma_wait;
5851f34d4Rajkumar Sivaprakasam
597c478bdstevel@tonic-gaterdma_registry_t	*rdma_mod_head = NULL;	/* head for RDMA modules */
607c478bdstevel@tonic-gatekrwlock_t	rdma_lock;		/* protects rdma_mod_head list */
617c478bdstevel@tonic-gateldi_ident_t rpcmod_li = NULL;	/* identifies us with ldi_ framework */
627c478bdstevel@tonic-gate
630a701b1Robert Gordonkmem_cache_t *clist_cache = NULL;
640a701b1Robert Gordon
657c478bdstevel@tonic-gate/*
667c478bdstevel@tonic-gate * Statics
677c478bdstevel@tonic-gate */
6851f34d4Rajkumar Sivaprakasamldi_handle_t rpcib_handle = NULL;
697c478bdstevel@tonic-gate
707c478bdstevel@tonic-gate/*
717c478bdstevel@tonic-gate * Externs
727c478bdstevel@tonic-gate */
737c478bdstevel@tonic-gateextern	kstat_named_t	*rdmarcstat_ptr;
747c478bdstevel@tonic-gateextern	uint_t		rdmarcstat_ndata;
757c478bdstevel@tonic-gateextern	kstat_named_t	*rdmarsstat_ptr;
767c478bdstevel@tonic-gateextern	uint_t		rdmarsstat_ndata;
777c478bdstevel@tonic-gate
787c478bdstevel@tonic-gatevoid rdma_kstat_init();
797c478bdstevel@tonic-gate
807c478bdstevel@tonic-gate/*
817c478bdstevel@tonic-gate * RDMATF module registration routine.
827c478bdstevel@tonic-gate * This routine is expected to be called by the init routine in
837c478bdstevel@tonic-gate * the plugin modules.
847c478bdstevel@tonic-gate */
857c478bdstevel@tonic-gaterdma_stat
867c478bdstevel@tonic-gaterdma_register_mod(rdma_mod_t *mod)
877c478bdstevel@tonic-gate{
887c478bdstevel@tonic-gate	rdma_registry_t **mp, *m;
897c478bdstevel@tonic-gate
907c478bdstevel@tonic-gate	if (mod->rdma_version != RDMATF_VERS) {
917c478bdstevel@tonic-gate		return (RDMA_BADVERS);
927c478bdstevel@tonic-gate	}
937c478bdstevel@tonic-gate
947c478bdstevel@tonic-gate	rw_enter(&rdma_lock, RW_WRITER);
957c478bdstevel@tonic-gate	/*
967c478bdstevel@tonic-gate	 * Ensure not already registered
977c478bdstevel@tonic-gate	 */
987c478bdstevel@tonic-gate	mp = &rdma_mod_head;
997c478bdstevel@tonic-gate	while (*mp != NULL) {
1007c478bdstevel@tonic-gate		if (strncmp((*mp)->r_mod->rdma_api, mod->rdma_api,
1017c478bdstevel@tonic-gate		    KNC_STRSIZE) == 0) {
10251f34d4Rajkumar Sivaprakasam			if ((*mp)->r_mod_state == RDMA_MOD_INACTIVE) {
10351f34d4Rajkumar Sivaprakasam				(*mp)->r_mod_state = RDMA_MOD_ACTIVE;
10451f34d4Rajkumar Sivaprakasam				(*mp)->r_mod->rdma_ops = mod->rdma_ops;
10551f34d4Rajkumar Sivaprakasam				(*mp)->r_mod->rdma_count = mod->rdma_count;
10651f34d4Rajkumar Sivaprakasam				goto announce_hca;
10751f34d4Rajkumar Sivaprakasam			}
1087c478bdstevel@tonic-gate			rw_exit(&rdma_lock);
1097c478bdstevel@tonic-gate			return (RDMA_REG_EXIST);
1107c478bdstevel@tonic-gate		}
1117c478bdstevel@tonic-gate		mp = &((*mp)->r_next);
1127c478bdstevel@tonic-gate	}
1137c478bdstevel@tonic-gate
1147c478bdstevel@tonic-gate	/*
1157c478bdstevel@tonic-gate	 * New one, create and add to registry
1167c478bdstevel@tonic-gate	 */
1177c478bdstevel@tonic-gate	m = kmem_alloc(sizeof (rdma_registry_t), KM_SLEEP);
1187c478bdstevel@tonic-gate	m->r_mod = kmem_alloc(sizeof (rdma_mod_t), KM_SLEEP);
1197c478bdstevel@tonic-gate	*m->r_mod = *mod;
1207c478bdstevel@tonic-gate	m->r_next = NULL;
1217c478bdstevel@tonic-gate	m->r_mod->rdma_api = kmem_zalloc(KNC_STRSIZE, KM_SLEEP);
1227c478bdstevel@tonic-gate	(void) strncpy(m->r_mod->rdma_api, mod->rdma_api, KNC_STRSIZE);
1237c478bdstevel@tonic-gate	m->r_mod->rdma_api[KNC_STRSIZE - 1] = '\0';
12451f34d4Rajkumar Sivaprakasam	m->r_mod_state = RDMA_MOD_ACTIVE;
1257c478bdstevel@tonic-gate	*mp = m;
12651f34d4Rajkumar Sivaprakasam
12751f34d4Rajkumar Sivaprakasamannounce_hca:
1287c478bdstevel@tonic-gate	rw_exit(&rdma_lock);
12951f34d4Rajkumar Sivaprakasam	/*
13051f34d4Rajkumar Sivaprakasam	 * Start the nfs service on the rdma xprts.
13151f34d4Rajkumar Sivaprakasam	 * (this notification mechanism will need to change when we support
13251f34d4Rajkumar Sivaprakasam	 * multiple hcas and have support for multiple rdma plugins).
13351f34d4Rajkumar Sivaprakasam	 */
13451f34d4Rajkumar Sivaprakasam	mutex_enter(&rdma_wait.svc_lock);
13551f34d4Rajkumar Sivaprakasam	rdma_wait.svc_stat = RDMA_HCA_ATTACH;
13651f34d4Rajkumar Sivaprakasam	cv_signal(&rdma_wait.svc_cv);
13751f34d4Rajkumar Sivaprakasam	mutex_exit(&rdma_wait.svc_lock);
1387c478bdstevel@tonic-gate
1397c478bdstevel@tonic-gate	return (RDMA_SUCCESS);
1407c478bdstevel@tonic-gate}
1417c478bdstevel@tonic-gate
1427c478bdstevel@tonic-gate/*
1437c478bdstevel@tonic-gate * RDMATF module unregistration routine.
1447c478bdstevel@tonic-gate * This routine is expected to be called by the fini routine in
1457c478bdstevel@tonic-gate * the plugin modules.
1467c478bdstevel@tonic-gate */
1477c478bdstevel@tonic-gaterdma_stat
1487c478bdstevel@tonic-gaterdma_unregister_mod(rdma_mod_t *mod)
1497c478bdstevel@tonic-gate{
1507c478bdstevel@tonic-gate	rdma_registry_t **m, *mmod = NULL;
1517c478bdstevel@tonic-gate
1527c478bdstevel@tonic-gate	rw_enter(&rdma_lock, RW_WRITER);
1537c478bdstevel@tonic-gate
1547c478bdstevel@tonic-gate	m = &rdma_mod_head;
1557c478bdstevel@tonic-gate	while (*m != NULL) {
1567c478bdstevel@tonic-gate		if (strncmp((*m)->r_mod->rdma_api, mod->rdma_api,
1577c478bdstevel@tonic-gate		    KNC_STRSIZE) != 0) {
1587c478bdstevel@tonic-gate			m = &((*m)->r_next);
1597c478bdstevel@tonic-gate			continue;
1607c478bdstevel@tonic-gate		}
1617c478bdstevel@tonic-gate		/*
1627c478bdstevel@tonic-gate		 * Check if any device attached, if so return error
1637c478bdstevel@tonic-gate		 */
16451f34d4Rajkumar Sivaprakasam		if (mod->rdma_count != 0) {
1657c478bdstevel@tonic-gate			rw_exit(&rdma_lock);
1667c478bdstevel@tonic-gate			return (RDMA_FAILED);
1677c478bdstevel@tonic-gate		}
1687c478bdstevel@tonic-gate		/*
16951f34d4Rajkumar Sivaprakasam		 * Found entry. Mark it inactive.
1707c478bdstevel@tonic-gate		 */
1717c478bdstevel@tonic-gate		mmod = *m;
17251f34d4Rajkumar Sivaprakasam		mmod->r_mod->rdma_count = 0;
17351f34d4Rajkumar Sivaprakasam		mmod->r_mod_state = RDMA_MOD_INACTIVE;
17451f34d4Rajkumar Sivaprakasam		break;
1757c478bdstevel@tonic-gate	}
1767c478bdstevel@tonic-gate
17751f34d4Rajkumar Sivaprakasam	rdma_modloaded = 0;
17851f34d4Rajkumar Sivaprakasam	rdma_dev_available = 0;
17951f34d4Rajkumar Sivaprakasam	rw_exit(&rdma_lock);
18051f34d4Rajkumar Sivaprakasam
18151f34d4Rajkumar Sivaprakasam	/*
18251f34d4Rajkumar Sivaprakasam	 * Stop the nfs service running on the rdma xprts.
18351f34d4Rajkumar Sivaprakasam	 * (this notification mechanism will need to change when we support
18451f34d4Rajkumar Sivaprakasam	 * multiple hcas and have support for multiple rdma plugins).
18551f34d4Rajkumar Sivaprakasam	 */
18651f34d4Rajkumar Sivaprakasam	mutex_enter(&rdma_wait.svc_lock);
18751f34d4Rajkumar Sivaprakasam	rdma_wait.svc_stat = RDMA_HCA_DETACH;
18851f34d4Rajkumar Sivaprakasam	cv_signal(&rdma_wait.svc_cv);
18951f34d4Rajkumar Sivaprakasam	mutex_exit(&rdma_wait.svc_lock);
19051f34d4Rajkumar Sivaprakasam
1917c478bdstevel@tonic-gate	/*
1927c478bdstevel@tonic-gate	 * Not found.
1937c478bdstevel@tonic-gate	 */
19451f34d4Rajkumar Sivaprakasam	return (RDMA_SUCCESS);
1957c478bdstevel@tonic-gate}
1967c478bdstevel@tonic-gate
1970a701b1Robert Gordonstruct clist *
1980a701b1Robert Gordonclist_alloc(void)
1990a701b1Robert Gordon{
2000a701b1Robert Gordon	struct clist *clp;
2010a701b1Robert Gordon
2020a701b1Robert Gordon	clp = kmem_cache_alloc(clist_cache, KM_SLEEP);
2030a701b1Robert Gordon
2040a701b1Robert Gordon	bzero(clp, sizeof (*clp));
2050a701b1Robert Gordon
2060a701b1Robert Gordon	return (clp);
2070a701b1Robert Gordon}
2080a701b1Robert Gordon
209f837ee4Siddheshwar Maheshuint32_t
210f837ee4Siddheshwar Maheshclist_len(struct clist *cl)
211f837ee4Siddheshwar Mahesh{
212f837ee4Siddheshwar Mahesh	uint32_t len = 0;
213f837ee4Siddheshwar Mahesh	while (cl) {
214f837ee4Siddheshwar Mahesh		len += cl->c_len;
215f837ee4Siddheshwar Mahesh		cl = cl->c_next;
216f837ee4Siddheshwar Mahesh	}
217f837ee4Siddheshwar Mahesh	return (len);
218f837ee4Siddheshwar Mahesh}
219f837ee4Siddheshwar Mahesh
220f837ee4Siddheshwar Maheshvoid
221f837ee4Siddheshwar Maheshclist_zero_len(struct clist *cl)
222f837ee4Siddheshwar Mahesh{
223f837ee4Siddheshwar Mahesh	while (cl != NULL) {
224f837ee4Siddheshwar Mahesh		if (cl->c_dmemhandle.mrc_rmr == 0)
225f837ee4Siddheshwar Mahesh			break;
226f837ee4Siddheshwar Mahesh		cl->c_len = 0;
227f837ee4Siddheshwar Mahesh		cl = cl->c_next;
228f837ee4Siddheshwar Mahesh	}
229f837ee4Siddheshwar Mahesh}
230f837ee4Siddheshwar Mahesh
2317c478bdstevel@tonic-gate/*
2327c478bdstevel@tonic-gate * Creates a new chunk list entry, and
2337c478bdstevel@tonic-gate * adds it to the end of a chunk list.
2347c478bdstevel@tonic-gate */
2357c478bdstevel@tonic-gatevoid
2367c478bdstevel@tonic-gateclist_add(struct clist **clp, uint32_t xdroff, int len,
23725c9576Toomas Soome    struct mrc *shandle, caddr_t saddr,
23825c9576Toomas Soome    struct mrc *dhandle, caddr_t daddr)
2397c478bdstevel@tonic-gate{
2407c478bdstevel@tonic-gate	struct clist *cl;
2417c478bdstevel@tonic-gate
2427c478bdstevel@tonic-gate	/* Find the end of the list */
2437c478bdstevel@tonic-gate
2447c478bdstevel@tonic-gate	while (*clp != NULL)
2457c478bdstevel@tonic-gate		clp = &((*clp)->c_next);
2467c478bdstevel@tonic-gate
2470a701b1Robert Gordon	cl = clist_alloc();
2487c478bdstevel@tonic-gate	cl->c_xdroff = xdroff;
2497c478bdstevel@tonic-gate	cl->c_len = len;
2500a701b1Robert Gordon	cl->w.c_saddr = (uint64_t)(uintptr_t)saddr;
2517c478bdstevel@tonic-gate	if (shandle)
2527c478bdstevel@tonic-gate		cl->c_smemhandle = *shandle;
2530a701b1Robert Gordon	cl->u.c_daddr = (uint64_t)(uintptr_t)daddr;
2547c478bdstevel@tonic-gate	if (dhandle)
2557c478bdstevel@tonic-gate		cl->c_dmemhandle = *dhandle;
2567c478bdstevel@tonic-gate	cl->c_next = NULL;
2577c478bdstevel@tonic-gate
2587c478bdstevel@tonic-gate	*clp = cl;
2597c478bdstevel@tonic-gate}
2607c478bdstevel@tonic-gate
2610a701b1Robert Gordonrdma_stat
2620a701b1Robert Gordonclist_register(CONN *conn, struct clist *cl, clist_dstsrc dstsrc)
2637c478bdstevel@tonic-gate{
2647c478bdstevel@tonic-gate	struct clist *c;
2657c478bdstevel@tonic-gate	int status;
2667c478bdstevel@tonic-gate
2677c478bdstevel@tonic-gate	for (c = cl; c; c = c->c_next) {
2680a701b1Robert Gordon		if (c->c_len <= 0)
2690a701b1Robert Gordon			continue;
270f837ee4Siddheshwar Mahesh
271f837ee4Siddheshwar Mahesh		c->c_regtype = dstsrc;
272f837ee4Siddheshwar Mahesh
2730a701b1Robert Gordon		switch (dstsrc) {
2740a701b1Robert Gordon		case CLIST_REG_SOURCE:
2757c478bdstevel@tonic-gate			status = RDMA_REGMEMSYNC(conn,
276f837ee4Siddheshwar Mahesh			    (caddr_t)(struct as *)c->c_adspc,
2770a701b1Robert Gordon			    (caddr_t)(uintptr_t)c->w.c_saddr3, c->c_len,
2780a701b1Robert Gordon			    &c->c_smemhandle, (void **)&c->c_ssynchandle,
2790a701b1Robert Gordon			    (void *)c->rb_longbuf.rb_private);
2800a701b1Robert Gordon			break;
2810a701b1Robert Gordon		case CLIST_REG_DST:
2827c478bdstevel@tonic-gate			status = RDMA_REGMEMSYNC(conn,
283f837ee4Siddheshwar Mahesh			    (caddr_t)(struct as *)c->c_adspc,
2840a701b1Robert Gordon			    (caddr_t)(uintptr_t)c->u.c_daddr3, c->c_len,
2850a701b1Robert Gordon			    &c->c_dmemhandle, (void **)&c->c_dsynchandle,
2860a701b1Robert Gordon			    (void *)c->rb_longbuf.rb_private);
2870a701b1Robert Gordon			break;
2880a701b1Robert Gordon		default:
2890a701b1Robert Gordon			return (RDMA_INVAL);
2907c478bdstevel@tonic-gate		}
2917c478bdstevel@tonic-gate		if (status != RDMA_SUCCESS) {
292f837ee4Siddheshwar Mahesh			(void) clist_deregister(conn, cl);
2937c478bdstevel@tonic-gate			return (status);
2947c478bdstevel@tonic-gate		}
2957c478bdstevel@tonic-gate	}
2967c478bdstevel@tonic-gate
2977c478bdstevel@tonic-gate	return (RDMA_SUCCESS);
2987c478bdstevel@tonic-gate}
2997c478bdstevel@tonic-gate
3000a701b1Robert Gordonrdma_stat
301f837ee4Siddheshwar Maheshclist_deregister(CONN *conn, struct clist *cl)
3027c478bdstevel@tonic-gate{
3037c478bdstevel@tonic-gate	struct clist *c;
3047c478bdstevel@tonic-gate
3057c478bdstevel@tonic-gate	for (c = cl; c; c = c->c_next) {
306f837ee4Siddheshwar Mahesh		switch (c->c_regtype) {
3070a701b1Robert Gordon		case CLIST_REG_SOURCE:
3087c478bdstevel@tonic-gate			if (c->c_smemhandle.mrc_rmr != 0) {
3097c478bdstevel@tonic-gate				(void) RDMA_DEREGMEMSYNC(conn,
3100a701b1Robert Gordon				    (caddr_t)(uintptr_t)c->w.c_saddr3,
3117c478bdstevel@tonic-gate				    c->c_smemhandle,
3120a701b1Robert Gordon				    (void *)(uintptr_t)c->c_ssynchandle,
3130a701b1Robert Gordon				    (void *)c->rb_longbuf.rb_private);
3147c478bdstevel@tonic-gate				c->c_smemhandle.mrc_rmr = 0;
31525c9576Toomas Soome				c->c_ssynchandle = 0;
3167c478bdstevel@tonic-gate			}
3170a701b1Robert Gordon			break;
3180a701b1Robert Gordon		case CLIST_REG_DST:
3197c478bdstevel@tonic-gate			if (c->c_dmemhandle.mrc_rmr != 0) {
3207c478bdstevel@tonic-gate				(void) RDMA_DEREGMEMSYNC(conn,
3210a701b1Robert Gordon				    (caddr_t)(uintptr_t)c->u.c_daddr3,
3227c478bdstevel@tonic-gate				    c->c_dmemhandle,
3230a701b1Robert Gordon				    (void *)(uintptr_t)c->c_dsynchandle,
3240a701b1Robert Gordon				    (void *)c->rb_longbuf.rb_private);
3257c478bdstevel@tonic-gate				c->c_dmemhandle.mrc_rmr = 0;
32625c9576Toomas Soome				c->c_dsynchandle = 0;
3277c478bdstevel@tonic-gate			}
3280a701b1Robert Gordon			break;
3290a701b1Robert Gordon		default:
330f837ee4Siddheshwar Mahesh			/* clist unregistered. continue */
331f837ee4Siddheshwar Mahesh			break;
3327c478bdstevel@tonic-gate		}
3337c478bdstevel@tonic-gate	}
3347c478bdstevel@tonic-gate
3357c478bdstevel@tonic-gate	return (RDMA_SUCCESS);
3367c478bdstevel@tonic-gate}
3377c478bdstevel@tonic-gate
3380a701b1Robert Gordonrdma_stat
3390a701b1Robert Gordonclist_syncmem(CONN *conn, struct clist *cl, clist_dstsrc dstsrc)
3400a701b1Robert Gordon{
3410a701b1Robert Gordon	struct clist *c;
3420a701b1Robert Gordon	rdma_stat status;
3430a701b1Robert Gordon
3440a701b1Robert Gordon	c = cl;
3450a701b1Robert Gordon	switch (dstsrc) {
3460a701b1Robert Gordon	case CLIST_REG_SOURCE:
3470a701b1Robert Gordon		while (c != NULL) {
3480a701b1Robert Gordon			if (c->c_ssynchandle) {
3490a701b1Robert Gordon				status = RDMA_SYNCMEM(conn,
3500a701b1Robert Gordon				    (void *)(uintptr_t)c->c_ssynchandle,
3510a701b1Robert Gordon				    (caddr_t)(uintptr_t)c->w.c_saddr3,
3520a701b1Robert Gordon				    c->c_len, 0);
3530a701b1Robert Gordon				if (status != RDMA_SUCCESS)
3540a701b1Robert Gordon					return (status);
3550a701b1Robert Gordon			}
3560a701b1Robert Gordon			c = c->c_next;
3570a701b1Robert Gordon		}
3580a701b1Robert Gordon		break;
3590a701b1Robert Gordon	case CLIST_REG_DST:
3600a701b1Robert Gordon		while (c != NULL) {
3610a701b1Robert Gordon			if (c->c_ssynchandle) {
3620a701b1Robert Gordon				status = RDMA_SYNCMEM(conn,
3630a701b1Robert Gordon				    (void *)(uintptr_t)c->c_dsynchandle,
3640a701b1Robert Gordon				    (caddr_t)(uintptr_t)c->u.c_daddr3,
3650a701b1Robert Gordon				    c->c_len, 1);
3660a701b1Robert Gordon				if (status != RDMA_SUCCESS)
3670a701b1Robert Gordon					return (status);
3680a701b1Robert Gordon			}
3690a701b1Robert Gordon			c = c->c_next;
3700a701b1Robert Gordon		}
3710a701b1Robert Gordon		break;
3720a701b1Robert Gordon	default:
3730a701b1Robert Gordon		return (RDMA_INVAL);
3740a701b1Robert Gordon	}
3750a701b1Robert Gordon
3760a701b1Robert Gordon	return (RDMA_SUCCESS);
3770a701b1Robert Gordon}
3780a701b1Robert Gordon
3797c478bdstevel@tonic-gate/*
3807c478bdstevel@tonic-gate * Frees up entries in chunk list
3817c478bdstevel@tonic-gate */
3827c478bdstevel@tonic-gatevoid
3837c478bdstevel@tonic-gateclist_free(struct clist *cl)
3847c478bdstevel@tonic-gate{
3857c478bdstevel@tonic-gate	struct clist *c = cl;
3867c478bdstevel@tonic-gate
3877c478bdstevel@tonic-gate	while (c != NULL) {
3887c478bdstevel@tonic-gate		cl = cl->c_next;
3890a701b1Robert Gordon		kmem_cache_free(clist_cache, c);
3907c478bdstevel@tonic-gate		c = cl;
3917c478bdstevel@tonic-gate	}
3927c478bdstevel@tonic-gate}
3937c478bdstevel@tonic-gate
3947c478bdstevel@tonic-gaterdma_stat
3957c478bdstevel@tonic-gaterdma_clnt_postrecv(CONN *conn, uint32_t xid)
3967c478bdstevel@tonic-gate{
3977c478bdstevel@tonic-gate	struct clist *cl = NULL;
3987c478bdstevel@tonic-gate	rdma_stat retval;
3990a701b1Robert Gordon	rdma_buf_t rbuf = {0};
4007c478bdstevel@tonic-gate
4017c478bdstevel@tonic-gate	rbuf.type = RECV_BUFFER;
4027c478bdstevel@tonic-gate	if (RDMA_BUF_ALLOC(conn, &rbuf)) {
4030a701b1Robert Gordon		return (RDMA_NORESOURCE);
4047c478bdstevel@tonic-gate	}
4050a701b1Robert Gordon
4060a701b1Robert Gordon	clist_add(&cl, 0, rbuf.len, &rbuf.handle, rbuf.addr,
4070a701b1Robert Gordon	    NULL, NULL);
4080a701b1Robert Gordon	retval = RDMA_CLNT_RECVBUF(conn, cl, xid);
4090a701b1Robert Gordon	clist_free(cl);
4100a701b1Robert Gordon
4117c478bdstevel@tonic-gate	return (retval);
4127c478bdstevel@tonic-gate}
4137c478bdstevel@tonic-gate
4147c478bdstevel@tonic-gaterdma_stat
4150a701b1Robert Gordonrdma_clnt_postrecv_remove(CONN *conn, uint32_t xid)
4160a701b1Robert Gordon{
4170a701b1Robert Gordon	return (RDMA_CLNT_RECVBUF_REMOVE(conn, xid));
4180a701b1Robert Gordon}
4190a701b1Robert Gordon
4200a701b1Robert Gordonrdma_stat
4217c478bdstevel@tonic-gaterdma_svc_postrecv(CONN *conn)
4227c478bdstevel@tonic-gate{
4237c478bdstevel@tonic-gate	struct clist *cl = NULL;
4247c478bdstevel@tonic-gate	rdma_stat retval;
4250a701b1Robert Gordon	rdma_buf_t rbuf = {0};
4267c478bdstevel@tonic-gate
4277c478bdstevel@tonic-gate	rbuf.type = RECV_BUFFER;
4287c478bdstevel@tonic-gate	if (RDMA_BUF_ALLOC(conn, &rbuf)) {
4297c478bdstevel@tonic-gate		retval = RDMA_NORESOURCE;
4307c478bdstevel@tonic-gate	} else {
4317c478bdstevel@tonic-gate		clist_add(&cl, 0, rbuf.len, &rbuf.handle, rbuf.addr,
4320a701b1Robert Gordon		    NULL, NULL);
4337c478bdstevel@tonic-gate		retval = RDMA_SVC_RECVBUF(conn, cl);
4347c478bdstevel@tonic-gate		clist_free(cl);
4357c478bdstevel@tonic-gate	}
4367c478bdstevel@tonic-gate	return (retval);
4377c478bdstevel@tonic-gate}
4387c478bdstevel@tonic-gate
4397c478bdstevel@tonic-gaterdma_stat
4400a701b1Robert Gordonrdma_buf_alloc(CONN *conn, rdma_buf_t *rbuf)
4417c478bdstevel@tonic-gate{
4420a701b1Robert Gordon	return (RDMA_BUF_ALLOC(conn, rbuf));
4437c478bdstevel@tonic-gate}
4447c478bdstevel@tonic-gate
4457c478bdstevel@tonic-gatevoid
4467c478bdstevel@tonic-gaterdma_buf_free(CONN *conn, rdma_buf_t *rbuf)
4477c478bdstevel@tonic-gate{
4487c478bdstevel@tonic-gate	if (!rbuf || rbuf->addr == NULL) {
4497c478bdstevel@tonic-gate		return;
4507c478bdstevel@tonic-gate	}
4510a701b1Robert Gordon	RDMA_BUF_FREE(conn, rbuf);
4520a701b1Robert Gordon	bzero(rbuf, sizeof (rdma_buf_t));
4537c478bdstevel@tonic-gate}
4547c478bdstevel@tonic-gate
4557c478bdstevel@tonic-gate/*
4567c478bdstevel@tonic-gate * Caller is holding rdma_modload_lock mutex
4577c478bdstevel@tonic-gate */
4587c478bdstevel@tonic-gateint
4597c478bdstevel@tonic-gaterdma_modload()
4607c478bdstevel@tonic-gate{
4617c478bdstevel@tonic-gate	int status;
4627c478bdstevel@tonic-gate	ASSERT(MUTEX_HELD(&rdma_modload_lock));
4637c478bdstevel@tonic-gate	/*
4647c478bdstevel@tonic-gate	 * Load all available RDMA plugins which right now is only IB plugin.
4657c478bdstevel@tonic-gate	 * If no IB hardware is present, then quit right away.
4667c478bdstevel@tonic-gate	 * ENODEV -- For no device on the system
4677c478bdstevel@tonic-gate	 * EPROTONOSUPPORT -- For module not avilable either due to failure to
4687c478bdstevel@tonic-gate	 * load or some other reason.
4697c478bdstevel@tonic-gate	 */
4707c478bdstevel@tonic-gate	rdma_modloaded = 1;
4717c478bdstevel@tonic-gate	if (ibt_hw_is_present() == 0) {
4727c478bdstevel@tonic-gate		rdma_dev_available = 0;
4737c478bdstevel@tonic-gate		return (ENODEV);
4747c478bdstevel@tonic-gate	}
4757c478bdstevel@tonic-gate
4767c478bdstevel@tonic-gate	rdma_dev_available = 1;
4777c478bdstevel@tonic-gate	if (rpcmod_li == NULL)
4787c478bdstevel@tonic-gate		return (EPROTONOSUPPORT);
4797c478bdstevel@tonic-gate
4807c478bdstevel@tonic-gate	status = ldi_open_by_name("/devices/ib/rpcib@0:rpcib",
4817c478bdstevel@tonic-gate	    FREAD | FWRITE, kcred,
4827c478bdstevel@tonic-gate	    &rpcib_handle, rpcmod_li);
48351f34d4Rajkumar Sivaprakasam
4847c478bdstevel@tonic-gate	if (status != 0)
4857c478bdstevel@tonic-gate		return (EPROTONOSUPPORT);
4867c478bdstevel@tonic-gate
4870a701b1Robert Gordon
48851f34d4Rajkumar Sivaprakasam	/*
48951f34d4Rajkumar Sivaprakasam	 * We will need to reload the plugin module after it was unregistered
49051f34d4Rajkumar Sivaprakasam	 * but the resources below need to allocated only the first time.
49151f34d4Rajkumar Sivaprakasam	 */
49251f34d4Rajkumar Sivaprakasam	if (!clist_cache) {
49351f34d4Rajkumar Sivaprakasam		clist_cache = kmem_cache_create("rdma_clist",
49451f34d4Rajkumar Sivaprakasam		    sizeof (struct clist), _POINTER_ALIGNMENT, NULL,
49551f34d4Rajkumar Sivaprakasam		    NULL, NULL, NULL, 0, 0);
49651f34d4Rajkumar Sivaprakasam		rdma_kstat_init();
49751f34d4Rajkumar Sivaprakasam	}
49851f34d4Rajkumar Sivaprakasam
49951f34d4Rajkumar Sivaprakasam	(void) ldi_close(rpcib_handle, FREAD|FWRITE, kcred);
5000a701b1Robert Gordon
5017c478bdstevel@tonic-gate	return (0);
5027c478bdstevel@tonic-gate}
5037c478bdstevel@tonic-gate
5047c478bdstevel@tonic-gatevoid
5057c478bdstevel@tonic-gaterdma_kstat_init(void)
5067c478bdstevel@tonic-gate{
5077c478bdstevel@tonic-gate	kstat_t *ksp;
5087c478bdstevel@tonic-gate
5097c478bdstevel@tonic-gate	/*
5107c478bdstevel@tonic-gate	 * The RDMA framework doesn't know how to deal with Zones, and is
5117c478bdstevel@tonic-gate	 * only available in the global zone.
5127c478bdstevel@tonic-gate	 */
5137c478bdstevel@tonic-gate	ASSERT(INGLOBALZONE(curproc));
5147c478bdstevel@tonic-gate	ksp = kstat_create_zone("unix", 0, "rpc_rdma_client", "rpc",
5157c478bdstevel@tonic-gate	    KSTAT_TYPE_NAMED, rdmarcstat_ndata,
5167c478bdstevel@tonic-gate	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE, GLOBAL_ZONEID);
5177c478bdstevel@tonic-gate	if (ksp) {
5187c478bdstevel@tonic-gate		ksp->ks_data = (void *) rdmarcstat_ptr;
5197c478bdstevel@tonic-gate		kstat_install(ksp);
5207c478bdstevel@tonic-gate	}
5217c478bdstevel@tonic-gate
5227c478bdstevel@tonic-gate	ksp = kstat_create_zone("unix", 0, "rpc_rdma_server", "rpc",
5237c478bdstevel@tonic-gate	    KSTAT_TYPE_NAMED, rdmarsstat_ndata,
5247c478bdstevel@tonic-gate	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE, GLOBAL_ZONEID);
5257c478bdstevel@tonic-gate	if (ksp) {
5267c478bdstevel@tonic-gate		ksp->ks_data = (void *) rdmarsstat_ptr;
5277c478bdstevel@tonic-gate		kstat_install(ksp);
5287c478bdstevel@tonic-gate	}
5297c478bdstevel@tonic-gate}
53051f34d4Rajkumar Sivaprakasam
53151f34d4Rajkumar Sivaprakasamrdma_stat
53251f34d4Rajkumar Sivaprakasamrdma_kwait(void)
53351f34d4Rajkumar Sivaprakasam{
53451f34d4Rajkumar Sivaprakasam	int ret;
53551f34d4Rajkumar Sivaprakasam	rdma_stat stat;
53651f34d4Rajkumar Sivaprakasam
53751f34d4Rajkumar Sivaprakasam	mutex_enter(&rdma_wait.svc_lock);
53851f34d4Rajkumar Sivaprakasam
53951f34d4Rajkumar Sivaprakasam	ret = cv_wait_sig(&rdma_wait.svc_cv, &rdma_wait.svc_lock);
54051f34d4Rajkumar Sivaprakasam
54151f34d4Rajkumar Sivaprakasam	/*
54251f34d4Rajkumar Sivaprakasam	 * If signalled by a hca attach/detach, pass the right
54351f34d4Rajkumar Sivaprakasam	 * stat back.
54451f34d4Rajkumar Sivaprakasam	 */
54551f34d4Rajkumar Sivaprakasam
54651f34d4Rajkumar Sivaprakasam	if (ret)
54751f34d4Rajkumar Sivaprakasam		stat =  rdma_wait.svc_stat;
54851f34d4Rajkumar Sivaprakasam	else
54951f34d4Rajkumar Sivaprakasam		stat = RDMA_INTR;
55051f34d4Rajkumar Sivaprakasam
55151f34d4Rajkumar Sivaprakasam	mutex_exit(&rdma_wait.svc_lock);
55251f34d4Rajkumar Sivaprakasam
55351f34d4Rajkumar Sivaprakasam	return (stat);
55451f34d4Rajkumar Sivaprakasam}
555