17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50a701b1eSRobert Gordon * Common Development and Distribution License (the "License"). 60a701b1eSRobert Gordon * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 220a701b1eSRobert Gordon * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ 267c478bd9Sstevel@tonic-gate /* All Rights Reserved */ 277c478bd9Sstevel@tonic-gate /* 287c478bd9Sstevel@tonic-gate * Portions of this source code were derived from Berkeley 297c478bd9Sstevel@tonic-gate * 4.3 BSD under license from the Regents of the University of 307c478bd9Sstevel@tonic-gate * California. 317c478bd9Sstevel@tonic-gate */ 327c478bd9Sstevel@tonic-gate 337c478bd9Sstevel@tonic-gate /* 347c478bd9Sstevel@tonic-gate * Server side of RPC over RDMA in the kernel. 357c478bd9Sstevel@tonic-gate */ 367c478bd9Sstevel@tonic-gate 377c478bd9Sstevel@tonic-gate #include <sys/param.h> 387c478bd9Sstevel@tonic-gate #include <sys/types.h> 397c478bd9Sstevel@tonic-gate #include <sys/user.h> 407c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 417c478bd9Sstevel@tonic-gate #include <sys/proc.h> 427c478bd9Sstevel@tonic-gate #include <sys/file.h> 437c478bd9Sstevel@tonic-gate #include <sys/errno.h> 447c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 457c478bd9Sstevel@tonic-gate #include <sys/debug.h> 467c478bd9Sstevel@tonic-gate #include <sys/systm.h> 477c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 487c478bd9Sstevel@tonic-gate #include <sys/kstat.h> 497c478bd9Sstevel@tonic-gate #include <sys/vtrace.h> 507c478bd9Sstevel@tonic-gate #include <sys/debug.h> 517c478bd9Sstevel@tonic-gate 527c478bd9Sstevel@tonic-gate #include <rpc/types.h> 537c478bd9Sstevel@tonic-gate #include <rpc/xdr.h> 547c478bd9Sstevel@tonic-gate #include <rpc/auth.h> 557c478bd9Sstevel@tonic-gate #include <rpc/clnt.h> 567c478bd9Sstevel@tonic-gate #include <rpc/rpc_msg.h> 577c478bd9Sstevel@tonic-gate #include <rpc/svc.h> 587c478bd9Sstevel@tonic-gate #include <rpc/rpc_rdma.h> 597c478bd9Sstevel@tonic-gate #include <sys/ddi.h> 607c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 617c478bd9Sstevel@tonic-gate 627c478bd9Sstevel@tonic-gate #include <inet/common.h> 637c478bd9Sstevel@tonic-gate #include <inet/ip.h> 647c478bd9Sstevel@tonic-gate #include <inet/ip6.h> 657c478bd9Sstevel@tonic-gate 660a701b1eSRobert Gordon #include <nfs/nfs.h> 670a701b1eSRobert Gordon #include <sys/sdt.h> 680a701b1eSRobert Gordon 690a701b1eSRobert Gordon #define SVC_RDMA_SUCCESS 0 700a701b1eSRobert Gordon #define SVC_RDMA_FAIL -1 710a701b1eSRobert Gordon 720a701b1eSRobert Gordon #define SVC_CREDIT_FACTOR (0.5) 730a701b1eSRobert Gordon 740a701b1eSRobert Gordon #define MSG_IS_RPCSEC_GSS(msg) \ 750a701b1eSRobert Gordon ((msg)->rm_reply.rp_acpt.ar_verf.oa_flavor == RPCSEC_GSS) 760a701b1eSRobert Gordon 770a701b1eSRobert Gordon 780a701b1eSRobert Gordon uint32_t rdma_bufs_granted = RDMA_BUFS_GRANT; 790a701b1eSRobert Gordon 807c478bd9Sstevel@tonic-gate /* 817c478bd9Sstevel@tonic-gate * RDMA transport specific data associated with SVCMASTERXPRT 827c478bd9Sstevel@tonic-gate */ 837c478bd9Sstevel@tonic-gate struct rdma_data { 847c478bd9Sstevel@tonic-gate SVCMASTERXPRT *rd_xprt; /* back ptr to SVCMASTERXPRT */ 857c478bd9Sstevel@tonic-gate struct rdma_svc_data rd_data; /* rdma data */ 867c478bd9Sstevel@tonic-gate rdma_mod_t *r_mod; /* RDMA module containing ops ptr */ 877c478bd9Sstevel@tonic-gate }; 887c478bd9Sstevel@tonic-gate 897c478bd9Sstevel@tonic-gate /* 907c478bd9Sstevel@tonic-gate * Plugin connection specific data stashed away in clone SVCXPRT 917c478bd9Sstevel@tonic-gate */ 927c478bd9Sstevel@tonic-gate struct clone_rdma_data { 937c478bd9Sstevel@tonic-gate CONN *conn; /* RDMA connection */ 947c478bd9Sstevel@tonic-gate rdma_buf_t rpcbuf; /* RPC req/resp buffer */ 950a701b1eSRobert Gordon struct clist *cl_reply; /* reply chunk buffer info */ 960a701b1eSRobert Gordon struct clist *cl_wlist; /* write list clist */ 977c478bd9Sstevel@tonic-gate }; 987c478bd9Sstevel@tonic-gate 997c478bd9Sstevel@tonic-gate #define MAXADDRLEN 128 /* max length for address mask */ 1007c478bd9Sstevel@tonic-gate 1017c478bd9Sstevel@tonic-gate /* 1027c478bd9Sstevel@tonic-gate * Routines exported through ops vector. 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate static bool_t svc_rdma_krecv(SVCXPRT *, mblk_t *, struct rpc_msg *); 1057c478bd9Sstevel@tonic-gate static bool_t svc_rdma_ksend(SVCXPRT *, struct rpc_msg *); 1067c478bd9Sstevel@tonic-gate static bool_t svc_rdma_kgetargs(SVCXPRT *, xdrproc_t, caddr_t); 1077c478bd9Sstevel@tonic-gate static bool_t svc_rdma_kfreeargs(SVCXPRT *, xdrproc_t, caddr_t); 1087c478bd9Sstevel@tonic-gate void svc_rdma_kdestroy(SVCMASTERXPRT *); 1097c478bd9Sstevel@tonic-gate static int svc_rdma_kdup(struct svc_req *, caddr_t, int, 1107c478bd9Sstevel@tonic-gate struct dupreq **, bool_t *); 1117c478bd9Sstevel@tonic-gate static void svc_rdma_kdupdone(struct dupreq *, caddr_t, 1127c478bd9Sstevel@tonic-gate void (*)(), int, int); 1137c478bd9Sstevel@tonic-gate static int32_t *svc_rdma_kgetres(SVCXPRT *, int); 1147c478bd9Sstevel@tonic-gate static void svc_rdma_kfreeres(SVCXPRT *); 1157c478bd9Sstevel@tonic-gate static void svc_rdma_kclone_destroy(SVCXPRT *); 1167c478bd9Sstevel@tonic-gate static void svc_rdma_kstart(SVCMASTERXPRT *); 1177c478bd9Sstevel@tonic-gate void svc_rdma_kstop(SVCMASTERXPRT *); 1187c478bd9Sstevel@tonic-gate 1190a701b1eSRobert Gordon static int svc_process_long_reply(SVCXPRT *, xdrproc_t, 1200a701b1eSRobert Gordon caddr_t, struct rpc_msg *, bool_t, int *, 1210a701b1eSRobert Gordon int *, int *, unsigned int *); 1220a701b1eSRobert Gordon 1230a701b1eSRobert Gordon static int svc_compose_rpcmsg(SVCXPRT *, CONN *, xdrproc_t, 1240a701b1eSRobert Gordon caddr_t, rdma_buf_t *, XDR **, struct rpc_msg *, 1250a701b1eSRobert Gordon bool_t, uint_t *); 1260a701b1eSRobert Gordon static bool_t rpcmsg_length(xdrproc_t, 1270a701b1eSRobert Gordon caddr_t, 1280a701b1eSRobert Gordon struct rpc_msg *, bool_t, int); 1290a701b1eSRobert Gordon 1307c478bd9Sstevel@tonic-gate /* 1317c478bd9Sstevel@tonic-gate * Server transport operations vector. 1327c478bd9Sstevel@tonic-gate */ 1337c478bd9Sstevel@tonic-gate struct svc_ops rdma_svc_ops = { 1347c478bd9Sstevel@tonic-gate svc_rdma_krecv, /* Get requests */ 1357c478bd9Sstevel@tonic-gate svc_rdma_kgetargs, /* Deserialize arguments */ 1367c478bd9Sstevel@tonic-gate svc_rdma_ksend, /* Send reply */ 1377c478bd9Sstevel@tonic-gate svc_rdma_kfreeargs, /* Free argument data space */ 1387c478bd9Sstevel@tonic-gate svc_rdma_kdestroy, /* Destroy transport handle */ 1397c478bd9Sstevel@tonic-gate svc_rdma_kdup, /* Check entry in dup req cache */ 1407c478bd9Sstevel@tonic-gate svc_rdma_kdupdone, /* Mark entry in dup req cache as done */ 1417c478bd9Sstevel@tonic-gate svc_rdma_kgetres, /* Get pointer to response buffer */ 1427c478bd9Sstevel@tonic-gate svc_rdma_kfreeres, /* Destroy pre-serialized response header */ 1437c478bd9Sstevel@tonic-gate svc_rdma_kclone_destroy, /* Destroy a clone xprt */ 1447c478bd9Sstevel@tonic-gate svc_rdma_kstart /* Tell `ready-to-receive' to rpcmod */ 1457c478bd9Sstevel@tonic-gate }; 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate /* 1487c478bd9Sstevel@tonic-gate * Server statistics 1497c478bd9Sstevel@tonic-gate * NOTE: This structure type is duplicated in the NFS fast path. 1507c478bd9Sstevel@tonic-gate */ 1517c478bd9Sstevel@tonic-gate struct { 1527c478bd9Sstevel@tonic-gate kstat_named_t rscalls; 1537c478bd9Sstevel@tonic-gate kstat_named_t rsbadcalls; 1547c478bd9Sstevel@tonic-gate kstat_named_t rsnullrecv; 1557c478bd9Sstevel@tonic-gate kstat_named_t rsbadlen; 1567c478bd9Sstevel@tonic-gate kstat_named_t rsxdrcall; 1577c478bd9Sstevel@tonic-gate kstat_named_t rsdupchecks; 1587c478bd9Sstevel@tonic-gate kstat_named_t rsdupreqs; 1597c478bd9Sstevel@tonic-gate kstat_named_t rslongrpcs; 1600a701b1eSRobert Gordon kstat_named_t rstotalreplies; 1610a701b1eSRobert Gordon kstat_named_t rstotallongreplies; 1620a701b1eSRobert Gordon kstat_named_t rstotalinlinereplies; 1637c478bd9Sstevel@tonic-gate } rdmarsstat = { 1647c478bd9Sstevel@tonic-gate { "calls", KSTAT_DATA_UINT64 }, 1657c478bd9Sstevel@tonic-gate { "badcalls", KSTAT_DATA_UINT64 }, 1667c478bd9Sstevel@tonic-gate { "nullrecv", KSTAT_DATA_UINT64 }, 1677c478bd9Sstevel@tonic-gate { "badlen", KSTAT_DATA_UINT64 }, 1687c478bd9Sstevel@tonic-gate { "xdrcall", KSTAT_DATA_UINT64 }, 1697c478bd9Sstevel@tonic-gate { "dupchecks", KSTAT_DATA_UINT64 }, 1707c478bd9Sstevel@tonic-gate { "dupreqs", KSTAT_DATA_UINT64 }, 1710a701b1eSRobert Gordon { "longrpcs", KSTAT_DATA_UINT64 }, 1720a701b1eSRobert Gordon { "totalreplies", KSTAT_DATA_UINT64 }, 1730a701b1eSRobert Gordon { "totallongreplies", KSTAT_DATA_UINT64 }, 1740a701b1eSRobert Gordon { "totalinlinereplies", KSTAT_DATA_UINT64 }, 1757c478bd9Sstevel@tonic-gate }; 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate kstat_named_t *rdmarsstat_ptr = (kstat_named_t *)&rdmarsstat; 1787c478bd9Sstevel@tonic-gate uint_t rdmarsstat_ndata = sizeof (rdmarsstat) / sizeof (kstat_named_t); 1797c478bd9Sstevel@tonic-gate 1800a701b1eSRobert Gordon #define RSSTAT_INCR(x) atomic_add_64(&rdmarsstat.x.value.ui64, 1) 1817c478bd9Sstevel@tonic-gate /* 1827c478bd9Sstevel@tonic-gate * Create a transport record. 1837c478bd9Sstevel@tonic-gate * The transport record, output buffer, and private data structure 1847c478bd9Sstevel@tonic-gate * are allocated. The output buffer is serialized into using xdrmem. 1857c478bd9Sstevel@tonic-gate * There is one transport record per user process which implements a 1867c478bd9Sstevel@tonic-gate * set of services. 1877c478bd9Sstevel@tonic-gate */ 1887c478bd9Sstevel@tonic-gate /* ARGSUSED */ 1897c478bd9Sstevel@tonic-gate int 1907c478bd9Sstevel@tonic-gate svc_rdma_kcreate(char *netid, SVC_CALLOUT_TABLE *sct, int id, 1910a701b1eSRobert Gordon rdma_xprt_group_t *started_xprts) 1927c478bd9Sstevel@tonic-gate { 1937c478bd9Sstevel@tonic-gate int error; 1947c478bd9Sstevel@tonic-gate SVCMASTERXPRT *xprt; 1957c478bd9Sstevel@tonic-gate struct rdma_data *rd; 1967c478bd9Sstevel@tonic-gate rdma_registry_t *rmod; 1977c478bd9Sstevel@tonic-gate rdma_xprt_record_t *xprt_rec; 1987c478bd9Sstevel@tonic-gate queue_t *q; 1997c478bd9Sstevel@tonic-gate /* 2007c478bd9Sstevel@tonic-gate * modload the RDMA plugins is not already done. 2017c478bd9Sstevel@tonic-gate */ 2027c478bd9Sstevel@tonic-gate if (!rdma_modloaded) { 2030a701b1eSRobert Gordon /*CONSTANTCONDITION*/ 2040a701b1eSRobert Gordon ASSERT(sizeof (struct clone_rdma_data) <= SVC_P2LEN); 2050a701b1eSRobert Gordon 2067c478bd9Sstevel@tonic-gate mutex_enter(&rdma_modload_lock); 2077c478bd9Sstevel@tonic-gate if (!rdma_modloaded) { 2087c478bd9Sstevel@tonic-gate error = rdma_modload(); 2097c478bd9Sstevel@tonic-gate } 2107c478bd9Sstevel@tonic-gate mutex_exit(&rdma_modload_lock); 2117c478bd9Sstevel@tonic-gate 2127c478bd9Sstevel@tonic-gate if (error) 2137c478bd9Sstevel@tonic-gate return (error); 2147c478bd9Sstevel@tonic-gate } 2157c478bd9Sstevel@tonic-gate 2167c478bd9Sstevel@tonic-gate /* 2177c478bd9Sstevel@tonic-gate * master_xprt_count is the count of master transport handles 2187c478bd9Sstevel@tonic-gate * that were successfully created and are ready to recieve for 2197c478bd9Sstevel@tonic-gate * RDMA based access. 2207c478bd9Sstevel@tonic-gate */ 2217c478bd9Sstevel@tonic-gate error = 0; 2227c478bd9Sstevel@tonic-gate xprt_rec = NULL; 2237c478bd9Sstevel@tonic-gate rw_enter(&rdma_lock, RW_READER); 2247c478bd9Sstevel@tonic-gate if (rdma_mod_head == NULL) { 2257c478bd9Sstevel@tonic-gate started_xprts->rtg_count = 0; 2267c478bd9Sstevel@tonic-gate rw_exit(&rdma_lock); 2277c478bd9Sstevel@tonic-gate if (rdma_dev_available) 2287c478bd9Sstevel@tonic-gate return (EPROTONOSUPPORT); 2297c478bd9Sstevel@tonic-gate else 2307c478bd9Sstevel@tonic-gate return (ENODEV); 2317c478bd9Sstevel@tonic-gate } 2327c478bd9Sstevel@tonic-gate 2337c478bd9Sstevel@tonic-gate /* 2347c478bd9Sstevel@tonic-gate * If we have reached here, then atleast one RDMA plugin has loaded. 2357c478bd9Sstevel@tonic-gate * Create a master_xprt, make it start listenining on the device, 2367c478bd9Sstevel@tonic-gate * if an error is generated, record it, we might need to shut 2377c478bd9Sstevel@tonic-gate * the master_xprt. 2387c478bd9Sstevel@tonic-gate * SVC_START() calls svc_rdma_kstart which calls plugin binding 2397c478bd9Sstevel@tonic-gate * routines. 2407c478bd9Sstevel@tonic-gate */ 2417c478bd9Sstevel@tonic-gate for (rmod = rdma_mod_head; rmod != NULL; rmod = rmod->r_next) { 2427c478bd9Sstevel@tonic-gate 2437c478bd9Sstevel@tonic-gate /* 2447c478bd9Sstevel@tonic-gate * One SVCMASTERXPRT per RDMA plugin. 2457c478bd9Sstevel@tonic-gate */ 2467c478bd9Sstevel@tonic-gate xprt = kmem_zalloc(sizeof (*xprt), KM_SLEEP); 2477c478bd9Sstevel@tonic-gate xprt->xp_ops = &rdma_svc_ops; 2487c478bd9Sstevel@tonic-gate xprt->xp_sct = sct; 2497c478bd9Sstevel@tonic-gate xprt->xp_type = T_RDMA; 2507c478bd9Sstevel@tonic-gate mutex_init(&xprt->xp_req_lock, NULL, MUTEX_DEFAULT, NULL); 2517c478bd9Sstevel@tonic-gate mutex_init(&xprt->xp_thread_lock, NULL, MUTEX_DEFAULT, NULL); 2527c478bd9Sstevel@tonic-gate xprt->xp_req_head = (mblk_t *)0; 2537c478bd9Sstevel@tonic-gate xprt->xp_req_tail = (mblk_t *)0; 2547c478bd9Sstevel@tonic-gate xprt->xp_threads = 0; 2557c478bd9Sstevel@tonic-gate xprt->xp_detached_threads = 0; 2567c478bd9Sstevel@tonic-gate 2577c478bd9Sstevel@tonic-gate rd = kmem_zalloc(sizeof (*rd), KM_SLEEP); 2587c478bd9Sstevel@tonic-gate xprt->xp_p2 = (caddr_t)rd; 2597c478bd9Sstevel@tonic-gate rd->rd_xprt = xprt; 2607c478bd9Sstevel@tonic-gate rd->r_mod = rmod->r_mod; 2617c478bd9Sstevel@tonic-gate 2627c478bd9Sstevel@tonic-gate q = &rd->rd_data.q; 2637c478bd9Sstevel@tonic-gate xprt->xp_wq = q; 2647c478bd9Sstevel@tonic-gate q->q_ptr = &rd->rd_xprt; 2657c478bd9Sstevel@tonic-gate xprt->xp_netid = NULL; 2667c478bd9Sstevel@tonic-gate 2677c478bd9Sstevel@tonic-gate if (netid != NULL) { 2687c478bd9Sstevel@tonic-gate xprt->xp_netid = kmem_alloc(strlen(netid) + 1, 2690a701b1eSRobert Gordon KM_SLEEP); 2707c478bd9Sstevel@tonic-gate (void) strcpy(xprt->xp_netid, netid); 2717c478bd9Sstevel@tonic-gate } 2727c478bd9Sstevel@tonic-gate 2737c478bd9Sstevel@tonic-gate xprt->xp_addrmask.maxlen = 2747c478bd9Sstevel@tonic-gate xprt->xp_addrmask.len = sizeof (struct sockaddr_in); 2757c478bd9Sstevel@tonic-gate xprt->xp_addrmask.buf = 2767c478bd9Sstevel@tonic-gate kmem_zalloc(xprt->xp_addrmask.len, KM_SLEEP); 2777c478bd9Sstevel@tonic-gate ((struct sockaddr_in *)xprt->xp_addrmask.buf)->sin_addr.s_addr = 2787c478bd9Sstevel@tonic-gate (uint32_t)~0; 2797c478bd9Sstevel@tonic-gate ((struct sockaddr_in *)xprt->xp_addrmask.buf)->sin_family = 2807c478bd9Sstevel@tonic-gate (ushort_t)~0; 2817c478bd9Sstevel@tonic-gate 2827c478bd9Sstevel@tonic-gate /* 2837c478bd9Sstevel@tonic-gate * Each of the plugins will have their own Service ID 2847c478bd9Sstevel@tonic-gate * to listener specific mapping, like port number for VI 2857c478bd9Sstevel@tonic-gate * and service name for IB. 2867c478bd9Sstevel@tonic-gate */ 2877c478bd9Sstevel@tonic-gate rd->rd_data.svcid = id; 2887c478bd9Sstevel@tonic-gate error = svc_xprt_register(xprt, id); 2897c478bd9Sstevel@tonic-gate if (error) { 2900a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__xprt__reg); 2917c478bd9Sstevel@tonic-gate goto cleanup; 2927c478bd9Sstevel@tonic-gate } 2937c478bd9Sstevel@tonic-gate 2947c478bd9Sstevel@tonic-gate SVC_START(xprt); 2957c478bd9Sstevel@tonic-gate if (!rd->rd_data.active) { 2967c478bd9Sstevel@tonic-gate svc_xprt_unregister(xprt); 2977c478bd9Sstevel@tonic-gate error = rd->rd_data.err_code; 2987c478bd9Sstevel@tonic-gate goto cleanup; 2997c478bd9Sstevel@tonic-gate } 3007c478bd9Sstevel@tonic-gate 3017c478bd9Sstevel@tonic-gate /* 3027c478bd9Sstevel@tonic-gate * This is set only when there is atleast one or more 3037c478bd9Sstevel@tonic-gate * transports successfully created. We insert the pointer 3047c478bd9Sstevel@tonic-gate * to the created RDMA master xprt into a separately maintained 3057c478bd9Sstevel@tonic-gate * list. This way we can easily reference it later to cleanup, 3067c478bd9Sstevel@tonic-gate * when NFS kRPC service pool is going away/unregistered. 3077c478bd9Sstevel@tonic-gate */ 3087c478bd9Sstevel@tonic-gate started_xprts->rtg_count ++; 3097c478bd9Sstevel@tonic-gate xprt_rec = kmem_alloc(sizeof (*xprt_rec), KM_SLEEP); 3107c478bd9Sstevel@tonic-gate xprt_rec->rtr_xprt_ptr = xprt; 3117c478bd9Sstevel@tonic-gate xprt_rec->rtr_next = started_xprts->rtg_listhead; 3127c478bd9Sstevel@tonic-gate started_xprts->rtg_listhead = xprt_rec; 3137c478bd9Sstevel@tonic-gate continue; 3147c478bd9Sstevel@tonic-gate cleanup: 3157c478bd9Sstevel@tonic-gate SVC_DESTROY(xprt); 3167c478bd9Sstevel@tonic-gate if (error == RDMA_FAILED) 3177c478bd9Sstevel@tonic-gate error = EPROTONOSUPPORT; 3187c478bd9Sstevel@tonic-gate } 3197c478bd9Sstevel@tonic-gate 3207c478bd9Sstevel@tonic-gate rw_exit(&rdma_lock); 3217c478bd9Sstevel@tonic-gate 3227c478bd9Sstevel@tonic-gate /* 3237c478bd9Sstevel@tonic-gate * Don't return any error even if a single plugin was started 3247c478bd9Sstevel@tonic-gate * successfully. 3257c478bd9Sstevel@tonic-gate */ 3267c478bd9Sstevel@tonic-gate if (started_xprts->rtg_count == 0) 3277c478bd9Sstevel@tonic-gate return (error); 3287c478bd9Sstevel@tonic-gate return (0); 3297c478bd9Sstevel@tonic-gate } 3307c478bd9Sstevel@tonic-gate 3317c478bd9Sstevel@tonic-gate /* 3327c478bd9Sstevel@tonic-gate * Cleanup routine for freeing up memory allocated by 3337c478bd9Sstevel@tonic-gate * svc_rdma_kcreate() 3347c478bd9Sstevel@tonic-gate */ 3357c478bd9Sstevel@tonic-gate void 3367c478bd9Sstevel@tonic-gate svc_rdma_kdestroy(SVCMASTERXPRT *xprt) 3377c478bd9Sstevel@tonic-gate { 3387c478bd9Sstevel@tonic-gate struct rdma_data *rd = (struct rdma_data *)xprt->xp_p2; 3397c478bd9Sstevel@tonic-gate 3407c478bd9Sstevel@tonic-gate 3417c478bd9Sstevel@tonic-gate mutex_destroy(&xprt->xp_req_lock); 3427c478bd9Sstevel@tonic-gate mutex_destroy(&xprt->xp_thread_lock); 3437c478bd9Sstevel@tonic-gate kmem_free(xprt->xp_netid, strlen(xprt->xp_netid) + 1); 3447c478bd9Sstevel@tonic-gate kmem_free(rd, sizeof (*rd)); 3457c478bd9Sstevel@tonic-gate kmem_free(xprt->xp_addrmask.buf, xprt->xp_addrmask.maxlen); 3467c478bd9Sstevel@tonic-gate kmem_free(xprt, sizeof (*xprt)); 3477c478bd9Sstevel@tonic-gate } 3487c478bd9Sstevel@tonic-gate 3497c478bd9Sstevel@tonic-gate 3507c478bd9Sstevel@tonic-gate static void 3517c478bd9Sstevel@tonic-gate svc_rdma_kstart(SVCMASTERXPRT *xprt) 3527c478bd9Sstevel@tonic-gate { 3537c478bd9Sstevel@tonic-gate struct rdma_svc_data *svcdata; 3547c478bd9Sstevel@tonic-gate rdma_mod_t *rmod; 3557c478bd9Sstevel@tonic-gate 3567c478bd9Sstevel@tonic-gate svcdata = &((struct rdma_data *)xprt->xp_p2)->rd_data; 3577c478bd9Sstevel@tonic-gate rmod = ((struct rdma_data *)xprt->xp_p2)->r_mod; 3587c478bd9Sstevel@tonic-gate 3597c478bd9Sstevel@tonic-gate /* 3607c478bd9Sstevel@tonic-gate * Create a listener for module at this port 3617c478bd9Sstevel@tonic-gate */ 3627c478bd9Sstevel@tonic-gate 3637c478bd9Sstevel@tonic-gate (*rmod->rdma_ops->rdma_svc_listen)(svcdata); 3647c478bd9Sstevel@tonic-gate } 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate void 3677c478bd9Sstevel@tonic-gate svc_rdma_kstop(SVCMASTERXPRT *xprt) 3687c478bd9Sstevel@tonic-gate { 3697c478bd9Sstevel@tonic-gate struct rdma_svc_data *svcdata; 3707c478bd9Sstevel@tonic-gate rdma_mod_t *rmod; 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate svcdata = &((struct rdma_data *)xprt->xp_p2)->rd_data; 3737c478bd9Sstevel@tonic-gate rmod = ((struct rdma_data *)xprt->xp_p2)->r_mod; 3747c478bd9Sstevel@tonic-gate 3757c478bd9Sstevel@tonic-gate /* 3767c478bd9Sstevel@tonic-gate * Call the stop listener routine for each plugin. 3777c478bd9Sstevel@tonic-gate */ 3787c478bd9Sstevel@tonic-gate (*rmod->rdma_ops->rdma_svc_stop)(svcdata); 3797c478bd9Sstevel@tonic-gate if (svcdata->active) 3800a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__kstop); 3817c478bd9Sstevel@tonic-gate } 3827c478bd9Sstevel@tonic-gate 3837c478bd9Sstevel@tonic-gate /* ARGSUSED */ 3847c478bd9Sstevel@tonic-gate static void 3857c478bd9Sstevel@tonic-gate svc_rdma_kclone_destroy(SVCXPRT *clone_xprt) 3867c478bd9Sstevel@tonic-gate { 3877c478bd9Sstevel@tonic-gate } 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate static bool_t 3907c478bd9Sstevel@tonic-gate svc_rdma_krecv(SVCXPRT *clone_xprt, mblk_t *mp, struct rpc_msg *msg) 3917c478bd9Sstevel@tonic-gate { 3920a701b1eSRobert Gordon XDR *xdrs; 3930a701b1eSRobert Gordon CONN *conn; 3940a701b1eSRobert Gordon 3950a701b1eSRobert Gordon rdma_recv_data_t *rdp = (rdma_recv_data_t *)mp->b_rptr; 3960a701b1eSRobert Gordon struct clone_rdma_data *crdp; 3970a701b1eSRobert Gordon struct clist *cl = NULL; 3980a701b1eSRobert Gordon struct clist *wcl = NULL; 3990a701b1eSRobert Gordon struct clist *cllong = NULL; 4000a701b1eSRobert Gordon 4010a701b1eSRobert Gordon rdma_stat status; 4020a701b1eSRobert Gordon uint32_t vers, op, pos, xid; 4030a701b1eSRobert Gordon uint32_t rdma_credit; 4040a701b1eSRobert Gordon uint32_t wcl_total_length = 0; 4050a701b1eSRobert Gordon bool_t wwl = FALSE; 4060a701b1eSRobert Gordon 4070a701b1eSRobert Gordon crdp = (struct clone_rdma_data *)clone_xprt->xp_p2buf; 4087c478bd9Sstevel@tonic-gate RSSTAT_INCR(rscalls); 4097c478bd9Sstevel@tonic-gate conn = rdp->conn; 4107c478bd9Sstevel@tonic-gate 4117c478bd9Sstevel@tonic-gate status = rdma_svc_postrecv(conn); 4127c478bd9Sstevel@tonic-gate if (status != RDMA_SUCCESS) { 4130a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__postrecv); 4140a701b1eSRobert Gordon goto badrpc_call; 4157c478bd9Sstevel@tonic-gate } 4167c478bd9Sstevel@tonic-gate 4177c478bd9Sstevel@tonic-gate xdrs = &clone_xprt->xp_xdrin; 4187c478bd9Sstevel@tonic-gate xdrmem_create(xdrs, rdp->rpcmsg.addr, rdp->rpcmsg.len, XDR_DECODE); 4197c478bd9Sstevel@tonic-gate xid = *(uint32_t *)rdp->rpcmsg.addr; 4207c478bd9Sstevel@tonic-gate XDR_SETPOS(xdrs, sizeof (uint32_t)); 4210a701b1eSRobert Gordon 4227c478bd9Sstevel@tonic-gate if (! xdr_u_int(xdrs, &vers) || 4230a701b1eSRobert Gordon ! xdr_u_int(xdrs, &rdma_credit) || 4247c478bd9Sstevel@tonic-gate ! xdr_u_int(xdrs, &op)) { 4250a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__uint); 4260a701b1eSRobert Gordon goto xdr_err; 4277c478bd9Sstevel@tonic-gate } 4280a701b1eSRobert Gordon 4290a701b1eSRobert Gordon /* Checking if the status of the recv operation was normal */ 4300a701b1eSRobert Gordon if (rdp->status != 0) { 4310a701b1eSRobert Gordon DTRACE_PROBE1(krpc__e__svcrdma__krecv__invalid__status, 4320a701b1eSRobert Gordon int, rdp->status); 4330a701b1eSRobert Gordon goto badrpc_call; 4347c478bd9Sstevel@tonic-gate } 4357c478bd9Sstevel@tonic-gate 4367c478bd9Sstevel@tonic-gate if (! xdr_do_clist(xdrs, &cl)) { 4370a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__do__clist); 4380a701b1eSRobert Gordon goto xdr_err; 4397c478bd9Sstevel@tonic-gate } 4407c478bd9Sstevel@tonic-gate 4410a701b1eSRobert Gordon if (!xdr_decode_wlist_svc(xdrs, &wcl, &wwl, &wcl_total_length, conn)) { 4420a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__decode__wlist); 4430a701b1eSRobert Gordon if (cl) 4440a701b1eSRobert Gordon clist_free(cl); 4450a701b1eSRobert Gordon goto xdr_err; 4460a701b1eSRobert Gordon } 4470a701b1eSRobert Gordon crdp->cl_wlist = wcl; 4480a701b1eSRobert Gordon 4490a701b1eSRobert Gordon crdp->cl_reply = NULL; 4500a701b1eSRobert Gordon (void) xdr_decode_reply_wchunk(xdrs, &crdp->cl_reply); 4510a701b1eSRobert Gordon 4527c478bd9Sstevel@tonic-gate /* 4537c478bd9Sstevel@tonic-gate * A chunk at 0 offset indicates that the RPC call message 4547c478bd9Sstevel@tonic-gate * is in a chunk. Get the RPC call message chunk. 4557c478bd9Sstevel@tonic-gate */ 4567c478bd9Sstevel@tonic-gate if (cl != NULL && op == RDMA_NOMSG) { 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate /* Remove RPC call message chunk from chunklist */ 4597c478bd9Sstevel@tonic-gate cllong = cl; 4607c478bd9Sstevel@tonic-gate cl = cl->c_next; 4617c478bd9Sstevel@tonic-gate cllong->c_next = NULL; 4627c478bd9Sstevel@tonic-gate 4630a701b1eSRobert Gordon 4647c478bd9Sstevel@tonic-gate /* Allocate and register memory for the RPC call msg chunk */ 4650a701b1eSRobert Gordon cllong->rb_longbuf.type = RDMA_LONG_BUFFER; 4660a701b1eSRobert Gordon cllong->rb_longbuf.len = cllong->c_len > LONG_REPLY_LEN ? 4670a701b1eSRobert Gordon cllong->c_len : LONG_REPLY_LEN; 4680a701b1eSRobert Gordon 4690a701b1eSRobert Gordon if (rdma_buf_alloc(conn, &cllong->rb_longbuf)) { 4707c478bd9Sstevel@tonic-gate clist_free(cllong); 4710a701b1eSRobert Gordon goto cll_malloc_err; 4727c478bd9Sstevel@tonic-gate } 4730a701b1eSRobert Gordon 4740a701b1eSRobert Gordon cllong->u.c_daddr3 = cllong->rb_longbuf.addr; 4750a701b1eSRobert Gordon 4760a701b1eSRobert Gordon if (cllong->u.c_daddr == NULL) { 4770a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__nomem); 4780a701b1eSRobert Gordon rdma_buf_free(conn, &cllong->rb_longbuf); 4790a701b1eSRobert Gordon clist_free(cllong); 4800a701b1eSRobert Gordon goto cll_malloc_err; 4810a701b1eSRobert Gordon } 4820a701b1eSRobert Gordon 4830a701b1eSRobert Gordon status = clist_register(conn, cllong, CLIST_REG_DST); 4847c478bd9Sstevel@tonic-gate if (status) { 4850a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__clist__reg); 4860a701b1eSRobert Gordon rdma_buf_free(conn, &cllong->rb_longbuf); 4877c478bd9Sstevel@tonic-gate clist_free(cllong); 4880a701b1eSRobert Gordon goto cll_malloc_err; 4897c478bd9Sstevel@tonic-gate } 4907c478bd9Sstevel@tonic-gate 4917c478bd9Sstevel@tonic-gate /* 4927c478bd9Sstevel@tonic-gate * Now read the RPC call message in 4937c478bd9Sstevel@tonic-gate */ 4947c478bd9Sstevel@tonic-gate status = RDMA_READ(conn, cllong, WAIT); 4957c478bd9Sstevel@tonic-gate if (status) { 4960a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__read); 4970a701b1eSRobert Gordon (void) clist_deregister(conn, cllong, CLIST_REG_DST); 4980a701b1eSRobert Gordon rdma_buf_free(conn, &cllong->rb_longbuf); 4997c478bd9Sstevel@tonic-gate clist_free(cllong); 5000a701b1eSRobert Gordon goto cll_malloc_err; 5017c478bd9Sstevel@tonic-gate } 5027c478bd9Sstevel@tonic-gate 5030a701b1eSRobert Gordon status = clist_syncmem(conn, cllong, CLIST_REG_DST); 5040a701b1eSRobert Gordon (void) clist_deregister(conn, cllong, CLIST_REG_DST); 5057c478bd9Sstevel@tonic-gate 5060a701b1eSRobert Gordon xdrrdma_create(xdrs, (caddr_t)(uintptr_t)cllong->u.c_daddr3, 5077c478bd9Sstevel@tonic-gate cllong->c_len, 0, cl, XDR_DECODE, conn); 5087c478bd9Sstevel@tonic-gate 5090a701b1eSRobert Gordon crdp->rpcbuf = cllong->rb_longbuf; 5100a701b1eSRobert Gordon crdp->rpcbuf.len = cllong->c_len; 5117c478bd9Sstevel@tonic-gate clist_free(cllong); 5127c478bd9Sstevel@tonic-gate RDMA_BUF_FREE(conn, &rdp->rpcmsg); 5137c478bd9Sstevel@tonic-gate } else { 5147c478bd9Sstevel@tonic-gate pos = XDR_GETPOS(xdrs); 5157c478bd9Sstevel@tonic-gate xdrrdma_create(xdrs, rdp->rpcmsg.addr + pos, 5160a701b1eSRobert Gordon rdp->rpcmsg.len - pos, 0, cl, XDR_DECODE, conn); 5170a701b1eSRobert Gordon crdp->rpcbuf = rdp->rpcmsg; 5180a701b1eSRobert Gordon 5190a701b1eSRobert Gordon /* Use xdrrdmablk_ops to indicate there is a read chunk list */ 5200a701b1eSRobert Gordon if (cl != NULL) { 5210a701b1eSRobert Gordon int32_t flg = XDR_RDMA_RLIST_REG; 5220a701b1eSRobert Gordon 5230a701b1eSRobert Gordon XDR_CONTROL(xdrs, XDR_RDMA_SET_FLAGS, &flg); 5240a701b1eSRobert Gordon xdrs->x_ops = &xdrrdmablk_ops; 5250a701b1eSRobert Gordon } 5267c478bd9Sstevel@tonic-gate } 5270a701b1eSRobert Gordon 5280a701b1eSRobert Gordon if (crdp->cl_wlist) { 5290a701b1eSRobert Gordon int32_t flg = XDR_RDMA_WLIST_REG; 5300a701b1eSRobert Gordon 5310a701b1eSRobert Gordon XDR_CONTROL(xdrs, XDR_RDMA_SET_WLIST, crdp->cl_wlist); 5320a701b1eSRobert Gordon XDR_CONTROL(xdrs, XDR_RDMA_SET_FLAGS, &flg); 5330a701b1eSRobert Gordon } 5340a701b1eSRobert Gordon 5357c478bd9Sstevel@tonic-gate if (! xdr_callmsg(xdrs, msg)) { 5360a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__krecv__callmsg); 5377c478bd9Sstevel@tonic-gate RSSTAT_INCR(rsxdrcall); 5380a701b1eSRobert Gordon goto callmsg_err; 5397c478bd9Sstevel@tonic-gate } 5407c478bd9Sstevel@tonic-gate 5417c478bd9Sstevel@tonic-gate /* 5427c478bd9Sstevel@tonic-gate * Point the remote transport address in the service_transport 5437c478bd9Sstevel@tonic-gate * handle at the address in the request. 5447c478bd9Sstevel@tonic-gate */ 5457c478bd9Sstevel@tonic-gate clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf; 5467c478bd9Sstevel@tonic-gate clone_xprt->xp_rtaddr.len = conn->c_raddr.len; 5477c478bd9Sstevel@tonic-gate clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len; 5487c478bd9Sstevel@tonic-gate clone_xprt->xp_xid = xid; 5490a701b1eSRobert Gordon crdp->conn = conn; 5500a701b1eSRobert Gordon 5517c478bd9Sstevel@tonic-gate freeb(mp); 5520a701b1eSRobert Gordon 5537c478bd9Sstevel@tonic-gate return (TRUE); 5540a701b1eSRobert Gordon 5550a701b1eSRobert Gordon callmsg_err: 5560a701b1eSRobert Gordon rdma_buf_free(conn, &crdp->rpcbuf); 5570a701b1eSRobert Gordon 5580a701b1eSRobert Gordon cll_malloc_err: 5590a701b1eSRobert Gordon if (cl) 5600a701b1eSRobert Gordon clist_free(cl); 5610a701b1eSRobert Gordon xdr_err: 5620a701b1eSRobert Gordon XDR_DESTROY(xdrs); 5630a701b1eSRobert Gordon 5640a701b1eSRobert Gordon badrpc_call: 5650a701b1eSRobert Gordon RDMA_BUF_FREE(conn, &rdp->rpcmsg); 5660a701b1eSRobert Gordon RDMA_REL_CONN(conn); 5670a701b1eSRobert Gordon freeb(mp); 5680a701b1eSRobert Gordon RSSTAT_INCR(rsbadcalls); 5690a701b1eSRobert Gordon return (FALSE); 5700a701b1eSRobert Gordon } 5710a701b1eSRobert Gordon 5720a701b1eSRobert Gordon static int 5730a701b1eSRobert Gordon svc_process_long_reply(SVCXPRT * clone_xprt, 5740a701b1eSRobert Gordon xdrproc_t xdr_results, caddr_t xdr_location, 5750a701b1eSRobert Gordon struct rpc_msg *msg, bool_t has_args, int *msglen, 5760a701b1eSRobert Gordon int *freelen, int *numchunks, unsigned int *final_len) 5770a701b1eSRobert Gordon { 5780a701b1eSRobert Gordon int status; 5790a701b1eSRobert Gordon XDR xdrslong; 5800a701b1eSRobert Gordon struct clist *wcl = NULL; 5810a701b1eSRobert Gordon int count = 0; 5820a701b1eSRobert Gordon int alloc_len; 5830a701b1eSRobert Gordon char *memp; 5840a701b1eSRobert Gordon rdma_buf_t long_rpc = {0}; 5850a701b1eSRobert Gordon struct clone_rdma_data *crdp; 5860a701b1eSRobert Gordon 5870a701b1eSRobert Gordon crdp = (struct clone_rdma_data *)clone_xprt->xp_p2buf; 5880a701b1eSRobert Gordon 5890a701b1eSRobert Gordon bzero(&xdrslong, sizeof (xdrslong)); 5900a701b1eSRobert Gordon 5910a701b1eSRobert Gordon /* Choose a size for the long rpc response */ 5920a701b1eSRobert Gordon if (MSG_IS_RPCSEC_GSS(msg)) { 5930a701b1eSRobert Gordon alloc_len = RNDUP(MAX_AUTH_BYTES + *msglen); 5940a701b1eSRobert Gordon } else { 5950a701b1eSRobert Gordon alloc_len = RNDUP(*msglen); 5960a701b1eSRobert Gordon } 5970a701b1eSRobert Gordon 5980a701b1eSRobert Gordon if (alloc_len <= 64 * 1024) { 5990a701b1eSRobert Gordon if (alloc_len > 32 * 1024) { 6000a701b1eSRobert Gordon alloc_len = 64 * 1024; 6010a701b1eSRobert Gordon } else { 6020a701b1eSRobert Gordon if (alloc_len > 16 * 1024) { 6030a701b1eSRobert Gordon alloc_len = 32 * 1024; 6040a701b1eSRobert Gordon } else { 6050a701b1eSRobert Gordon alloc_len = 16 * 1024; 6060a701b1eSRobert Gordon } 6070a701b1eSRobert Gordon } 6080a701b1eSRobert Gordon } 6090a701b1eSRobert Gordon 6100a701b1eSRobert Gordon long_rpc.type = RDMA_LONG_BUFFER; 6110a701b1eSRobert Gordon long_rpc.len = alloc_len; 6120a701b1eSRobert Gordon if (rdma_buf_alloc(crdp->conn, &long_rpc)) { 6130a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 6140a701b1eSRobert Gordon } 6150a701b1eSRobert Gordon 6160a701b1eSRobert Gordon memp = long_rpc.addr; 6170a701b1eSRobert Gordon xdrmem_create(&xdrslong, memp, alloc_len, XDR_ENCODE); 6180a701b1eSRobert Gordon 6190a701b1eSRobert Gordon msg->rm_xid = clone_xprt->xp_xid; 6200a701b1eSRobert Gordon 6210a701b1eSRobert Gordon if (!(xdr_replymsg(&xdrslong, msg) && 6220a701b1eSRobert Gordon (!has_args || SVCAUTH_WRAP(&clone_xprt->xp_auth, &xdrslong, 6230a701b1eSRobert Gordon xdr_results, xdr_location)))) { 6240a701b1eSRobert Gordon rdma_buf_free(crdp->conn, &long_rpc); 6250a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__longrep__authwrap); 6260a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 6270a701b1eSRobert Gordon } 6280a701b1eSRobert Gordon 6290a701b1eSRobert Gordon *final_len = XDR_GETPOS(&xdrslong); 6300a701b1eSRobert Gordon 6310a701b1eSRobert Gordon *numchunks = 0; 6320a701b1eSRobert Gordon *freelen = 0; 6330a701b1eSRobert Gordon 6340a701b1eSRobert Gordon wcl = crdp->cl_reply; 6350a701b1eSRobert Gordon wcl->rb_longbuf = long_rpc; 6360a701b1eSRobert Gordon 6370a701b1eSRobert Gordon count = *final_len; 6380a701b1eSRobert Gordon while (wcl != NULL) { 6390a701b1eSRobert Gordon if (wcl->c_dmemhandle.mrc_rmr == 0) 6400a701b1eSRobert Gordon break; 6410a701b1eSRobert Gordon 6420a701b1eSRobert Gordon if (wcl->c_len > count) { 6430a701b1eSRobert Gordon wcl->c_len = count; 6440a701b1eSRobert Gordon } 6450a701b1eSRobert Gordon wcl->w.c_saddr3 = (caddr_t)memp; 6460a701b1eSRobert Gordon 6470a701b1eSRobert Gordon count -= wcl->c_len; 6480a701b1eSRobert Gordon *numchunks += 1; 6490a701b1eSRobert Gordon if (count == 0) 6500a701b1eSRobert Gordon break; 6510a701b1eSRobert Gordon memp += wcl->c_len; 6520a701b1eSRobert Gordon wcl = wcl->c_next; 6530a701b1eSRobert Gordon } 6540a701b1eSRobert Gordon 6550a701b1eSRobert Gordon wcl = crdp->cl_reply; 6560a701b1eSRobert Gordon 6570a701b1eSRobert Gordon /* 6580a701b1eSRobert Gordon * MUST fail if there are still more data 6590a701b1eSRobert Gordon */ 6600a701b1eSRobert Gordon if (count > 0) { 6610a701b1eSRobert Gordon rdma_buf_free(crdp->conn, &long_rpc); 6620a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__longrep__dlen__clist); 6630a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 6640a701b1eSRobert Gordon } 6650a701b1eSRobert Gordon 6660a701b1eSRobert Gordon if (clist_register(crdp->conn, wcl, CLIST_REG_SOURCE) != RDMA_SUCCESS) { 6670a701b1eSRobert Gordon rdma_buf_free(crdp->conn, &long_rpc); 6680a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__longrep__clistreg); 6690a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 6700a701b1eSRobert Gordon } 6710a701b1eSRobert Gordon 6720a701b1eSRobert Gordon status = clist_syncmem(crdp->conn, wcl, CLIST_REG_SOURCE); 6730a701b1eSRobert Gordon 6740a701b1eSRobert Gordon if (status) { 6750a701b1eSRobert Gordon (void) clist_deregister(crdp->conn, wcl, CLIST_REG_SOURCE); 6760a701b1eSRobert Gordon rdma_buf_free(crdp->conn, &long_rpc); 6770a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__longrep__syncmem); 6780a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 6790a701b1eSRobert Gordon } 6800a701b1eSRobert Gordon 6810a701b1eSRobert Gordon status = RDMA_WRITE(crdp->conn, wcl, WAIT); 6820a701b1eSRobert Gordon 6830a701b1eSRobert Gordon (void) clist_deregister(crdp->conn, wcl, CLIST_REG_SOURCE); 6840a701b1eSRobert Gordon rdma_buf_free(crdp->conn, &wcl->rb_longbuf); 6850a701b1eSRobert Gordon 6860a701b1eSRobert Gordon if (status != RDMA_SUCCESS) { 6870a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__longrep__write); 6880a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 6890a701b1eSRobert Gordon } 6900a701b1eSRobert Gordon 6910a701b1eSRobert Gordon return (SVC_RDMA_SUCCESS); 6920a701b1eSRobert Gordon } 6930a701b1eSRobert Gordon 6940a701b1eSRobert Gordon 6950a701b1eSRobert Gordon static int 6960a701b1eSRobert Gordon svc_compose_rpcmsg(SVCXPRT * clone_xprt, CONN * conn, xdrproc_t xdr_results, 6970a701b1eSRobert Gordon caddr_t xdr_location, rdma_buf_t *rpcreply, XDR ** xdrs, 6980a701b1eSRobert Gordon struct rpc_msg *msg, bool_t has_args, uint_t *len) 6990a701b1eSRobert Gordon { 7000a701b1eSRobert Gordon /* 7010a701b1eSRobert Gordon * Get a pre-allocated buffer for rpc reply 7020a701b1eSRobert Gordon */ 7030a701b1eSRobert Gordon rpcreply->type = SEND_BUFFER; 7040a701b1eSRobert Gordon if (rdma_buf_alloc(conn, rpcreply)) { 7050a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__rpcmsg__reply__nofreebufs); 7060a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 7070a701b1eSRobert Gordon } 7080a701b1eSRobert Gordon 7090a701b1eSRobert Gordon xdrrdma_create(*xdrs, rpcreply->addr, rpcreply->len, 7100a701b1eSRobert Gordon 0, NULL, XDR_ENCODE, conn); 7110a701b1eSRobert Gordon 7120a701b1eSRobert Gordon msg->rm_xid = clone_xprt->xp_xid; 7130a701b1eSRobert Gordon 7140a701b1eSRobert Gordon if (has_args) { 7150a701b1eSRobert Gordon if (!(xdr_replymsg(*xdrs, msg) && 7160a701b1eSRobert Gordon (!has_args || 7170a701b1eSRobert Gordon SVCAUTH_WRAP(&clone_xprt->xp_auth, *xdrs, 7180a701b1eSRobert Gordon xdr_results, xdr_location)))) { 7190a701b1eSRobert Gordon rdma_buf_free(conn, rpcreply); 7200a701b1eSRobert Gordon DTRACE_PROBE( 7210a701b1eSRobert Gordon krpc__e__svcrdma__rpcmsg__reply__authwrap1); 7220a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 7230a701b1eSRobert Gordon } 7240a701b1eSRobert Gordon } else { 7250a701b1eSRobert Gordon if (!xdr_replymsg(*xdrs, msg)) { 7260a701b1eSRobert Gordon rdma_buf_free(conn, rpcreply); 7270a701b1eSRobert Gordon DTRACE_PROBE( 7280a701b1eSRobert Gordon krpc__e__svcrdma__rpcmsg__reply__authwrap2); 7290a701b1eSRobert Gordon return (SVC_RDMA_FAIL); 7300a701b1eSRobert Gordon } 7310a701b1eSRobert Gordon } 7320a701b1eSRobert Gordon 7330a701b1eSRobert Gordon *len = XDR_GETPOS(*xdrs); 7340a701b1eSRobert Gordon 7350a701b1eSRobert Gordon return (SVC_RDMA_SUCCESS); 7367c478bd9Sstevel@tonic-gate } 7377c478bd9Sstevel@tonic-gate 7387c478bd9Sstevel@tonic-gate /* 7397c478bd9Sstevel@tonic-gate * Send rpc reply. 7407c478bd9Sstevel@tonic-gate */ 7417c478bd9Sstevel@tonic-gate static bool_t 7420a701b1eSRobert Gordon svc_rdma_ksend(SVCXPRT * clone_xprt, struct rpc_msg *msg) 7437c478bd9Sstevel@tonic-gate { 7440a701b1eSRobert Gordon XDR *xdrs_rpc = &(clone_xprt->xp_xdrout); 7450a701b1eSRobert Gordon XDR xdrs_rhdr; 7460a701b1eSRobert Gordon CONN *conn = NULL; 7470a701b1eSRobert Gordon rdma_buf_t rbuf_resp = {0}, rbuf_rpc_resp = {0}; 7480a701b1eSRobert Gordon 7490a701b1eSRobert Gordon struct clone_rdma_data *crdp; 7500a701b1eSRobert Gordon struct clist *cl_read = NULL; 7510a701b1eSRobert Gordon struct clist *cl_send = NULL; 7520a701b1eSRobert Gordon struct clist *cl_write = NULL; 7530a701b1eSRobert Gordon xdrproc_t xdr_results; /* results XDR encoding function */ 7540a701b1eSRobert Gordon caddr_t xdr_location; /* response results pointer */ 7550a701b1eSRobert Gordon 7567c478bd9Sstevel@tonic-gate int retval = FALSE; 7570a701b1eSRobert Gordon int status, msglen, num_wreply_segments = 0; 7580a701b1eSRobert Gordon uint32_t rdma_credit = 0; 7590a701b1eSRobert Gordon int freelen = 0; 7600a701b1eSRobert Gordon bool_t has_args; 7610a701b1eSRobert Gordon uint_t final_resp_len, rdma_response_op, vers; 7627c478bd9Sstevel@tonic-gate 7630a701b1eSRobert Gordon bzero(&xdrs_rhdr, sizeof (XDR)); 7640a701b1eSRobert Gordon crdp = (struct clone_rdma_data *)clone_xprt->xp_p2buf; 7650a701b1eSRobert Gordon conn = crdp->conn; 7667c478bd9Sstevel@tonic-gate 7677c478bd9Sstevel@tonic-gate /* 7687c478bd9Sstevel@tonic-gate * If there is a result procedure specified in the reply message, 7697c478bd9Sstevel@tonic-gate * it will be processed in the xdr_replymsg and SVCAUTH_WRAP. 7707c478bd9Sstevel@tonic-gate * We need to make sure it won't be processed twice, so we null 7717c478bd9Sstevel@tonic-gate * it for xdr_replymsg here. 7727c478bd9Sstevel@tonic-gate */ 7737c478bd9Sstevel@tonic-gate has_args = FALSE; 7747c478bd9Sstevel@tonic-gate if (msg->rm_reply.rp_stat == MSG_ACCEPTED && 7757c478bd9Sstevel@tonic-gate msg->rm_reply.rp_acpt.ar_stat == SUCCESS) { 7767c478bd9Sstevel@tonic-gate if ((xdr_results = msg->acpted_rply.ar_results.proc) != NULL) { 7777c478bd9Sstevel@tonic-gate has_args = TRUE; 7787c478bd9Sstevel@tonic-gate xdr_location = msg->acpted_rply.ar_results.where; 7797c478bd9Sstevel@tonic-gate msg->acpted_rply.ar_results.proc = xdr_void; 7807c478bd9Sstevel@tonic-gate msg->acpted_rply.ar_results.where = NULL; 7817c478bd9Sstevel@tonic-gate } 7827c478bd9Sstevel@tonic-gate } 7837c478bd9Sstevel@tonic-gate 7847c478bd9Sstevel@tonic-gate /* 7850a701b1eSRobert Gordon * Given the limit on the inline response size (RPC_MSG_SZ), 7860a701b1eSRobert Gordon * there is a need to make a guess as to the overall size of 7870a701b1eSRobert Gordon * the response. If the resultant size is beyond the inline 7880a701b1eSRobert Gordon * size, then the server needs to use the "reply chunk list" 7890a701b1eSRobert Gordon * provided by the client (if the client provided one). An 7900a701b1eSRobert Gordon * example of this type of response would be a READDIR 7910a701b1eSRobert Gordon * response (e.g. a small directory read would fit in RPC_MSG_SZ 7920a701b1eSRobert Gordon * and that is the preference but it may not fit) 7930a701b1eSRobert Gordon * 7940a701b1eSRobert Gordon * Combine the encoded size and the size of the true results 7950a701b1eSRobert Gordon * and then make the decision about where to encode and send results. 7960a701b1eSRobert Gordon * 7970a701b1eSRobert Gordon * One important note, this calculation is ignoring the size 7980a701b1eSRobert Gordon * of the encoding of the authentication overhead. The reason 7990a701b1eSRobert Gordon * for this is rooted in the complexities of access to the 8000a701b1eSRobert Gordon * encoded size of RPCSEC_GSS related authentiation, 8010a701b1eSRobert Gordon * integrity, and privacy. 8020a701b1eSRobert Gordon * 8030a701b1eSRobert Gordon * If it turns out that the encoded authentication bumps the 8040a701b1eSRobert Gordon * response over the RPC_MSG_SZ limit, then it may need to 8050a701b1eSRobert Gordon * attempt to encode for the reply chunk list. 8060a701b1eSRobert Gordon */ 8070a701b1eSRobert Gordon 8080a701b1eSRobert Gordon /* 8090a701b1eSRobert Gordon * Calculating the "sizeof" the RPC response header and the 8100a701b1eSRobert Gordon * encoded results. 8117c478bd9Sstevel@tonic-gate */ 8127c478bd9Sstevel@tonic-gate msglen = xdr_sizeof(xdr_replymsg, msg); 8130a701b1eSRobert Gordon 8140a701b1eSRobert Gordon if (msglen > 0) { 8150a701b1eSRobert Gordon RSSTAT_INCR(rstotalreplies); 8160a701b1eSRobert Gordon } 8170a701b1eSRobert Gordon if (has_args) 8187c478bd9Sstevel@tonic-gate msglen += xdrrdma_sizeof(xdr_results, xdr_location, 8190a701b1eSRobert Gordon rdma_minchunk, NULL, NULL); 8207c478bd9Sstevel@tonic-gate 8210a701b1eSRobert Gordon DTRACE_PROBE1(krpc__i__svcrdma__ksend__msglen, int, msglen); 8227c478bd9Sstevel@tonic-gate 8230a701b1eSRobert Gordon status = SVC_RDMA_SUCCESS; 8247c478bd9Sstevel@tonic-gate 8250a701b1eSRobert Gordon if (msglen < RPC_MSG_SZ) { 8267c478bd9Sstevel@tonic-gate /* 8270a701b1eSRobert Gordon * Looks like the response will fit in the inline 8280a701b1eSRobert Gordon * response; let's try 8297c478bd9Sstevel@tonic-gate */ 8300a701b1eSRobert Gordon RSSTAT_INCR(rstotalinlinereplies); 8317c478bd9Sstevel@tonic-gate 8320a701b1eSRobert Gordon rdma_response_op = RDMA_MSG; 8337c478bd9Sstevel@tonic-gate 8340a701b1eSRobert Gordon status = svc_compose_rpcmsg(clone_xprt, conn, xdr_results, 8350a701b1eSRobert Gordon xdr_location, &rbuf_rpc_resp, &xdrs_rpc, msg, 8360a701b1eSRobert Gordon has_args, &final_resp_len); 8370a701b1eSRobert Gordon 8380a701b1eSRobert Gordon DTRACE_PROBE1(krpc__i__srdma__ksend__compose_status, 8390a701b1eSRobert Gordon int, status); 8400a701b1eSRobert Gordon DTRACE_PROBE1(krpc__i__srdma__ksend__compose_len, 8410a701b1eSRobert Gordon int, final_resp_len); 8420a701b1eSRobert Gordon 8430a701b1eSRobert Gordon if (status == SVC_RDMA_SUCCESS && crdp->cl_reply) { 8440a701b1eSRobert Gordon clist_free(crdp->cl_reply); 8450a701b1eSRobert Gordon crdp->cl_reply = NULL; 8467c478bd9Sstevel@tonic-gate } 8470a701b1eSRobert Gordon } 8487c478bd9Sstevel@tonic-gate 8490a701b1eSRobert Gordon /* 8500a701b1eSRobert Gordon * If the encode failed (size?) or the message really is 8510a701b1eSRobert Gordon * larger than what is allowed, try the response chunk list. 8520a701b1eSRobert Gordon */ 8530a701b1eSRobert Gordon if (status != SVC_RDMA_SUCCESS || msglen >= RPC_MSG_SZ) { 8547c478bd9Sstevel@tonic-gate /* 8550a701b1eSRobert Gordon * attempting to use a reply chunk list when there 8560a701b1eSRobert Gordon * isn't one won't get very far... 8577c478bd9Sstevel@tonic-gate */ 8580a701b1eSRobert Gordon if (crdp->cl_reply == NULL) { 8590a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__ksend__noreplycl); 8600a701b1eSRobert Gordon goto out; 8617c478bd9Sstevel@tonic-gate } 8627c478bd9Sstevel@tonic-gate 8630a701b1eSRobert Gordon RSSTAT_INCR(rstotallongreplies); 8647c478bd9Sstevel@tonic-gate 8650a701b1eSRobert Gordon msglen = xdr_sizeof(xdr_replymsg, msg); 8660a701b1eSRobert Gordon msglen += xdrrdma_sizeof(xdr_results, xdr_location, 0, 8670a701b1eSRobert Gordon NULL, NULL); 8687c478bd9Sstevel@tonic-gate 8690a701b1eSRobert Gordon status = svc_process_long_reply(clone_xprt, xdr_results, 8700a701b1eSRobert Gordon xdr_location, msg, has_args, &msglen, &freelen, 8710a701b1eSRobert Gordon &num_wreply_segments, &final_resp_len); 8727c478bd9Sstevel@tonic-gate 8730a701b1eSRobert Gordon DTRACE_PROBE1(krpc__i__svcrdma__ksend__longreplen, 8740a701b1eSRobert Gordon int, final_resp_len); 8750a701b1eSRobert Gordon 8760a701b1eSRobert Gordon if (status != SVC_RDMA_SUCCESS) { 8770a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__ksend__compose__failed); 8787c478bd9Sstevel@tonic-gate goto out; 8797c478bd9Sstevel@tonic-gate } 8800a701b1eSRobert Gordon 8810a701b1eSRobert Gordon rdma_response_op = RDMA_NOMSG; 8827c478bd9Sstevel@tonic-gate } 8837c478bd9Sstevel@tonic-gate 8840a701b1eSRobert Gordon DTRACE_PROBE1(krpc__i__svcrdma__ksend__rdmamsg__len, 8850a701b1eSRobert Gordon int, final_resp_len); 8860a701b1eSRobert Gordon 8870a701b1eSRobert Gordon rbuf_resp.type = SEND_BUFFER; 8880a701b1eSRobert Gordon if (rdma_buf_alloc(conn, &rbuf_resp)) { 8890a701b1eSRobert Gordon rdma_buf_free(conn, &rbuf_rpc_resp); 8900a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__ksend__nofreebufs); 8917c478bd9Sstevel@tonic-gate goto out; 8927c478bd9Sstevel@tonic-gate } 8937c478bd9Sstevel@tonic-gate 8940a701b1eSRobert Gordon rdma_credit = rdma_bufs_granted; 8950a701b1eSRobert Gordon 8960a701b1eSRobert Gordon vers = RPCRDMA_VERS; 8970a701b1eSRobert Gordon xdrmem_create(&xdrs_rhdr, rbuf_resp.addr, rbuf_resp.len, XDR_ENCODE); 8980a701b1eSRobert Gordon (*(uint32_t *)rbuf_resp.addr) = msg->rm_xid; 8990a701b1eSRobert Gordon /* Skip xid and set the xdr position accordingly. */ 9000a701b1eSRobert Gordon XDR_SETPOS(&xdrs_rhdr, sizeof (uint32_t)); 9010a701b1eSRobert Gordon if (!xdr_u_int(&xdrs_rhdr, &vers) || 9020a701b1eSRobert Gordon !xdr_u_int(&xdrs_rhdr, &rdma_credit) || 9030a701b1eSRobert Gordon !xdr_u_int(&xdrs_rhdr, &rdma_response_op)) { 9040a701b1eSRobert Gordon rdma_buf_free(conn, &rbuf_rpc_resp); 9050a701b1eSRobert Gordon rdma_buf_free(conn, &rbuf_resp); 9060a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__ksend__uint); 9070a701b1eSRobert Gordon goto out; 9087c478bd9Sstevel@tonic-gate } 9097c478bd9Sstevel@tonic-gate 9107c478bd9Sstevel@tonic-gate /* 9110a701b1eSRobert Gordon * Now XDR the read chunk list, actually always NULL 9127c478bd9Sstevel@tonic-gate */ 9130a701b1eSRobert Gordon (void) xdr_encode_rlist_svc(&xdrs_rhdr, cl_read); 9140a701b1eSRobert Gordon 9157c478bd9Sstevel@tonic-gate /* 9160a701b1eSRobert Gordon * encode write list -- we already drove RDMA_WRITEs 9177c478bd9Sstevel@tonic-gate */ 9180a701b1eSRobert Gordon cl_write = crdp->cl_wlist; 9190a701b1eSRobert Gordon if (!xdr_encode_wlist(&xdrs_rhdr, cl_write)) { 9200a701b1eSRobert Gordon DTRACE_PROBE(krpc__e__svcrdma__ksend__enc__wlist); 9210a701b1eSRobert Gordon rdma_buf_free(conn, &rbuf_rpc_resp); 9220a701b1eSRobert Gordon rdma_buf_free(conn, &rbuf_resp); 9237c478bd9Sstevel@tonic-gate goto out; 9247c478bd9Sstevel@tonic-gate } 9257c478bd9Sstevel@tonic-gate 9267c478bd9Sstevel@tonic-gate /* 9270a701b1eSRobert Gordon * XDR encode the RDMA_REPLY write chunk 9287c478bd9Sstevel@tonic-gate */ 9290a701b1eSRobert Gordon if (!xdr_encode_reply_wchunk(&xdrs_rhdr, crdp->cl_reply, 9300a701b1eSRobert Gordon num_wreply_segments)) { 9310a701b1eSRobert Gordon rdma_buf_free(conn, &rbuf_rpc_resp); 9320a701b1eSRobert Gordon rdma_buf_free(conn, &rbuf_resp); 9330a701b1eSRobert Gordon goto out; 9340a701b1eSRobert Gordon } 9357c478bd9Sstevel@tonic-gate 9360a701b1eSRobert Gordon clist_add(&cl_send, 0, XDR_GETPOS(&xdrs_rhdr), &rbuf_resp.handle, 9370a701b1eSRobert Gordon rbuf_resp.addr, NULL, NULL); 9387c478bd9Sstevel@tonic-gate 9390a701b1eSRobert Gordon if (rdma_response_op == RDMA_MSG) { 9400a701b1eSRobert Gordon clist_add(&cl_send, 0, final_resp_len, &rbuf_rpc_resp.handle, 9410a701b1eSRobert Gordon rbuf_rpc_resp.addr, NULL, NULL); 9427c478bd9Sstevel@tonic-gate } 9437c478bd9Sstevel@tonic-gate 9440a701b1eSRobert Gordon status = RDMA_SEND(conn, cl_send, msg->rm_xid); 9457c478bd9Sstevel@tonic-gate 9460a701b1eSRobert Gordon if (status == RDMA_SUCCESS) { 9470a701b1eSRobert Gordon retval = TRUE; 9487c478bd9Sstevel@tonic-gate } 9497c478bd9Sstevel@tonic-gate 9507c478bd9Sstevel@tonic-gate out: 9517c478bd9Sstevel@tonic-gate /* 9527c478bd9Sstevel@tonic-gate * Free up sendlist chunks 9537c478bd9Sstevel@tonic-gate */ 9540a701b1eSRobert Gordon if (cl_send != NULL) 9550a701b1eSRobert Gordon clist_free(cl_send); 9567c478bd9Sstevel@tonic-gate 9577c478bd9Sstevel@tonic-gate /* 9587c478bd9Sstevel@tonic-gate * Destroy private data for xdr rdma 9597c478bd9Sstevel@tonic-gate */ 9600a701b1eSRobert Gordon if (clone_xprt->xp_xdrout.x_ops != NULL) { 9610a701b1eSRobert Gordon XDR_DESTROY(&(clone_xprt->xp_xdrout)); 9620a701b1eSRobert Gordon } 9630a701b1eSRobert Gordon 9640a701b1eSRobert Gordon if (crdp->cl_reply) { 9650a701b1eSRobert Gordon clist_free(crdp->cl_reply); 9660a701b1eSRobert Gordon crdp->cl_reply = NULL; 9670a701b1eSRobert Gordon } 9687c478bd9Sstevel@tonic-gate 9697c478bd9Sstevel@tonic-gate /* 9707c478bd9Sstevel@tonic-gate * This is completely disgusting. If public is set it is 9717c478bd9Sstevel@tonic-gate * a pointer to a structure whose first field is the address 9727c478bd9Sstevel@tonic-gate * of the function to free that structure and any related 9737c478bd9Sstevel@tonic-gate * stuff. (see rrokfree in nfs_xdr.c). 9747c478bd9Sstevel@tonic-gate */ 9750a701b1eSRobert Gordon if (xdrs_rpc->x_public) { 9767c478bd9Sstevel@tonic-gate /* LINTED pointer alignment */ 9770a701b1eSRobert Gordon (**((int (**)()) xdrs_rpc->x_public)) (xdrs_rpc->x_public); 9780a701b1eSRobert Gordon } 9790a701b1eSRobert Gordon 9800a701b1eSRobert Gordon if (xdrs_rhdr.x_ops != NULL) { 9810a701b1eSRobert Gordon XDR_DESTROY(&xdrs_rhdr); 9827c478bd9Sstevel@tonic-gate } 9837c478bd9Sstevel@tonic-gate 9847c478bd9Sstevel@tonic-gate return (retval); 9857c478bd9Sstevel@tonic-gate } 9867c478bd9Sstevel@tonic-gate 9877c478bd9Sstevel@tonic-gate /* 9887c478bd9Sstevel@tonic-gate * Deserialize arguments. 9897c478bd9Sstevel@tonic-gate */ 9907c478bd9Sstevel@tonic-gate static bool_t 9917c478bd9Sstevel@tonic-gate svc_rdma_kgetargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, caddr_t args_ptr) 9927c478bd9Sstevel@tonic-gate { 9937c478bd9Sstevel@tonic-gate if ((SVCAUTH_UNWRAP(&clone_xprt->xp_auth, &clone_xprt->xp_xdrin, 9947c478bd9Sstevel@tonic-gate xdr_args, args_ptr)) != TRUE) 9957c478bd9Sstevel@tonic-gate return (FALSE); 9967c478bd9Sstevel@tonic-gate return (TRUE); 9977c478bd9Sstevel@tonic-gate } 9987c478bd9Sstevel@tonic-gate 9997c478bd9Sstevel@tonic-gate static bool_t 10007c478bd9Sstevel@tonic-gate svc_rdma_kfreeargs(SVCXPRT *clone_xprt, xdrproc_t xdr_args, 10017c478bd9Sstevel@tonic-gate caddr_t args_ptr) 10027c478bd9Sstevel@tonic-gate { 10030a701b1eSRobert Gordon struct clone_rdma_data *crdp; 10047c478bd9Sstevel@tonic-gate bool_t retval; 10057c478bd9Sstevel@tonic-gate 10060a701b1eSRobert Gordon crdp = (struct clone_rdma_data *)clone_xprt->xp_p2buf; 10070a701b1eSRobert Gordon 10080a701b1eSRobert Gordon /* 10090a701b1eSRobert Gordon * Free the args if needed then XDR_DESTROY 10100a701b1eSRobert Gordon */ 10117c478bd9Sstevel@tonic-gate if (args_ptr) { 10127c478bd9Sstevel@tonic-gate XDR *xdrs = &clone_xprt->xp_xdrin; 10137c478bd9Sstevel@tonic-gate 10147c478bd9Sstevel@tonic-gate xdrs->x_op = XDR_FREE; 10157c478bd9Sstevel@tonic-gate retval = (*xdr_args)(xdrs, args_ptr); 10167c478bd9Sstevel@tonic-gate } 10170a701b1eSRobert Gordon 10187c478bd9Sstevel@tonic-gate XDR_DESTROY(&(clone_xprt->xp_xdrin)); 10190a701b1eSRobert Gordon rdma_buf_free(crdp->conn, &crdp->rpcbuf); 10200a701b1eSRobert Gordon if (crdp->cl_reply) { 10210a701b1eSRobert Gordon clist_free(crdp->cl_reply); 10220a701b1eSRobert Gordon crdp->cl_reply = NULL; 10230a701b1eSRobert Gordon } 10240a701b1eSRobert Gordon RDMA_REL_CONN(crdp->conn); 10250a701b1eSRobert Gordon 10267c478bd9Sstevel@tonic-gate return (retval); 10277c478bd9Sstevel@tonic-gate } 10287c478bd9Sstevel@tonic-gate 10297c478bd9Sstevel@tonic-gate /* ARGSUSED */ 10307c478bd9Sstevel@tonic-gate static int32_t * 10317c478bd9Sstevel@tonic-gate svc_rdma_kgetres(SVCXPRT *clone_xprt, int size) 10327c478bd9Sstevel@tonic-gate { 10337c478bd9Sstevel@tonic-gate return (NULL); 10347c478bd9Sstevel@tonic-gate } 10357c478bd9Sstevel@tonic-gate 10367c478bd9Sstevel@tonic-gate /* ARGSUSED */ 10377c478bd9Sstevel@tonic-gate static void 10387c478bd9Sstevel@tonic-gate svc_rdma_kfreeres(SVCXPRT *clone_xprt) 10397c478bd9Sstevel@tonic-gate { 10407c478bd9Sstevel@tonic-gate } 10417c478bd9Sstevel@tonic-gate 10427c478bd9Sstevel@tonic-gate /* 10437c478bd9Sstevel@tonic-gate * the dup cacheing routines below provide a cache of non-failure 10447c478bd9Sstevel@tonic-gate * transaction id's. rpc service routines can use this to detect 10457c478bd9Sstevel@tonic-gate * retransmissions and re-send a non-failure response. 10467c478bd9Sstevel@tonic-gate */ 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate /* 10497c478bd9Sstevel@tonic-gate * MAXDUPREQS is the number of cached items. It should be adjusted 10507c478bd9Sstevel@tonic-gate * to the service load so that there is likely to be a response entry 10517c478bd9Sstevel@tonic-gate * when the first retransmission comes in. 10527c478bd9Sstevel@tonic-gate */ 10537c478bd9Sstevel@tonic-gate #define MAXDUPREQS 1024 10547c478bd9Sstevel@tonic-gate 10557c478bd9Sstevel@tonic-gate /* 10567c478bd9Sstevel@tonic-gate * This should be appropriately scaled to MAXDUPREQS. 10577c478bd9Sstevel@tonic-gate */ 10587c478bd9Sstevel@tonic-gate #define DRHASHSZ 257 10597c478bd9Sstevel@tonic-gate 10607c478bd9Sstevel@tonic-gate #if ((DRHASHSZ & (DRHASHSZ - 1)) == 0) 10617c478bd9Sstevel@tonic-gate #define XIDHASH(xid) ((xid) & (DRHASHSZ - 1)) 10627c478bd9Sstevel@tonic-gate #else 10637c478bd9Sstevel@tonic-gate #define XIDHASH(xid) ((xid) % DRHASHSZ) 10647c478bd9Sstevel@tonic-gate #endif 10657c478bd9Sstevel@tonic-gate #define DRHASH(dr) XIDHASH((dr)->dr_xid) 10667c478bd9Sstevel@tonic-gate #define REQTOXID(req) ((req)->rq_xprt->xp_xid) 10677c478bd9Sstevel@tonic-gate 10687c478bd9Sstevel@tonic-gate static int rdmandupreqs = 0; 1069*59418bdeSgt int rdmamaxdupreqs = MAXDUPREQS; 10707c478bd9Sstevel@tonic-gate static kmutex_t rdmadupreq_lock; 10717c478bd9Sstevel@tonic-gate static struct dupreq *rdmadrhashtbl[DRHASHSZ]; 10727c478bd9Sstevel@tonic-gate static int rdmadrhashstat[DRHASHSZ]; 10737c478bd9Sstevel@tonic-gate 10747c478bd9Sstevel@tonic-gate static void unhash(struct dupreq *); 10757c478bd9Sstevel@tonic-gate 10767c478bd9Sstevel@tonic-gate /* 10777c478bd9Sstevel@tonic-gate * rdmadrmru points to the head of a circular linked list in lru order. 10787c478bd9Sstevel@tonic-gate * rdmadrmru->dr_next == drlru 10797c478bd9Sstevel@tonic-gate */ 10807c478bd9Sstevel@tonic-gate struct dupreq *rdmadrmru; 10817c478bd9Sstevel@tonic-gate 10827c478bd9Sstevel@tonic-gate /* 10837c478bd9Sstevel@tonic-gate * svc_rdma_kdup searches the request cache and returns 0 if the 10847c478bd9Sstevel@tonic-gate * request is not found in the cache. If it is found, then it 10857c478bd9Sstevel@tonic-gate * returns the state of the request (in progress or done) and 10867c478bd9Sstevel@tonic-gate * the status or attributes that were part of the original reply. 10877c478bd9Sstevel@tonic-gate */ 10887c478bd9Sstevel@tonic-gate static int 10897c478bd9Sstevel@tonic-gate svc_rdma_kdup(struct svc_req *req, caddr_t res, int size, struct dupreq **drpp, 10907c478bd9Sstevel@tonic-gate bool_t *dupcachedp) 10917c478bd9Sstevel@tonic-gate { 10927c478bd9Sstevel@tonic-gate struct dupreq *dr; 10937c478bd9Sstevel@tonic-gate uint32_t xid; 10947c478bd9Sstevel@tonic-gate uint32_t drhash; 10957c478bd9Sstevel@tonic-gate int status; 10967c478bd9Sstevel@tonic-gate 10977c478bd9Sstevel@tonic-gate xid = REQTOXID(req); 10987c478bd9Sstevel@tonic-gate mutex_enter(&rdmadupreq_lock); 10997c478bd9Sstevel@tonic-gate RSSTAT_INCR(rsdupchecks); 11007c478bd9Sstevel@tonic-gate /* 11017c478bd9Sstevel@tonic-gate * Check to see whether an entry already exists in the cache. 11027c478bd9Sstevel@tonic-gate */ 11037c478bd9Sstevel@tonic-gate dr = rdmadrhashtbl[XIDHASH(xid)]; 11047c478bd9Sstevel@tonic-gate while (dr != NULL) { 11057c478bd9Sstevel@tonic-gate if (dr->dr_xid == xid && 11067c478bd9Sstevel@tonic-gate dr->dr_proc == req->rq_proc && 11077c478bd9Sstevel@tonic-gate dr->dr_prog == req->rq_prog && 11087c478bd9Sstevel@tonic-gate dr->dr_vers == req->rq_vers && 11097c478bd9Sstevel@tonic-gate dr->dr_addr.len == req->rq_xprt->xp_rtaddr.len && 11107c478bd9Sstevel@tonic-gate bcmp((caddr_t)dr->dr_addr.buf, 11117c478bd9Sstevel@tonic-gate (caddr_t)req->rq_xprt->xp_rtaddr.buf, 11127c478bd9Sstevel@tonic-gate dr->dr_addr.len) == 0) { 11137c478bd9Sstevel@tonic-gate status = dr->dr_status; 11147c478bd9Sstevel@tonic-gate if (status == DUP_DONE) { 11157c478bd9Sstevel@tonic-gate bcopy(dr->dr_resp.buf, res, size); 11167c478bd9Sstevel@tonic-gate if (dupcachedp != NULL) 11177c478bd9Sstevel@tonic-gate *dupcachedp = (dr->dr_resfree != NULL); 11187c478bd9Sstevel@tonic-gate } else { 11197c478bd9Sstevel@tonic-gate dr->dr_status = DUP_INPROGRESS; 11207c478bd9Sstevel@tonic-gate *drpp = dr; 11217c478bd9Sstevel@tonic-gate } 11227c478bd9Sstevel@tonic-gate RSSTAT_INCR(rsdupreqs); 11237c478bd9Sstevel@tonic-gate mutex_exit(&rdmadupreq_lock); 11247c478bd9Sstevel@tonic-gate return (status); 11257c478bd9Sstevel@tonic-gate } 11267c478bd9Sstevel@tonic-gate dr = dr->dr_chain; 11277c478bd9Sstevel@tonic-gate } 11287c478bd9Sstevel@tonic-gate 11297c478bd9Sstevel@tonic-gate /* 11307c478bd9Sstevel@tonic-gate * There wasn't an entry, either allocate a new one or recycle 11317c478bd9Sstevel@tonic-gate * an old one. 11327c478bd9Sstevel@tonic-gate */ 11337c478bd9Sstevel@tonic-gate if (rdmandupreqs < rdmamaxdupreqs) { 11347c478bd9Sstevel@tonic-gate dr = kmem_alloc(sizeof (*dr), KM_NOSLEEP); 11357c478bd9Sstevel@tonic-gate if (dr == NULL) { 11367c478bd9Sstevel@tonic-gate mutex_exit(&rdmadupreq_lock); 11377c478bd9Sstevel@tonic-gate return (DUP_ERROR); 11387c478bd9Sstevel@tonic-gate } 11397c478bd9Sstevel@tonic-gate dr->dr_resp.buf = NULL; 11407c478bd9Sstevel@tonic-gate dr->dr_resp.maxlen = 0; 11417c478bd9Sstevel@tonic-gate dr->dr_addr.buf = NULL; 11427c478bd9Sstevel@tonic-gate dr->dr_addr.maxlen = 0; 11437c478bd9Sstevel@tonic-gate if (rdmadrmru) { 11447c478bd9Sstevel@tonic-gate dr->dr_next = rdmadrmru->dr_next; 11457c478bd9Sstevel@tonic-gate rdmadrmru->dr_next = dr; 11467c478bd9Sstevel@tonic-gate } else { 11477c478bd9Sstevel@tonic-gate dr->dr_next = dr; 11487c478bd9Sstevel@tonic-gate } 11497c478bd9Sstevel@tonic-gate rdmandupreqs++; 11507c478bd9Sstevel@tonic-gate } else { 11517c478bd9Sstevel@tonic-gate dr = rdmadrmru->dr_next; 11527c478bd9Sstevel@tonic-gate while (dr->dr_status == DUP_INPROGRESS) { 11537c478bd9Sstevel@tonic-gate dr = dr->dr_next; 11547c478bd9Sstevel@tonic-gate if (dr == rdmadrmru->dr_next) { 11557c478bd9Sstevel@tonic-gate mutex_exit(&rdmadupreq_lock); 11567c478bd9Sstevel@tonic-gate return (DUP_ERROR); 11577c478bd9Sstevel@tonic-gate } 11587c478bd9Sstevel@tonic-gate } 11597c478bd9Sstevel@tonic-gate unhash(dr); 11607c478bd9Sstevel@tonic-gate if (dr->dr_resfree) { 11617c478bd9Sstevel@tonic-gate (*dr->dr_resfree)(dr->dr_resp.buf); 11627c478bd9Sstevel@tonic-gate } 11637c478bd9Sstevel@tonic-gate } 11647c478bd9Sstevel@tonic-gate dr->dr_resfree = NULL; 11657c478bd9Sstevel@tonic-gate rdmadrmru = dr; 11667c478bd9Sstevel@tonic-gate 11677c478bd9Sstevel@tonic-gate dr->dr_xid = REQTOXID(req); 11687c478bd9Sstevel@tonic-gate dr->dr_prog = req->rq_prog; 11697c478bd9Sstevel@tonic-gate dr->dr_vers = req->rq_vers; 11707c478bd9Sstevel@tonic-gate dr->dr_proc = req->rq_proc; 11717c478bd9Sstevel@tonic-gate if (dr->dr_addr.maxlen < req->rq_xprt->xp_rtaddr.len) { 11727c478bd9Sstevel@tonic-gate if (dr->dr_addr.buf != NULL) 11737c478bd9Sstevel@tonic-gate kmem_free(dr->dr_addr.buf, dr->dr_addr.maxlen); 11747c478bd9Sstevel@tonic-gate dr->dr_addr.maxlen = req->rq_xprt->xp_rtaddr.len; 11757c478bd9Sstevel@tonic-gate dr->dr_addr.buf = kmem_alloc(dr->dr_addr.maxlen, KM_NOSLEEP); 11767c478bd9Sstevel@tonic-gate if (dr->dr_addr.buf == NULL) { 11777c478bd9Sstevel@tonic-gate dr->dr_addr.maxlen = 0; 11787c478bd9Sstevel@tonic-gate dr->dr_status = DUP_DROP; 11797c478bd9Sstevel@tonic-gate mutex_exit(&rdmadupreq_lock); 11807c478bd9Sstevel@tonic-gate return (DUP_ERROR); 11817c478bd9Sstevel@tonic-gate } 11827c478bd9Sstevel@tonic-gate } 11837c478bd9Sstevel@tonic-gate dr->dr_addr.len = req->rq_xprt->xp_rtaddr.len; 11847c478bd9Sstevel@tonic-gate bcopy(req->rq_xprt->xp_rtaddr.buf, dr->dr_addr.buf, dr->dr_addr.len); 11857c478bd9Sstevel@tonic-gate if (dr->dr_resp.maxlen < size) { 11867c478bd9Sstevel@tonic-gate if (dr->dr_resp.buf != NULL) 11877c478bd9Sstevel@tonic-gate kmem_free(dr->dr_resp.buf, dr->dr_resp.maxlen); 11887c478bd9Sstevel@tonic-gate dr->dr_resp.maxlen = (unsigned int)size; 11897c478bd9Sstevel@tonic-gate dr->dr_resp.buf = kmem_alloc(size, KM_NOSLEEP); 11907c478bd9Sstevel@tonic-gate if (dr->dr_resp.buf == NULL) { 11917c478bd9Sstevel@tonic-gate dr->dr_resp.maxlen = 0; 11927c478bd9Sstevel@tonic-gate dr->dr_status = DUP_DROP; 11937c478bd9Sstevel@tonic-gate mutex_exit(&rdmadupreq_lock); 11947c478bd9Sstevel@tonic-gate return (DUP_ERROR); 11957c478bd9Sstevel@tonic-gate } 11967c478bd9Sstevel@tonic-gate } 11977c478bd9Sstevel@tonic-gate dr->dr_status = DUP_INPROGRESS; 11987c478bd9Sstevel@tonic-gate 11997c478bd9Sstevel@tonic-gate drhash = (uint32_t)DRHASH(dr); 12007c478bd9Sstevel@tonic-gate dr->dr_chain = rdmadrhashtbl[drhash]; 12017c478bd9Sstevel@tonic-gate rdmadrhashtbl[drhash] = dr; 12027c478bd9Sstevel@tonic-gate rdmadrhashstat[drhash]++; 12037c478bd9Sstevel@tonic-gate mutex_exit(&rdmadupreq_lock); 12047c478bd9Sstevel@tonic-gate *drpp = dr; 12057c478bd9Sstevel@tonic-gate return (DUP_NEW); 12067c478bd9Sstevel@tonic-gate } 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate /* 12097c478bd9Sstevel@tonic-gate * svc_rdma_kdupdone marks the request done (DUP_DONE or DUP_DROP) 12107c478bd9Sstevel@tonic-gate * and stores the response. 12117c478bd9Sstevel@tonic-gate */ 12127c478bd9Sstevel@tonic-gate static void 12137c478bd9Sstevel@tonic-gate svc_rdma_kdupdone(struct dupreq *dr, caddr_t res, void (*dis_resfree)(), 12147c478bd9Sstevel@tonic-gate int size, int status) 12157c478bd9Sstevel@tonic-gate { 12167c478bd9Sstevel@tonic-gate ASSERT(dr->dr_resfree == NULL); 12177c478bd9Sstevel@tonic-gate if (status == DUP_DONE) { 12187c478bd9Sstevel@tonic-gate bcopy(res, dr->dr_resp.buf, size); 12197c478bd9Sstevel@tonic-gate dr->dr_resfree = dis_resfree; 12207c478bd9Sstevel@tonic-gate } 12217c478bd9Sstevel@tonic-gate dr->dr_status = status; 12227c478bd9Sstevel@tonic-gate } 12237c478bd9Sstevel@tonic-gate 12247c478bd9Sstevel@tonic-gate /* 12257c478bd9Sstevel@tonic-gate * This routine expects that the mutex, rdmadupreq_lock, is already held. 12267c478bd9Sstevel@tonic-gate */ 12277c478bd9Sstevel@tonic-gate static void 12287c478bd9Sstevel@tonic-gate unhash(struct dupreq *dr) 12297c478bd9Sstevel@tonic-gate { 12307c478bd9Sstevel@tonic-gate struct dupreq *drt; 12317c478bd9Sstevel@tonic-gate struct dupreq *drtprev = NULL; 12327c478bd9Sstevel@tonic-gate uint32_t drhash; 12337c478bd9Sstevel@tonic-gate 12347c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&rdmadupreq_lock)); 12357c478bd9Sstevel@tonic-gate 12367c478bd9Sstevel@tonic-gate drhash = (uint32_t)DRHASH(dr); 12377c478bd9Sstevel@tonic-gate drt = rdmadrhashtbl[drhash]; 12387c478bd9Sstevel@tonic-gate while (drt != NULL) { 12397c478bd9Sstevel@tonic-gate if (drt == dr) { 12407c478bd9Sstevel@tonic-gate rdmadrhashstat[drhash]--; 12417c478bd9Sstevel@tonic-gate if (drtprev == NULL) { 12427c478bd9Sstevel@tonic-gate rdmadrhashtbl[drhash] = drt->dr_chain; 12437c478bd9Sstevel@tonic-gate } else { 12447c478bd9Sstevel@tonic-gate drtprev->dr_chain = drt->dr_chain; 12457c478bd9Sstevel@tonic-gate } 12467c478bd9Sstevel@tonic-gate return; 12477c478bd9Sstevel@tonic-gate } 12487c478bd9Sstevel@tonic-gate drtprev = drt; 12497c478bd9Sstevel@tonic-gate drt = drt->dr_chain; 12507c478bd9Sstevel@tonic-gate } 12517c478bd9Sstevel@tonic-gate } 12520a701b1eSRobert Gordon 12530a701b1eSRobert Gordon bool_t 12540a701b1eSRobert Gordon rdma_get_wchunk(struct svc_req *req, iovec_t *iov, struct clist *wlist) 12550a701b1eSRobert Gordon { 12560a701b1eSRobert Gordon struct clist *clist; 12570a701b1eSRobert Gordon uint32_t tlen; 12580a701b1eSRobert Gordon 12590a701b1eSRobert Gordon if (req->rq_xprt->xp_type != T_RDMA) { 12600a701b1eSRobert Gordon return (FALSE); 12610a701b1eSRobert Gordon } 12620a701b1eSRobert Gordon 12630a701b1eSRobert Gordon tlen = 0; 12640a701b1eSRobert Gordon clist = wlist; 12650a701b1eSRobert Gordon while (clist) { 12660a701b1eSRobert Gordon tlen += clist->c_len; 12670a701b1eSRobert Gordon clist = clist->c_next; 12680a701b1eSRobert Gordon } 12690a701b1eSRobert Gordon 12700a701b1eSRobert Gordon /* 12710a701b1eSRobert Gordon * set iov to addr+len of first segment of first wchunk of 12720a701b1eSRobert Gordon * wlist sent by client. krecv() already malloc'd a buffer 12730a701b1eSRobert Gordon * large enough, but registration is deferred until we write 12740a701b1eSRobert Gordon * the buffer back to (NFS) client using RDMA_WRITE. 12750a701b1eSRobert Gordon */ 12760a701b1eSRobert Gordon iov->iov_base = (caddr_t)(uintptr_t)wlist->w.c_saddr; 12770a701b1eSRobert Gordon iov->iov_len = tlen; 12780a701b1eSRobert Gordon 12790a701b1eSRobert Gordon return (TRUE); 12800a701b1eSRobert Gordon } 1281