17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 50a701b1eSRobert Gordon * Common Development and Distribution License (the "License"). 60a701b1eSRobert Gordon * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*1cfa752fSRamaswamy Tummala * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 237c478bd9Sstevel@tonic-gate */ 240a701b1eSRobert Gordon /* 250a701b1eSRobert Gordon * Copyright (c) 2007, The Ohio State University. All rights reserved. 260a701b1eSRobert Gordon * 270a701b1eSRobert Gordon * Portions of this source code is developed by the team members of 280a701b1eSRobert Gordon * The Ohio State University's Network-Based Computing Laboratory (NBCL), 290a701b1eSRobert Gordon * headed by Professor Dhabaleswar K. (DK) Panda. 300a701b1eSRobert Gordon * 310a701b1eSRobert Gordon * Acknowledgements to contributions from developors: 320a701b1eSRobert Gordon * Ranjit Noronha: noronha@cse.ohio-state.edu 330a701b1eSRobert Gordon * Lei Chai : chail@cse.ohio-state.edu 340a701b1eSRobert Gordon * Weikuan Yu : yuw@cse.ohio-state.edu 350a701b1eSRobert Gordon * 360a701b1eSRobert Gordon */ 370a701b1eSRobert Gordon 387c478bd9Sstevel@tonic-gate 397c478bd9Sstevel@tonic-gate #ifndef _IB_H 407c478bd9Sstevel@tonic-gate #define _IB_H 417c478bd9Sstevel@tonic-gate 427c478bd9Sstevel@tonic-gate /* 437c478bd9Sstevel@tonic-gate * ib.h, rpcib plugin interface. 447c478bd9Sstevel@tonic-gate */ 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate #include <sys/types.h> 477c478bd9Sstevel@tonic-gate #include <sys/ddi.h> 487c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 497c478bd9Sstevel@tonic-gate #include <sys/conf.h> 507c478bd9Sstevel@tonic-gate #include <sys/stat.h> 517c478bd9Sstevel@tonic-gate #include <rpc/rpc.h> 527c478bd9Sstevel@tonic-gate #include <rpc/rpc_rdma.h> 537c478bd9Sstevel@tonic-gate #include <sys/ib/ibtl/ibti.h> 540a701b1eSRobert Gordon #include <sys/avl.h> 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate #ifdef __cplusplus 577c478bd9Sstevel@tonic-gate extern "C" { 587c478bd9Sstevel@tonic-gate #endif 597c478bd9Sstevel@tonic-gate 600a701b1eSRobert Gordon #define MAX_BUFS 1024 /* max no. of buffers per pool */ 610a701b1eSRobert Gordon 627c478bd9Sstevel@tonic-gate #define DEF_CQ_SIZE 4096 - 1 /* default CQ size */ 637c478bd9Sstevel@tonic-gate /* 647c478bd9Sstevel@tonic-gate * Tavor returns the next higher power of 2 657c478bd9Sstevel@tonic-gate * CQ entries than the requested size. 667c478bd9Sstevel@tonic-gate * For instance, if you request (2^12 - 1) 677c478bd9Sstevel@tonic-gate * CQ entries, Tavor returns 2^12 entries. 687c478bd9Sstevel@tonic-gate * 4K CQ entries suffice. Hence, 4096 - 1. 697c478bd9Sstevel@tonic-gate */ 707c478bd9Sstevel@tonic-gate #define DEF_SQ_SIZE 128 /* default SendQ size */ 717c478bd9Sstevel@tonic-gate #define DEF_RQ_SIZE 256 /* default RecvQ size */ 727c478bd9Sstevel@tonic-gate #define DSEG_MAX 2 737c478bd9Sstevel@tonic-gate #define RQ_DSEG_MAX 1 /* default RQ data seg */ 747c478bd9Sstevel@tonic-gate #define IBSRM_HB 0x8000 /* high order bit of pkey */ 757c478bd9Sstevel@tonic-gate 767c478bd9Sstevel@tonic-gate /* max no. of refresh attempts on IBT_CM_CONN_STALE error */ 777c478bd9Sstevel@tonic-gate #define REFRESH_ATTEMPTS 3 787c478bd9Sstevel@tonic-gate 797c478bd9Sstevel@tonic-gate typedef struct rib_hca_s rib_hca_t; 807c478bd9Sstevel@tonic-gate typedef struct rib_qp_s rib_qp_t; 817c478bd9Sstevel@tonic-gate typedef struct rib_cq_s rib_cq_t; 827c478bd9Sstevel@tonic-gate 837c478bd9Sstevel@tonic-gate /* 847c478bd9Sstevel@tonic-gate * Notification for RDMA_DONE is based on xid 857c478bd9Sstevel@tonic-gate */ 867c478bd9Sstevel@tonic-gate struct rdma_done_list { 877c478bd9Sstevel@tonic-gate uint32_t xid; /* XID waiting for RDMA_DONE */ 887c478bd9Sstevel@tonic-gate kcondvar_t rdma_done_cv; /* cv for RDMA_DONE */ 897c478bd9Sstevel@tonic-gate struct rdma_done_list *next; 907c478bd9Sstevel@tonic-gate struct rdma_done_list *prev; 917c478bd9Sstevel@tonic-gate }; 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate /* 947c478bd9Sstevel@tonic-gate * State of the plugin. 957c478bd9Sstevel@tonic-gate * ACCEPT = accepting new connections and requests 967c478bd9Sstevel@tonic-gate * NO_ACCEPT = not accepting new connection and requests 977c478bd9Sstevel@tonic-gate */ 987c478bd9Sstevel@tonic-gate #define ACCEPT 1 997c478bd9Sstevel@tonic-gate #define NO_ACCEPT 2 1007c478bd9Sstevel@tonic-gate 1017c478bd9Sstevel@tonic-gate /* 1027c478bd9Sstevel@tonic-gate * Send Wait states 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate #define SEND_WAIT -1 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate /* 1077c478bd9Sstevel@tonic-gate * Reply states 1087c478bd9Sstevel@tonic-gate */ 1097c478bd9Sstevel@tonic-gate #define REPLY_WAIT -1 1107c478bd9Sstevel@tonic-gate 1117c478bd9Sstevel@tonic-gate typedef void * rib_pvoid; 1127c478bd9Sstevel@tonic-gate typedef rib_pvoid RIB_SYNCMEM_HANDLE; 1137c478bd9Sstevel@tonic-gate 1147c478bd9Sstevel@tonic-gate /* 1157c478bd9Sstevel@tonic-gate * IB buffer pool management structure 1167c478bd9Sstevel@tonic-gate */ 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate /* 1197c478bd9Sstevel@tonic-gate * Buffer pool info 1207c478bd9Sstevel@tonic-gate */ 1217c478bd9Sstevel@tonic-gate typedef struct { 1227c478bd9Sstevel@tonic-gate kmutex_t buflock; /* lock for this structure */ 1237c478bd9Sstevel@tonic-gate caddr_t buf; /* pool address */ 1247c478bd9Sstevel@tonic-gate uint32_t bufhandle; /* rkey for this pool */ 1257c478bd9Sstevel@tonic-gate ulong_t bufsize; /* size of pool */ 1267c478bd9Sstevel@tonic-gate int rsize; /* size of each element */ 1277c478bd9Sstevel@tonic-gate int numelems; /* no. of elements allocated */ 1287c478bd9Sstevel@tonic-gate int buffree; /* no. of free elements */ 1297c478bd9Sstevel@tonic-gate void *buflist[1]; /* free elements in pool */ 1307c478bd9Sstevel@tonic-gate } bufpool_t; 1317c478bd9Sstevel@tonic-gate 1327c478bd9Sstevel@tonic-gate typedef struct { 1337c478bd9Sstevel@tonic-gate bufpool_t *bpool; 1347c478bd9Sstevel@tonic-gate ibt_mr_hdl_t *mr_hdl; 1357c478bd9Sstevel@tonic-gate ibt_mr_desc_t *mr_desc; /* vaddr, lkey, rkey */ 1367c478bd9Sstevel@tonic-gate } rib_bufpool_t; 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate /* 1397c478bd9Sstevel@tonic-gate * ATS relsted defines and structures. 1407c478bd9Sstevel@tonic-gate */ 1417c478bd9Sstevel@tonic-gate #define ATS_AR_DATA_LEN 16 1427c478bd9Sstevel@tonic-gate 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate /* 1457c478bd9Sstevel@tonic-gate * Service types supported by RPCIB 1467c478bd9Sstevel@tonic-gate * For now only NFS is supported. 1477c478bd9Sstevel@tonic-gate */ 1487c478bd9Sstevel@tonic-gate #define NFS 1 1497c478bd9Sstevel@tonic-gate #define NLM 2 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate /* 1527c478bd9Sstevel@tonic-gate * Tracks consumer state (client or server). 1537c478bd9Sstevel@tonic-gate */ 1547c478bd9Sstevel@tonic-gate typedef enum { 1557c478bd9Sstevel@tonic-gate RIB_SERVER, 1567c478bd9Sstevel@tonic-gate RIB_CLIENT 1577c478bd9Sstevel@tonic-gate } rib_mode_t; 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate /* 1607c478bd9Sstevel@tonic-gate * CQ structure 1617c478bd9Sstevel@tonic-gate */ 1627c478bd9Sstevel@tonic-gate struct rib_cq_s { 1637c478bd9Sstevel@tonic-gate rib_hca_t *rib_hca; 1647c478bd9Sstevel@tonic-gate ibt_cq_hdl_t rib_cq_hdl; 1657c478bd9Sstevel@tonic-gate }; 1667c478bd9Sstevel@tonic-gate 1677f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States /* 1687f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States * Each registered service's data structure. 1697f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States */ 1707f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States typedef struct rib_service_s rib_service_t; 1717f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States struct rib_service_s { 1727f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States uint32_t srv_type; /* i.e, NFS, NLM, v4CBD */ 1737f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States ibt_srv_hdl_t srv_hdl; /* from ibt_register call */ 1747f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States ib_svc_id_t srv_id; 1757f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States rib_service_t *next; 1767f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States }; 1777f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States 1787c478bd9Sstevel@tonic-gate /* 1797c478bd9Sstevel@tonic-gate * RPCIB plugin state 1807c478bd9Sstevel@tonic-gate */ 1817c478bd9Sstevel@tonic-gate typedef struct rpcib_state { 1827c478bd9Sstevel@tonic-gate ibt_clnt_hdl_t ibt_clnt_hdl; 1837c478bd9Sstevel@tonic-gate uint32_t hca_count; 1847c478bd9Sstevel@tonic-gate uint32_t nhca_inited; 1857f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States rib_hca_t *hcas_list; 1867f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States krwlock_t hcas_list_lock; /* protects hcas_list */ 1877c478bd9Sstevel@tonic-gate int refcount; 1887c478bd9Sstevel@tonic-gate kmutex_t open_hca_lock; 1897c478bd9Sstevel@tonic-gate queue_t *q; /* up queue for a serv_type */ 1907c478bd9Sstevel@tonic-gate void *private; 1917f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States rib_service_t *service_list; 1927f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States krwlock_t service_list_lock; 1937f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States kmutex_t listen_lock; 1947c478bd9Sstevel@tonic-gate } rpcib_state_t; 1957c478bd9Sstevel@tonic-gate 1967c478bd9Sstevel@tonic-gate /* 1977c478bd9Sstevel@tonic-gate * Connection lists 1987c478bd9Sstevel@tonic-gate */ 1997c478bd9Sstevel@tonic-gate typedef struct { 2007c478bd9Sstevel@tonic-gate krwlock_t conn_lock; /* list lock */ 2017c478bd9Sstevel@tonic-gate CONN *conn_hd; /* list head */ 2027c478bd9Sstevel@tonic-gate } rib_conn_list_t; 2037c478bd9Sstevel@tonic-gate 2047c478bd9Sstevel@tonic-gate enum hca_state { 20551f34d4bSRajkumar Sivaprakasam HCA_DETACHED, /* hca in detached state */ 2067c478bd9Sstevel@tonic-gate HCA_INITED, /* hca in up and running state */ 2077c478bd9Sstevel@tonic-gate }; 2087c478bd9Sstevel@tonic-gate 2097f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States typedef struct rib_hca_service_s rib_hca_service_t; 2107f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States struct rib_hca_service_s { 2117f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States ib_svc_id_t srv_id; 2127f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States ib_gid_t gid; 2137f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States ibt_sbind_hdl_t sbind_hdl; 2147f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States rib_hca_service_t *next; 2157f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States }; 2167f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States 2177c478bd9Sstevel@tonic-gate /* 2187c478bd9Sstevel@tonic-gate * RPCIB per HCA structure 2197c478bd9Sstevel@tonic-gate */ 2207c478bd9Sstevel@tonic-gate struct rib_hca_s { 2217c478bd9Sstevel@tonic-gate ibt_clnt_hdl_t ibt_clnt_hdl; 2227c478bd9Sstevel@tonic-gate 2237c478bd9Sstevel@tonic-gate /* 2247c478bd9Sstevel@tonic-gate * per HCA. 2257c478bd9Sstevel@tonic-gate */ 2267c478bd9Sstevel@tonic-gate ibt_hca_hdl_t hca_hdl; /* HCA handle */ 2277c478bd9Sstevel@tonic-gate ibt_hca_attr_t hca_attrs; /* HCA attributes */ 2287c478bd9Sstevel@tonic-gate ibt_pd_hdl_t pd_hdl; 2297f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States rib_hca_service_t *bound_services; 2307f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States krwlock_t bound_services_lock; 2317c478bd9Sstevel@tonic-gate ib_guid_t hca_guid; 2327c478bd9Sstevel@tonic-gate uint32_t hca_nports; 2337c478bd9Sstevel@tonic-gate ibt_hca_portinfo_t *hca_ports; 2347c478bd9Sstevel@tonic-gate size_t hca_pinfosz; 2357c478bd9Sstevel@tonic-gate enum hca_state state; /* state of HCA */ 2367c478bd9Sstevel@tonic-gate krwlock_t state_lock; /* protects state field */ 2377c478bd9Sstevel@tonic-gate bool_t inuse; /* indicates HCA usage */ 2387c478bd9Sstevel@tonic-gate kmutex_t inuse_lock; /* protects inuse field */ 2397c478bd9Sstevel@tonic-gate 2407c478bd9Sstevel@tonic-gate rib_conn_list_t cl_conn_list; /* client conn list */ 2417c478bd9Sstevel@tonic-gate rib_conn_list_t srv_conn_list; /* server conn list */ 2427c478bd9Sstevel@tonic-gate 2437c478bd9Sstevel@tonic-gate rib_cq_t *clnt_scq; 2447c478bd9Sstevel@tonic-gate rib_cq_t *clnt_rcq; 2457c478bd9Sstevel@tonic-gate rib_cq_t *svc_scq; 2467c478bd9Sstevel@tonic-gate rib_cq_t *svc_rcq; 2477c478bd9Sstevel@tonic-gate kmutex_t cb_lock; 2487c478bd9Sstevel@tonic-gate kcondvar_t cb_cv; 2497c478bd9Sstevel@tonic-gate 2507c478bd9Sstevel@tonic-gate rib_bufpool_t *recv_pool; /* recv buf pool */ 2517c478bd9Sstevel@tonic-gate rib_bufpool_t *send_pool; /* send buf pool */ 2527c478bd9Sstevel@tonic-gate 2537c478bd9Sstevel@tonic-gate void *iblock; /* interrupt cookie */ 2540a701b1eSRobert Gordon 2550a701b1eSRobert Gordon kmem_cache_t *server_side_cache; /* long reply pool */ 2560a701b1eSRobert Gordon avl_tree_t avl_tree; 2570a701b1eSRobert Gordon kmutex_t avl_lock; 2580a701b1eSRobert Gordon krwlock_t avl_rw_lock; 2590a701b1eSRobert Gordon volatile bool_t avl_init; 2607f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States kmutex_t cache_allocation_lock; 261065714dcSSiddheshwar Mahesh ddi_taskq_t *cleanup_helper; 2620a701b1eSRobert Gordon ib_svc_id_t srv_id; 2630a701b1eSRobert Gordon ibt_srv_hdl_t srv_hdl; 2640a701b1eSRobert Gordon uint_t reg_state; 2657f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States 2667f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States volatile uint64_t cache_allocation; 2677f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States uint64_t cache_hits; 2687f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States uint64_t cache_misses; 2697f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States uint64_t cache_cold_misses; 2707f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States uint64_t cache_hot_misses; 2717f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States uint64_t cache_misses_above_the_limit; 2727f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States 2737f379ad1SFaramarz Jalalian - Sun Microsystems - Irvine United States struct rib_hca_s *next; 2747c478bd9Sstevel@tonic-gate }; 2757c478bd9Sstevel@tonic-gate 2767c478bd9Sstevel@tonic-gate 2777c478bd9Sstevel@tonic-gate /* 2787c478bd9Sstevel@tonic-gate * Structure on wait state of a post send 2797c478bd9Sstevel@tonic-gate */ 2807c478bd9Sstevel@tonic-gate struct send_wid { 2817c478bd9Sstevel@tonic-gate uint32_t xid; 2827c478bd9Sstevel@tonic-gate int cv_sig; 2837c478bd9Sstevel@tonic-gate kmutex_t sendwait_lock; 2847c478bd9Sstevel@tonic-gate kcondvar_t wait_cv; 2857c478bd9Sstevel@tonic-gate uint_t status; 2867c478bd9Sstevel@tonic-gate rib_qp_t *qp; 2877c478bd9Sstevel@tonic-gate int nsbufs; /* # of send buffers posted */ 2887c478bd9Sstevel@tonic-gate uint64_t sbufaddr[DSEG_MAX]; /* posted send buffers */ 2890a701b1eSRobert Gordon caddr_t c; 2900a701b1eSRobert Gordon caddr_t c1; 2910a701b1eSRobert Gordon int l1; 2920a701b1eSRobert Gordon caddr_t c2; 2930a701b1eSRobert Gordon int l2; 2940a701b1eSRobert Gordon int wl, rl; 2957c478bd9Sstevel@tonic-gate }; 2967c478bd9Sstevel@tonic-gate 2977c478bd9Sstevel@tonic-gate /* 2987c478bd9Sstevel@tonic-gate * Structure on reply descriptor for recv queue. 2997c478bd9Sstevel@tonic-gate * Different from the above posting of a descriptor. 3007c478bd9Sstevel@tonic-gate */ 3017c478bd9Sstevel@tonic-gate struct reply { 3027c478bd9Sstevel@tonic-gate uint32_t xid; 3037c478bd9Sstevel@tonic-gate uint_t status; 3047c478bd9Sstevel@tonic-gate uint64_t vaddr_cq; /* buf addr from CQ */ 3057c478bd9Sstevel@tonic-gate uint_t bytes_xfer; 3067c478bd9Sstevel@tonic-gate kcondvar_t wait_cv; 3077c478bd9Sstevel@tonic-gate struct reply *next; 3087c478bd9Sstevel@tonic-gate struct reply *prev; 3097c478bd9Sstevel@tonic-gate }; 3107c478bd9Sstevel@tonic-gate 3117c478bd9Sstevel@tonic-gate struct svc_recv { 3127c478bd9Sstevel@tonic-gate rib_qp_t *qp; 3137c478bd9Sstevel@tonic-gate uint64_t vaddr; 3147c478bd9Sstevel@tonic-gate uint_t bytes_xfer; 3157c478bd9Sstevel@tonic-gate }; 3167c478bd9Sstevel@tonic-gate 3177c478bd9Sstevel@tonic-gate struct recv_wid { 3187c478bd9Sstevel@tonic-gate uint32_t xid; 3197c478bd9Sstevel@tonic-gate rib_qp_t *qp; 3207c478bd9Sstevel@tonic-gate uint64_t addr; /* posted buf addr */ 3217c478bd9Sstevel@tonic-gate }; 3227c478bd9Sstevel@tonic-gate 3237c478bd9Sstevel@tonic-gate /* 3247c478bd9Sstevel@tonic-gate * Per QP data structure 3257c478bd9Sstevel@tonic-gate */ 3267c478bd9Sstevel@tonic-gate struct rib_qp_s { 3277c478bd9Sstevel@tonic-gate rib_hca_t *hca; 3287c478bd9Sstevel@tonic-gate rib_mode_t mode; /* RIB_SERVER or RIB_CLIENT */ 3297c478bd9Sstevel@tonic-gate CONN rdmaconn; 3307c478bd9Sstevel@tonic-gate ibt_channel_hdl_t qp_hdl; 3317c478bd9Sstevel@tonic-gate uint_t port_num; 3327c478bd9Sstevel@tonic-gate ib_qpn_t qpn; 3337c478bd9Sstevel@tonic-gate int chan_flags; 3347c478bd9Sstevel@tonic-gate clock_t timeout; 3357c478bd9Sstevel@tonic-gate ibt_rc_chan_query_attr_t qp_q_attrs; 3367c478bd9Sstevel@tonic-gate rib_cq_t *send_cq; /* send CQ */ 3377c478bd9Sstevel@tonic-gate rib_cq_t *recv_cq; /* recv CQ */ 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate /* 3407c478bd9Sstevel@tonic-gate * Number of pre-posted rbufs 3417c478bd9Sstevel@tonic-gate */ 3427c478bd9Sstevel@tonic-gate uint_t n_posted_rbufs; 3437c478bd9Sstevel@tonic-gate kcondvar_t posted_rbufs_cv; 3447c478bd9Sstevel@tonic-gate kmutex_t posted_rbufs_lock; 3457c478bd9Sstevel@tonic-gate 346065714dcSSiddheshwar Mahesh /* 347065714dcSSiddheshwar Mahesh * Number of SENDs pending completion 348065714dcSSiddheshwar Mahesh */ 349065714dcSSiddheshwar Mahesh 350065714dcSSiddheshwar Mahesh uint_t n_send_rbufs; 351065714dcSSiddheshwar Mahesh kcondvar_t send_rbufs_cv; 352065714dcSSiddheshwar Mahesh kmutex_t send_rbufs_lock; 353065714dcSSiddheshwar Mahesh 3547c478bd9Sstevel@tonic-gate /* 3557c478bd9Sstevel@tonic-gate * RPC reply 3567c478bd9Sstevel@tonic-gate */ 3577c478bd9Sstevel@tonic-gate uint_t rep_list_size; 3587c478bd9Sstevel@tonic-gate struct reply *replylist; 3597c478bd9Sstevel@tonic-gate kmutex_t replylist_lock; 3607c478bd9Sstevel@tonic-gate 3617c478bd9Sstevel@tonic-gate /* 3627c478bd9Sstevel@tonic-gate * server only, RDMA_DONE 3637c478bd9Sstevel@tonic-gate */ 3647c478bd9Sstevel@tonic-gate struct rdma_done_list *rdlist; 3657c478bd9Sstevel@tonic-gate kmutex_t rdlist_lock; 3667c478bd9Sstevel@tonic-gate 3677c478bd9Sstevel@tonic-gate kmutex_t cb_lock; 3687c478bd9Sstevel@tonic-gate kcondvar_t cb_conn_cv; 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate caddr_t q; /* upstream queue */ 3710a701b1eSRobert Gordon struct send_wid wd; 3727c478bd9Sstevel@tonic-gate }; 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate #define ctoqp(conn) ((rib_qp_t *)((conn)->c_private)) 3757c478bd9Sstevel@tonic-gate #define qptoc(rqp) ((CONN *)&((rqp)->rdmaconn)) 3767c478bd9Sstevel@tonic-gate 3777c478bd9Sstevel@tonic-gate /* 3787c478bd9Sstevel@tonic-gate * Timeout for various calls 3797c478bd9Sstevel@tonic-gate */ 3807c478bd9Sstevel@tonic-gate #define CONN_WAIT_TIME 40 3817c478bd9Sstevel@tonic-gate #define SEND_WAIT_TIME 40 /* time for send completion */ 3827c478bd9Sstevel@tonic-gate 3837c478bd9Sstevel@tonic-gate #define REPLY_WAIT_TIME 40 /* time to get reply from remote QP */ 3847c478bd9Sstevel@tonic-gate 3857c478bd9Sstevel@tonic-gate #ifdef __cplusplus 3867c478bd9Sstevel@tonic-gate } 3877c478bd9Sstevel@tonic-gate #endif 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate #endif /* !_IB_H */ 390