1*30e7468fSPeter Dunlap /* 2*30e7468fSPeter Dunlap * CDDL HEADER START 3*30e7468fSPeter Dunlap * 4*30e7468fSPeter Dunlap * The contents of this file are subject to the terms of the 5*30e7468fSPeter Dunlap * Common Development and Distribution License (the "License"). 6*30e7468fSPeter Dunlap * You may not use this file except in compliance with the License. 7*30e7468fSPeter Dunlap * 8*30e7468fSPeter Dunlap * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*30e7468fSPeter Dunlap * or http://www.opensolaris.org/os/licensing. 10*30e7468fSPeter Dunlap * See the License for the specific language governing permissions 11*30e7468fSPeter Dunlap * and limitations under the License. 12*30e7468fSPeter Dunlap * 13*30e7468fSPeter Dunlap * When distributing Covered Code, include this CDDL HEADER in each 14*30e7468fSPeter Dunlap * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*30e7468fSPeter Dunlap * If applicable, add the following below this CDDL HEADER, with the 16*30e7468fSPeter Dunlap * fields enclosed by brackets "[]" replaced with your own identifying 17*30e7468fSPeter Dunlap * information: Portions Copyright [yyyy] [name of copyright owner] 18*30e7468fSPeter Dunlap * 19*30e7468fSPeter Dunlap * CDDL HEADER END 20*30e7468fSPeter Dunlap */ 21*30e7468fSPeter Dunlap /* 22*30e7468fSPeter Dunlap * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23*30e7468fSPeter Dunlap * Use is subject to license terms. 24*30e7468fSPeter Dunlap */ 25*30e7468fSPeter Dunlap 26*30e7468fSPeter Dunlap #ifndef _ISER_RESOURCE_H 27*30e7468fSPeter Dunlap #define _ISER_RESOURCE_H 28*30e7468fSPeter Dunlap 29*30e7468fSPeter Dunlap #ifdef __cplusplus 30*30e7468fSPeter Dunlap extern "C" { 31*30e7468fSPeter Dunlap #endif 32*30e7468fSPeter Dunlap 33*30e7468fSPeter Dunlap #include <sys/types.h> 34*30e7468fSPeter Dunlap #include <sys/ib/ibtl/ibti.h> 35*30e7468fSPeter Dunlap #include <sys/ib/ibtl/ibtl_types.h> 36*30e7468fSPeter Dunlap #include <sys/iscsi_protocol.h> 37*30e7468fSPeter Dunlap 38*30e7468fSPeter Dunlap #define ISER_CACHE_NAMELEN 31 /* KMEM_CACHE_NAMELEN */ 39*30e7468fSPeter Dunlap 40*30e7468fSPeter Dunlap /* Default message lengths */ 41*30e7468fSPeter Dunlap #define ISER_MAX_CTRLPDU_LEN 0x4000 42*30e7468fSPeter Dunlap #define ISER_MAX_TEXTPDU_LEN 0x4000 43*30e7468fSPeter Dunlap 44*30e7468fSPeter Dunlap /* Default data buffer length */ 45*30e7468fSPeter Dunlap #define ISER_DEFAULT_BUFLEN 0x20000 46*30e7468fSPeter Dunlap 47*30e7468fSPeter Dunlap /* 48*30e7468fSPeter Dunlap * iser_resource.h 49*30e7468fSPeter Dunlap * Definitions and functions related to set up buffer allocation from 50*30e7468fSPeter Dunlap * IBT memory regions and managment of work requessts. 51*30e7468fSPeter Dunlap */ 52*30e7468fSPeter Dunlap 53*30e7468fSPeter Dunlap struct iser_hca_s; 54*30e7468fSPeter Dunlap 55*30e7468fSPeter Dunlap /* 56*30e7468fSPeter Dunlap * Memory regions 57*30e7468fSPeter Dunlap */ 58*30e7468fSPeter Dunlap typedef struct iser_mr_s { 59*30e7468fSPeter Dunlap ibt_mr_hdl_t is_mrhdl; 60*30e7468fSPeter Dunlap ib_vaddr_t is_mrva; 61*30e7468fSPeter Dunlap ib_memlen_t is_mrlen; 62*30e7468fSPeter Dunlap ibt_lkey_t is_mrlkey; 63*30e7468fSPeter Dunlap ibt_rkey_t is_mrrkey; 64*30e7468fSPeter Dunlap avl_node_t is_avl_ln; 65*30e7468fSPeter Dunlap } iser_mr_t; 66*30e7468fSPeter Dunlap 67*30e7468fSPeter Dunlap typedef struct iser_vmem_mr_pool_s { 68*30e7468fSPeter Dunlap iser_hca_t *ivmp_hca; 69*30e7468fSPeter Dunlap ibt_mr_flags_t ivmp_mr_flags; 70*30e7468fSPeter Dunlap ib_memlen_t ivmp_chunksize; 71*30e7468fSPeter Dunlap vmem_t *ivmp_vmem; 72*30e7468fSPeter Dunlap uint64_t ivmp_total_size; 73*30e7468fSPeter Dunlap uint64_t ivmp_max_total_size; 74*30e7468fSPeter Dunlap avl_tree_t ivmp_mr_list; 75*30e7468fSPeter Dunlap kmutex_t ivmp_mutex; 76*30e7468fSPeter Dunlap } iser_vmem_mr_pool_t; 77*30e7468fSPeter Dunlap 78*30e7468fSPeter Dunlap #define ISER_MR_QUANTSIZE 0x400 79*30e7468fSPeter Dunlap #define ISER_MIN_CHUNKSIZE 0x100000 /* 1MB */ 80*30e7468fSPeter Dunlap 81*30e7468fSPeter Dunlap #ifdef _LP64 82*30e7468fSPeter Dunlap #define ISER_BUF_MR_CHUNKSIZE 0x8000000 /* 128MB */ 83*30e7468fSPeter Dunlap #define ISER_BUF_POOL_MAX 0x40000000 /* 1GB */ 84*30e7468fSPeter Dunlap #else 85*30e7468fSPeter Dunlap /* Memory is very limited on 32-bit kernels */ 86*30e7468fSPeter Dunlap #define ISER_BUF_MR_CHUNKSIZE 0x400000 /* 4MB */ 87*30e7468fSPeter Dunlap #define ISER_BUF_POOL_MAX 0x4000000 /* 64MB */ 88*30e7468fSPeter Dunlap #endif 89*30e7468fSPeter Dunlap #define ISER_BUF_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE | \ 90*30e7468fSPeter Dunlap IBT_MR_ENABLE_REMOTE_READ | IBT_MR_ENABLE_REMOTE_WRITE 91*30e7468fSPeter Dunlap #ifdef _LP64 92*30e7468fSPeter Dunlap #define ISER_MSG_MR_CHUNKSIZE 0x2000000 /* 32MB */ 93*30e7468fSPeter Dunlap #define ISER_MSG_POOL_MAX 0x10000000 /* 256MB */ 94*30e7468fSPeter Dunlap #else 95*30e7468fSPeter Dunlap #define ISER_MSG_MR_CHUNKSIZE 0x100000 /* 1MB */ 96*30e7468fSPeter Dunlap #define ISER_MSG_POOL_MAX 0x2000000 /* 32MB */ 97*30e7468fSPeter Dunlap #endif 98*30e7468fSPeter Dunlap #define ISER_MSG_MR_FLAGS IBT_MR_ENABLE_LOCAL_WRITE 99*30e7468fSPeter Dunlap 100*30e7468fSPeter Dunlap iser_vmem_mr_pool_t *iser_vmem_create(const char *name, iser_hca_t *hca, 101*30e7468fSPeter Dunlap ib_memlen_t chunksize, uint64_t max_total_size, 102*30e7468fSPeter Dunlap ibt_mr_flags_t arena_mr_flags); 103*30e7468fSPeter Dunlap void iser_vmem_destroy(iser_vmem_mr_pool_t *vmr_pool); 104*30e7468fSPeter Dunlap void *iser_vmem_alloc(iser_vmem_mr_pool_t *vmr_pool, size_t size); 105*30e7468fSPeter Dunlap void iser_vmem_free(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size); 106*30e7468fSPeter Dunlap idm_status_t iser_vmem_mr(iser_vmem_mr_pool_t *vmr_pool, 107*30e7468fSPeter Dunlap void *vaddr, size_t size, iser_mr_t *mr); 108*30e7468fSPeter Dunlap 109*30e7468fSPeter Dunlap /* 110*30e7468fSPeter Dunlap * iSER work request structure encodes an iSER Send Queue work request 111*30e7468fSPeter Dunlap * context, with pointers to relevant resources related to the work request. 112*30e7468fSPeter Dunlap * We hold a pointer to either an IDM PDU handle, an iSER message handle 113*30e7468fSPeter Dunlap * or an IDM buffer handle. These are allocated from a kmem_cache when 114*30e7468fSPeter Dunlap * we post send WR's, and freed back when the completion is polled. 115*30e7468fSPeter Dunlap */ 116*30e7468fSPeter Dunlap typedef enum { 117*30e7468fSPeter Dunlap ISER_WR_SEND, 118*30e7468fSPeter Dunlap ISER_WR_RDMAW, 119*30e7468fSPeter Dunlap ISER_WR_RDMAR, 120*30e7468fSPeter Dunlap ISER_WR_UNDEFINED 121*30e7468fSPeter Dunlap } iser_wr_type_t; 122*30e7468fSPeter Dunlap 123*30e7468fSPeter Dunlap typedef struct iser_wr_s { 124*30e7468fSPeter Dunlap iser_wr_type_t iw_type; 125*30e7468fSPeter Dunlap struct iser_msg_s *iw_msg; 126*30e7468fSPeter Dunlap struct idm_buf_s *iw_buf; 127*30e7468fSPeter Dunlap struct idm_pdu_s *iw_pdu; 128*30e7468fSPeter Dunlap } iser_wr_t; 129*30e7468fSPeter Dunlap 130*30e7468fSPeter Dunlap int iser_wr_cache_constructor(void *mr, void *arg, int flags); 131*30e7468fSPeter Dunlap void iser_wr_cache_destructor(void *mr, void *arg); 132*30e7468fSPeter Dunlap iser_wr_t *iser_wr_get(); 133*30e7468fSPeter Dunlap void iser_wr_free(iser_wr_t *iser_wr); 134*30e7468fSPeter Dunlap 135*30e7468fSPeter Dunlap /* 136*30e7468fSPeter Dunlap * iSER message structure for iSCSI Control PDUs, constructor and 137*30e7468fSPeter Dunlap * destructor routines, and utility routines for allocating and 138*30e7468fSPeter Dunlap * freeing message handles. 139*30e7468fSPeter Dunlap */ 140*30e7468fSPeter Dunlap typedef struct iser_msg_s { 141*30e7468fSPeter Dunlap struct iser_msg_s *nextp; /* for building lists */ 142*30e7468fSPeter Dunlap kmem_cache_t *cache; /* back pointer for cleanup */ 143*30e7468fSPeter Dunlap ibt_wr_ds_t msg_ds; /* SGEs for hdr and text */ 144*30e7468fSPeter Dunlap ibt_mr_hdl_t mrhdl[2]; /* MR handles for each SGE */ 145*30e7468fSPeter Dunlap } iser_msg_t; 146*30e7468fSPeter Dunlap 147*30e7468fSPeter Dunlap int iser_msg_cache_constructor(void *mr, void *arg, int flags); 148*30e7468fSPeter Dunlap void iser_msg_cache_destructor(void *mr, void *arg); 149*30e7468fSPeter Dunlap iser_msg_t *iser_msg_get(iser_hca_t *hca, int num, int *ret); 150*30e7468fSPeter Dunlap void iser_msg_free(iser_msg_t *msg); 151*30e7468fSPeter Dunlap 152*30e7468fSPeter Dunlap /* 153*30e7468fSPeter Dunlap * iSER data buffer structure for iSER RDMA operations, constructor and 154*30e7468fSPeter Dunlap * destructor routines, and utility routines for allocating and freeing 155*30e7468fSPeter Dunlap * buffer handles. 156*30e7468fSPeter Dunlap */ 157*30e7468fSPeter Dunlap typedef struct iser_buf_s { 158*30e7468fSPeter Dunlap kmem_cache_t *cache; /* back pointer for cleanup */ 159*30e7468fSPeter Dunlap void *buf; /* buffer */ 160*30e7468fSPeter Dunlap uint64_t buflen; 161*30e7468fSPeter Dunlap iser_mr_t *iser_mr; /* MR handle for this buffer */ 162*30e7468fSPeter Dunlap ibt_wr_ds_t buf_ds; /* SGE for this buffer */ 163*30e7468fSPeter Dunlap ibt_send_wr_t buf_wr; /* DEBUG, copy of wr from request */ 164*30e7468fSPeter Dunlap ibt_wc_t buf_wc; /* DEBUG, copy of wc from completion */ 165*30e7468fSPeter Dunlap timespec_t buf_constructed; 166*30e7468fSPeter Dunlap timespec_t buf_destructed; 167*30e7468fSPeter Dunlap } iser_buf_t; 168*30e7468fSPeter Dunlap 169*30e7468fSPeter Dunlap int iser_buf_cache_constructor(void *mr, void *arg, int flags); 170*30e7468fSPeter Dunlap void iser_buf_cache_destructor(void *mr, void *arg); 171*30e7468fSPeter Dunlap 172*30e7468fSPeter Dunlap void iser_init_hca_caches(struct iser_hca_s *hca); 173*30e7468fSPeter Dunlap void iser_fini_hca_caches(struct iser_hca_s *hca); 174*30e7468fSPeter Dunlap 175*30e7468fSPeter Dunlap /* Routines to register in-place memory passed on an existing idb */ 176*30e7468fSPeter Dunlap int iser_reg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb); 177*30e7468fSPeter Dunlap void iser_dereg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb); 178*30e7468fSPeter Dunlap 179*30e7468fSPeter Dunlap #ifdef __cplusplus 180*30e7468fSPeter Dunlap } 181*30e7468fSPeter Dunlap #endif 182*30e7468fSPeter Dunlap 183*30e7468fSPeter Dunlap #endif /* _ISER_RESOURCE_H */ 184