1*30e7468fSPeter Dunlap /*
2*30e7468fSPeter Dunlap  * CDDL HEADER START
3*30e7468fSPeter Dunlap  *
4*30e7468fSPeter Dunlap  * The contents of this file are subject to the terms of the
5*30e7468fSPeter Dunlap  * Common Development and Distribution License (the "License").
6*30e7468fSPeter Dunlap  * You may not use this file except in compliance with the License.
7*30e7468fSPeter Dunlap  *
8*30e7468fSPeter Dunlap  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9*30e7468fSPeter Dunlap  * or http://www.opensolaris.org/os/licensing.
10*30e7468fSPeter Dunlap  * See the License for the specific language governing permissions
11*30e7468fSPeter Dunlap  * and limitations under the License.
12*30e7468fSPeter Dunlap  *
13*30e7468fSPeter Dunlap  * When distributing Covered Code, include this CDDL HEADER in each
14*30e7468fSPeter Dunlap  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15*30e7468fSPeter Dunlap  * If applicable, add the following below this CDDL HEADER, with the
16*30e7468fSPeter Dunlap  * fields enclosed by brackets "[]" replaced with your own identifying
17*30e7468fSPeter Dunlap  * information: Portions Copyright [yyyy] [name of copyright owner]
18*30e7468fSPeter Dunlap  *
19*30e7468fSPeter Dunlap  * CDDL HEADER END
20*30e7468fSPeter Dunlap  */
21*30e7468fSPeter Dunlap /*
22*30e7468fSPeter Dunlap  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23*30e7468fSPeter Dunlap  * Use is subject to license terms.
24*30e7468fSPeter Dunlap  */
25*30e7468fSPeter Dunlap 
26*30e7468fSPeter Dunlap #ifndef _ISER_RESOURCE_H
27*30e7468fSPeter Dunlap #define	_ISER_RESOURCE_H
28*30e7468fSPeter Dunlap 
29*30e7468fSPeter Dunlap #ifdef	__cplusplus
30*30e7468fSPeter Dunlap extern "C" {
31*30e7468fSPeter Dunlap #endif
32*30e7468fSPeter Dunlap 
33*30e7468fSPeter Dunlap #include <sys/types.h>
34*30e7468fSPeter Dunlap #include <sys/ib/ibtl/ibti.h>
35*30e7468fSPeter Dunlap #include <sys/ib/ibtl/ibtl_types.h>
36*30e7468fSPeter Dunlap #include <sys/iscsi_protocol.h>
37*30e7468fSPeter Dunlap 
38*30e7468fSPeter Dunlap #define	ISER_CACHE_NAMELEN	31	/* KMEM_CACHE_NAMELEN */
39*30e7468fSPeter Dunlap 
40*30e7468fSPeter Dunlap /* Default message lengths */
41*30e7468fSPeter Dunlap #define	ISER_MAX_CTRLPDU_LEN	0x4000
42*30e7468fSPeter Dunlap #define	ISER_MAX_TEXTPDU_LEN	0x4000
43*30e7468fSPeter Dunlap 
44*30e7468fSPeter Dunlap /* Default data buffer length */
45*30e7468fSPeter Dunlap #define	ISER_DEFAULT_BUFLEN	0x20000
46*30e7468fSPeter Dunlap 
47*30e7468fSPeter Dunlap /*
48*30e7468fSPeter Dunlap  * iser_resource.h
49*30e7468fSPeter Dunlap  * Definitions and functions related to set up buffer allocation from
50*30e7468fSPeter Dunlap  * IBT memory regions and managment of work requessts.
51*30e7468fSPeter Dunlap  */
52*30e7468fSPeter Dunlap 
53*30e7468fSPeter Dunlap struct iser_hca_s;
54*30e7468fSPeter Dunlap 
55*30e7468fSPeter Dunlap /*
56*30e7468fSPeter Dunlap  * Memory regions
57*30e7468fSPeter Dunlap  */
58*30e7468fSPeter Dunlap typedef struct iser_mr_s {
59*30e7468fSPeter Dunlap 	ibt_mr_hdl_t		is_mrhdl;
60*30e7468fSPeter Dunlap 	ib_vaddr_t		is_mrva;
61*30e7468fSPeter Dunlap 	ib_memlen_t		is_mrlen;
62*30e7468fSPeter Dunlap 	ibt_lkey_t		is_mrlkey;
63*30e7468fSPeter Dunlap 	ibt_rkey_t		is_mrrkey;
64*30e7468fSPeter Dunlap 	avl_node_t		is_avl_ln;
65*30e7468fSPeter Dunlap } iser_mr_t;
66*30e7468fSPeter Dunlap 
67*30e7468fSPeter Dunlap typedef struct iser_vmem_mr_pool_s {
68*30e7468fSPeter Dunlap 	iser_hca_t		*ivmp_hca;
69*30e7468fSPeter Dunlap 	ibt_mr_flags_t		ivmp_mr_flags;
70*30e7468fSPeter Dunlap 	ib_memlen_t		ivmp_chunksize;
71*30e7468fSPeter Dunlap 	vmem_t			*ivmp_vmem;
72*30e7468fSPeter Dunlap 	uint64_t		ivmp_total_size;
73*30e7468fSPeter Dunlap 	uint64_t		ivmp_max_total_size;
74*30e7468fSPeter Dunlap 	avl_tree_t		ivmp_mr_list;
75*30e7468fSPeter Dunlap 	kmutex_t		ivmp_mutex;
76*30e7468fSPeter Dunlap } iser_vmem_mr_pool_t;
77*30e7468fSPeter Dunlap 
78*30e7468fSPeter Dunlap #define	ISER_MR_QUANTSIZE	0x400
79*30e7468fSPeter Dunlap #define	ISER_MIN_CHUNKSIZE	0x100000	/* 1MB */
80*30e7468fSPeter Dunlap 
81*30e7468fSPeter Dunlap #ifdef _LP64
82*30e7468fSPeter Dunlap #define	ISER_BUF_MR_CHUNKSIZE	0x8000000	/* 128MB */
83*30e7468fSPeter Dunlap #define	ISER_BUF_POOL_MAX	0x40000000	/* 1GB */
84*30e7468fSPeter Dunlap #else
85*30e7468fSPeter Dunlap /* Memory is very limited on 32-bit kernels */
86*30e7468fSPeter Dunlap #define	ISER_BUF_MR_CHUNKSIZE	0x400000	/* 4MB */
87*30e7468fSPeter Dunlap #define	ISER_BUF_POOL_MAX	0x4000000	/* 64MB */
88*30e7468fSPeter Dunlap #endif
89*30e7468fSPeter Dunlap #define	ISER_BUF_MR_FLAGS	IBT_MR_ENABLE_LOCAL_WRITE | \
90*30e7468fSPeter Dunlap 	IBT_MR_ENABLE_REMOTE_READ | IBT_MR_ENABLE_REMOTE_WRITE
91*30e7468fSPeter Dunlap #ifdef _LP64
92*30e7468fSPeter Dunlap #define	ISER_MSG_MR_CHUNKSIZE	0x2000000	/* 32MB */
93*30e7468fSPeter Dunlap #define	ISER_MSG_POOL_MAX	0x10000000	/* 256MB */
94*30e7468fSPeter Dunlap #else
95*30e7468fSPeter Dunlap #define	ISER_MSG_MR_CHUNKSIZE	0x100000	/* 1MB */
96*30e7468fSPeter Dunlap #define	ISER_MSG_POOL_MAX	0x2000000	/* 32MB */
97*30e7468fSPeter Dunlap #endif
98*30e7468fSPeter Dunlap #define	ISER_MSG_MR_FLAGS	IBT_MR_ENABLE_LOCAL_WRITE
99*30e7468fSPeter Dunlap 
100*30e7468fSPeter Dunlap iser_vmem_mr_pool_t *iser_vmem_create(const char *name, iser_hca_t *hca,
101*30e7468fSPeter Dunlap     ib_memlen_t chunksize, uint64_t max_total_size,
102*30e7468fSPeter Dunlap     ibt_mr_flags_t arena_mr_flags);
103*30e7468fSPeter Dunlap void iser_vmem_destroy(iser_vmem_mr_pool_t *vmr_pool);
104*30e7468fSPeter Dunlap void *iser_vmem_alloc(iser_vmem_mr_pool_t *vmr_pool, size_t size);
105*30e7468fSPeter Dunlap void iser_vmem_free(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size);
106*30e7468fSPeter Dunlap idm_status_t iser_vmem_mr(iser_vmem_mr_pool_t *vmr_pool,
107*30e7468fSPeter Dunlap     void *vaddr, size_t size, iser_mr_t *mr);
108*30e7468fSPeter Dunlap 
109*30e7468fSPeter Dunlap /*
110*30e7468fSPeter Dunlap  * iSER work request structure encodes an iSER Send Queue work request
111*30e7468fSPeter Dunlap  * context, with pointers to relevant resources related to the work request.
112*30e7468fSPeter Dunlap  * We hold a pointer to either an IDM PDU handle, an iSER message handle
113*30e7468fSPeter Dunlap  * or an IDM buffer handle. These are allocated from a kmem_cache when
114*30e7468fSPeter Dunlap  * we post send WR's, and freed back when the completion is polled.
115*30e7468fSPeter Dunlap  */
116*30e7468fSPeter Dunlap typedef enum {
117*30e7468fSPeter Dunlap 	ISER_WR_SEND,
118*30e7468fSPeter Dunlap 	ISER_WR_RDMAW,
119*30e7468fSPeter Dunlap 	ISER_WR_RDMAR,
120*30e7468fSPeter Dunlap 	ISER_WR_UNDEFINED
121*30e7468fSPeter Dunlap } iser_wr_type_t;
122*30e7468fSPeter Dunlap 
123*30e7468fSPeter Dunlap typedef struct iser_wr_s {
124*30e7468fSPeter Dunlap 	iser_wr_type_t		iw_type;
125*30e7468fSPeter Dunlap 	struct iser_msg_s	*iw_msg;
126*30e7468fSPeter Dunlap 	struct idm_buf_s	*iw_buf;
127*30e7468fSPeter Dunlap 	struct idm_pdu_s	*iw_pdu;
128*30e7468fSPeter Dunlap } iser_wr_t;
129*30e7468fSPeter Dunlap 
130*30e7468fSPeter Dunlap int iser_wr_cache_constructor(void *mr, void *arg, int flags);
131*30e7468fSPeter Dunlap void iser_wr_cache_destructor(void *mr, void *arg);
132*30e7468fSPeter Dunlap iser_wr_t *iser_wr_get();
133*30e7468fSPeter Dunlap void iser_wr_free(iser_wr_t *iser_wr);
134*30e7468fSPeter Dunlap 
135*30e7468fSPeter Dunlap /*
136*30e7468fSPeter Dunlap  * iSER message structure for iSCSI Control PDUs, constructor and
137*30e7468fSPeter Dunlap  * destructor routines, and utility routines for allocating and
138*30e7468fSPeter Dunlap  * freeing message handles.
139*30e7468fSPeter Dunlap  */
140*30e7468fSPeter Dunlap typedef struct iser_msg_s {
141*30e7468fSPeter Dunlap 	struct iser_msg_s	*nextp;	  /* for building lists */
142*30e7468fSPeter Dunlap 	kmem_cache_t		*cache;	  /* back pointer for cleanup */
143*30e7468fSPeter Dunlap 	ibt_wr_ds_t		msg_ds;	  /* SGEs for hdr and text */
144*30e7468fSPeter Dunlap 	ibt_mr_hdl_t		mrhdl[2]; /* MR handles for each SGE */
145*30e7468fSPeter Dunlap } iser_msg_t;
146*30e7468fSPeter Dunlap 
147*30e7468fSPeter Dunlap int iser_msg_cache_constructor(void *mr, void *arg, int flags);
148*30e7468fSPeter Dunlap void iser_msg_cache_destructor(void *mr, void *arg);
149*30e7468fSPeter Dunlap iser_msg_t *iser_msg_get(iser_hca_t *hca, int num, int *ret);
150*30e7468fSPeter Dunlap void iser_msg_free(iser_msg_t *msg);
151*30e7468fSPeter Dunlap 
152*30e7468fSPeter Dunlap /*
153*30e7468fSPeter Dunlap  * iSER data buffer structure for iSER RDMA operations, constructor and
154*30e7468fSPeter Dunlap  * destructor routines, and utility routines for allocating and freeing
155*30e7468fSPeter Dunlap  * buffer handles.
156*30e7468fSPeter Dunlap  */
157*30e7468fSPeter Dunlap typedef struct iser_buf_s {
158*30e7468fSPeter Dunlap 	kmem_cache_t	*cache;	/* back pointer for cleanup */
159*30e7468fSPeter Dunlap 	void		*buf;	/* buffer */
160*30e7468fSPeter Dunlap 	uint64_t	buflen;
161*30e7468fSPeter Dunlap 	iser_mr_t	*iser_mr; /* MR handle for this buffer */
162*30e7468fSPeter Dunlap 	ibt_wr_ds_t	buf_ds;	/* SGE for this buffer */
163*30e7468fSPeter Dunlap 	ibt_send_wr_t	buf_wr;	/* DEBUG, copy of wr from request */
164*30e7468fSPeter Dunlap 	ibt_wc_t	buf_wc; /* DEBUG, copy of wc from completion */
165*30e7468fSPeter Dunlap 	timespec_t	buf_constructed;
166*30e7468fSPeter Dunlap 	timespec_t	buf_destructed;
167*30e7468fSPeter Dunlap } iser_buf_t;
168*30e7468fSPeter Dunlap 
169*30e7468fSPeter Dunlap int iser_buf_cache_constructor(void *mr, void *arg, int flags);
170*30e7468fSPeter Dunlap void iser_buf_cache_destructor(void *mr, void *arg);
171*30e7468fSPeter Dunlap 
172*30e7468fSPeter Dunlap void iser_init_hca_caches(struct iser_hca_s *hca);
173*30e7468fSPeter Dunlap void iser_fini_hca_caches(struct iser_hca_s *hca);
174*30e7468fSPeter Dunlap 
175*30e7468fSPeter Dunlap /* Routines to register in-place memory passed on an existing idb */
176*30e7468fSPeter Dunlap int iser_reg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb);
177*30e7468fSPeter Dunlap void iser_dereg_rdma_mem(struct iser_hca_s *hca, idm_buf_t *idb);
178*30e7468fSPeter Dunlap 
179*30e7468fSPeter Dunlap #ifdef	__cplusplus
180*30e7468fSPeter Dunlap }
181*30e7468fSPeter Dunlap #endif
182*30e7468fSPeter Dunlap 
183*30e7468fSPeter Dunlap #endif /* _ISER_RESOURCE_H */
184