1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #ifndef	_SYS_IB_ADAPTERS_HERMON_MR_H
27 #define	_SYS_IB_ADAPTERS_HERMON_MR_H
28 
29 /*
30  * hermon_mr.h
31  *    Contains all of the prototypes, #defines, and structures necessary
32  *    for the Hermon Memory Region/Window routines.
33  *    Specifically it contains #defines, macros, and prototypes for each of
34  *    the required memory region/window verbs that can be accessed through
35  *    the IBTF's CI interfaces.  In particular each of the prototypes defined
36  *    below is called from a corresponding CI interface routine (as specified
37  *    in the hermon_ci.c file).
38  */
39 
40 #include <sys/types.h>
41 #include <sys/conf.h>
42 #include <sys/ddi.h>
43 #include <sys/sunddi.h>
44 
45 #ifdef __cplusplus
46 extern "C" {
47 #endif
48 
49 /*
50  * The following defines specify the default number of MPT entries to
51  * configure. This value is controllable through the "hermon_log_num_mpt"
52  * configuration variable.
53  */
54 #define	HERMON_NUM_DMPT_SHIFT		0x16
55 
56 /*
57  * The following defines specify the default number of MPT entries to
58  * configure. This value is controllable through the "hermon_log_num_mtt"
59  * configuration variable. This default value expects an averages of 8
60  * MTTs per MPT. We also define a log MTT size, since it's not likely
61  * to change.
62  */
63 #define	HERMON_NUM_MTT_SHIFT		0x1d
64 #define	HERMON_MTT_SIZE_SHIFT		0x3
65 
66 /*
67  * This define is the maximum size of a memory region or window (log 2), which
68  * is used to initialize the "hermon_log_max_mrw_sz" configuration variable.
69  */
70 #define	HERMON_MAX_MEM_MPT_SHIFT			0x24
71 
72 /*
73  * Defines used by hermon_mr_deregister() to specify how much/to what extent
74  * a given memory regions resources should be freed up.  HERMON_MR_DEREG_ALL
75  * says what it means, free up all the resources associated with the region.
76  * HERMON_MR_DEREG_NO_HW2SW_MPT indicates that it is unnecessary to attempt
77  * the ownership transfer (from hardware to software) for the given MPT entry.
78  * And HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND indicates that it is not only
79  * unnecessary to attempt the ownership transfer for MPT, but it is also
80  * unnecessary to attempt to unbind the memory.
81  * In general, these last two are specified when hermon_mr_deregister() is
82  * called from hermon_mr_reregister(), where the MPT ownership transfer or
83  * memory unbinding may have already been successfully performed.
84  */
85 #define	HERMON_MR_DEREG_ALL			3
86 #define	HERMON_MR_DEREG_NO_HW2SW_MPT		2
87 #define	HERMON_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND	1
88 
89 /*
90  * The following define is used by hermon_mr_rereg_xlat_helper() to determine
91  * whether or not a given DMA handle can be reused.  If the DMA handle was
92  * previously initialized for IOMMU bypass mapping, then it can not be reused
93  * to reregister a region for DDI_DMA_STREAMING access.
94  */
95 #define	HERMON_MR_REUSE_DMAHDL(mr, flags)				\
96 	(((mr)->mr_bindinfo.bi_bypass != HERMON_BINDMEM_BYPASS) ||	\
97 	    !((flags) & IBT_MR_NONCOHERENT))
98 
99 /*
100  * The hermon_sw_refcnt_t structure is used internally by the Hermon driver to
101  * track all the information necessary to manage shared memory regions.  Since
102  * a shared memory region _will_ have its own distinct MPT entry, but will
103  * _share_ its MTT entries with another region, it is necessary to track the
104  * number of times a given MTT structure is shared.  This ensures that it will
105  * not be prematurely freed up and that can be destroyed only when it is
106  * appropriate to do so.
107  *
108  * Each hermon_sw_refcnt_t structure contains a lock and a reference count
109  * variable which are used to track the necessary information.
110  *
111  * The following macros (below) are used to manipulate and query the MTT
112  * reference count parameters.  HERMON_MTT_REFCNT_INIT() is used to initialize
113  * a newly allocated hermon_sw_refcnt_t struct (setting the "swrc_refcnt" to 1).
114  * And the HERMON_MTT_IS_NOT_SHARED() and HERMON_MTT_IS_SHARED() macros are
115  * used to query the current status of hermon_sw_refcnt_t struct to determine
116  * if its "swrc_refcnt" is one or not.
117  */
118 typedef struct hermon_sw_refcnt_s {
119 	kmutex_t		swrc_lock;
120 	uint_t			swrc_refcnt;
121 } hermon_sw_refcnt_t;
122 _NOTE(DATA_READABLE_WITHOUT_LOCK(hermon_sw_refcnt_t::swrc_refcnt))
123 _NOTE(MUTEX_PROTECTS_DATA(hermon_sw_refcnt_t::swrc_lock,
124     hermon_sw_refcnt_t::swrc_refcnt))
125 #define	HERMON_MTT_REFCNT_INIT(swrc_tmp)	((swrc_tmp)->swrc_refcnt = 1)
126 #define	HERMON_MTT_IS_NOT_SHARED(swrc_tmp)	((swrc_tmp)->swrc_refcnt == 1)
127 #define	HERMON_MTT_IS_SHARED(swrc_tmp)		((swrc_tmp)->swrc_refcnt != 1)
128 
129 
130 /*
131  * The hermon_bind_info_t structure is used internally by the Hermon driver to
132  * track all the information necessary to perform the DMA mappings necessary
133  * for memory registration.  It is specifically passed into both the
134  * hermon_mr_mem_bind() and hermon_mr_mtt_write() functions which perform most
135  * of the necessary operations for Hermon memory registration.
136  *
137  * This structure is used to pass all the information necessary for a call
138  * to either ddi_dma_addr_bind_handle() or ddi_dma_buf_bind_handle().  Note:
139  * the fields which need to be valid for each type of binding are slightly
140  * different and that it indicated by the value in the "bi_type" field.  The
141  * "bi_type" field may be set to either of the following defined values:
142  * HERMON_BINDHDL_VADDR (to indicate an "addr" bind) or HERMON_BINDHDL_BUF (to
143  * indicate a "buf" bind).
144  *
145  * Upon return from hermon_mr_mem_bind(), the hermon_bind_info_t struct will
146  * have its "bi_dmahdl", "bi_dmacookie", and "bi_cookiecnt" fields filled in.
147  * It is these values which are of particular interest to the
148  * hermon_mr_mtt_write() routine (they hold the PCI mapped addresses).
149  *
150  * Once initialized and used in this way, the hermon_bind_info_t will not to be
151  * modified in anyway until it is subsequently passed to hermon_mr_mem_unbind()
152  * where the memory and resources will be unbound and reclaimed.  Note:  the
153  * "bi_free_dmahdl" flag indicated whether the ddi_dma_handle_t should be
154  * freed as part of the hermon_mr_mem_unbind() operation or whether it will
155  * be freed later elsewhere.
156  */
157 typedef struct hermon_bind_info_s {
158 	uint64_t		bi_addr;
159 	uint64_t		bi_len;
160 	struct as		*bi_as;
161 	struct buf		*bi_buf;
162 	ddi_dma_handle_t	bi_dmahdl;
163 	ddi_dma_cookie_t	bi_dmacookie;
164 	uint_t			bi_cookiecnt;
165 	uint_t			bi_type;
166 	uint_t			bi_flags;
167 	uint_t			bi_bypass;
168 	uint_t			bi_free_dmahdl;
169 } hermon_bind_info_t;
170 #define	HERMON_BINDHDL_NONE		0
171 #define	HERMON_BINDHDL_VADDR		1
172 #define	HERMON_BINDHDL_BUF		2
173 #define	HERMON_BINDHDL_UBUF		3
174 #define	HERMON_BINDHDL_LKEY		4
175 
176 /*
177  * The hermon_sw_mr_s structure is also referred to using the "hermon_mrhdl_t"
178  * typedef (see hermon_typedef.h).  It encodes all the information necessary
179  * to track the various resources needed to register, reregister, deregister,
180  * and perform all the myriad other operations on both memory regions _and_
181  * memory windows.
182  *
183  * A pointer to this structure is returned from many of the IBTF's CI verbs
184  * interfaces for memory registration.
185  *
186  * It contains pointers to the various resources allocated for a memory
187  * region, i.e. MPT resource, MTT resource, and MTT reference count resource.
188  * In addition it contains the hermon_bind_info_t struct used for the memory
189  * bind operation on a given memory region.
190  *
191  * It also has a pointers to the associated PD handle, placeholders for access
192  * flags, memory keys, and suggested page size for the region.  It also has
193  * the necessary backpointer to the resource that corresponds to the structure
194  * itself.  And lastly, it contains a placeholder for a callback which should
195  * be called on memory region unpinning.
196  */
197 struct hermon_sw_mr_s {
198 	kmutex_t		mr_lock;
199 	hermon_rsrc_t		*mr_mptrsrcp;
200 	hermon_rsrc_t		*mr_mttrsrcp;
201 	hermon_rsrc_t		*mr_mttrefcntp;
202 	hermon_pdhdl_t		mr_pdhdl;
203 	hermon_bind_info_t	mr_bindinfo;
204 	ibt_mr_attr_flags_t	mr_accflag;
205 	uint32_t		mr_lkey;
206 	uint32_t		mr_rkey;
207 	uint32_t		mr_logmttpgsz;
208 	hermon_mpt_rsrc_type_t	mr_mpt_type;
209 	uint64_t		mr_mttaddr;	/* for cMPTs */
210 	uint64_t		mr_log2_pgsz;
211 				/* entity_size (in bytes), for cMPTS */
212 	hermon_rsrc_t		*mr_rsrcp;
213 	uint_t			mr_is_fmr;
214 	uint8_t			mr_fmr_key;	/* per FMR 8-bit key */
215 	hermon_fmr_list_t	*mr_fmr;
216 	uint_t			mr_is_umem;
217 	ddi_umem_cookie_t	mr_umemcookie;
218 	void 			(*mr_umem_cbfunc)(void *, void *);
219 	void			*mr_umem_cbarg1;
220 	void			*mr_umem_cbarg2;
221 };
222 _NOTE(DATA_READABLE_WITHOUT_LOCK(hermon_sw_mr_s::mr_bindinfo
223     hermon_sw_mr_s::mr_lkey
224     hermon_sw_mr_s::mr_mttaddr
225     hermon_sw_mr_s::mr_is_umem
226     hermon_sw_mr_s::mr_is_fmr
227     hermon_sw_mr_s::mr_fmr))
228 _NOTE(MUTEX_PROTECTS_DATA(hermon_sw_mr_s::mr_lock,
229     hermon_sw_mr_s::mr_mptrsrcp
230     hermon_sw_mr_s::mr_mttrsrcp
231     hermon_sw_mr_s::mr_mttrefcntp
232     hermon_sw_mr_s::mr_bindinfo
233     hermon_sw_mr_s::mr_lkey
234     hermon_sw_mr_s::mr_rkey
235     hermon_sw_mr_s::mr_logmttpgsz
236     hermon_sw_mr_s::mr_rsrcp
237     hermon_sw_mr_s::mr_is_umem
238     hermon_sw_mr_s::mr_umemcookie
239     hermon_sw_mr_s::mr_umem_cbfunc
240     hermon_sw_mr_s::mr_umem_cbarg1
241     hermon_sw_mr_s::mr_umem_cbarg2))
242 
243 /*
244  * The hermon_mr_options_t structure is used in several of the Hermon memory
245  * registration routines to provide additional option functionality.  When
246  * a NULL pointer is passed in place of a pointer to this struct, it is a
247  * way of specifying the "default" behavior.  Using this structure, however,
248  * is a way of controlling any extended behavior.
249  *
250  * Currently, the only defined "extended" behaviors are for specifying whether
251  * a given memory region should bypass the PCI IOMMU (HERMON_BINDMEM_BYPASS)
252  * or be mapped into the IOMMU (HERMON_BINDMEM_NORMAL), for specifying whether
253  * a given ddi_dma_handle_t should be used in the bind operation, and for
254  * specifying whether a memory registration should attempt to return an IB
255  * vaddr which is "zero-based" (aids in alignment contraints for QPs).
256  *
257  * This defaults today to always bypassing the IOMMU (can be changed by using
258  * the "hermon_iommu_bypass" configuration variable), to always allocating
259  * a new dma handle, and to using the virtual address passed in (i.e. not
260  * "zero-based").
261  */
262 typedef struct hermon_mr_options_s {
263 	ddi_dma_handle_t	mro_bind_dmahdl;
264 	uint_t			mro_bind_type;
265 	uint_t			mro_bind_override_addr;
266 } hermon_mr_options_t;
267 #define	HERMON_BINDMEM_NORMAL		1
268 #define	HERMON_BINDMEM_BYPASS		0
269 
270 #define	HERMON_NO_MPT_OWNERSHIP		0	/* for cMPTs */
271 #define	HERMON_PASS_MPT_OWNERSHIP	1
272 
273 /*
274  * Memory Allocation/Deallocation
275  *
276  * Although this is not strictly related to "memory regions", this is
277  * the most logical place to define the struct used for the memory
278  * allocation/deallocation CI entry points.
279  *
280  * ibc_mem_alloc_s structure is used to store DMA handles for
281  * for these allocations.
282  */
283 struct ibc_mem_alloc_s {
284 	ddi_dma_handle_t ibc_dma_hdl;
285 	ddi_acc_handle_t ibc_acc_hdl;
286 };
287 _NOTE(SCHEME_PROTECTS_DATA("safe sharing",
288     ibc_mem_alloc_s::ibc_dma_hdl
289     ibc_mem_alloc_s::ibc_acc_hdl))
290 
291 int hermon_dma_mr_register(hermon_state_t *state, hermon_pdhdl_t pdhdl,
292     ibt_dmr_attr_t *attr_p, hermon_mrhdl_t *mrhdl);
293 int hermon_mr_register(hermon_state_t *state, hermon_pdhdl_t pdhdl,
294     ibt_mr_attr_t *attr_p, hermon_mrhdl_t *mrhdl, hermon_mr_options_t *op,
295     hermon_mpt_rsrc_type_t mpt_type);
296 int hermon_mr_register_buf(hermon_state_t *state, hermon_pdhdl_t pdhdl,
297     ibt_smr_attr_t *attrp, struct buf *buf, hermon_mrhdl_t *mrhdl,
298     hermon_mr_options_t *op, hermon_mpt_rsrc_type_t mpt_type);
299 int hermon_mr_mtt_bind(hermon_state_t *state, hermon_bind_info_t *bind,
300     ddi_dma_handle_t bind_dmahdl, hermon_rsrc_t **mtt, uint_t *mtt_pgsz_bits,
301     uint_t is_buffer);
302 int hermon_mr_mtt_unbind(hermon_state_t *state, hermon_bind_info_t *bind,
303     hermon_rsrc_t *mtt);
304 int hermon_mr_register_shared(hermon_state_t *state, hermon_mrhdl_t mrhdl,
305     hermon_pdhdl_t pdhdl, ibt_smr_attr_t *attr_p, hermon_mrhdl_t *mrhdl_new);
306 int hermon_mr_deregister(hermon_state_t *state, hermon_mrhdl_t *mrhdl,
307     uint_t level, uint_t sleep);
308 int hermon_mr_query(hermon_state_t *state, hermon_mrhdl_t mrhdl,
309     ibt_mr_query_attr_t *attr);
310 int hermon_mr_reregister(hermon_state_t *state, hermon_mrhdl_t mrhdl,
311     hermon_pdhdl_t pdhdl, ibt_mr_attr_t *attr_p, hermon_mrhdl_t *mrhdl_new,
312     hermon_mr_options_t *op);
313 int hermon_mr_reregister_buf(hermon_state_t *state, hermon_mrhdl_t mr,
314     hermon_pdhdl_t pd, ibt_smr_attr_t *mr_attr, struct buf *buf,
315     hermon_mrhdl_t *mrhdl_new, hermon_mr_options_t *op);
316 int hermon_mr_sync(hermon_state_t *state, ibt_mr_sync_t *mr_segs,
317     size_t num_segs);
318 int hermon_mw_alloc(hermon_state_t *state, hermon_pdhdl_t pdhdl,
319     ibt_mw_flags_t flags, hermon_mwhdl_t *mwhdl);
320 int hermon_mw_free(hermon_state_t *state, hermon_mwhdl_t *mwhdl, uint_t sleep);
321 uint32_t hermon_mr_keycalc(uint32_t indx);
322 uint32_t hermon_mr_key_swap(uint32_t indx);
323 uint32_t hermon_index_to_mkey(uint32_t indx);
324 int hermon_mr_alloc_fmr(hermon_state_t *state, hermon_pdhdl_t pd,
325     hermon_fmrhdl_t fmr_pool, hermon_mrhdl_t *mrhdl);
326 int hermon_mr_dealloc_fmr(hermon_state_t *state, hermon_mrhdl_t *mrhdl);
327 int hermon_mr_register_physical_fmr(hermon_state_t *state,
328     ibt_pmr_attr_t *mem_pattr_p, hermon_mrhdl_t mr, ibt_pmr_desc_t *mem_desc_p);
329 int hermon_mr_alloc_lkey(hermon_state_t *state, hermon_pdhdl_t pd,
330     ibt_lkey_flags_t flags, uint_t sz, hermon_mrhdl_t *mr);
331 int hermon_mr_fexch_mpt_init(hermon_state_t *state, hermon_pdhdl_t pd,
332     uint32_t mpt_indx, uint_t nummtt, uint64_t mtt_addr, uint_t sleep);
333 int hermon_mr_fexch_mpt_fini(hermon_state_t *state, hermon_pdhdl_t pd,
334     uint32_t mpt_indx, uint_t sleep);
335 
336 
337 #ifdef __cplusplus
338 }
339 #endif
340 
341 #endif	/* _SYS_IB_ADAPTERS_HERMON_MR_H */
342