1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
24 */
25
26 /*
27 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31 /*
32 *
33 * MODULE: dapl_rmr_bind.c
34 *
35 * PURPOSE: Memory management
36 * Description: Interfaces in this file are completely described in
37 * the DAPL 1.1 API, Chapter 6, section 6
38 *
39 * $Id: dapl_rmr_bind.c,v 1.14 2003/07/11 18:23:31 jlentini Exp $
40 */
41
42 #include "dapl.h"
43 #include "dapl_rmr_util.h"
44 #include "dapl_ep_util.h"
45 #include "dapl_cookie.h"
46 #include "dapl_adapter_util.h"
47
48 /*
49 *
50 * Function Prototypes
51 *
52 */
53
54 static DAT_RETURN
55 dapli_rmr_bind_fuse(
56 IN DAPL_RMR *rmr,
57 IN const DAT_LMR_TRIPLET *lmr_triplet,
58 IN DAT_MEM_PRIV_FLAGS mem_priv,
59 IN DAPL_EP *ep,
60 IN DAT_RMR_COOKIE user_cookie,
61 IN DAT_COMPLETION_FLAGS completion_flags,
62 OUT DAT_RMR_CONTEXT *rmr_context);
63
64 static DAT_RETURN
65 dapli_rmr_bind_unfuse(
66 IN DAPL_RMR *rmr,
67 IN const DAT_LMR_TRIPLET *lmr_triplet,
68 IN DAPL_EP *ep,
69 IN DAT_RMR_COOKIE user_cookie,
70 IN DAT_COMPLETION_FLAGS completion_flags);
71
72
73 /*
74 *
75 * Function Definitions
76 *
77 */
78
79 static DAT_RETURN
dapli_rmr_bind_fuse(IN DAPL_RMR * rmr,IN const DAT_LMR_TRIPLET * lmr_triplet,IN DAT_MEM_PRIV_FLAGS mem_priv,IN DAPL_EP * ep_ptr,IN DAT_RMR_COOKIE user_cookie,IN DAT_COMPLETION_FLAGS completion_flags,OUT DAT_RMR_CONTEXT * rmr_context)80 dapli_rmr_bind_fuse(
81 IN DAPL_RMR *rmr,
82 IN const DAT_LMR_TRIPLET* lmr_triplet,
83 IN DAT_MEM_PRIV_FLAGS mem_priv,
84 IN DAPL_EP *ep_ptr,
85 IN DAT_RMR_COOKIE user_cookie,
86 IN DAT_COMPLETION_FLAGS completion_flags,
87 OUT DAT_RMR_CONTEXT *rmr_context)
88 {
89 DAPL_LMR *lmr;
90 DAPL_COOKIE *cookie;
91 DAT_RETURN dat_status;
92
93 dat_status = dapls_hash_search(
94 rmr->header.owner_ia->hca_ptr->lmr_hash_table,
95 lmr_triplet->lmr_context,
96 (DAPL_HASH_DATA *) &lmr);
97 if (DAT_SUCCESS != dat_status) {
98 dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG2);
99 goto bail;
100 }
101
102 /*
103 * if the ep in unconnected return an error. IB requires that the
104 * QP be connected to change a memory window binding since:
105 *
106 * - memory window bind operations are WQEs placed on a QP's
107 * send queue
108 *
109 * - QP's only process WQEs on the send queue when the QP is in
110 * the RTS state
111 */
112 if (DAT_EP_STATE_CONNECTED != ep_ptr->param.ep_state) {
113 dat_status = DAT_ERROR(DAT_INVALID_STATE,
114 dapls_ep_state_subtype(ep_ptr));
115 goto bail;
116 }
117
118 if (DAT_FALSE == dapl_mr_bounds_check(
119 dapl_mr_get_address(lmr->param.region_desc, lmr->param.mem_type),
120 lmr->param.length,
121 lmr_triplet->virtual_address,
122 lmr_triplet->segment_length)) {
123 dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
124 DAT_INVALID_ARG2);
125 goto bail;
126 }
127
128 /* If the LMR, RMR, and EP are not in the same PZ, there is an error */
129 if ((ep_ptr->param.pz_handle != lmr->param.pz_handle) ||
130 (ep_ptr->param.pz_handle != rmr->param.pz_handle)) {
131 dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG4);
132 goto bail;
133 }
134
135 if (!dapl_rmr_validate_completion_flag(DAT_COMPLETION_SUPPRESS_FLAG,
136 ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
137 !dapl_rmr_validate_completion_flag(DAT_COMPLETION_UNSIGNALLED_FLAG,
138 ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
139 !dapl_rmr_validate_completion_flag(
140 DAT_COMPLETION_BARRIER_FENCE_FLAG,
141 ep_ptr->param.ep_attr.request_completion_flags, completion_flags)) {
142 dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG4);
143 goto bail;
144 }
145
146 dat_status = dapls_rmr_cookie_alloc(&ep_ptr->req_buffer,
147 rmr, user_cookie, &cookie);
148 if (DAT_SUCCESS != dat_status) {
149 goto bail;
150 }
151
152 dat_status = dapls_ib_mw_bind(rmr,
153 lmr_triplet->lmr_context,
154 ep_ptr,
155 cookie,
156 lmr_triplet->virtual_address,
157 lmr_triplet->segment_length,
158 mem_priv,
159 completion_flags);
160 if (DAT_SUCCESS != dat_status) {
161 dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie);
162 goto bail;
163 }
164
165 dapl_os_atomic_inc(&lmr->lmr_ref_count);
166
167 /* if the RMR was previously bound */
168 if (NULL != rmr->lmr) {
169 dapl_os_atomic_dec(&rmr->lmr->lmr_ref_count);
170 }
171
172 rmr->param.mem_priv = mem_priv;
173 rmr->param.lmr_triplet = *lmr_triplet;
174 rmr->ep = ep_ptr;
175 rmr->lmr = lmr;
176
177 dapl_os_atomic_inc(&ep_ptr->req_count);
178
179 if (NULL != rmr_context) { *rmr_context = rmr->param.rmr_context; }
180 bail:
181 return (dat_status);
182 }
183
184
185 static DAT_RETURN
dapli_rmr_bind_unfuse(IN DAPL_RMR * rmr,IN const DAT_LMR_TRIPLET * lmr_triplet,IN DAPL_EP * ep_ptr,IN DAT_RMR_COOKIE user_cookie,IN DAT_COMPLETION_FLAGS completion_flags)186 dapli_rmr_bind_unfuse(
187 IN DAPL_RMR *rmr,
188 IN const DAT_LMR_TRIPLET *lmr_triplet,
189 IN DAPL_EP *ep_ptr,
190 IN DAT_RMR_COOKIE user_cookie,
191 IN DAT_COMPLETION_FLAGS completion_flags)
192 {
193 DAPL_COOKIE *cookie;
194 DAT_RETURN dat_status;
195
196 dat_status = DAT_SUCCESS;
197 /*
198 * if the ep in unconnected return an error. IB requires that the
199 * QP be connected to change a memory window binding since:
200 *
201 * - memory window bind operations are WQEs placed on a QP's
202 * send queue
203 *
204 * - QP's only process WQEs on the send queue when the QP is in
205 * the RTS state
206 */
207 if (DAT_EP_STATE_CONNECTED != ep_ptr->param.ep_state) {
208 dat_status = DAT_ERROR(DAT_INVALID_STATE,
209 dapls_ep_state_subtype(ep_ptr));
210 goto bail1;
211 }
212
213 /* If the RMR and EP are not in the same PZ, there is an error */
214 if (ep_ptr->param.pz_handle != rmr->param.pz_handle) {
215 dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG2);
216 goto bail1;
217 }
218
219 if (!dapl_rmr_validate_completion_flag(DAT_COMPLETION_SUPPRESS_FLAG,
220 ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
221 !dapl_rmr_validate_completion_flag(DAT_COMPLETION_UNSIGNALLED_FLAG,
222 ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
223 !dapl_rmr_validate_completion_flag(
224 DAT_COMPLETION_BARRIER_FENCE_FLAG,
225 ep_ptr->param.ep_attr.request_completion_flags, completion_flags)) {
226 dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG2);
227 goto bail1;
228 }
229
230 dat_status = dapls_rmr_cookie_alloc(&ep_ptr->req_buffer, rmr,
231 user_cookie, &cookie);
232 if (DAT_SUCCESS != dat_status) {
233 goto bail1;
234 }
235
236
237 dat_status = dapls_ib_mw_unbind(rmr, lmr_triplet->lmr_context,
238 ep_ptr, cookie, completion_flags);
239
240 if (DAT_SUCCESS != dat_status) {
241 dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie);
242 goto bail1;
243 }
244
245 /* if the RMR was previously bound */
246 if (NULL != rmr->lmr) {
247 dapl_os_atomic_dec(&rmr->lmr->lmr_ref_count);
248 }
249
250 rmr->param.mem_priv = DAT_MEM_PRIV_NONE_FLAG;
251 rmr->param.lmr_triplet.lmr_context = 0;
252 rmr->param.lmr_triplet.virtual_address = 0;
253 rmr->param.lmr_triplet.segment_length = 0;
254 rmr->ep = ep_ptr;
255 rmr->lmr = NULL;
256
257 dapl_os_atomic_inc(&ep_ptr->req_count);
258
259 bail1:
260 return (dat_status);
261 }
262
263
264 /*
265 * dapl_rmr_bind
266 *
267 * DAPL Requirements Version xxx, 6.6.4.4
268 *
269 * Bind the RMR to the specified memory region within the LMR and
270 * provide a new rmr_context value.
271 *
272 * Input:
273 * Output:
274 */
275 DAT_RETURN
dapl_rmr_bind(IN DAT_RMR_HANDLE rmr_handle,IN const DAT_LMR_TRIPLET * lmr_triplet,IN DAT_MEM_PRIV_FLAGS mem_priv,IN DAT_EP_HANDLE ep_handle,IN DAT_RMR_COOKIE user_cookie,IN DAT_COMPLETION_FLAGS completion_flags,OUT DAT_RMR_CONTEXT * rmr_context)276 dapl_rmr_bind(
277 IN DAT_RMR_HANDLE rmr_handle,
278 IN const DAT_LMR_TRIPLET *lmr_triplet,
279 IN DAT_MEM_PRIV_FLAGS mem_priv,
280 IN DAT_EP_HANDLE ep_handle,
281 IN DAT_RMR_COOKIE user_cookie,
282 IN DAT_COMPLETION_FLAGS completion_flags,
283 OUT DAT_RMR_CONTEXT *rmr_context)
284 {
285 DAPL_RMR *rmr;
286 DAPL_EP *ep_ptr;
287
288 if (DAPL_BAD_HANDLE(rmr_handle, DAPL_MAGIC_RMR)) {
289 return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RMR));
290 }
291 if (DAPL_BAD_HANDLE(ep_handle, DAPL_MAGIC_EP)) {
292 return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP));
293 }
294
295 rmr = (DAPL_RMR *) rmr_handle;
296 ep_ptr = (DAPL_EP *) ep_handle;
297
298 /* if the rmr should be bound */
299 if (0 != lmr_triplet->segment_length) {
300 return (dapli_rmr_bind_fuse(rmr,
301 lmr_triplet,
302 mem_priv,
303 ep_ptr,
304 user_cookie,
305 completion_flags,
306 rmr_context));
307 } else { /* the rmr should be unbound */
308 return (dapli_rmr_bind_unfuse(rmr,
309 lmr_triplet,
310 ep_ptr,
311 user_cookie,
312 completion_flags));
313 }
314 }
315