1*9e39c5baSBill Taylor /*
2*9e39c5baSBill Taylor * CDDL HEADER START
3*9e39c5baSBill Taylor *
4*9e39c5baSBill Taylor * The contents of this file are subject to the terms of the
5*9e39c5baSBill Taylor * Common Development and Distribution License (the "License").
6*9e39c5baSBill Taylor * You may not use this file except in compliance with the License.
7*9e39c5baSBill Taylor *
8*9e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9*9e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing.
10*9e39c5baSBill Taylor * See the License for the specific language governing permissions
11*9e39c5baSBill Taylor * and limitations under the License.
12*9e39c5baSBill Taylor *
13*9e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each
14*9e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15*9e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the
16*9e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
17*9e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
18*9e39c5baSBill Taylor *
19*9e39c5baSBill Taylor * CDDL HEADER END
20*9e39c5baSBill Taylor */
21*9e39c5baSBill Taylor
22*9e39c5baSBill Taylor /*
23*9e39c5baSBill Taylor * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
24*9e39c5baSBill Taylor */
25*9e39c5baSBill Taylor
26*9e39c5baSBill Taylor /*
27*9e39c5baSBill Taylor * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
28*9e39c5baSBill Taylor * Use is subject to license terms.
29*9e39c5baSBill Taylor */
30*9e39c5baSBill Taylor
31*9e39c5baSBill Taylor
32*9e39c5baSBill Taylor /*
33*9e39c5baSBill Taylor *
34*9e39c5baSBill Taylor * MODULE: dapl_ia_open.c
35*9e39c5baSBill Taylor *
36*9e39c5baSBill Taylor * PURPOSE: Interface Adapter management
37*9e39c5baSBill Taylor * Description: Interfaces in this file are completely described in
38*9e39c5baSBill Taylor * the DAPL 1.1 API, Chapter 6, section 2
39*9e39c5baSBill Taylor *
40*9e39c5baSBill Taylor * $Id: dapl_ia_open.c,v 1.30 2003/07/31 14:04:17 jlentini Exp $
41*9e39c5baSBill Taylor */
42*9e39c5baSBill Taylor
43*9e39c5baSBill Taylor #include "dapl.h"
44*9e39c5baSBill Taylor #include "dapl_provider.h"
45*9e39c5baSBill Taylor #include "dapl_evd_util.h"
46*9e39c5baSBill Taylor #include "dapl_hca_util.h"
47*9e39c5baSBill Taylor #include "dapl_ia_util.h"
48*9e39c5baSBill Taylor #include "dapl_adapter_util.h"
49*9e39c5baSBill Taylor #include <sys/systeminfo.h>
50*9e39c5baSBill Taylor #include <libdevinfo.h>
51*9e39c5baSBill Taylor
52*9e39c5baSBill Taylor
53*9e39c5baSBill Taylor /*
54*9e39c5baSBill Taylor * LOCAL PROTOTYPES
55*9e39c5baSBill Taylor */
56*9e39c5baSBill Taylor #if defined(IBHOSTS_NAMING)
57*9e39c5baSBill Taylor void dapli_assign_hca_ip_address(
58*9e39c5baSBill Taylor DAPL_HCA *hca_ptr,
59*9e39c5baSBill Taylor char *device_name);
60*9e39c5baSBill Taylor #endif /* IBHOSTS_NAMING */
61*9e39c5baSBill Taylor
62*9e39c5baSBill Taylor static void dapli_hca_cleanup(DAPL_HCA *hca_ptr, DAT_BOOLEAN dec_ref);
63*9e39c5baSBill Taylor
64*9e39c5baSBill Taylor /*
65*9e39c5baSBill Taylor * Determine whether the platform supports RO (Relaxed ordering)
66*9e39c5baSBill Taylor * Return B_TRUE if it does support RO and B_FALSE if it does not support RO
67*9e39c5baSBill Taylor *
68*9e39c5baSBill Taylor * udapl_ro_disallowed is an out paramter returning whether or not
69*9e39c5baSBill Taylor * relaxed ordering should be disabled (regardless of whether the platform
70*9e39c5baSBill Taylor * is capable of supporting relaxed ordering)
71*9e39c5baSBill Taylor *
72*9e39c5baSBill Taylor */
73*9e39c5baSBill Taylor static boolean_t
dapl_ro_disallowed(void)74*9e39c5baSBill Taylor dapl_ro_disallowed(void)
75*9e39c5baSBill Taylor {
76*9e39c5baSBill Taylor static const char * const non_ro_capable_platforms[] = {
77*9e39c5baSBill Taylor "i86pc",
78*9e39c5baSBill Taylor "i86xpv",
79*9e39c5baSBill Taylor "SUNW,Sun-Fire-V215",
80*9e39c5baSBill Taylor "SUNW,Sun-Fire-V245",
81*9e39c5baSBill Taylor "SUNW,Sun-Fire-V445",
82*9e39c5baSBill Taylor "SUNW,Sun-Fire-T1000",
83*9e39c5baSBill Taylor "SUNW,Sun-Fire-T200",
84*9e39c5baSBill Taylor "SUNW,Sun-Blade-T6300",
85*9e39c5baSBill Taylor "SUNW,Sun-Blade-T6320",
86*9e39c5baSBill Taylor "SUNW,SPARC-Enterprise-T1000",
87*9e39c5baSBill Taylor "SUNW,SPARC-Enterprise-T2000",
88*9e39c5baSBill Taylor "SUNW,SPARC-Enterprise-T5120",
89*9e39c5baSBill Taylor "SUNW,SPARC-Enterprise-T5220",
90*9e39c5baSBill Taylor NULL
91*9e39c5baSBill Taylor };
92*9e39c5baSBill Taylor char platform[256 + 1];
93*9e39c5baSBill Taylor register int i;
94*9e39c5baSBill Taylor register const char *cp;
95*9e39c5baSBill Taylor int ret;
96*9e39c5baSBill Taylor di_node_t root_node, node;
97*9e39c5baSBill Taylor boolean_t ro_disallowed;
98*9e39c5baSBill Taylor static const char *ro_disallowed_property =
99*9e39c5baSBill Taylor "pci-relaxed-ordering-disallowed";
100*9e39c5baSBill Taylor int bool;
101*9e39c5baSBill Taylor int *boolp = &bool;
102*9e39c5baSBill Taylor
103*9e39c5baSBill Taylor ret = sysinfo(SI_PLATFORM, platform, sizeof (platform));
104*9e39c5baSBill Taylor if ((ret != -1) && (ret <= sizeof (platform))) {
105*9e39c5baSBill Taylor for (i = 0; (cp = non_ro_capable_platforms[i]) != NULL; ++i) {
106*9e39c5baSBill Taylor if (strcmp(platform, cp) == 0)
107*9e39c5baSBill Taylor return (B_TRUE);
108*9e39c5baSBill Taylor }
109*9e39c5baSBill Taylor }
110*9e39c5baSBill Taylor
111*9e39c5baSBill Taylor /*
112*9e39c5baSBill Taylor * This function only finds and looks at the FIRST udapl node.
113*9e39c5baSBill Taylor * It is assumed that there can only be one such node.
114*9e39c5baSBill Taylor */
115*9e39c5baSBill Taylor if ((root_node = di_init("/", DINFOSUBTREE | DINFOPROP)) == DI_NODE_NIL)
116*9e39c5baSBill Taylor return (B_FALSE);
117*9e39c5baSBill Taylor
118*9e39c5baSBill Taylor node = di_drv_first_node("daplt", root_node);
119*9e39c5baSBill Taylor if (node != DI_NODE_NIL) {
120*9e39c5baSBill Taylor ret = di_prop_lookup_ints(DDI_DEV_T_ANY, node,
121*9e39c5baSBill Taylor ro_disallowed_property, &boolp);
122*9e39c5baSBill Taylor switch (ret) {
123*9e39c5baSBill Taylor case 0:
124*9e39c5baSBill Taylor case 1:
125*9e39c5baSBill Taylor ro_disallowed = B_TRUE;
126*9e39c5baSBill Taylor break;
127*9e39c5baSBill Taylor default:
128*9e39c5baSBill Taylor ro_disallowed = B_FALSE;
129*9e39c5baSBill Taylor break;
130*9e39c5baSBill Taylor }
131*9e39c5baSBill Taylor
132*9e39c5baSBill Taylor }
133*9e39c5baSBill Taylor else
134*9e39c5baSBill Taylor ro_disallowed = B_FALSE;
135*9e39c5baSBill Taylor
136*9e39c5baSBill Taylor di_fini(root_node);
137*9e39c5baSBill Taylor
138*9e39c5baSBill Taylor return (ro_disallowed);
139*9e39c5baSBill Taylor }
140*9e39c5baSBill Taylor
141*9e39c5baSBill Taylor /*
142*9e39c5baSBill Taylor * dapl_ia_open
143*9e39c5baSBill Taylor *
144*9e39c5baSBill Taylor * DAPL Requirements Version xxx, 6.2.1.1
145*9e39c5baSBill Taylor *
146*9e39c5baSBill Taylor * Open a provider and return a handle. The handle enables the user
147*9e39c5baSBill Taylor * to invoke operations on this provider.
148*9e39c5baSBill Taylor *
149*9e39c5baSBill Taylor * The dat_ia_open call is actually part of the DAT registration module.
150*9e39c5baSBill Taylor * That function maps the DAT_NAME parameter of dat_ia_open to a DAT_PROVIDER,
151*9e39c5baSBill Taylor * and calls this function.
152*9e39c5baSBill Taylor *
153*9e39c5baSBill Taylor * Input:
154*9e39c5baSBill Taylor * provider
155*9e39c5baSBill Taylor * async_evd_qlen
156*9e39c5baSBill Taylor * async_evd_handle_ptr
157*9e39c5baSBill Taylor *
158*9e39c5baSBill Taylor * Output:
159*9e39c5baSBill Taylor * async_evd_handle
160*9e39c5baSBill Taylor * ia_handle
161*9e39c5baSBill Taylor *
162*9e39c5baSBill Taylor * Return Values:
163*9e39c5baSBill Taylor * DAT_SUCCESS
164*9e39c5baSBill Taylor * DAT_INSUFFICIENT_RESOURCES
165*9e39c5baSBill Taylor * DAT_INVALID_PARAMETER
166*9e39c5baSBill Taylor * DAT_INVALID_HANDLE
167*9e39c5baSBill Taylor * DAT_NAME_NOT_FOUND (returned by dat registry if necessary)
168*9e39c5baSBill Taylor */
169*9e39c5baSBill Taylor DAT_RETURN
dapl_ia_open(IN const DAT_NAME_PTR name,IN DAT_COUNT async_evd_qlen,INOUT DAT_EVD_HANDLE * async_evd_handle_ptr,OUT DAT_IA_HANDLE * ia_handle_ptr,IN boolean_t ro_aware_client)170*9e39c5baSBill Taylor dapl_ia_open(
171*9e39c5baSBill Taylor IN const DAT_NAME_PTR name,
172*9e39c5baSBill Taylor IN DAT_COUNT async_evd_qlen,
173*9e39c5baSBill Taylor INOUT DAT_EVD_HANDLE *async_evd_handle_ptr,
174*9e39c5baSBill Taylor OUT DAT_IA_HANDLE *ia_handle_ptr,
175*9e39c5baSBill Taylor IN boolean_t ro_aware_client)
176*9e39c5baSBill Taylor {
177*9e39c5baSBill Taylor DAT_RETURN dat_status;
178*9e39c5baSBill Taylor DAT_PROVIDER *provider;
179*9e39c5baSBill Taylor DAPL_HCA *hca_ptr;
180*9e39c5baSBill Taylor DAPL_IA *ia_ptr;
181*9e39c5baSBill Taylor DAPL_EVD *evd_ptr;
182*9e39c5baSBill Taylor boolean_t ro_disallowed;
183*9e39c5baSBill Taylor
184*9e39c5baSBill Taylor dat_status = DAT_SUCCESS;
185*9e39c5baSBill Taylor hca_ptr = NULL;
186*9e39c5baSBill Taylor ia_ptr = NULL;
187*9e39c5baSBill Taylor
188*9e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_API,
189*9e39c5baSBill Taylor "dapl_ia_open(%s, %d, %p, %p, %d)\n",
190*9e39c5baSBill Taylor name,
191*9e39c5baSBill Taylor async_evd_qlen,
192*9e39c5baSBill Taylor async_evd_handle_ptr,
193*9e39c5baSBill Taylor ia_handle_ptr,
194*9e39c5baSBill Taylor ro_aware_client);
195*9e39c5baSBill Taylor
196*9e39c5baSBill Taylor dat_status = dapl_provider_list_search(name, &provider);
197*9e39c5baSBill Taylor if (DAT_SUCCESS != dat_status) {
198*9e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_API,
199*9e39c5baSBill Taylor "dapl_ia_open: dapl_provider_list_search(\"%s\") returned "
200*9e39c5baSBill Taylor "%d\n",
201*9e39c5baSBill Taylor name,
202*9e39c5baSBill Taylor dat_status);
203*9e39c5baSBill Taylor
204*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1);
205*9e39c5baSBill Taylor goto bail;
206*9e39c5baSBill Taylor }
207*9e39c5baSBill Taylor
208*9e39c5baSBill Taylor /* ia_handle_ptr and async_evd_handle_ptr cannot be NULL */
209*9e39c5baSBill Taylor if (ia_handle_ptr == NULL) {
210*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG4);
211*9e39c5baSBill Taylor goto bail;
212*9e39c5baSBill Taylor }
213*9e39c5baSBill Taylor if (async_evd_handle_ptr == NULL) {
214*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG3);
215*9e39c5baSBill Taylor goto bail;
216*9e39c5baSBill Taylor }
217*9e39c5baSBill Taylor
218*9e39c5baSBill Taylor /* initialize the caller's OUT param */
219*9e39c5baSBill Taylor *ia_handle_ptr = DAT_HANDLE_NULL;
220*9e39c5baSBill Taylor
221*9e39c5baSBill Taylor /* get the hca_ptr */
222*9e39c5baSBill Taylor hca_ptr = (DAPL_HCA *)provider->extension;
223*9e39c5baSBill Taylor
224*9e39c5baSBill Taylor /*
225*9e39c5baSBill Taylor * Open the HCA if it has not been done before.
226*9e39c5baSBill Taylor */
227*9e39c5baSBill Taylor dapl_os_lock(&hca_ptr->lock);
228*9e39c5baSBill Taylor if (hca_ptr->ib_hca_handle == IB_INVALID_HANDLE) {
229*9e39c5baSBill Taylor /* register with the HW */
230*9e39c5baSBill Taylor dat_status = dapls_ib_open_hca(hca_ptr,
231*9e39c5baSBill Taylor &hca_ptr->ib_hca_handle);
232*9e39c5baSBill Taylor
233*9e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
234*9e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
235*9e39c5baSBill Taylor "dapls_ib_open_hca failed %d\n", dat_status);
236*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
237*9e39c5baSBill Taylor goto bail;
238*9e39c5baSBill Taylor }
239*9e39c5baSBill Taylor
240*9e39c5baSBill Taylor /* create a cq domain for this HCA */
241*9e39c5baSBill Taylor dat_status = dapls_ib_cqd_create(hca_ptr);
242*9e39c5baSBill Taylor
243*9e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
244*9e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
245*9e39c5baSBill Taylor "ERR: Cannot allocate CQD: err %x\n", dat_status);
246*9e39c5baSBill Taylor dapli_hca_cleanup(hca_ptr, DAT_FALSE);
247*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
248*9e39c5baSBill Taylor goto bail;
249*9e39c5baSBill Taylor }
250*9e39c5baSBill Taylor /*
251*9e39c5baSBill Taylor * Obtain the IP address associated with this name and HCA.
252*9e39c5baSBill Taylor */
253*9e39c5baSBill Taylor
254*9e39c5baSBill Taylor #ifdef IBHOSTS_NAMING
255*9e39c5baSBill Taylor dapli_assign_hca_ip_address(hca_ptr, name);
256*9e39c5baSBill Taylor #endif /* IBHOSTS_NAMING */
257*9e39c5baSBill Taylor
258*9e39c5baSBill Taylor /*
259*9e39c5baSBill Taylor * Obtain IA attributes from the HCA to limit certain
260*9e39c5baSBill Taylor * operations.
261*9e39c5baSBill Taylor * If using DAPL_ATS naming, ib_query_hca will also set the ip
262*9e39c5baSBill Taylor * address.
263*9e39c5baSBill Taylor */
264*9e39c5baSBill Taylor dat_status = dapls_ib_query_hca(hca_ptr,
265*9e39c5baSBill Taylor &hca_ptr->ia_attr,
266*9e39c5baSBill Taylor NULL,
267*9e39c5baSBill Taylor &hca_ptr->hca_address, NULL);
268*9e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
269*9e39c5baSBill Taylor dapli_hca_cleanup(hca_ptr, DAT_FALSE);
270*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
271*9e39c5baSBill Taylor goto bail;
272*9e39c5baSBill Taylor }
273*9e39c5baSBill Taylor }
274*9e39c5baSBill Taylor
275*9e39c5baSBill Taylor /* is the IA going to use the ConnectX? */
276*9e39c5baSBill Taylor if (hca_ptr->hermon_resize_cq != 0) {
277*9e39c5baSBill Taylor /*
278*9e39c5baSBill Taylor * We are running with a ConnectX.
279*9e39c5baSBill Taylor * Determine whether platform is RO capable.
280*9e39c5baSBill Taylor * If platform support RO and client does not
281*9e39c5baSBill Taylor * support RO and we are not disabling RO, reject the open.
282*9e39c5baSBill Taylor */
283*9e39c5baSBill Taylor ro_disallowed = dapl_ro_disallowed();
284*9e39c5baSBill Taylor
285*9e39c5baSBill Taylor if (! ro_aware_client && ! ro_disallowed) {
286*9e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_API,
287*9e39c5baSBill Taylor "dapl_ia_open: failing ro_disallowed %d "
288*9e39c5baSBill Taylor "ro_aware_client %d \n",
289*9e39c5baSBill Taylor ro_disallowed, ro_aware_client);
290*9e39c5baSBill Taylor
291*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
292*9e39c5baSBill Taylor DAT_INVALID_RO_COOKIE);
293*9e39c5baSBill Taylor dapli_hca_cleanup(hca_ptr, DAT_FALSE);
294*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
295*9e39c5baSBill Taylor goto bail;
296*9e39c5baSBill Taylor }
297*9e39c5baSBill Taylor } else {
298*9e39c5baSBill Taylor /* We are not running with a Connect X */
299*9e39c5baSBill Taylor ro_disallowed = B_TRUE;
300*9e39c5baSBill Taylor }
301*9e39c5baSBill Taylor
302*9e39c5baSBill Taylor
303*9e39c5baSBill Taylor /* Take a reference on the hca_handle */
304*9e39c5baSBill Taylor dapl_os_atomic_inc(&hca_ptr->handle_ref_count);
305*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
306*9e39c5baSBill Taylor
307*9e39c5baSBill Taylor /* Allocate and initialize ia structure */
308*9e39c5baSBill Taylor ia_ptr = dapl_ia_alloc(provider, hca_ptr);
309*9e39c5baSBill Taylor if (!ia_ptr) {
310*9e39c5baSBill Taylor dapl_os_lock(&hca_ptr->lock);
311*9e39c5baSBill Taylor dapli_hca_cleanup(hca_ptr, DAT_TRUE);
312*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
313*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
314*9e39c5baSBill Taylor DAT_RESOURCE_MEMORY);
315*9e39c5baSBill Taylor goto bail;
316*9e39c5baSBill Taylor }
317*9e39c5baSBill Taylor
318*9e39c5baSBill Taylor /*
319*9e39c5baSBill Taylor * Note when we should be disabling relaxed ordering.
320*9e39c5baSBill Taylor * If the property indicates that we should not use relaxed ordering
321*9e39c5baSBill Taylor * we remember that fact. If the platform is supposed to be
322*9e39c5baSBill Taylor * non relaxed ordering capable, we disable relaxed ordering as
323*9e39c5baSBill Taylor * well, just in case the property or the list indicating that
324*9e39c5baSBill Taylor * this platform is not relaxed ordering capable is mistaken.
325*9e39c5baSBill Taylor */
326*9e39c5baSBill Taylor if (ro_disallowed)
327*9e39c5baSBill Taylor ia_ptr->dapl_flags |= DAPL_DISABLE_RO;
328*9e39c5baSBill Taylor
329*9e39c5baSBill Taylor /*
330*9e39c5baSBill Taylor * we need an async EVD for this IA
331*9e39c5baSBill Taylor * use the one passed in (if non-NULL) or create one
332*9e39c5baSBill Taylor */
333*9e39c5baSBill Taylor
334*9e39c5baSBill Taylor evd_ptr = (DAPL_EVD *) *async_evd_handle_ptr;
335*9e39c5baSBill Taylor if (evd_ptr) {
336*9e39c5baSBill Taylor if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD) ||
337*9e39c5baSBill Taylor ! (evd_ptr->evd_flags & DAT_EVD_ASYNC_FLAG)) {
338*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_HANDLE,
339*9e39c5baSBill Taylor DAT_INVALID_HANDLE_EVD_ASYNC);
340*9e39c5baSBill Taylor goto bail;
341*9e39c5baSBill Taylor }
342*9e39c5baSBill Taylor /*
343*9e39c5baSBill Taylor * InfiniBand allows only 1 asychronous event handler per HCA
344*9e39c5baSBill Taylor * (see InfiniBand Spec, release 1.1, vol I, section 11.5.2,
345*9e39c5baSBill Taylor * page 559).
346*9e39c5baSBill Taylor *
347*9e39c5baSBill Taylor * We only need to make sure that this EVD's CQ belongs to
348*9e39c5baSBill Taylor * the same HCA as is being opened.
349*9e39c5baSBill Taylor */
350*9e39c5baSBill Taylor
351*9e39c5baSBill Taylor if (evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle !=
352*9e39c5baSBill Taylor hca_ptr->ib_hca_handle) {
353*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_HANDLE,
354*9e39c5baSBill Taylor DAT_INVALID_HANDLE_EVD_ASYNC);
355*9e39c5baSBill Taylor goto bail;
356*9e39c5baSBill Taylor }
357*9e39c5baSBill Taylor
358*9e39c5baSBill Taylor ia_ptr->cleanup_async_error_evd = DAT_FALSE;
359*9e39c5baSBill Taylor ia_ptr->async_error_evd = evd_ptr;
360*9e39c5baSBill Taylor } else {
361*9e39c5baSBill Taylor /*
362*9e39c5baSBill Taylor * Verify we have >0 length, and let the provider check the
363*9e39c5baSBill Taylor * size
364*9e39c5baSBill Taylor */
365*9e39c5baSBill Taylor if (async_evd_qlen <= 0) {
366*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
367*9e39c5baSBill Taylor DAT_INVALID_ARG2);
368*9e39c5baSBill Taylor goto bail;
369*9e39c5baSBill Taylor }
370*9e39c5baSBill Taylor dat_status = dapls_evd_internal_create(ia_ptr,
371*9e39c5baSBill Taylor NULL, /* CNO ptr */
372*9e39c5baSBill Taylor async_evd_qlen,
373*9e39c5baSBill Taylor DAT_EVD_ASYNC_FLAG,
374*9e39c5baSBill Taylor &evd_ptr);
375*9e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
376*9e39c5baSBill Taylor goto bail;
377*9e39c5baSBill Taylor }
378*9e39c5baSBill Taylor
379*9e39c5baSBill Taylor dapl_os_atomic_inc(&evd_ptr->evd_ref_count);
380*9e39c5baSBill Taylor
381*9e39c5baSBill Taylor dapl_os_lock(&hca_ptr->lock);
382*9e39c5baSBill Taylor if (hca_ptr->async_evd != (DAPL_EVD *) 0) {
383*9e39c5baSBill Taylor #if 0
384*9e39c5baSBill Taylor /*
385*9e39c5baSBill Taylor * The async EVD for this HCA has already been assigned.
386*9e39c5baSBill Taylor * It's an error to try and assign another one.
387*9e39c5baSBill Taylor *
388*9e39c5baSBill Taylor * However, we need to somehow allow multiple IAs
389*9e39c5baSBill Taylor * off of the same HCA. The right way to do this
390*9e39c5baSBill Taylor * is by dispatching events off the HCA to the
391*9e39c5baSBill Taylor * appropriate IA, but we aren't there yet. So for
392*9e39c5baSBill Taylor * now we create the EVD but don't connect it to
393*9e39c5baSBill Taylor * anything.
394*9e39c5baSBill Taylor */
395*9e39c5baSBill Taylor dapl_os_atomic_dec(&evd_ptr->evd_ref_count);
396*9e39c5baSBill Taylor dapl_evd_free(evd_ptr);
397*9e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
398*9e39c5baSBill Taylor DAT_INVALID_ARG4);
399*9e39c5baSBill Taylor goto bail;
400*9e39c5baSBill Taylor #endif
401*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
402*9e39c5baSBill Taylor } else {
403*9e39c5baSBill Taylor hca_ptr->async_evd = evd_ptr;
404*9e39c5baSBill Taylor dapl_os_unlock(&hca_ptr->lock);
405*9e39c5baSBill Taylor
406*9e39c5baSBill Taylor /*
407*9e39c5baSBill Taylor * Register the handlers associated with the async EVD.
408*9e39c5baSBill Taylor */
409*9e39c5baSBill Taylor dat_status = dapls_ia_setup_callbacks(ia_ptr, evd_ptr);
410*9e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
411*9e39c5baSBill Taylor /* Assign the EVD so it gets cleaned up */
412*9e39c5baSBill Taylor ia_ptr->cleanup_async_error_evd = DAT_TRUE;
413*9e39c5baSBill Taylor ia_ptr->async_error_evd = evd_ptr;
414*9e39c5baSBill Taylor goto bail;
415*9e39c5baSBill Taylor }
416*9e39c5baSBill Taylor }
417*9e39c5baSBill Taylor
418*9e39c5baSBill Taylor ia_ptr->cleanup_async_error_evd = DAT_TRUE;
419*9e39c5baSBill Taylor ia_ptr->async_error_evd = evd_ptr;
420*9e39c5baSBill Taylor }
421*9e39c5baSBill Taylor
422*9e39c5baSBill Taylor dat_status = DAT_SUCCESS;
423*9e39c5baSBill Taylor *ia_handle_ptr = ia_ptr;
424*9e39c5baSBill Taylor *async_evd_handle_ptr = evd_ptr;
425*9e39c5baSBill Taylor
426*9e39c5baSBill Taylor bail:
427*9e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
428*9e39c5baSBill Taylor if (ia_ptr) {
429*9e39c5baSBill Taylor /* This will release the async EVD if needed. */
430*9e39c5baSBill Taylor (void) dapl_ia_close(ia_ptr, DAT_CLOSE_ABRUPT_FLAG);
431*9e39c5baSBill Taylor }
432*9e39c5baSBill Taylor }
433*9e39c5baSBill Taylor
434*9e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_RTN,
435*9e39c5baSBill Taylor "dapl_ia_open () returns 0x%x\n",
436*9e39c5baSBill Taylor dat_status);
437*9e39c5baSBill Taylor
438*9e39c5baSBill Taylor return (dat_status);
439*9e39c5baSBill Taylor }
440*9e39c5baSBill Taylor
441*9e39c5baSBill Taylor /*
442*9e39c5baSBill Taylor * dapli_hca_cleanup
443*9e39c5baSBill Taylor *
444*9e39c5baSBill Taylor * Clean up partially allocated HCA stuff. Strictly to make cleanup
445*9e39c5baSBill Taylor * simple.
446*9e39c5baSBill Taylor */
447*9e39c5baSBill Taylor void
dapli_hca_cleanup(DAPL_HCA * hca_ptr,DAT_BOOLEAN dec_ref)448*9e39c5baSBill Taylor dapli_hca_cleanup(
449*9e39c5baSBill Taylor DAPL_HCA *hca_ptr,
450*9e39c5baSBill Taylor DAT_BOOLEAN dec_ref)
451*9e39c5baSBill Taylor {
452*9e39c5baSBill Taylor (void) dapls_ib_close_hca(hca_ptr->ib_hca_handle);
453*9e39c5baSBill Taylor hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
454*9e39c5baSBill Taylor if (dec_ref == DAT_TRUE) {
455*9e39c5baSBill Taylor dapl_os_atomic_dec(&hca_ptr->handle_ref_count);
456*9e39c5baSBill Taylor }
457*9e39c5baSBill Taylor }
458*9e39c5baSBill Taylor
459*9e39c5baSBill Taylor #if defined(IBHOSTS_NAMING)
460*9e39c5baSBill Taylor
461*9e39c5baSBill Taylor char *dapli_get_adapter_num(
462*9e39c5baSBill Taylor char *device_name);
463*9e39c5baSBill Taylor
464*9e39c5baSBill Taylor void dapli_setup_dummy_addr(
465*9e39c5baSBill Taylor IN DAPL_HCA *hca_ptr,
466*9e39c5baSBill Taylor IN char *hca_name);
467*9e39c5baSBill Taylor /*
468*9e39c5baSBill Taylor * dapli_assign_hca_ip_address
469*9e39c5baSBill Taylor *
470*9e39c5baSBill Taylor * Obtain the IP address of the passed in name, which represents a
471*9e39c5baSBill Taylor * port on the hca. There are three methods here to obtain the
472*9e39c5baSBill Taylor * appropriate IP address, each with their own shortcoming:
473*9e39c5baSBill Taylor * 1) IPOIB_NAMING. Requires the implementation of the IPoIB
474*9e39c5baSBill Taylor * interface defined in include/dapl/ipoib_names.h. This is
475*9e39c5baSBill Taylor * not the recommended interface as IPoIB is limited at
476*9e39c5baSBill Taylor * the point we need to obtain an IP address on the
477*9e39c5baSBill Taylor * passive side of a connection. The code supporting this
478*9e39c5baSBill Taylor * implementation has been removed.
479*9e39c5baSBill Taylor *
480*9e39c5baSBill Taylor * 2) IBHOSTS. An entry exists in DNS and in the /etc/dapl/ibhosts
481*9e39c5baSBill Taylor * file. The immediate drawback here is that we must dictate
482*9e39c5baSBill Taylor * how to name the interface, which is a stated DAPL non-goal.
483*9e39c5baSBill Taylor * In the broader perspective, this method requires us to xmit
484*9e39c5baSBill Taylor * the IP address in the private data of a connection, which has
485*9e39c5baSBill Taylor * other fun problems. This is the default method and is known to
486*9e39c5baSBill Taylor * work, but it has problems.
487*9e39c5baSBill Taylor *
488*9e39c5baSBill Taylor * 3) Obtain the IP address from the driver, which has registered
489*9e39c5baSBill Taylor * the address with the SA for retrieval.
490*9e39c5baSBill Taylor *
491*9e39c5baSBill Taylor *
492*9e39c5baSBill Taylor * Input:
493*9e39c5baSBill Taylor * hca_ptr Pointer to HCA structure
494*9e39c5baSBill Taylor * device_name Name of device as reported by the provider
495*9e39c5baSBill Taylor *
496*9e39c5baSBill Taylor * Output:
497*9e39c5baSBill Taylor * none
498*9e39c5baSBill Taylor *
499*9e39c5baSBill Taylor * Returns:
500*9e39c5baSBill Taylor * char * to string number
501*9e39c5baSBill Taylor */
502*9e39c5baSBill Taylor void
dapli_assign_hca_ip_address(DAPL_HCA * hca_ptr,char * device_name)503*9e39c5baSBill Taylor dapli_assign_hca_ip_address(
504*9e39c5baSBill Taylor DAPL_HCA *hca_ptr,
505*9e39c5baSBill Taylor char *device_name)
506*9e39c5baSBill Taylor {
507*9e39c5baSBill Taylor char *adapter_num;
508*9e39c5baSBill Taylor #define NAMELEN 128
509*9e39c5baSBill Taylor struct addrinfo *addr;
510*9e39c5baSBill Taylor char hostname[NAMELEN];
511*9e39c5baSBill Taylor char *str;
512*9e39c5baSBill Taylor int rc;
513*9e39c5baSBill Taylor
514*9e39c5baSBill Taylor /*
515*9e39c5baSBill Taylor * Obtain the IP address of the adapter. This is a simple
516*9e39c5baSBill Taylor * scheme that creates a name that must appear available to
517*9e39c5baSBill Taylor * DNS, e.g. it must be in the local site DNS or in the local
518*9e39c5baSBill Taylor * /etc/hosts file, etc.
519*9e39c5baSBill Taylor *
520*9e39c5baSBill Taylor * <hostname>-ib<index>
521*9e39c5baSBill Taylor *
522*9e39c5baSBill Taylor * This scheme obviously doesn't work with adapters from
523*9e39c5baSBill Taylor * multiple vendors, but will suffice in common installations.
524*9e39c5baSBill Taylor */
525*9e39c5baSBill Taylor
526*9e39c5baSBill Taylor rc = gethostname(hostname, NAMELEN);
527*9e39c5baSBill Taylor /*
528*9e39c5baSBill Taylor * Strip off domain info if it exists (e.g. mynode.mydomain.com)
529*9e39c5baSBill Taylor */
530*9e39c5baSBill Taylor for (str = hostname; *str && *str != '.'; ) {
531*9e39c5baSBill Taylor str++;
532*9e39c5baSBill Taylor }
533*9e39c5baSBill Taylor if (*str == '.') {
534*9e39c5baSBill Taylor *str = '\0';
535*9e39c5baSBill Taylor }
536*9e39c5baSBill Taylor dapl_os_strcat(hostname, "-ib");
537*9e39c5baSBill Taylor adapter_num = dapli_get_adapter_num(device_name);
538*9e39c5baSBill Taylor dapl_os_strcat(hostname, adapter_num);
539*9e39c5baSBill Taylor
540*9e39c5baSBill Taylor rc = dapls_osd_getaddrinfo(hostname, &addr);
541*9e39c5baSBill Taylor
542*9e39c5baSBill Taylor if (rc != 0) {
543*9e39c5baSBill Taylor /* Not registered in DNS, provide a dummy value */
544*9e39c5baSBill Taylor dapli_setup_dummy_addr(hca_ptr, hostname);
545*9e39c5baSBill Taylor } else {
546*9e39c5baSBill Taylor /*
547*9e39c5baSBill Taylor * hca_address is defined as a DAT_SOCK_ADDR6 whereas ai_addr
548*9e39c5baSBill Taylor * is a sockaddr
549*9e39c5baSBill Taylor */
550*9e39c5baSBill Taylor (void) dapl_os_memcpy((void *)&hca_ptr->hca_address,
551*9e39c5baSBill Taylor (void *)(addr->ai_addr), sizeof (DAT_SOCK_ADDR6));
552*9e39c5baSBill Taylor }
553*9e39c5baSBill Taylor }
554*9e39c5baSBill Taylor
555*9e39c5baSBill Taylor
556*9e39c5baSBill Taylor /*
557*9e39c5baSBill Taylor * dapli_stup_dummy_addr
558*9e39c5baSBill Taylor *
559*9e39c5baSBill Taylor * Set up a dummy local address for the HCA. Things are not going
560*9e39c5baSBill Taylor * to work too well if this happens.
561*9e39c5baSBill Taylor * We call this routine if:
562*9e39c5baSBill Taylor * - remote host adapter name is not in DNS
563*9e39c5baSBill Taylor * - IPoIB implementation is not correctly set up
564*9e39c5baSBill Taylor * - Similar nonsense.
565*9e39c5baSBill Taylor *
566*9e39c5baSBill Taylor * Input:
567*9e39c5baSBill Taylor * hca_ptr
568*9e39c5baSBill Taylor * rhost_name Name of remote adapter
569*9e39c5baSBill Taylor *
570*9e39c5baSBill Taylor * Output:
571*9e39c5baSBill Taylor * none
572*9e39c5baSBill Taylor *
573*9e39c5baSBill Taylor * Returns:
574*9e39c5baSBill Taylor * none
575*9e39c5baSBill Taylor */
576*9e39c5baSBill Taylor void
dapli_setup_dummy_addr(IN DAPL_HCA * hca_ptr,IN char * rhost_name)577*9e39c5baSBill Taylor dapli_setup_dummy_addr(
578*9e39c5baSBill Taylor IN DAPL_HCA *hca_ptr,
579*9e39c5baSBill Taylor IN char *rhost_name)
580*9e39c5baSBill Taylor {
581*9e39c5baSBill Taylor struct sockaddr_in *si;
582*9e39c5baSBill Taylor
583*9e39c5baSBill Taylor /* Not registered in DNS, provide a dummy value */
584*9e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR, "WARNING: <%s> not registered in DNS,"
585*9e39c5baSBill Taylor " using dummy IP value\n", rhost_name);
586*9e39c5baSBill Taylor si = (struct sockaddr_in *)&hca_ptr->hca_address;
587*9e39c5baSBill Taylor si->sin_family = AF_INET;
588*9e39c5baSBill Taylor si->sin_addr.s_addr = 0x01020304;
589*9e39c5baSBill Taylor }
590*9e39c5baSBill Taylor
591*9e39c5baSBill Taylor
592*9e39c5baSBill Taylor /*
593*9e39c5baSBill Taylor * dapls_get_adapter_num
594*9e39c5baSBill Taylor *
595*9e39c5baSBill Taylor * Given a device name, return a string of the device number
596*9e39c5baSBill Taylor *
597*9e39c5baSBill Taylor * Input:
598*9e39c5baSBill Taylor * device_name Name of device as reported by the provider
599*9e39c5baSBill Taylor *
600*9e39c5baSBill Taylor * Output:
601*9e39c5baSBill Taylor * none
602*9e39c5baSBill Taylor *
603*9e39c5baSBill Taylor * Returns:
604*9e39c5baSBill Taylor * char * to string number
605*9e39c5baSBill Taylor */
606*9e39c5baSBill Taylor char *
dapli_get_adapter_num(char * device_name)607*9e39c5baSBill Taylor dapli_get_adapter_num(
608*9e39c5baSBill Taylor char *device_name)
609*9e39c5baSBill Taylor {
610*9e39c5baSBill Taylor static char *zero = "0";
611*9e39c5baSBill Taylor char *p;
612*9e39c5baSBill Taylor
613*9e39c5baSBill Taylor /*
614*9e39c5baSBill Taylor * Optimisticaly simple algorithm: the device number appears at
615*9e39c5baSBill Taylor * the end of the device name string. Device that do not end
616*9e39c5baSBill Taylor * in a number are by default "0".
617*9e39c5baSBill Taylor */
618*9e39c5baSBill Taylor
619*9e39c5baSBill Taylor for (p = device_name; *p; p++) {
620*9e39c5baSBill Taylor if (isdigit(*p)) {
621*9e39c5baSBill Taylor return (p);
622*9e39c5baSBill Taylor }
623*9e39c5baSBill Taylor }
624*9e39c5baSBill Taylor
625*9e39c5baSBill Taylor return (zero);
626*9e39c5baSBill Taylor }
627*9e39c5baSBill Taylor #endif /* IBHOSTS_NAMING */
628*9e39c5baSBill Taylor
629*9e39c5baSBill Taylor
630*9e39c5baSBill Taylor /*
631*9e39c5baSBill Taylor * Local variables:
632*9e39c5baSBill Taylor * c-indent-level: 4
633*9e39c5baSBill Taylor * c-basic-offset: 4
634*9e39c5baSBill Taylor * tab-width: 8
635*9e39c5baSBill Taylor * End:
636*9e39c5baSBill Taylor */
637