19e39c5baSBill Taylor /*
29e39c5baSBill Taylor * CDDL HEADER START
39e39c5baSBill Taylor *
49e39c5baSBill Taylor * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor *
89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor * See the License for the specific language governing permissions
119e39c5baSBill Taylor * and limitations under the License.
129e39c5baSBill Taylor *
139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor *
199e39c5baSBill Taylor * CDDL HEADER END
209e39c5baSBill Taylor */
219e39c5baSBill Taylor
229e39c5baSBill Taylor /*
231cfa752fSRamaswamy Tummala * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor */
259e39c5baSBill Taylor
269e39c5baSBill Taylor #include <sys/types.h>
279e39c5baSBill Taylor #include <sys/socket.h>
289e39c5baSBill Taylor #include <sys/sockio.h>
299e39c5baSBill Taylor #include <sys/param.h>
309e39c5baSBill Taylor #include <sys/stat.h>
319e39c5baSBill Taylor #include <netinet/in.h>
329e39c5baSBill Taylor #include <arpa/inet.h>
339e39c5baSBill Taylor #include <net/if.h>
349e39c5baSBill Taylor #include <unistd.h>
359e39c5baSBill Taylor #include <stdlib.h>
369e39c5baSBill Taylor #include <string.h>
379e39c5baSBill Taylor #include <errno.h>
381cfa752fSRamaswamy Tummala #include <strings.h>
391cfa752fSRamaswamy Tummala #include <fcntl.h>
401cfa752fSRamaswamy Tummala #include <libdladm.h>
411cfa752fSRamaswamy Tummala #include <libdlib.h>
421cfa752fSRamaswamy Tummala #include <libdllink.h>
431cfa752fSRamaswamy Tummala #include <sys/ib/ibnex/ibnex_devctl.h>
449e39c5baSBill Taylor
459e39c5baSBill Taylor #include "dapl.h"
469e39c5baSBill Taylor #include "dapl_adapter_util.h"
479e39c5baSBill Taylor #include "dapl_tavor_ibtf_impl.h"
489e39c5baSBill Taylor #include "dapl_hca_util.h"
499e39c5baSBill Taylor #include "dapl_name_service.h"
509e39c5baSBill Taylor #define MAX_HCAS 64
519e39c5baSBill Taylor #define PROP_HCA_GUID "hca-guid"
529e39c5baSBill Taylor #define PROP_PORT_NUM "port-number"
539e39c5baSBill Taylor #define PROP_PORT_PKEY "port-pkey"
549e39c5baSBill Taylor
559e39c5baSBill Taylor #define DEVDAPLT "/dev/daplt"
569e39c5baSBill Taylor
579e39c5baSBill Taylor /* function prototypes */
581cfa752fSRamaswamy Tummala static DAT_RETURN dapli_process_tavor_node(char *dev_path, int *hca_idx,
599e39c5baSBill Taylor int try_blueflame);
601cfa752fSRamaswamy Tummala static DAT_RETURN dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr,
619e39c5baSBill Taylor int hca_idx);
629e39c5baSBill Taylor
639e39c5baSBill Taylor #if defined(IBHOSTS_NAMING)
649e39c5baSBill Taylor #include <stdio.h>
659e39c5baSBill Taylor static int dapli_process_fake_ibds(DAPL_HCA **hca_list, int hca_idx);
669e39c5baSBill Taylor #endif /* IBHOSTS_NAMING */
679e39c5baSBill Taylor
689e39c5baSBill Taylor static DAPL_OS_LOCK g_tavor_state_lock;
699e39c5baSBill Taylor static struct dapls_ib_hca_state g_tavor_state[MAX_HCAS];
709e39c5baSBill Taylor DAPL_OS_LOCK g_tavor_uar_lock;
71*e3b27002SToomas Soome DAPL_OS_LOCK dapls_ib_dbp_lock;
729e39c5baSBill Taylor
739e39c5baSBill Taylor DAT_RETURN
dapli_init_hca(IN DAPL_HCA * hca_ptr)749e39c5baSBill Taylor dapli_init_hca(
759e39c5baSBill Taylor IN DAPL_HCA *hca_ptr)
769e39c5baSBill Taylor {
771cfa752fSRamaswamy Tummala DAT_RETURN dat_status = DAT_SUCCESS;
781cfa752fSRamaswamy Tummala int hca_idx = 0;
791cfa752fSRamaswamy Tummala int check_for_bf = 0;
801cfa752fSRamaswamy Tummala datalink_class_t class;
811cfa752fSRamaswamy Tummala datalink_id_t linkid;
821cfa752fSRamaswamy Tummala dladm_ib_attr_t ib_attr;
831cfa752fSRamaswamy Tummala ibnex_ctl_query_hca_t query_hca;
841cfa752fSRamaswamy Tummala int ibnex_fd = -1;
851cfa752fSRamaswamy Tummala dladm_handle_t dlh;
861cfa752fSRamaswamy Tummala char hca_device_path[MAXPATHLEN];
871cfa752fSRamaswamy Tummala
881cfa752fSRamaswamy Tummala if (dladm_open(&dlh) != DLADM_STATUS_OK) {
899e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
901cfa752fSRamaswamy Tummala "init_hca: dladm_open failed\n");
919e39c5baSBill Taylor return (DAT_INTERNAL_ERROR);
929e39c5baSBill Taylor }
939e39c5baSBill Taylor
941cfa752fSRamaswamy Tummala if ((ibnex_fd = open(IBNEX_DEVCTL_DEV, O_RDONLY)) < 0) {
951cfa752fSRamaswamy Tummala dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
961cfa752fSRamaswamy Tummala dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
971cfa752fSRamaswamy Tummala "init_hca: could not open ib nexus (%s)\n",
981cfa752fSRamaswamy Tummala strerror(errno));
991cfa752fSRamaswamy Tummala goto bail;
1009e39c5baSBill Taylor }
1019e39c5baSBill Taylor
1021cfa752fSRamaswamy Tummala if ((dladm_name2info(dlh, hca_ptr->name, &linkid, NULL, &class,
1031cfa752fSRamaswamy Tummala NULL) != DLADM_STATUS_OK) ||
1041cfa752fSRamaswamy Tummala (class != DATALINK_CLASS_PART) ||
1051cfa752fSRamaswamy Tummala (dladm_part_info(dlh, linkid, &ib_attr,
1061cfa752fSRamaswamy Tummala DLADM_OPT_ACTIVE) != DLADM_STATUS_OK)) {
1079e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
1081cfa752fSRamaswamy Tummala dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
1091cfa752fSRamaswamy Tummala "init_hca: %s not found - couldn't get partition info\n",
1101cfa752fSRamaswamy Tummala hca_ptr->name);
1119e39c5baSBill Taylor goto bail;
1129e39c5baSBill Taylor }
1139e39c5baSBill Taylor
1141cfa752fSRamaswamy Tummala bzero(&query_hca, sizeof (query_hca));
1151cfa752fSRamaswamy Tummala query_hca.hca_guid = ib_attr.dia_hca_guid;
1161cfa752fSRamaswamy Tummala query_hca.hca_device_path = hca_device_path;
1171cfa752fSRamaswamy Tummala query_hca.hca_device_path_alloc_sz = sizeof (hca_device_path);
1181cfa752fSRamaswamy Tummala if (ioctl(ibnex_fd, IBNEX_CTL_QUERY_HCA, &query_hca) == -1) {
1191cfa752fSRamaswamy Tummala dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
1201cfa752fSRamaswamy Tummala dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
1211cfa752fSRamaswamy Tummala "init_hca: %s not found; query_hca failed\n",
1221cfa752fSRamaswamy Tummala hca_ptr->name);
1231cfa752fSRamaswamy Tummala goto bail;
1241cfa752fSRamaswamy Tummala }
1251cfa752fSRamaswamy Tummala
1261cfa752fSRamaswamy Tummala if (strcmp(query_hca.hca_info.hca_driver_name, "tavor") == 0)
1279e39c5baSBill Taylor dapls_init_funcs_tavor(hca_ptr);
1281cfa752fSRamaswamy Tummala else if (strcmp(query_hca.hca_info.hca_driver_name, "arbel") == 0)
1299e39c5baSBill Taylor dapls_init_funcs_arbel(hca_ptr);
1301cfa752fSRamaswamy Tummala else if (strcmp(query_hca.hca_info.hca_driver_name, "hermon") == 0) {
1319e39c5baSBill Taylor dapls_init_funcs_hermon(hca_ptr);
1329e39c5baSBill Taylor check_for_bf = 1;
1339e39c5baSBill Taylor } else {
1349e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
1359e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
1361cfa752fSRamaswamy Tummala "init_hca: %s not found\n", hca_ptr->name);
1379e39c5baSBill Taylor goto bail;
1389e39c5baSBill Taylor }
1399e39c5baSBill Taylor
1401cfa752fSRamaswamy Tummala dat_status = dapli_process_tavor_node(hca_device_path, &hca_idx,
1411cfa752fSRamaswamy Tummala check_for_bf);
1429e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
1439e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
1441cfa752fSRamaswamy Tummala "init_hcas: %s process_tavor_node failed(0x%x)\n",
1451cfa752fSRamaswamy Tummala hca_ptr->name, dat_status);
1469e39c5baSBill Taylor goto bail;
1479e39c5baSBill Taylor }
1489e39c5baSBill Taylor
1499e39c5baSBill Taylor #if defined(IBHOSTS_NAMING)
1509e39c5baSBill Taylor if (dapli_process_fake_ibds(hca_ptr, hca_idx) == 0) {
1519e39c5baSBill Taylor /* no entries were found */
1529e39c5baSBill Taylor dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
1539e39c5baSBill Taylor }
1549e39c5baSBill Taylor #else
1551cfa752fSRamaswamy Tummala dat_status = dapli_process_ia(&ib_attr, hca_ptr, hca_idx);
1569e39c5baSBill Taylor #endif
1579e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
1589e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
1591cfa752fSRamaswamy Tummala "init_hcas: %s process_ia failed(0x%x)\n",
1601cfa752fSRamaswamy Tummala hca_ptr->name, dat_status);
1619e39c5baSBill Taylor goto bail;
1629e39c5baSBill Taylor }
1639e39c5baSBill Taylor
1649e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
1651cfa752fSRamaswamy Tummala "init_hcas: done %s\n", hca_ptr->name);
1669e39c5baSBill Taylor
1679e39c5baSBill Taylor bail:
1681cfa752fSRamaswamy Tummala if (ibnex_fd != -1)
1691cfa752fSRamaswamy Tummala (void) close(ibnex_fd);
1701cfa752fSRamaswamy Tummala dladm_close(dlh);
1719e39c5baSBill Taylor return (dat_status);
1729e39c5baSBill Taylor }
1739e39c5baSBill Taylor
1749e39c5baSBill Taylor static DAT_RETURN
dapli_process_tavor_node(char * dev_path,int * hca_idx,int try_blueflame)1751cfa752fSRamaswamy Tummala dapli_process_tavor_node(char *dev_path, int *hca_idx, int try_blueflame)
1769e39c5baSBill Taylor {
1779e39c5baSBill Taylor char path_buf[MAXPATHLEN];
1781ed53a3fSBill Taylor int i, idx, fd;
1799e39c5baSBill Taylor #ifndef _LP64
1809e39c5baSBill Taylor int tmpfd;
1819e39c5baSBill Taylor #endif
1829e39c5baSBill Taylor size_t pagesize;
1839e39c5baSBill Taylor void *mapaddr;
1849e39c5baSBill Taylor pid_t cur_pid;
1859e39c5baSBill Taylor off64_t uarpg_offset;
1869e39c5baSBill Taylor
1879e39c5baSBill Taylor dapl_os_lock(&g_tavor_state_lock);
1889e39c5baSBill Taylor
1899e39c5baSBill Taylor for (idx = 0; idx < MAX_HCAS; idx++) {
1909e39c5baSBill Taylor /*
1919e39c5baSBill Taylor * page size == 0 means this entry is not occupied
1929e39c5baSBill Taylor */
1939e39c5baSBill Taylor if (g_tavor_state[idx].uarpg_size == 0) {
1949e39c5baSBill Taylor break;
1959e39c5baSBill Taylor }
1969e39c5baSBill Taylor }
1979e39c5baSBill Taylor if (idx == MAX_HCAS) {
1989e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
1999e39c5baSBill Taylor "process_tavor: all hcas are being used!\n");
2009e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
2019e39c5baSBill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
2029e39c5baSBill Taylor }
2039e39c5baSBill Taylor
2041ed53a3fSBill Taylor for (i = 0; i < idx; i++) {
2051ed53a3fSBill Taylor if (strcmp(dev_path, g_tavor_state[i].hca_path) == 0) {
2061ed53a3fSBill Taylor /* no need for a refcnt */
2071ed53a3fSBill Taylor idx = i;
2081ed53a3fSBill Taylor goto done;
2091ed53a3fSBill Taylor }
2101ed53a3fSBill Taylor }
2111ed53a3fSBill Taylor
2129e39c5baSBill Taylor /* Add 16 to accomodate the prefix "/devices" and suffix ":devctl" */
2139e39c5baSBill Taylor if (strlen("/devices") + strlen(dev_path) + strlen(":devctl") + 1 >
2149e39c5baSBill Taylor MAXPATHLEN) {
2159e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
2169e39c5baSBill Taylor "process_tavor: devfs path %s is too long\n",
2179e39c5baSBill Taylor dev_path);
2189e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
2199e39c5baSBill Taylor return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
2209e39c5baSBill Taylor }
2219e39c5baSBill Taylor (void) dapl_os_strcpy(path_buf, "/devices");
2229e39c5baSBill Taylor (void) dapl_os_strcat(path_buf, dev_path);
2239e39c5baSBill Taylor (void) dapl_os_strcat(path_buf, ":devctl");
2241ed53a3fSBill Taylor (void) dapl_os_strcpy(g_tavor_state[idx].hca_path, dev_path);
2259e39c5baSBill Taylor
2269e39c5baSBill Taylor pagesize = (size_t)sysconf(_SC_PAGESIZE);
2279e39c5baSBill Taylor if (pagesize == 0) {
2289e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
2299e39c5baSBill Taylor "process_tavor: page_size == 0\n");
2309e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
2319e39c5baSBill Taylor return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
2329e39c5baSBill Taylor }
2339e39c5baSBill Taylor cur_pid = getpid();
2349e39c5baSBill Taylor
2359e39c5baSBill Taylor fd = open(path_buf, O_RDWR);
2369e39c5baSBill Taylor if (fd < 0) {
2379e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
2389e39c5baSBill Taylor "process_tavor: cannot open %s: %s\n",
2399e39c5baSBill Taylor path_buf, strerror(errno));
2409e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
2419e39c5baSBill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
2429e39c5baSBill Taylor }
2439e39c5baSBill Taylor #ifndef _LP64
2449e39c5baSBill Taylor /*
2459e39c5baSBill Taylor * libc can't handle fd's greater than 255, in order to
2469e39c5baSBill Taylor * ensure that these values remain available make fd > 255.
2479e39c5baSBill Taylor * Note: not needed for LP64
2489e39c5baSBill Taylor */
2499e39c5baSBill Taylor tmpfd = fcntl(fd, F_DUPFD, 256);
2509e39c5baSBill Taylor if (tmpfd < 0) {
2519e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
2529e39c5baSBill Taylor "process_tavor: cannot F_DUPFD: %s\n", strerror(errno));
2539e39c5baSBill Taylor } else {
2549e39c5baSBill Taylor (void) close(fd);
2559e39c5baSBill Taylor fd = tmpfd;
2569e39c5baSBill Taylor }
2579e39c5baSBill Taylor #endif /* _LP64 */
2589e39c5baSBill Taylor
2599e39c5baSBill Taylor if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
2609e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
2619e39c5baSBill Taylor "process_tavor: cannot F_SETFD: %s\n", strerror(errno));
2629e39c5baSBill Taylor (void) close(fd);
2639e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
2649e39c5baSBill Taylor return (DAT_ERROR(DAT_INTERNAL_ERROR, 0));
2659e39c5baSBill Taylor }
2669e39c5baSBill Taylor
2679e39c5baSBill Taylor uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
2689e39c5baSBill Taylor MLNX_UMAP_UARPG_RSRC) * pagesize;
2699e39c5baSBill Taylor
2709e39c5baSBill Taylor mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE,
2719e39c5baSBill Taylor MAP_SHARED, fd, uarpg_offset);
2729e39c5baSBill Taylor if (mapaddr == MAP_FAILED) {
2739e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
2749e39c5baSBill Taylor "process_tavor: mmap failed %s\n", strerror(errno));
2759e39c5baSBill Taylor (void) close(fd);
2769e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
2779e39c5baSBill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
2789e39c5baSBill Taylor }
2799e39c5baSBill Taylor
2809e39c5baSBill Taylor g_tavor_state[idx].hca_fd = fd;
2819e39c5baSBill Taylor g_tavor_state[idx].uarpg_baseaddr = mapaddr;
2829e39c5baSBill Taylor g_tavor_state[idx].uarpg_size = pagesize;
2839e39c5baSBill Taylor
2849e39c5baSBill Taylor if (try_blueflame == 0)
2859e39c5baSBill Taylor goto done;
2869e39c5baSBill Taylor
2879e39c5baSBill Taylor /* Try to do the Hermon Blueflame page mapping */
2889e39c5baSBill Taylor uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
2899e39c5baSBill Taylor MLNX_UMAP_BLUEFLAMEPG_RSRC) * pagesize;
2909e39c5baSBill Taylor
2919e39c5baSBill Taylor mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE,
2929e39c5baSBill Taylor MAP_SHARED, fd, uarpg_offset);
2939e39c5baSBill Taylor if (mapaddr == MAP_FAILED) {
2949e39c5baSBill Taylor /* This is not considered to be fatal. Charge on! */
2959e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_WARN,
2969e39c5baSBill Taylor "process_tavor: mmap of blueflame page failed %s\n",
2979e39c5baSBill Taylor strerror(errno));
2989e39c5baSBill Taylor } else {
2999e39c5baSBill Taylor g_tavor_state[idx].bf_pg_baseaddr = mapaddr;
3001ed53a3fSBill Taylor g_tavor_state[idx].bf_toggle = 0;
3019e39c5baSBill Taylor }
3029e39c5baSBill Taylor done:
3039e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
3049e39c5baSBill Taylor
3059e39c5baSBill Taylor *hca_idx = idx;
3069e39c5baSBill Taylor
3079e39c5baSBill Taylor return (DAT_SUCCESS);
3089e39c5baSBill Taylor }
3099e39c5baSBill Taylor
3109e39c5baSBill Taylor static DAT_RETURN
dapli_process_ia(dladm_ib_attr_t * ib_attr,DAPL_HCA * hca_ptr,int hca_idx)3111cfa752fSRamaswamy Tummala dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr, int hca_idx)
3129e39c5baSBill Taylor {
3139e39c5baSBill Taylor struct lifreq lifreq;
3141cfa752fSRamaswamy Tummala int sfd, retval, af;
3159e39c5baSBill Taylor char addr_buf[64];
3169e39c5baSBill Taylor
3171cfa752fSRamaswamy Tummala if (ib_attr->dia_hca_guid == 0 || ib_attr->dia_portnum == 0 ||
3181cfa752fSRamaswamy Tummala ib_attr->dia_pkey == 0) {
3199e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
3201cfa752fSRamaswamy Tummala "process_ia: invalid properties: guid 0x%016llx, "
3211cfa752fSRamaswamy Tummala "port %d, pkey 0x%08x\n", ib_attr->dia_hca_guid,
3221cfa752fSRamaswamy Tummala ib_attr->dia_portnum, (uint_t)ib_attr->dia_pkey);
3239e39c5baSBill Taylor return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
3249e39c5baSBill Taylor }
3259e39c5baSBill Taylor
3269e39c5baSBill Taylor /*
3279e39c5baSBill Taylor * if an interface has both v4 and v6 addresses plumbed,
3289e39c5baSBill Taylor * we'll take the v4 address.
3299e39c5baSBill Taylor */
3309e39c5baSBill Taylor af = AF_INET;
3319e39c5baSBill Taylor again:
3329e39c5baSBill Taylor sfd = socket(af, SOCK_DGRAM, 0);
3339e39c5baSBill Taylor if (sfd < 0) {
3349e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
3351cfa752fSRamaswamy Tummala "process_ia: socket failed: %s\n", strerror(errno));
3369e39c5baSBill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
3379e39c5baSBill Taylor }
3389e39c5baSBill Taylor
3399e39c5baSBill Taylor /* check if name will fit in lifr_name */
3401cfa752fSRamaswamy Tummala if (dapl_os_strlen(hca_ptr->name) >= LIFNAMSIZ) {
3419e39c5baSBill Taylor (void) close(sfd);
3429e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
3431cfa752fSRamaswamy Tummala "process_ia: if name overflow %s\n",
3441cfa752fSRamaswamy Tummala hca_ptr->name);
3459e39c5baSBill Taylor return (DAT_ERROR(DAT_INVALID_PARAMETER, 0));
3469e39c5baSBill Taylor }
3479e39c5baSBill Taylor
3481cfa752fSRamaswamy Tummala (void) dapl_os_strcpy(lifreq.lifr_name, hca_ptr->name);
3499e39c5baSBill Taylor retval = ioctl(sfd, SIOCGLIFADDR, (caddr_t)&lifreq);
3509e39c5baSBill Taylor if (retval < 0) {
3519e39c5baSBill Taylor (void) close(sfd);
3529e39c5baSBill Taylor if (af == AF_INET6) {
3539e39c5baSBill Taylor /*
3549e39c5baSBill Taylor * the interface is not plumbed.
3559e39c5baSBill Taylor */
3569e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
3571cfa752fSRamaswamy Tummala "process_ia: %s: ip address not found\n",
3589e39c5baSBill Taylor lifreq.lifr_name);
3599e39c5baSBill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
3609e39c5baSBill Taylor } else {
3619e39c5baSBill Taylor /*
3629e39c5baSBill Taylor * we've failed to find a v4 address. now
3639e39c5baSBill Taylor * let's try v6.
3649e39c5baSBill Taylor */
3659e39c5baSBill Taylor af = AF_INET6;
3669e39c5baSBill Taylor goto again;
3679e39c5baSBill Taylor }
3689e39c5baSBill Taylor }
3699e39c5baSBill Taylor (void) close(sfd);
3709e39c5baSBill Taylor
3719e39c5baSBill Taylor hca_ptr->tavor_idx = hca_idx;
3721cfa752fSRamaswamy Tummala hca_ptr->node_GUID = ib_attr->dia_hca_guid;
3731cfa752fSRamaswamy Tummala hca_ptr->port_num = ib_attr->dia_portnum;
3741cfa752fSRamaswamy Tummala hca_ptr->partition_key = ib_attr->dia_pkey;
3759e39c5baSBill Taylor (void) dapl_os_memcpy((void *)&hca_ptr->hca_address,
3769e39c5baSBill Taylor (void *)&lifreq.lifr_addr, sizeof (hca_ptr->hca_address));
3779e39c5baSBill Taylor hca_ptr->max_inline_send = dapls_tavor_max_inline();
3789e39c5baSBill Taylor
3799e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
3801cfa752fSRamaswamy Tummala "process_ia: interface %s, hca guid 0x%016llx, port %d, "
3811cfa752fSRamaswamy Tummala "pkey 0x%08x, ip addr %s\n", lifreq.lifr_name, hca_ptr->node_GUID,
3821cfa752fSRamaswamy Tummala hca_ptr->port_num, hca_ptr->partition_key, dapls_inet_ntop(
3839e39c5baSBill Taylor (struct sockaddr *)&hca_ptr->hca_address, addr_buf, 64));
3849e39c5baSBill Taylor return (DAT_SUCCESS);
3859e39c5baSBill Taylor }
3869e39c5baSBill Taylor
3879e39c5baSBill Taylor void
dapls_ib_state_init(void)3889e39c5baSBill Taylor dapls_ib_state_init(void)
3899e39c5baSBill Taylor {
3909e39c5baSBill Taylor int i;
3919e39c5baSBill Taylor
3929e39c5baSBill Taylor (void) dapl_os_lock_init(&g_tavor_state_lock);
3939e39c5baSBill Taylor (void) dapl_os_lock_init(&g_tavor_uar_lock);
3949e39c5baSBill Taylor (void) dapl_os_lock_init(&dapls_ib_dbp_lock);
3959e39c5baSBill Taylor
3969e39c5baSBill Taylor for (i = 0; i < MAX_HCAS; i++) {
3979e39c5baSBill Taylor g_tavor_state[i].hca_fd = 0;
3989e39c5baSBill Taylor g_tavor_state[i].uarpg_baseaddr = NULL;
3999e39c5baSBill Taylor g_tavor_state[i].uarpg_size = 0;
4009e39c5baSBill Taylor g_tavor_state[i].bf_pg_baseaddr = NULL;
4019e39c5baSBill Taylor }
4029e39c5baSBill Taylor }
4039e39c5baSBill Taylor
4049e39c5baSBill Taylor void
dapls_ib_state_fini(void)4059e39c5baSBill Taylor dapls_ib_state_fini(void)
4069e39c5baSBill Taylor {
4079e39c5baSBill Taylor int i, count = 0;
4089e39c5baSBill Taylor
4099e39c5baSBill Taylor /*
4109e39c5baSBill Taylor * Uinitialize the per hca instance state
4119e39c5baSBill Taylor */
4129e39c5baSBill Taylor dapl_os_lock(&g_tavor_state_lock);
4139e39c5baSBill Taylor for (i = 0; i < MAX_HCAS; i++) {
4149e39c5baSBill Taylor if (g_tavor_state[i].uarpg_size == 0) {
4159e39c5baSBill Taylor dapl_os_assert(g_tavor_state[i].uarpg_baseaddr ==
4169e39c5baSBill Taylor NULL);
4179e39c5baSBill Taylor continue;
4189e39c5baSBill Taylor }
4199e39c5baSBill Taylor if (munmap(g_tavor_state[i].uarpg_baseaddr,
4209e39c5baSBill Taylor g_tavor_state[i].uarpg_size) < 0) {
4219e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
4229e39c5baSBill Taylor "ib_state_fini: "
4239e39c5baSBill Taylor "munmap(0x%p, 0x%llx) failed(%d)\n",
4249e39c5baSBill Taylor g_tavor_state[i].uarpg_baseaddr,
4259e39c5baSBill Taylor g_tavor_state[i].uarpg_size, errno);
4269e39c5baSBill Taylor }
4279e39c5baSBill Taylor if ((g_tavor_state[i].bf_pg_baseaddr != NULL) &&
4289e39c5baSBill Taylor (munmap(g_tavor_state[i].bf_pg_baseaddr,
4299e39c5baSBill Taylor g_tavor_state[i].uarpg_size) < 0)) {
4309e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
4319e39c5baSBill Taylor "ib_state_fini: "
4329e39c5baSBill Taylor "munmap(0x%p, 0x%llx) of blueflame failed(%d)\n",
4339e39c5baSBill Taylor g_tavor_state[i].bf_pg_baseaddr,
4349e39c5baSBill Taylor g_tavor_state[i].uarpg_size, errno);
4359e39c5baSBill Taylor }
4369e39c5baSBill Taylor
4379e39c5baSBill Taylor (void) close(g_tavor_state[i].hca_fd);
4389e39c5baSBill Taylor count++;
4399e39c5baSBill Taylor }
4409e39c5baSBill Taylor dapl_os_unlock(&g_tavor_state_lock);
4419e39c5baSBill Taylor
4429e39c5baSBill Taylor dapl_os_lock_destroy(&g_tavor_uar_lock);
4439e39c5baSBill Taylor dapl_os_lock_destroy(&g_tavor_state_lock);
4449e39c5baSBill Taylor dapl_os_lock_destroy(&dapls_ib_dbp_lock);
4459e39c5baSBill Taylor
4469e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
4479e39c5baSBill Taylor "ib_state_fini: cleaned %d hcas\n", count);
4489e39c5baSBill Taylor }
4499e39c5baSBill Taylor
4509e39c5baSBill Taylor /*
4519e39c5baSBill Taylor * dapls_ib_open_hca
4529e39c5baSBill Taylor *
4539e39c5baSBill Taylor * Open HCA
4549e39c5baSBill Taylor *
4559e39c5baSBill Taylor * Input:
4569e39c5baSBill Taylor * *hca_ptr pointer to hca device
4579e39c5baSBill Taylor * *ib_hca_handle_p pointer to provide HCA handle
4589e39c5baSBill Taylor *
4599e39c5baSBill Taylor * Output:
4609e39c5baSBill Taylor * none
4619e39c5baSBill Taylor *
4629e39c5baSBill Taylor * Return:
4639e39c5baSBill Taylor * DAT_SUCCESS
4649e39c5baSBill Taylor * DAT_INSUFFICIENT_RESOURCES
4659e39c5baSBill Taylor *
4669e39c5baSBill Taylor */
4679e39c5baSBill Taylor DAT_RETURN
dapls_ib_open_hca(IN DAPL_HCA * hca_ptr,OUT ib_hca_handle_t * ib_hca_handle_p)4689e39c5baSBill Taylor dapls_ib_open_hca(
4699e39c5baSBill Taylor IN DAPL_HCA *hca_ptr,
4709e39c5baSBill Taylor OUT ib_hca_handle_t *ib_hca_handle_p)
4719e39c5baSBill Taylor {
4729e39c5baSBill Taylor dapl_ia_create_t args;
4739e39c5baSBill Taylor DAT_RETURN dat_status;
4749e39c5baSBill Taylor struct dapls_ib_hca_handle *hca_p;
4759e39c5baSBill Taylor int fd;
4769e39c5baSBill Taylor #ifndef _LP64
4779e39c5baSBill Taylor int tmpfd;
4789e39c5baSBill Taylor #endif
4799e39c5baSBill Taylor int retval;
4809e39c5baSBill Taylor struct sockaddr *s;
4819e39c5baSBill Taylor struct sockaddr_in6 *v6addr;
4829e39c5baSBill Taylor struct sockaddr_in *v4addr;
4839e39c5baSBill Taylor dapl_ia_addr_t *sap;
4849e39c5baSBill Taylor
4859e39c5baSBill Taylor dat_status = dapli_init_hca(hca_ptr);
4869e39c5baSBill Taylor if (dat_status != DAT_SUCCESS) {
4879e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
4889e39c5baSBill Taylor "dapls_ib_open_hca: init_hca failed %d\n", dat_status);
4899e39c5baSBill Taylor return (dat_status);
4909e39c5baSBill Taylor }
4919e39c5baSBill Taylor
4929e39c5baSBill Taylor fd = open(DEVDAPLT, O_RDONLY);
4939e39c5baSBill Taylor if (fd < 0) {
4949e39c5baSBill Taylor return (DAT_INSUFFICIENT_RESOURCES);
4959e39c5baSBill Taylor }
4969e39c5baSBill Taylor
4979e39c5baSBill Taylor #ifndef _LP64
4989e39c5baSBill Taylor /*
4999e39c5baSBill Taylor * libc can't handle fd's greater than 255, in order to
5009e39c5baSBill Taylor * ensure that these values remain available make fd > 255.
5019e39c5baSBill Taylor * Note: not needed for LP64
5029e39c5baSBill Taylor */
5039e39c5baSBill Taylor tmpfd = fcntl(fd, F_DUPFD, 256);
5049e39c5baSBill Taylor if (tmpfd < 0) {
5059e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
5069e39c5baSBill Taylor "dapls_ib_open_hca: cannot F_DUPFD: %s\n",
5079e39c5baSBill Taylor strerror(errno));
5089e39c5baSBill Taylor } else {
5099e39c5baSBill Taylor (void) close(fd);
5109e39c5baSBill Taylor fd = tmpfd;
5119e39c5baSBill Taylor }
5129e39c5baSBill Taylor #endif /* _LP64 */
5139e39c5baSBill Taylor
5149e39c5baSBill Taylor if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
5159e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
5169e39c5baSBill Taylor "dapls_ib_open_hca: cannot F_SETFD: %s\n", strerror(errno));
5179e39c5baSBill Taylor (void) close(fd);
5189e39c5baSBill Taylor return (DAT_INTERNAL_ERROR);
5199e39c5baSBill Taylor }
5209e39c5baSBill Taylor
5219e39c5baSBill Taylor hca_p = (struct dapls_ib_hca_handle *)dapl_os_alloc(
5229e39c5baSBill Taylor sizeof (struct dapls_ib_hca_handle));
5239e39c5baSBill Taylor if (hca_p == NULL) {
5249e39c5baSBill Taylor (void) close(fd);
5259e39c5baSBill Taylor return (DAT_INSUFFICIENT_RESOURCES);
5269e39c5baSBill Taylor }
5279e39c5baSBill Taylor
5289e39c5baSBill Taylor args.ia_guid = hca_ptr->node_GUID;
5299e39c5baSBill Taylor args.ia_port = hca_ptr->port_num;
5309e39c5baSBill Taylor args.ia_pkey = hca_ptr->partition_key;
5319e39c5baSBill Taylor args.ia_version = DAPL_IF_VERSION;
5329e39c5baSBill Taylor (void) dapl_os_memzero((void *)args.ia_sadata, DAPL_ATS_NBYTES);
5339e39c5baSBill Taylor
5349e39c5baSBill Taylor /* pass down local ip address to be stored in SA */
5359e39c5baSBill Taylor s = (struct sockaddr *)&hca_ptr->hca_address;
5369e39c5baSBill Taylor /* LINTED: E_BAD_PTR_CAST_ALIGN */
5379e39c5baSBill Taylor sap = (dapl_ia_addr_t *)args.ia_sadata;
5389e39c5baSBill Taylor switch (s->sa_family) {
5399e39c5baSBill Taylor case AF_INET:
5409e39c5baSBill Taylor /* LINTED: E_BAD_PTR_CAST_ALIGN */
5419e39c5baSBill Taylor v4addr = (struct sockaddr_in *)s;
5429e39c5baSBill Taylor sap->iad_v4 = v4addr->sin_addr;
5439e39c5baSBill Taylor break;
5449e39c5baSBill Taylor case AF_INET6:
5459e39c5baSBill Taylor /* LINTED: E_BAD_PTR_CAST_ALIGN */
5469e39c5baSBill Taylor v6addr = (struct sockaddr_in6 *)s;
5479e39c5baSBill Taylor sap->iad_v6 = v6addr->sin6_addr;
5489e39c5baSBill Taylor break;
5499e39c5baSBill Taylor default:
5509e39c5baSBill Taylor break; /* fall through */
5519e39c5baSBill Taylor }
5529e39c5baSBill Taylor
5539e39c5baSBill Taylor retval = ioctl(fd, DAPL_IA_CREATE, &args);
5549e39c5baSBill Taylor if (retval != 0) {
5559e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
5569e39c5baSBill Taylor "open_hca: ia_create failed, fd %d, "
5579e39c5baSBill Taylor "guid 0x%016llx, port %d, pkey 0x%x, version %d\n",
5589e39c5baSBill Taylor fd, args.ia_guid, args.ia_port, args.ia_pkey,
5599e39c5baSBill Taylor args.ia_version);
5609e39c5baSBill Taylor
5619e39c5baSBill Taylor dapl_os_free(hca_p, sizeof (*hca_p));
5629e39c5baSBill Taylor (void) close(fd);
5639e39c5baSBill Taylor return (dapls_convert_error(errno, retval));
5649e39c5baSBill Taylor }
5659e39c5baSBill Taylor
5669e39c5baSBill Taylor hca_p->ia_fd = fd;
5679e39c5baSBill Taylor hca_p->ia_rnum = args.ia_resnum;
5689e39c5baSBill Taylor hca_p->hca_fd = g_tavor_state[hca_ptr->tavor_idx].hca_fd;
5699e39c5baSBill Taylor hca_p->ia_uar = g_tavor_state[hca_ptr->tavor_idx].uarpg_baseaddr;
5709e39c5baSBill Taylor hca_p->ia_bf = g_tavor_state[hca_ptr->tavor_idx].bf_pg_baseaddr;
5711ed53a3fSBill Taylor hca_p->ia_bf_toggle = &g_tavor_state[hca_ptr->tavor_idx].bf_toggle;
5729e39c5baSBill Taylor *ib_hca_handle_p = hca_p;
5739e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
5749e39c5baSBill Taylor "open_hca: ia_created, hca_p 0x%p, fd %d, "
5759e39c5baSBill Taylor "rnum %d, guid 0x%016llx, port %d, pkey 0x%x\n",
5769e39c5baSBill Taylor hca_p, hca_p->ia_fd, hca_p->ia_rnum, hca_ptr->node_GUID,
5779e39c5baSBill Taylor hca_ptr->port_num, hca_ptr->partition_key);
5789e39c5baSBill Taylor
5799e39c5baSBill Taylor return (DAT_SUCCESS);
5809e39c5baSBill Taylor }
5819e39c5baSBill Taylor
5829e39c5baSBill Taylor /*
5839e39c5baSBill Taylor * dapls_ib_close_hca
5849e39c5baSBill Taylor *
5859e39c5baSBill Taylor * Open HCA
5869e39c5baSBill Taylor *
5879e39c5baSBill Taylor * Input:
5889e39c5baSBill Taylor * ib_hca_handle provide HCA handle
5899e39c5baSBill Taylor *
5909e39c5baSBill Taylor * Output:
5919e39c5baSBill Taylor * none
5929e39c5baSBill Taylor *
5939e39c5baSBill Taylor * Return:
5949e39c5baSBill Taylor * DAT_SUCCESS
5959e39c5baSBill Taylor * DAT_INSUFFICIENT_RESOURCES
5969e39c5baSBill Taylor *
5979e39c5baSBill Taylor */
5989e39c5baSBill Taylor DAT_RETURN
dapls_ib_close_hca(IN ib_hca_handle_t ib_hca_handle)5999e39c5baSBill Taylor dapls_ib_close_hca(
6009e39c5baSBill Taylor IN ib_hca_handle_t ib_hca_handle)
6019e39c5baSBill Taylor {
6029e39c5baSBill Taylor if (ib_hca_handle == NULL) {
6039e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
6049e39c5baSBill Taylor "close_hca: ib_hca_handle == NULL\n");
6059e39c5baSBill Taylor return (DAT_SUCCESS);
6069e39c5baSBill Taylor }
6079e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
6089e39c5baSBill Taylor "close_hca: closing hca 0x%p, fd %d, rnum %d\n",
6099e39c5baSBill Taylor ib_hca_handle, ib_hca_handle->ia_fd, ib_hca_handle->ia_rnum);
6109e39c5baSBill Taylor
6119e39c5baSBill Taylor (void) close(ib_hca_handle->ia_fd);
6129e39c5baSBill Taylor dapl_os_free((void *)ib_hca_handle,
6139e39c5baSBill Taylor sizeof (struct dapls_ib_hca_handle));
6149e39c5baSBill Taylor return (DAT_SUCCESS);
6159e39c5baSBill Taylor }
6169e39c5baSBill Taylor
6179e39c5baSBill Taylor #if defined(IBHOSTS_NAMING)
6189e39c5baSBill Taylor #define LINE_LEN 256
6199e39c5baSBill Taylor static int
dapli_process_fake_ibds(DAPL_HCA * hca_ptr,int hca_idx)6209e39c5baSBill Taylor dapli_process_fake_ibds(DAPL_HCA *hca_ptr, int hca_idx)
6219e39c5baSBill Taylor {
6229e39c5baSBill Taylor char line_buf[LINE_LEN];
6239e39c5baSBill Taylor char host_buf[LINE_LEN];
6249e39c5baSBill Taylor char localhost[LINE_LEN];
6259e39c5baSBill Taylor ib_guid_t prefix;
6269e39c5baSBill Taylor ib_guid_t guid;
6279e39c5baSBill Taylor FILE *fp;
6289e39c5baSBill Taylor int count = 0;
6299e39c5baSBill Taylor DAPL_HCA *hca_ptr;
6309e39c5baSBill Taylor
6319e39c5baSBill Taylor fp = fopen("/etc/dapl/ibhosts", "r");
6329e39c5baSBill Taylor if (fp == NULL) {
6339e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
6349e39c5baSBill Taylor "fake_ibds: ibhosts not found!\n");
6359e39c5baSBill Taylor return (0);
6369e39c5baSBill Taylor }
6379e39c5baSBill Taylor if (gethostname(localhost, LINE_LEN) != 0) {
6389e39c5baSBill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR,
6399e39c5baSBill Taylor "fake_ibds: hostname not found!\n");
6409e39c5baSBill Taylor return (0);
6419e39c5baSBill Taylor }
6429e39c5baSBill Taylor while (!feof(fp)) {
6439e39c5baSBill Taylor (void) fgets(line_buf, LINE_LEN, fp);
6449e39c5baSBill Taylor sscanf(line_buf, "%s %llx %llx", host_buf, &prefix, &guid);
6459e39c5baSBill Taylor (void) sprintf(line_buf, "%s-ib%d", localhost, count + 1);
6469e39c5baSBill Taylor if (strncmp(line_buf, host_buf, strlen(line_buf)) == 0) {
6479e39c5baSBill Taylor guid &= 0xfffffffffffffff0;
6489e39c5baSBill Taylor hca_ptr->tavor_idx = hca_idx;
6499e39c5baSBill Taylor hca_ptr->node_GUID = guid;
6509e39c5baSBill Taylor hca_ptr->port_num = count + 1;
6519e39c5baSBill Taylor hca_ptr->partition_key = 0x0000ffff;
6529e39c5baSBill Taylor count++;
6539e39c5baSBill Taylor }
6549e39c5baSBill Taylor if (count >= 2) break;
6559e39c5baSBill Taylor }
6569e39c5baSBill Taylor (void) fclose(fp);
6579e39c5baSBill Taylor return (count);
6589e39c5baSBill Taylor }
6599e39c5baSBill Taylor
6609e39c5baSBill Taylor #endif /* IBHOSTS_NAMING */
661