130e7468fSPeter Dunlap /*
230e7468fSPeter Dunlap * CDDL HEADER START
330e7468fSPeter Dunlap *
430e7468fSPeter Dunlap * The contents of this file are subject to the terms of the
530e7468fSPeter Dunlap * Common Development and Distribution License (the "License").
630e7468fSPeter Dunlap * You may not use this file except in compliance with the License.
730e7468fSPeter Dunlap *
830e7468fSPeter Dunlap * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
930e7468fSPeter Dunlap * or http://www.opensolaris.org/os/licensing.
1030e7468fSPeter Dunlap * See the License for the specific language governing permissions
1130e7468fSPeter Dunlap * and limitations under the License.
1230e7468fSPeter Dunlap *
1330e7468fSPeter Dunlap * When distributing Covered Code, include this CDDL HEADER in each
1430e7468fSPeter Dunlap * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1530e7468fSPeter Dunlap * If applicable, add the following below this CDDL HEADER, with the
1630e7468fSPeter Dunlap * fields enclosed by brackets "[]" replaced with your own identifying
1730e7468fSPeter Dunlap * information: Portions Copyright [yyyy] [name of copyright owner]
1830e7468fSPeter Dunlap *
1930e7468fSPeter Dunlap * CDDL HEADER END
2030e7468fSPeter Dunlap */
2130e7468fSPeter Dunlap /*
22*2360e12dSPeter Gill * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
2330e7468fSPeter Dunlap */
2430e7468fSPeter Dunlap
2530e7468fSPeter Dunlap #include <sys/types.h>
2630e7468fSPeter Dunlap #include <sys/ddi.h>
2730e7468fSPeter Dunlap #include <sys/types.h>
2830e7468fSPeter Dunlap #include <sys/socket.h>
2930e7468fSPeter Dunlap #include <netinet/in.h>
3030e7468fSPeter Dunlap #include <sys/sunddi.h>
3130e7468fSPeter Dunlap #include <sys/sysmacros.h>
3230e7468fSPeter Dunlap #include <sys/iscsi_protocol.h>
3330e7468fSPeter Dunlap
3430e7468fSPeter Dunlap #include <sys/ib/clients/iser/iser.h>
3530e7468fSPeter Dunlap #include <sys/ib/clients/iser/iser_idm.h>
3630e7468fSPeter Dunlap
3730e7468fSPeter Dunlap /*
3830e7468fSPeter Dunlap * iser_ib.c
3930e7468fSPeter Dunlap * Routines for InfiniBand transport for iSER
4030e7468fSPeter Dunlap *
4130e7468fSPeter Dunlap * This file contains the routines to interface with the IBT API to attach and
4230e7468fSPeter Dunlap * allocate IB resources, handle async events, and post recv work requests.
4330e7468fSPeter Dunlap *
4430e7468fSPeter Dunlap */
4530e7468fSPeter Dunlap
4630e7468fSPeter Dunlap static iser_hca_t *iser_ib_gid2hca(ib_gid_t gid);
4730e7468fSPeter Dunlap static iser_hca_t *iser_ib_guid2hca(ib_guid_t guid);
4830e7468fSPeter Dunlap
4930e7468fSPeter Dunlap static iser_hca_t *iser_ib_alloc_hca(ib_guid_t guid);
5030e7468fSPeter Dunlap static int iser_ib_free_hca(iser_hca_t *hca);
5130e7468fSPeter Dunlap static int iser_ib_update_hcaports(iser_hca_t *hca);
5230e7468fSPeter Dunlap static int iser_ib_init_hcas(void);
5330e7468fSPeter Dunlap static int iser_ib_fini_hcas(void);
5430e7468fSPeter Dunlap
5530e7468fSPeter Dunlap static iser_sbind_t *iser_ib_get_bind(
5630e7468fSPeter Dunlap iser_svc_t *iser_svc, ib_guid_t hca_guid, ib_gid_t gid);
5730e7468fSPeter Dunlap static int iser_ib_activate_port(
5830e7468fSPeter Dunlap idm_svc_t *idm_svc, ib_guid_t guid, ib_gid_t gid);
5930e7468fSPeter Dunlap static void iser_ib_deactivate_port(ib_guid_t hca_guid, ib_gid_t gid);
6030e7468fSPeter Dunlap
6130e7468fSPeter Dunlap static void iser_ib_init_qp(iser_chan_t *chan, uint_t sq_size, uint_t rq_size);
6230e7468fSPeter Dunlap static void iser_ib_fini_qp(iser_qp_t *qp);
6330e7468fSPeter Dunlap
6430e7468fSPeter Dunlap static int iser_ib_setup_cq(ibt_hca_hdl_t hca_hdl, uint_t cq_size,
6530e7468fSPeter Dunlap ibt_cq_hdl_t *cq_hdl);
6630e7468fSPeter Dunlap
6730e7468fSPeter Dunlap static void iser_ib_setup_chanargs(uint8_t hca_port, ibt_cq_hdl_t scq_hdl,
6830e7468fSPeter Dunlap ibt_cq_hdl_t rcq_hdl, uint_t sq_size, uint_t rq_size,
6930e7468fSPeter Dunlap ibt_pd_hdl_t hca_pdhdl, ibt_rc_chan_alloc_args_t *cargs);
7030e7468fSPeter Dunlap
7130e7468fSPeter Dunlap static void iser_ib_handle_portup_event(ibt_hca_hdl_t hdl,
7230e7468fSPeter Dunlap ibt_async_event_t *event);
7330e7468fSPeter Dunlap static void iser_ib_handle_portdown_event(ibt_hca_hdl_t hdl,
7430e7468fSPeter Dunlap ibt_async_event_t *event);
7530e7468fSPeter Dunlap static void iser_ib_handle_hca_detach_event(ibt_hca_hdl_t hdl,
7630e7468fSPeter Dunlap ibt_async_event_t *event);
7730e7468fSPeter Dunlap
78cf8c0ebaSPeter Dunlap static void iser_ib_post_recv_task(void *arg);
79cf8c0ebaSPeter Dunlap
8030e7468fSPeter Dunlap static struct ibt_clnt_modinfo_s iser_ib_modinfo = {
8130e7468fSPeter Dunlap IBTI_V_CURR,
8230e7468fSPeter Dunlap IBT_STORAGE_DEV,
8330e7468fSPeter Dunlap iser_ib_async_handler,
8430e7468fSPeter Dunlap NULL,
8530e7468fSPeter Dunlap "iSER"
8630e7468fSPeter Dunlap };
8730e7468fSPeter Dunlap
8830e7468fSPeter Dunlap /*
8930e7468fSPeter Dunlap * iser_ib_init
9030e7468fSPeter Dunlap *
9130e7468fSPeter Dunlap * This function registers the HCA drivers with IBTF and registers and binds
9230e7468fSPeter Dunlap * iSER as a service with IBTF.
9330e7468fSPeter Dunlap */
9430e7468fSPeter Dunlap int
iser_ib_init(void)9530e7468fSPeter Dunlap iser_ib_init(void)
9630e7468fSPeter Dunlap {
9730e7468fSPeter Dunlap int status;
9830e7468fSPeter Dunlap
9930e7468fSPeter Dunlap /* Register with IBTF */
10030e7468fSPeter Dunlap status = ibt_attach(&iser_ib_modinfo, iser_state->is_dip, iser_state,
10130e7468fSPeter Dunlap &iser_state->is_ibhdl);
10230e7468fSPeter Dunlap if (status != DDI_SUCCESS) {
10330e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_init: ibt_attach failed (0x%x)",
10430e7468fSPeter Dunlap status);
10530e7468fSPeter Dunlap return (DDI_FAILURE);
10630e7468fSPeter Dunlap }
10730e7468fSPeter Dunlap
10830e7468fSPeter Dunlap /* Create the global work request kmem_cache */
10930e7468fSPeter Dunlap iser_state->iser_wr_cache = kmem_cache_create("iser_wr_cache",
11030e7468fSPeter Dunlap sizeof (iser_wr_t), 0, NULL, NULL, NULL,
11130e7468fSPeter Dunlap iser_state, NULL, KM_SLEEP);
11230e7468fSPeter Dunlap
11330e7468fSPeter Dunlap /* Populate our list of HCAs */
11430e7468fSPeter Dunlap status = iser_ib_init_hcas();
11530e7468fSPeter Dunlap if (status != DDI_SUCCESS) {
11630e7468fSPeter Dunlap /* HCAs failed to initialize, tear it down */
11730e7468fSPeter Dunlap kmem_cache_destroy(iser_state->iser_wr_cache);
11830e7468fSPeter Dunlap (void) ibt_detach(iser_state->is_ibhdl);
11930e7468fSPeter Dunlap iser_state->is_ibhdl = NULL;
12030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_init: failed to initialize HCAs");
12130e7468fSPeter Dunlap return (DDI_FAILURE);
12230e7468fSPeter Dunlap }
12330e7468fSPeter Dunlap
12430e7468fSPeter Dunlap /* Target will register iSER as a service with IBTF when required */
12530e7468fSPeter Dunlap
12630e7468fSPeter Dunlap /* Target will bind this service when it comes online */
12730e7468fSPeter Dunlap
12830e7468fSPeter Dunlap return (DDI_SUCCESS);
12930e7468fSPeter Dunlap }
13030e7468fSPeter Dunlap
13130e7468fSPeter Dunlap /*
13230e7468fSPeter Dunlap * iser_ib_fini
13330e7468fSPeter Dunlap *
13430e7468fSPeter Dunlap * This function unbinds and degisters the iSER service from IBTF
13530e7468fSPeter Dunlap */
13630e7468fSPeter Dunlap int
iser_ib_fini(void)13730e7468fSPeter Dunlap iser_ib_fini(void)
13830e7468fSPeter Dunlap {
13930e7468fSPeter Dunlap /* IDM would have already disabled all the services */
14030e7468fSPeter Dunlap
14130e7468fSPeter Dunlap /* Teardown the HCA list and associated resources */
14230e7468fSPeter Dunlap if (iser_ib_fini_hcas() != DDI_SUCCESS)
14330e7468fSPeter Dunlap return (DDI_FAILURE);
14430e7468fSPeter Dunlap
14530e7468fSPeter Dunlap /* Teardown the global work request kmem_cache */
14630e7468fSPeter Dunlap kmem_cache_destroy(iser_state->iser_wr_cache);
14730e7468fSPeter Dunlap
14830e7468fSPeter Dunlap /* Deregister with IBTF */
14930e7468fSPeter Dunlap if (iser_state->is_ibhdl != NULL) {
15030e7468fSPeter Dunlap (void) ibt_detach(iser_state->is_ibhdl);
15130e7468fSPeter Dunlap iser_state->is_ibhdl = NULL;
15230e7468fSPeter Dunlap }
15330e7468fSPeter Dunlap
15430e7468fSPeter Dunlap return (DDI_SUCCESS);
15530e7468fSPeter Dunlap }
15630e7468fSPeter Dunlap
15730e7468fSPeter Dunlap /*
15830e7468fSPeter Dunlap * iser_ib_register_service
15930e7468fSPeter Dunlap *
16030e7468fSPeter Dunlap * This function registers the iSER service using the RDMA-Aware Service ID.
16130e7468fSPeter Dunlap */
16230e7468fSPeter Dunlap int
iser_ib_register_service(idm_svc_t * idm_svc)16330e7468fSPeter Dunlap iser_ib_register_service(idm_svc_t *idm_svc)
16430e7468fSPeter Dunlap {
16530e7468fSPeter Dunlap ibt_srv_desc_t srvdesc;
16630e7468fSPeter Dunlap iser_svc_t *iser_svc;
16730e7468fSPeter Dunlap int status;
16830e7468fSPeter Dunlap
16930e7468fSPeter Dunlap bzero(&srvdesc, sizeof (ibt_srv_desc_t));
17030e7468fSPeter Dunlap
17130e7468fSPeter Dunlap /* Set up IBTI client callback handler from the CM */
17230e7468fSPeter Dunlap srvdesc.sd_handler = iser_ib_cm_handler;
17330e7468fSPeter Dunlap
17430e7468fSPeter Dunlap srvdesc.sd_flags = IBT_SRV_NO_FLAGS;
17530e7468fSPeter Dunlap
17630e7468fSPeter Dunlap iser_svc = (iser_svc_t *)idm_svc->is_iser_svc;
17730e7468fSPeter Dunlap
17830e7468fSPeter Dunlap /* Register the service on the specified port */
17930e7468fSPeter Dunlap status = ibt_register_service(
18030e7468fSPeter Dunlap iser_state->is_ibhdl, &srvdesc,
18130e7468fSPeter Dunlap iser_svc->is_svcid, 1, &iser_svc->is_srvhdl, NULL);
18230e7468fSPeter Dunlap
18330e7468fSPeter Dunlap return (status);
18430e7468fSPeter Dunlap }
18530e7468fSPeter Dunlap
18630e7468fSPeter Dunlap /*
18730e7468fSPeter Dunlap * iser_ib_bind_service
18830e7468fSPeter Dunlap *
18932c66a4dSPriya Krishnan * This function binds a given iSER service on all available HCA ports. The
19032c66a4dSPriya Krishnan * current specification does not allow user to specify transport bindings
19132c66a4dSPriya Krishnan * for each iscsi target. The ULP invokes this function to bind the target
19232c66a4dSPriya Krishnan * to all available iser ports after checking for the presence of an IB HCA.
19332c66a4dSPriya Krishnan * iSER is "configured" whenever an IB-capable IP address exists. The lack
19432c66a4dSPriya Krishnan * of active IB ports is a less-fatal condition, and sockets would be used
19532c66a4dSPriya Krishnan * as the transport even though an Infiniband HCA is configured but unusable.
19632c66a4dSPriya Krishnan *
19730e7468fSPeter Dunlap */
19830e7468fSPeter Dunlap int
iser_ib_bind_service(idm_svc_t * idm_svc)19930e7468fSPeter Dunlap iser_ib_bind_service(idm_svc_t *idm_svc)
20030e7468fSPeter Dunlap {
20130e7468fSPeter Dunlap iser_hca_t *hca;
20230e7468fSPeter Dunlap ib_gid_t gid;
20330e7468fSPeter Dunlap int num_ports = 0;
20430e7468fSPeter Dunlap int num_binds = 0;
20532c66a4dSPriya Krishnan int num_inactive_binds = 0; /* if HCA ports inactive */
20630e7468fSPeter Dunlap int status;
20730e7468fSPeter Dunlap int i;
20830e7468fSPeter Dunlap
20930e7468fSPeter Dunlap ASSERT(idm_svc != NULL);
21030e7468fSPeter Dunlap ASSERT(idm_svc->is_iser_svc != NULL);
21130e7468fSPeter Dunlap
21230e7468fSPeter Dunlap /* Register the iSER service on all available ports */
21330e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
21430e7468fSPeter Dunlap
21530e7468fSPeter Dunlap for (hca = list_head(&iser_state->is_hcalist);
21630e7468fSPeter Dunlap hca != NULL;
21730e7468fSPeter Dunlap hca = list_next(&iser_state->is_hcalist, hca)) {
21830e7468fSPeter Dunlap
21930e7468fSPeter Dunlap for (i = 0; i < hca->hca_num_ports; i++) {
22030e7468fSPeter Dunlap num_ports++;
22130e7468fSPeter Dunlap if (hca->hca_port_info[i].p_linkstate !=
22230e7468fSPeter Dunlap IBT_PORT_ACTIVE) {
22330e7468fSPeter Dunlap /*
22430e7468fSPeter Dunlap * Move on. We will attempt to bind service
22530e7468fSPeter Dunlap * in our async handler if the port comes up
22630e7468fSPeter Dunlap * at a later time.
22730e7468fSPeter Dunlap */
22832c66a4dSPriya Krishnan num_inactive_binds++;
22930e7468fSPeter Dunlap continue;
23030e7468fSPeter Dunlap }
23130e7468fSPeter Dunlap
23230e7468fSPeter Dunlap gid = hca->hca_port_info[i].p_sgid_tbl[0];
23330e7468fSPeter Dunlap
23430e7468fSPeter Dunlap /* If the port is already bound, skip */
23530e7468fSPeter Dunlap if (iser_ib_get_bind(
23630e7468fSPeter Dunlap idm_svc->is_iser_svc, hca->hca_guid, gid) == NULL) {
23730e7468fSPeter Dunlap
23830e7468fSPeter Dunlap status = iser_ib_activate_port(
23930e7468fSPeter Dunlap idm_svc, hca->hca_guid, gid);
24030e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
24130e7468fSPeter Dunlap ISER_LOG(CE_NOTE,
24230e7468fSPeter Dunlap "iser_ib_bind_service: "
24330e7468fSPeter Dunlap "iser_ib_activate_port failure "
24430e7468fSPeter Dunlap "(0x%x)", status);
24530e7468fSPeter Dunlap continue;
24630e7468fSPeter Dunlap }
24730e7468fSPeter Dunlap }
24830e7468fSPeter Dunlap num_binds++;
24930e7468fSPeter Dunlap }
25030e7468fSPeter Dunlap }
25130e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
25230e7468fSPeter Dunlap
25330e7468fSPeter Dunlap if (num_binds) {
25430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_bind_service: Service available on "
25530e7468fSPeter Dunlap "(%d) of (%d) ports", num_binds, num_ports);
25630e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
25732c66a4dSPriya Krishnan } else if (num_inactive_binds) {
25832c66a4dSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_bind_service: Could not bind "
25932c66a4dSPriya Krishnan "service, HCA ports are not active.");
26032c66a4dSPriya Krishnan /*
26132c66a4dSPriya Krishnan * still considered success, the async handler will bind
26232c66a4dSPriya Krishnan * the service when the port comes up at a later time
26332c66a4dSPriya Krishnan */
26432c66a4dSPriya Krishnan return (ISER_STATUS_SUCCESS);
26530e7468fSPeter Dunlap } else {
26630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_bind_service: Did not bind service");
26730e7468fSPeter Dunlap return (ISER_STATUS_FAIL);
26830e7468fSPeter Dunlap }
26930e7468fSPeter Dunlap }
27030e7468fSPeter Dunlap
27130e7468fSPeter Dunlap /*
27230e7468fSPeter Dunlap * iser_ib_unbind_service
27330e7468fSPeter Dunlap *
27430e7468fSPeter Dunlap * This function unbinds a given service on a all HCA ports
27530e7468fSPeter Dunlap */
27630e7468fSPeter Dunlap void
iser_ib_unbind_service(idm_svc_t * idm_svc)27730e7468fSPeter Dunlap iser_ib_unbind_service(idm_svc_t *idm_svc)
27830e7468fSPeter Dunlap {
27930e7468fSPeter Dunlap iser_svc_t *iser_svc;
28030e7468fSPeter Dunlap iser_sbind_t *is_sbind, *next_sb;
28130e7468fSPeter Dunlap
28230e7468fSPeter Dunlap if (idm_svc != NULL && idm_svc->is_iser_svc != NULL) {
28330e7468fSPeter Dunlap
28430e7468fSPeter Dunlap iser_svc = idm_svc->is_iser_svc;
28530e7468fSPeter Dunlap
28630e7468fSPeter Dunlap for (is_sbind = list_head(&iser_svc->is_sbindlist);
28730e7468fSPeter Dunlap is_sbind != NULL;
28830e7468fSPeter Dunlap is_sbind = next_sb) {
28930e7468fSPeter Dunlap next_sb = list_next(&iser_svc->is_sbindlist, is_sbind);
290aedf2b3bSsrivijitha dugganapalli (void) ibt_unbind_service(iser_svc->is_srvhdl,
29130e7468fSPeter Dunlap is_sbind->is_sbindhdl);
29230e7468fSPeter Dunlap list_remove(&iser_svc->is_sbindlist, is_sbind);
29330e7468fSPeter Dunlap kmem_free(is_sbind, sizeof (iser_sbind_t));
29430e7468fSPeter Dunlap }
29530e7468fSPeter Dunlap }
29630e7468fSPeter Dunlap }
29730e7468fSPeter Dunlap
29830e7468fSPeter Dunlap /* ARGSUSED */
29930e7468fSPeter Dunlap void
iser_ib_deregister_service(idm_svc_t * idm_svc)30030e7468fSPeter Dunlap iser_ib_deregister_service(idm_svc_t *idm_svc)
30130e7468fSPeter Dunlap {
30230e7468fSPeter Dunlap iser_svc_t *iser_svc;
30330e7468fSPeter Dunlap
30430e7468fSPeter Dunlap if (idm_svc != NULL && idm_svc->is_iser_svc != NULL) {
30530e7468fSPeter Dunlap
30630e7468fSPeter Dunlap iser_svc = (iser_svc_t *)idm_svc->is_iser_svc;
307aedf2b3bSsrivijitha dugganapalli (void) ibt_deregister_service(iser_state->is_ibhdl,
30830e7468fSPeter Dunlap iser_svc->is_srvhdl);
309aedf2b3bSsrivijitha dugganapalli (void) ibt_release_ip_sid(iser_svc->is_svcid);
31030e7468fSPeter Dunlap }
31130e7468fSPeter Dunlap }
31230e7468fSPeter Dunlap
31330e7468fSPeter Dunlap /*
31430e7468fSPeter Dunlap * iser_ib_get_paths
31530e7468fSPeter Dunlap * This function finds the IB path between the local and the remote address.
31630e7468fSPeter Dunlap *
31730e7468fSPeter Dunlap */
31830e7468fSPeter Dunlap int
iser_ib_get_paths(ibt_ip_addr_t * local_ip,ibt_ip_addr_t * remote_ip,ibt_path_info_t * path,ibt_path_ip_src_t * path_src_ip)31930e7468fSPeter Dunlap iser_ib_get_paths(ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip,
32030e7468fSPeter Dunlap ibt_path_info_t *path, ibt_path_ip_src_t *path_src_ip)
32130e7468fSPeter Dunlap {
32230e7468fSPeter Dunlap ibt_ip_path_attr_t ipattr;
32330e7468fSPeter Dunlap int status;
32430e7468fSPeter Dunlap
32530e7468fSPeter Dunlap (void) bzero(&ipattr, sizeof (ibt_ip_path_attr_t));
32630e7468fSPeter Dunlap ipattr.ipa_dst_ip = remote_ip;
32730e7468fSPeter Dunlap ipattr.ipa_src_ip = *local_ip;
32830e7468fSPeter Dunlap ipattr.ipa_max_paths = 1;
32930e7468fSPeter Dunlap ipattr.ipa_ndst = 1;
33030e7468fSPeter Dunlap
33130e7468fSPeter Dunlap (void) bzero(path, sizeof (ibt_path_info_t));
33230e7468fSPeter Dunlap status = ibt_get_ip_paths(iser_state->is_ibhdl, IBT_PATH_NO_FLAGS,
33330e7468fSPeter Dunlap &ipattr, path, NULL, path_src_ip);
33430e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
33530e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "ibt_get_ip_paths: ibt_get_ip_paths "
33630e7468fSPeter Dunlap "failure: status (%d)", status);
33730e7468fSPeter Dunlap return (status);
33830e7468fSPeter Dunlap }
33930e7468fSPeter Dunlap
34030e7468fSPeter Dunlap if (local_ip != NULL) {
34130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_get_paths success: IP[%x to %x]",
34230e7468fSPeter Dunlap local_ip->un.ip4addr, remote_ip->un.ip4addr);
34330e7468fSPeter Dunlap } else {
34430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_get_paths success: "
34530e7468fSPeter Dunlap "IP[INADDR_ANY to %x]", remote_ip->un.ip4addr);
34630e7468fSPeter Dunlap }
34730e7468fSPeter Dunlap
34830e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
34930e7468fSPeter Dunlap }
35030e7468fSPeter Dunlap
35130e7468fSPeter Dunlap /*
3521d73e59fSPriya Krishnan * iser_ib_alloc_channel_nopathlookup
35330e7468fSPeter Dunlap *
3541d73e59fSPriya Krishnan * This function allocates a reliable connected channel. This function does
3551d73e59fSPriya Krishnan * not invoke ibt_get_ip_paths() to do the path lookup. The HCA GUID and
3561d73e59fSPriya Krishnan * port are input to this function.
35730e7468fSPeter Dunlap */
35830e7468fSPeter Dunlap iser_chan_t *
iser_ib_alloc_channel_nopathlookup(ib_guid_t hca_guid,uint8_t hca_port)3591d73e59fSPriya Krishnan iser_ib_alloc_channel_nopathlookup(ib_guid_t hca_guid, uint8_t hca_port)
36030e7468fSPeter Dunlap {
3611d73e59fSPriya Krishnan iser_hca_t *hca;
3621d73e59fSPriya Krishnan iser_chan_t *chan;
36330e7468fSPeter Dunlap
3641d73e59fSPriya Krishnan /* Lookup the hca using the gid in the path info */
3651d73e59fSPriya Krishnan hca = iser_ib_guid2hca(hca_guid);
3661d73e59fSPriya Krishnan if (hca == NULL) {
3671d73e59fSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_alloc_channel_nopathlookup: failed "
3681d73e59fSPriya Krishnan "to lookup HCA(%llx) handle", (longlong_t)hca_guid);
3691d73e59fSPriya Krishnan return (NULL);
3701d73e59fSPriya Krishnan }
37130e7468fSPeter Dunlap
3721d73e59fSPriya Krishnan chan = iser_ib_alloc_rc_channel(hca, hca_port);
3731d73e59fSPriya Krishnan if (chan == NULL) {
3741d73e59fSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_alloc_channel_nopathlookup: failed "
3751d73e59fSPriya Krishnan "to alloc channel on HCA(%llx) %d",
3761d73e59fSPriya Krishnan (longlong_t)hca_guid, hca_port);
3771d73e59fSPriya Krishnan return (NULL);
3781d73e59fSPriya Krishnan }
37930e7468fSPeter Dunlap
3801d73e59fSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_alloc_channel_pathlookup success: "
3811d73e59fSPriya Krishnan "chanhdl (0x%p), HCA(%llx) %d",
3821d73e59fSPriya Krishnan (void *)chan->ic_chanhdl, (longlong_t)hca_guid, hca_port);
3831d73e59fSPriya Krishnan
3841d73e59fSPriya Krishnan return (chan);
3851d73e59fSPriya Krishnan }
3861d73e59fSPriya Krishnan
3871d73e59fSPriya Krishnan /*
3881d73e59fSPriya Krishnan * iser_ib_alloc_channel_pathlookup
3891d73e59fSPriya Krishnan *
3901d73e59fSPriya Krishnan * This function allocates a reliable connected channel but first invokes
3911d73e59fSPriya Krishnan * ibt_get_ip_paths() with the given local and remote addres to get the
3921d73e59fSPriya Krishnan * HCA lgid and the port number.
3931d73e59fSPriya Krishnan */
3941d73e59fSPriya Krishnan iser_chan_t *
iser_ib_alloc_channel_pathlookup(ibt_ip_addr_t * local_ip,ibt_ip_addr_t * remote_ip)3951d73e59fSPriya Krishnan iser_ib_alloc_channel_pathlookup(
3961d73e59fSPriya Krishnan ibt_ip_addr_t *local_ip, ibt_ip_addr_t *remote_ip)
3971d73e59fSPriya Krishnan {
3981d73e59fSPriya Krishnan ibt_path_info_t ibt_path;
3991d73e59fSPriya Krishnan ibt_path_ip_src_t path_src_ip;
4001d73e59fSPriya Krishnan ib_gid_t lgid;
4011d73e59fSPriya Krishnan uint8_t hca_port; /* from path */
4021d73e59fSPriya Krishnan iser_hca_t *hca;
4031d73e59fSPriya Krishnan iser_chan_t *chan;
4041d73e59fSPriya Krishnan int status;
40530e7468fSPeter Dunlap
40630e7468fSPeter Dunlap /* Lookup a path to the given destination */
4071d73e59fSPriya Krishnan status = iser_ib_get_paths(
4081d73e59fSPriya Krishnan local_ip, remote_ip, &ibt_path, &path_src_ip);
40930e7468fSPeter Dunlap
41030e7468fSPeter Dunlap if (status != ISER_STATUS_SUCCESS) {
4111d73e59fSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_alloc_channel_pathlookup: faild "
4121d73e59fSPriya Krishnan "Path lookup IP:[%llx to %llx] failed: status (%d)",
4131d73e59fSPriya Krishnan (longlong_t)local_ip->un.ip4addr,
4141d73e59fSPriya Krishnan (longlong_t)remote_ip->un.ip4addr,
41530e7468fSPeter Dunlap status);
41630e7468fSPeter Dunlap return (NULL);
41730e7468fSPeter Dunlap }
41830e7468fSPeter Dunlap
41930e7468fSPeter Dunlap /* get the local gid from the path info */
4201d73e59fSPriya Krishnan lgid = ibt_path.pi_prim_cep_path.cep_adds_vect.av_sgid;
42130e7468fSPeter Dunlap
42230e7468fSPeter Dunlap /* get the hca port from the path info */
4231d73e59fSPriya Krishnan hca_port = ibt_path.pi_prim_cep_path.cep_hca_port_num;
42430e7468fSPeter Dunlap
42530e7468fSPeter Dunlap /* Lookup the hca using the gid in the path info */
42630e7468fSPeter Dunlap hca = iser_ib_gid2hca(lgid);
42730e7468fSPeter Dunlap if (hca == NULL) {
4281d73e59fSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_alloc_channel_pathlookup: failed "
4291d73e59fSPriya Krishnan "to lookup HCA (%llx) handle",
4301d73e59fSPriya Krishnan (longlong_t)hca->hca_guid);
43130e7468fSPeter Dunlap return (NULL);
43230e7468fSPeter Dunlap }
43330e7468fSPeter Dunlap
4341d73e59fSPriya Krishnan chan = iser_ib_alloc_rc_channel(hca, hca_port);
4351d73e59fSPriya Krishnan if (chan == NULL) {
4361d73e59fSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_alloc_channel_pathlookup: failed "
4371d73e59fSPriya Krishnan "to alloc channel from IP:[%llx to %llx] on HCA (%llx) %d",
4381d73e59fSPriya Krishnan (longlong_t)local_ip->un.ip4addr,
4391d73e59fSPriya Krishnan (longlong_t)remote_ip->un.ip4addr,
4401d73e59fSPriya Krishnan (longlong_t)hca->hca_guid, hca_port);
4411d73e59fSPriya Krishnan return (NULL);
4421d73e59fSPriya Krishnan }
4431d73e59fSPriya Krishnan
4441d73e59fSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_alloc_channel_pathlookup success: "
4451d73e59fSPriya Krishnan "chanhdl (0x%p), IP:[%llx to %llx], lgid (%llx:%llx), HCA(%llx) %d",
4461d73e59fSPriya Krishnan (void *)chan->ic_chanhdl,
4471d73e59fSPriya Krishnan (longlong_t)local_ip->un.ip4addr,
4481d73e59fSPriya Krishnan (longlong_t)remote_ip->un.ip4addr,
4491d73e59fSPriya Krishnan (longlong_t)lgid.gid_prefix, (longlong_t)lgid.gid_guid,
4501d73e59fSPriya Krishnan (longlong_t)hca->hca_guid, hca_port);
4511d73e59fSPriya Krishnan
4521d73e59fSPriya Krishnan chan->ic_ibt_path = ibt_path;
45330e7468fSPeter Dunlap chan->ic_localip = path_src_ip.ip_primary;
45430e7468fSPeter Dunlap chan->ic_remoteip = *remote_ip;
45530e7468fSPeter Dunlap
4561d73e59fSPriya Krishnan return (chan);
4571d73e59fSPriya Krishnan }
4581d73e59fSPriya Krishnan
4591d73e59fSPriya Krishnan /*
4601d73e59fSPriya Krishnan * iser_ib_alloc_rc_channel
4611d73e59fSPriya Krishnan *
4621d73e59fSPriya Krishnan * This function allocates a reliable communication channel using the specified
4631d73e59fSPriya Krishnan * channel attributes.
4641d73e59fSPriya Krishnan */
4651d73e59fSPriya Krishnan iser_chan_t *
iser_ib_alloc_rc_channel(iser_hca_t * hca,uint8_t hca_port)4661d73e59fSPriya Krishnan iser_ib_alloc_rc_channel(iser_hca_t *hca, uint8_t hca_port)
4671d73e59fSPriya Krishnan {
4681d73e59fSPriya Krishnan
4691d73e59fSPriya Krishnan iser_chan_t *chan;
4701d73e59fSPriya Krishnan ibt_rc_chan_alloc_args_t chanargs;
4711d73e59fSPriya Krishnan uint_t sq_size, rq_size;
4721d73e59fSPriya Krishnan int status;
4731d73e59fSPriya Krishnan
4741d73e59fSPriya Krishnan chan = kmem_zalloc(sizeof (iser_chan_t), KM_SLEEP);
4751d73e59fSPriya Krishnan
47692adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_init(&chan->ic_chan_lock, NULL, MUTEX_DRIVER, NULL);
4771d73e59fSPriya Krishnan mutex_init(&chan->ic_sq_post_lock, NULL, MUTEX_DRIVER, NULL);
4781d73e59fSPriya Krishnan
4791d73e59fSPriya Krishnan /* Set up the iSER channel handle with HCA */
4801d73e59fSPriya Krishnan chan->ic_hca = hca;
4811d73e59fSPriya Krishnan
48230e7468fSPeter Dunlap /*
48330e7468fSPeter Dunlap * Determine the queue sizes, based upon the HCA query data.
48430e7468fSPeter Dunlap * For our Work Queues, we will use either our default value,
48530e7468fSPeter Dunlap * or the HCA's maximum value, whichever is smaller.
48630e7468fSPeter Dunlap */
48730e7468fSPeter Dunlap sq_size = min(hca->hca_attr.hca_max_chan_sz, ISER_IB_SENDQ_SIZE);
48830e7468fSPeter Dunlap rq_size = min(hca->hca_attr.hca_max_chan_sz, ISER_IB_RECVQ_SIZE);
48930e7468fSPeter Dunlap
49030e7468fSPeter Dunlap /*
49130e7468fSPeter Dunlap * For our Completion Queues, we again check the device maximum.
49230e7468fSPeter Dunlap * We want to end up with CQs that are the next size up from the
49330e7468fSPeter Dunlap * WQs they are servicing so that they have some overhead.
49430e7468fSPeter Dunlap */
49530e7468fSPeter Dunlap if (hca->hca_attr.hca_max_cq_sz >= (sq_size + 1)) {
49630e7468fSPeter Dunlap chan->ic_sendcq_sz = sq_size + 1;
49730e7468fSPeter Dunlap } else {
49830e7468fSPeter Dunlap chan->ic_sendcq_sz = hca->hca_attr.hca_max_cq_sz;
49930e7468fSPeter Dunlap sq_size = chan->ic_sendcq_sz - 1;
50030e7468fSPeter Dunlap }
50130e7468fSPeter Dunlap
50230e7468fSPeter Dunlap if (hca->hca_attr.hca_max_cq_sz >= (rq_size + 1)) {
50330e7468fSPeter Dunlap chan->ic_recvcq_sz = rq_size + 1;
50430e7468fSPeter Dunlap } else {
50530e7468fSPeter Dunlap chan->ic_recvcq_sz = hca->hca_attr.hca_max_cq_sz;
50630e7468fSPeter Dunlap rq_size = chan->ic_recvcq_sz - 1;
50730e7468fSPeter Dunlap }
50830e7468fSPeter Dunlap
50930e7468fSPeter Dunlap /* Initialize the iSER channel's QP handle */
51030e7468fSPeter Dunlap iser_ib_init_qp(chan, sq_size, rq_size);
51130e7468fSPeter Dunlap
51230e7468fSPeter Dunlap /* Set up the Send Completion Queue */
51330e7468fSPeter Dunlap status = iser_ib_setup_cq(hca->hca_hdl, chan->ic_sendcq_sz,
51430e7468fSPeter Dunlap &chan->ic_sendcq);
51530e7468fSPeter Dunlap if (status != ISER_STATUS_SUCCESS) {
51630e7468fSPeter Dunlap iser_ib_fini_qp(&chan->ic_qp);
51792adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_destroy(&chan->ic_chan_lock);
51830e7468fSPeter Dunlap mutex_destroy(&chan->ic_sq_post_lock);
51930e7468fSPeter Dunlap kmem_free(chan, sizeof (iser_chan_t));
52030e7468fSPeter Dunlap return (NULL);
52130e7468fSPeter Dunlap }
52230e7468fSPeter Dunlap ibt_set_cq_handler(chan->ic_sendcq, iser_ib_sendcq_handler, chan);
523aedf2b3bSsrivijitha dugganapalli (void) ibt_enable_cq_notify(chan->ic_sendcq, IBT_NEXT_COMPLETION);
52430e7468fSPeter Dunlap
52530e7468fSPeter Dunlap /* Set up the Receive Completion Queue */
52630e7468fSPeter Dunlap status = iser_ib_setup_cq(hca->hca_hdl, chan->ic_recvcq_sz,
52730e7468fSPeter Dunlap &chan->ic_recvcq);
52830e7468fSPeter Dunlap if (status != ISER_STATUS_SUCCESS) {
52930e7468fSPeter Dunlap (void) ibt_free_cq(chan->ic_sendcq);
53030e7468fSPeter Dunlap iser_ib_fini_qp(&chan->ic_qp);
53192adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_destroy(&chan->ic_chan_lock);
53230e7468fSPeter Dunlap mutex_destroy(&chan->ic_sq_post_lock);
53330e7468fSPeter Dunlap kmem_free(chan, sizeof (iser_chan_t));
53430e7468fSPeter Dunlap return (NULL);
53530e7468fSPeter Dunlap }
53630e7468fSPeter Dunlap ibt_set_cq_handler(chan->ic_recvcq, iser_ib_recvcq_handler, chan);
537aedf2b3bSsrivijitha dugganapalli (void) ibt_enable_cq_notify(chan->ic_recvcq, IBT_NEXT_COMPLETION);
53830e7468fSPeter Dunlap
53930e7468fSPeter Dunlap /* Setup the channel arguments */
54030e7468fSPeter Dunlap iser_ib_setup_chanargs(hca_port, chan->ic_sendcq, chan->ic_recvcq,
54130e7468fSPeter Dunlap sq_size, rq_size, hca->hca_pdhdl, &chanargs);
54230e7468fSPeter Dunlap
54330e7468fSPeter Dunlap status = ibt_alloc_rc_channel(hca->hca_hdl,
54430e7468fSPeter Dunlap IBT_ACHAN_NO_FLAGS, &chanargs, &chan->ic_chanhdl, NULL);
54530e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
54630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_alloc_rc_channel: failed "
54730e7468fSPeter Dunlap "ibt_alloc_rc_channel: status (%d)", status);
54830e7468fSPeter Dunlap (void) ibt_free_cq(chan->ic_sendcq);
54930e7468fSPeter Dunlap (void) ibt_free_cq(chan->ic_recvcq);
55030e7468fSPeter Dunlap iser_ib_fini_qp(&chan->ic_qp);
55192adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_destroy(&chan->ic_chan_lock);
55230e7468fSPeter Dunlap mutex_destroy(&chan->ic_sq_post_lock);
55330e7468fSPeter Dunlap kmem_free(chan, sizeof (iser_chan_t));
55430e7468fSPeter Dunlap return (NULL);
55530e7468fSPeter Dunlap }
55630e7468fSPeter Dunlap
55730e7468fSPeter Dunlap /* Set the 'channel' as the client private data */
55830e7468fSPeter Dunlap (void) ibt_set_chan_private(chan->ic_chanhdl, chan);
55930e7468fSPeter Dunlap
56030e7468fSPeter Dunlap return (chan);
56130e7468fSPeter Dunlap }
56230e7468fSPeter Dunlap
56330e7468fSPeter Dunlap /*
56430e7468fSPeter Dunlap * iser_ib_open_rc_channel
56530e7468fSPeter Dunlap * This function opens a RC connection on the given allocated RC channel
56630e7468fSPeter Dunlap */
56730e7468fSPeter Dunlap int
iser_ib_open_rc_channel(iser_chan_t * chan)56830e7468fSPeter Dunlap iser_ib_open_rc_channel(iser_chan_t *chan)
56930e7468fSPeter Dunlap {
57030e7468fSPeter Dunlap ibt_ip_cm_info_t ipcm_info;
57130e7468fSPeter Dunlap iser_private_data_t iser_priv_data;
57230e7468fSPeter Dunlap ibt_chan_open_args_t ocargs;
57330e7468fSPeter Dunlap ibt_rc_returns_t ocreturns;
57430e7468fSPeter Dunlap int status;
57530e7468fSPeter Dunlap
57692adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_enter(&chan->ic_chan_lock);
57730e7468fSPeter Dunlap
57830e7468fSPeter Dunlap /*
57930e7468fSPeter Dunlap * For connection establishment, the initiator sends a CM REQ using the
58030e7468fSPeter Dunlap * iSER RDMA-Aware Service ID. Included are the source and destination
58130e7468fSPeter Dunlap * IP addresses, and the src port.
58230e7468fSPeter Dunlap */
58330e7468fSPeter Dunlap bzero(&ipcm_info, sizeof (ibt_ip_cm_info_t));
58430e7468fSPeter Dunlap ipcm_info.src_addr = chan->ic_localip;
58530e7468fSPeter Dunlap ipcm_info.dst_addr = chan->ic_remoteip;
58630e7468fSPeter Dunlap ipcm_info.src_port = chan->ic_lport;
58730e7468fSPeter Dunlap
58830e7468fSPeter Dunlap /*
58930e7468fSPeter Dunlap * The CM Private Data field defines the iSER connection parameters
59030e7468fSPeter Dunlap * such as zero based virtual address exception (ZBVAE) and Send with
59130e7468fSPeter Dunlap * invalidate Exception (SIE).
59230e7468fSPeter Dunlap *
59330e7468fSPeter Dunlap * Solaris IBT does not currently support ZBVAE or SIE.
59430e7468fSPeter Dunlap */
59530e7468fSPeter Dunlap iser_priv_data.rsvd1 = 0;
59630e7468fSPeter Dunlap iser_priv_data.sie = 1;
59730e7468fSPeter Dunlap iser_priv_data.zbvae = 1;
59830e7468fSPeter Dunlap
59930e7468fSPeter Dunlap status = ibt_format_ip_private_data(&ipcm_info,
60030e7468fSPeter Dunlap sizeof (iser_private_data_t), &iser_priv_data);
60130e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
60230e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_open_rc_channel failed: %d", status);
60392adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_exit(&chan->ic_chan_lock);
60430e7468fSPeter Dunlap return (status);
60530e7468fSPeter Dunlap }
60630e7468fSPeter Dunlap
60730e7468fSPeter Dunlap /*
60830e7468fSPeter Dunlap * Set the SID we are attempting to connect to, based upon the
60930e7468fSPeter Dunlap * remote port number.
61030e7468fSPeter Dunlap */
61130e7468fSPeter Dunlap chan->ic_ibt_path.pi_sid = ibt_get_ip_sid(IPPROTO_TCP, chan->ic_rport);
61230e7468fSPeter Dunlap
61330e7468fSPeter Dunlap /* Set up the args for the channel open */
61430e7468fSPeter Dunlap bzero(&ocargs, sizeof (ibt_chan_open_args_t));
61530e7468fSPeter Dunlap ocargs.oc_path = &chan->ic_ibt_path;
61630e7468fSPeter Dunlap ocargs.oc_cm_handler = iser_ib_cm_handler;
61730e7468fSPeter Dunlap ocargs.oc_cm_clnt_private = iser_state;
61830e7468fSPeter Dunlap ocargs.oc_rdma_ra_out = 4;
61930e7468fSPeter Dunlap ocargs.oc_rdma_ra_in = 4;
62030e7468fSPeter Dunlap ocargs.oc_path_retry_cnt = 2;
62130e7468fSPeter Dunlap ocargs.oc_path_rnr_retry_cnt = 2;
62230e7468fSPeter Dunlap ocargs.oc_priv_data_len = sizeof (iser_private_data_t);
62330e7468fSPeter Dunlap ocargs.oc_priv_data = &iser_priv_data;
62430e7468fSPeter Dunlap
62530e7468fSPeter Dunlap bzero(&ocreturns, sizeof (ibt_rc_returns_t));
62630e7468fSPeter Dunlap
62730e7468fSPeter Dunlap status = ibt_open_rc_channel(chan->ic_chanhdl,
62830e7468fSPeter Dunlap IBT_OCHAN_NO_FLAGS, IBT_BLOCKING, &ocargs, &ocreturns);
62930e7468fSPeter Dunlap
63030e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
63130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_open_rc_channel failed: %d", status);
63292adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_exit(&chan->ic_chan_lock);
63330e7468fSPeter Dunlap return (status);
63430e7468fSPeter Dunlap }
63530e7468fSPeter Dunlap
63692adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_exit(&chan->ic_chan_lock);
63730e7468fSPeter Dunlap return (IDM_STATUS_SUCCESS);
63830e7468fSPeter Dunlap }
63930e7468fSPeter Dunlap
64030e7468fSPeter Dunlap /*
64130e7468fSPeter Dunlap * iser_ib_close_rc_channel
64230e7468fSPeter Dunlap * This function closes the RC channel related to this iser_chan handle.
64330e7468fSPeter Dunlap * We invoke this in a non-blocking, no callbacks context.
64430e7468fSPeter Dunlap */
64530e7468fSPeter Dunlap void
iser_ib_close_rc_channel(iser_chan_t * chan)64630e7468fSPeter Dunlap iser_ib_close_rc_channel(iser_chan_t *chan)
64730e7468fSPeter Dunlap {
64830e7468fSPeter Dunlap int status;
64930e7468fSPeter Dunlap
65092adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_enter(&chan->ic_chan_lock);
65130e7468fSPeter Dunlap status = ibt_close_rc_channel(chan->ic_chanhdl, IBT_BLOCKING, NULL,
65230e7468fSPeter Dunlap 0, NULL, NULL, 0);
65330e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
65430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_close_rc_channel: "
65530e7468fSPeter Dunlap "ibt_close_rc_channel failed: status (%d)", status);
65630e7468fSPeter Dunlap }
65792adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_exit(&chan->ic_chan_lock);
65830e7468fSPeter Dunlap }
65930e7468fSPeter Dunlap
66030e7468fSPeter Dunlap /*
66130e7468fSPeter Dunlap * iser_ib_free_rc_channel
66230e7468fSPeter Dunlap *
66330e7468fSPeter Dunlap * This function tears down an RC channel's QP initialization and frees it.
66430e7468fSPeter Dunlap * Note that we do not need synchronization here; the channel has been
66530e7468fSPeter Dunlap * closed already, so we should only have completion polling occuring. Once
66630e7468fSPeter Dunlap * complete, we are free to free the IBTF channel, WQ and CQ resources, and
66730e7468fSPeter Dunlap * our own related resources.
66830e7468fSPeter Dunlap */
66930e7468fSPeter Dunlap void
iser_ib_free_rc_channel(iser_chan_t * chan)67030e7468fSPeter Dunlap iser_ib_free_rc_channel(iser_chan_t *chan)
67130e7468fSPeter Dunlap {
67230e7468fSPeter Dunlap iser_qp_t *iser_qp;
67330e7468fSPeter Dunlap
67430e7468fSPeter Dunlap iser_qp = &chan->ic_qp;
67530e7468fSPeter Dunlap
67630e7468fSPeter Dunlap /* Ensure the SQ is empty */
67730e7468fSPeter Dunlap while (chan->ic_sq_post_count != 0) {
67830e7468fSPeter Dunlap mutex_exit(&chan->ic_conn->ic_lock);
67930e7468fSPeter Dunlap delay(drv_usectohz(ISER_DELAY_HALF_SECOND));
68030e7468fSPeter Dunlap mutex_enter(&chan->ic_conn->ic_lock);
68130e7468fSPeter Dunlap }
68230e7468fSPeter Dunlap mutex_destroy(&chan->ic_sq_post_lock);
68330e7468fSPeter Dunlap
68430e7468fSPeter Dunlap /* Ensure the RQ is empty */
68530e7468fSPeter Dunlap (void) ibt_flush_channel(chan->ic_chanhdl);
68630e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
68730e7468fSPeter Dunlap while (iser_qp->rq_level != 0) {
68830e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
68930e7468fSPeter Dunlap mutex_exit(&chan->ic_conn->ic_lock);
69030e7468fSPeter Dunlap delay(drv_usectohz(ISER_DELAY_HALF_SECOND));
69130e7468fSPeter Dunlap mutex_enter(&chan->ic_conn->ic_lock);
69230e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
69330e7468fSPeter Dunlap }
69430e7468fSPeter Dunlap
69530e7468fSPeter Dunlap /* Free our QP handle */
69630e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
69730e7468fSPeter Dunlap (void) iser_ib_fini_qp(iser_qp);
69830e7468fSPeter Dunlap
69930e7468fSPeter Dunlap /* Free the IBT channel resources */
70030e7468fSPeter Dunlap (void) ibt_free_channel(chan->ic_chanhdl);
70130e7468fSPeter Dunlap chan->ic_chanhdl = NULL;
70230e7468fSPeter Dunlap
70330e7468fSPeter Dunlap /* Free the CQs */
704aedf2b3bSsrivijitha dugganapalli (void) ibt_free_cq(chan->ic_sendcq);
705aedf2b3bSsrivijitha dugganapalli (void) ibt_free_cq(chan->ic_recvcq);
70630e7468fSPeter Dunlap
70730e7468fSPeter Dunlap /* Free the chan handle */
70892adbba7SPeter Cudhea - Sun Microsystems - Burlington, MA United States mutex_destroy(&chan->ic_chan_lock);
70930e7468fSPeter Dunlap kmem_free(chan, sizeof (iser_chan_t));
71030e7468fSPeter Dunlap }
71130e7468fSPeter Dunlap
71230e7468fSPeter Dunlap /*
71330e7468fSPeter Dunlap * iser_ib_post_recv
71430e7468fSPeter Dunlap *
71530e7468fSPeter Dunlap * This function handles keeping the RQ full on a given channel.
71630e7468fSPeter Dunlap * This routine will mostly be run on a taskq, and will check the
71730e7468fSPeter Dunlap * current fill level of the RQ, and post as many WRs as necessary
71830e7468fSPeter Dunlap * to fill it again.
71930e7468fSPeter Dunlap */
720cf8c0ebaSPeter Dunlap
721cf8c0ebaSPeter Dunlap int
iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl)722cf8c0ebaSPeter Dunlap iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl)
723cf8c0ebaSPeter Dunlap {
724cf8c0ebaSPeter Dunlap iser_chan_t *chan;
725cf8c0ebaSPeter Dunlap int status;
726cf8c0ebaSPeter Dunlap
727cf8c0ebaSPeter Dunlap /* Pull our iSER channel handle from the private data */
728cf8c0ebaSPeter Dunlap chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
729cf8c0ebaSPeter Dunlap
73072cf3143Speter dunlap /*
73172cf3143Speter dunlap * Caller must check that chan->ic_conn->ic_stage indicates
73272cf3143Speter dunlap * the connection is active (not closing, not closed) and
73372cf3143Speter dunlap * it must hold the mutex cross the check and the call to this function
73472cf3143Speter dunlap */
73572cf3143Speter dunlap ASSERT(mutex_owned(&chan->ic_conn->ic_lock));
736*2360e12dSPeter Gill ASSERT((chan->ic_conn->ic_stage >= ISER_CONN_STAGE_ALLOCATED) &&
73772cf3143Speter dunlap (chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN));
738cf8c0ebaSPeter Dunlap idm_conn_hold(chan->ic_conn->ic_idmc);
739cf8c0ebaSPeter Dunlap status = ddi_taskq_dispatch(iser_taskq, iser_ib_post_recv_task,
740cf8c0ebaSPeter Dunlap (void *)chanhdl, DDI_NOSLEEP);
741cf8c0ebaSPeter Dunlap if (status != DDI_SUCCESS) {
742cf8c0ebaSPeter Dunlap idm_conn_rele(chan->ic_conn->ic_idmc);
743cf8c0ebaSPeter Dunlap }
744cf8c0ebaSPeter Dunlap
745cf8c0ebaSPeter Dunlap return (status);
746cf8c0ebaSPeter Dunlap }
747cf8c0ebaSPeter Dunlap
748cf8c0ebaSPeter Dunlap static void
iser_ib_post_recv_task(void * arg)749cf8c0ebaSPeter Dunlap iser_ib_post_recv_task(void *arg)
750cf8c0ebaSPeter Dunlap {
751cf8c0ebaSPeter Dunlap ibt_channel_hdl_t chanhdl = arg;
752cf8c0ebaSPeter Dunlap iser_chan_t *chan;
753cf8c0ebaSPeter Dunlap
754cf8c0ebaSPeter Dunlap /* Pull our iSER channel handle from the private data */
755cf8c0ebaSPeter Dunlap chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
756cf8c0ebaSPeter Dunlap
757cf8c0ebaSPeter Dunlap iser_ib_post_recv(chanhdl);
758cf8c0ebaSPeter Dunlap idm_conn_rele(chan->ic_conn->ic_idmc);
759cf8c0ebaSPeter Dunlap }
760cf8c0ebaSPeter Dunlap
76130e7468fSPeter Dunlap void
iser_ib_post_recv(ibt_channel_hdl_t chanhdl)762cf8c0ebaSPeter Dunlap iser_ib_post_recv(ibt_channel_hdl_t chanhdl)
76330e7468fSPeter Dunlap {
76430e7468fSPeter Dunlap iser_chan_t *chan;
76530e7468fSPeter Dunlap iser_hca_t *hca;
76630e7468fSPeter Dunlap iser_msg_t *msg;
76730e7468fSPeter Dunlap ibt_recv_wr_t *wrlist, wr[ISER_IB_RQ_POST_MAX];
76830e7468fSPeter Dunlap int rq_space, msg_ret;
76930e7468fSPeter Dunlap int total_num, npost;
77030e7468fSPeter Dunlap uint_t nposted;
77130e7468fSPeter Dunlap int status, i;
77230e7468fSPeter Dunlap iser_qp_t *iser_qp;
77330e7468fSPeter Dunlap
77430e7468fSPeter Dunlap /* Pull our iSER channel handle from the private data */
77530e7468fSPeter Dunlap chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
77630e7468fSPeter Dunlap
777cf8c0ebaSPeter Dunlap ASSERT(chan != NULL);
778cf8c0ebaSPeter Dunlap
77930e7468fSPeter Dunlap mutex_enter(&chan->ic_conn->ic_lock);
78030e7468fSPeter Dunlap
78130e7468fSPeter Dunlap /* Bail out if the connection is closed; no need for more recv WRs */
78230e7468fSPeter Dunlap if ((chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSING) ||
78330e7468fSPeter Dunlap (chan->ic_conn->ic_stage == ISER_CONN_STAGE_CLOSED)) {
78430e7468fSPeter Dunlap mutex_exit(&chan->ic_conn->ic_lock);
78530e7468fSPeter Dunlap return;
78630e7468fSPeter Dunlap }
78730e7468fSPeter Dunlap
78830e7468fSPeter Dunlap /* get the QP handle from the iser_chan */
78930e7468fSPeter Dunlap iser_qp = &chan->ic_qp;
79030e7468fSPeter Dunlap
7911d73e59fSPriya Krishnan hca = chan->ic_hca;
79230e7468fSPeter Dunlap
79330e7468fSPeter Dunlap if (hca == NULL) {
79430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_post_recv: unable to retrieve "
79530e7468fSPeter Dunlap "HCA handle");
79630e7468fSPeter Dunlap mutex_exit(&chan->ic_conn->ic_lock);
79730e7468fSPeter Dunlap return;
79830e7468fSPeter Dunlap }
79930e7468fSPeter Dunlap
80030e7468fSPeter Dunlap /* check for space to post on the RQ */
80130e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
80230e7468fSPeter Dunlap rq_space = iser_qp->rq_depth - iser_qp->rq_level;
80330e7468fSPeter Dunlap if (rq_space == 0) {
80430e7468fSPeter Dunlap /* The RQ is full, clear the pending flag and return */
80530e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_FALSE;
80630e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
80730e7468fSPeter Dunlap mutex_exit(&chan->ic_conn->ic_lock);
80830e7468fSPeter Dunlap return;
80930e7468fSPeter Dunlap }
81030e7468fSPeter Dunlap
81130e7468fSPeter Dunlap /* Keep track of the lowest value for rq_min_post_level */
81230e7468fSPeter Dunlap if (iser_qp->rq_level < iser_qp->rq_min_post_level)
81330e7468fSPeter Dunlap iser_qp->rq_min_post_level = iser_qp->rq_level;
81430e7468fSPeter Dunlap
81530e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
81630e7468fSPeter Dunlap
81730e7468fSPeter Dunlap /* we've room to post, so pull from the msg cache */
81830e7468fSPeter Dunlap msg = iser_msg_get(hca, rq_space, &msg_ret);
81930e7468fSPeter Dunlap if (msg == NULL) {
82030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_post_recv: no message handles "
82130e7468fSPeter Dunlap "available in msg cache currently");
82230e7468fSPeter Dunlap /*
82330e7468fSPeter Dunlap * There are no messages on the cache. Wait a half-
82430e7468fSPeter Dunlap * second, then try again.
82530e7468fSPeter Dunlap */
82630e7468fSPeter Dunlap delay(drv_usectohz(ISER_DELAY_HALF_SECOND));
827cf8c0ebaSPeter Dunlap status = iser_ib_post_recv_async(chanhdl);
82830e7468fSPeter Dunlap if (status != DDI_SUCCESS) {
82930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_post_recv: failed to "
83030e7468fSPeter Dunlap "redispatch routine");
83130e7468fSPeter Dunlap /* Failed to dispatch, clear pending flag */
83230e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
83330e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_FALSE;
83430e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
83530e7468fSPeter Dunlap }
83630e7468fSPeter Dunlap mutex_exit(&chan->ic_conn->ic_lock);
83730e7468fSPeter Dunlap return;
83830e7468fSPeter Dunlap }
83930e7468fSPeter Dunlap
84030e7468fSPeter Dunlap if (msg_ret != rq_space) {
84130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_post_recv: requested number of "
84230e7468fSPeter Dunlap "messages not allocated: requested (%d) allocated (%d)",
84330e7468fSPeter Dunlap rq_space, msg_ret);
84430e7468fSPeter Dunlap /* We got some, but not all, of our requested depth */
84530e7468fSPeter Dunlap rq_space = msg_ret;
84630e7468fSPeter Dunlap }
84730e7468fSPeter Dunlap
84830e7468fSPeter Dunlap /*
84930e7468fSPeter Dunlap * Now, walk through the allocated WRs and post them,
85030e7468fSPeter Dunlap * ISER_IB_RQ_POST_MAX (or less) at a time.
85130e7468fSPeter Dunlap */
85230e7468fSPeter Dunlap wrlist = &wr[0];
85330e7468fSPeter Dunlap total_num = rq_space;
85430e7468fSPeter Dunlap
85530e7468fSPeter Dunlap while (total_num) {
85630e7468fSPeter Dunlap /* determine the number to post on this iteration */
85730e7468fSPeter Dunlap npost = (total_num > ISER_IB_RQ_POST_MAX) ?
85830e7468fSPeter Dunlap ISER_IB_RQ_POST_MAX : total_num;
85930e7468fSPeter Dunlap
86030e7468fSPeter Dunlap /* build a list of WRs from the msg list */
86130e7468fSPeter Dunlap for (i = 0; i < npost; i++) {
86230e7468fSPeter Dunlap wrlist[i].wr_id = (ibt_wrid_t)(uintptr_t)msg;
86330e7468fSPeter Dunlap wrlist[i].wr_nds = ISER_IB_SGLIST_SIZE;
86430e7468fSPeter Dunlap wrlist[i].wr_sgl = &msg->msg_ds;
86530e7468fSPeter Dunlap msg = msg->nextp;
86630e7468fSPeter Dunlap }
86730e7468fSPeter Dunlap
86830e7468fSPeter Dunlap /* post the list to the RQ */
86930e7468fSPeter Dunlap nposted = 0;
87030e7468fSPeter Dunlap status = ibt_post_recv(chanhdl, wrlist, npost, &nposted);
87130e7468fSPeter Dunlap if ((status != IBT_SUCCESS) || (nposted != npost)) {
87230e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_post_recv: ibt_post_recv "
87330e7468fSPeter Dunlap "failed: requested (%d) posted (%d) status (%d)",
87430e7468fSPeter Dunlap npost, nposted, status);
87530e7468fSPeter Dunlap total_num -= nposted;
87630e7468fSPeter Dunlap break;
87730e7468fSPeter Dunlap }
87830e7468fSPeter Dunlap
87930e7468fSPeter Dunlap /* decrement total number to post by the number posted */
88030e7468fSPeter Dunlap total_num -= nposted;
88130e7468fSPeter Dunlap }
88230e7468fSPeter Dunlap
88330e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
88430e7468fSPeter Dunlap if (total_num != 0) {
88530e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_post_recv: unable to fill RQ, "
88630e7468fSPeter Dunlap "failed to post (%d) WRs", total_num);
88730e7468fSPeter Dunlap iser_qp->rq_level += rq_space - total_num;
88830e7468fSPeter Dunlap } else {
88930e7468fSPeter Dunlap iser_qp->rq_level += rq_space;
89030e7468fSPeter Dunlap }
89130e7468fSPeter Dunlap
89230e7468fSPeter Dunlap /*
89330e7468fSPeter Dunlap * Now that we've filled the RQ, check that all of the recv WRs
89430e7468fSPeter Dunlap * haven't just been immediately consumed. If so, taskqpending is
89530e7468fSPeter Dunlap * still B_TRUE, so we need to fire off a taskq thread to post
89630e7468fSPeter Dunlap * more WRs.
89730e7468fSPeter Dunlap */
89830e7468fSPeter Dunlap if (iser_qp->rq_level == 0) {
89930e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
900cf8c0ebaSPeter Dunlap status = iser_ib_post_recv_async(chanhdl);
90130e7468fSPeter Dunlap if (status != DDI_SUCCESS) {
90230e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_post_recv: failed to "
90330e7468fSPeter Dunlap "dispatch followup routine");
90430e7468fSPeter Dunlap /* Failed to dispatch, clear pending flag */
90530e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock);
90630e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_FALSE;
90730e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
90830e7468fSPeter Dunlap }
90930e7468fSPeter Dunlap } else {
91030e7468fSPeter Dunlap /*
91130e7468fSPeter Dunlap * We're done, we've filled the RQ. Clear the taskq
91230e7468fSPeter Dunlap * flag so that we can run again.
91330e7468fSPeter Dunlap */
91430e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_FALSE;
91530e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock);
91630e7468fSPeter Dunlap }
91730e7468fSPeter Dunlap
91830e7468fSPeter Dunlap mutex_exit(&chan->ic_conn->ic_lock);
91930e7468fSPeter Dunlap }
92030e7468fSPeter Dunlap
92130e7468fSPeter Dunlap /*
92230e7468fSPeter Dunlap * iser_ib_handle_portup_event()
92330e7468fSPeter Dunlap * This handles the IBT_EVENT_PORT_UP unaffiliated asynchronous event.
92430e7468fSPeter Dunlap *
92530e7468fSPeter Dunlap * To facilitate a seamless bringover of the port and configure the CM service
92630e7468fSPeter Dunlap * for inbound iSER service requests on this newly active port, the existing
92730e7468fSPeter Dunlap * IDM services will be checked for iSER support.
92830e7468fSPeter Dunlap * If an iSER service was already created, then this service will simply be
92930e7468fSPeter Dunlap * bound to the gid of the newly active port. If on the other hand, the CM
93030e7468fSPeter Dunlap * service did not exist, i.e. only socket communication, then a new CM
93130e7468fSPeter Dunlap * service will be first registered with the saved service parameters and
93230e7468fSPeter Dunlap * then bound to the newly active port.
93330e7468fSPeter Dunlap *
93430e7468fSPeter Dunlap */
93530e7468fSPeter Dunlap /* ARGSUSED */
93630e7468fSPeter Dunlap static void
iser_ib_handle_portup_event(ibt_hca_hdl_t hdl,ibt_async_event_t * event)93730e7468fSPeter Dunlap iser_ib_handle_portup_event(ibt_hca_hdl_t hdl, ibt_async_event_t *event)
93830e7468fSPeter Dunlap {
93930e7468fSPeter Dunlap iser_hca_t *hca;
94030e7468fSPeter Dunlap ib_gid_t gid;
94130e7468fSPeter Dunlap idm_svc_t *idm_svc;
94230e7468fSPeter Dunlap int status;
94330e7468fSPeter Dunlap
94430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portup_event: HCA(0x%llx) port(%d)",
94530e7468fSPeter Dunlap (longlong_t)event->ev_hca_guid, event->ev_port);
94630e7468fSPeter Dunlap
94730e7468fSPeter Dunlap /*
94830e7468fSPeter Dunlap * Query all ports on the HCA and update the port information
94930e7468fSPeter Dunlap * maintainted in the iser_hca_t structure
95030e7468fSPeter Dunlap */
95130e7468fSPeter Dunlap hca = iser_ib_guid2hca(event->ev_hca_guid);
95230e7468fSPeter Dunlap if (hca == NULL) {
95330e7468fSPeter Dunlap
95430e7468fSPeter Dunlap /* HCA is just made available, first port on that HCA */
95530e7468fSPeter Dunlap hca = iser_ib_alloc_hca(event->ev_hca_guid);
956a9194ad3SPriya Krishnan if (hca == NULL) {
957a9194ad3SPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_handle_portup_event "
958a9194ad3SPriya Krishnan "iser_ib_alloc_hca failed: HCA(0x%llx) port(%d)",
959a9194ad3SPriya Krishnan (longlong_t)event->ev_hca_guid, event->ev_port);
960a9194ad3SPriya Krishnan return;
961a9194ad3SPriya Krishnan }
96230e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
96330e7468fSPeter Dunlap list_insert_tail(&iser_state->is_hcalist, hca);
96430e7468fSPeter Dunlap iser_state->is_num_hcas++;
96530e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
96630e7468fSPeter Dunlap
96730e7468fSPeter Dunlap } else {
96830e7468fSPeter Dunlap
96930e7468fSPeter Dunlap status = iser_ib_update_hcaports(hca);
97030e7468fSPeter Dunlap
97130e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
97230e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portup_event "
97330e7468fSPeter Dunlap "status(0x%x): iser_ib_update_hcaports failed: "
97430e7468fSPeter Dunlap "HCA(0x%llx) port(%d)", status,
97530e7468fSPeter Dunlap (longlong_t)event->ev_hca_guid, event->ev_port);
97630e7468fSPeter Dunlap return;
97730e7468fSPeter Dunlap }
97830e7468fSPeter Dunlap }
97930e7468fSPeter Dunlap
98030e7468fSPeter Dunlap gid = hca->hca_port_info[event->ev_port - 1].p_sgid_tbl[0];
98130e7468fSPeter Dunlap
98230e7468fSPeter Dunlap /*
98330e7468fSPeter Dunlap * Iterate through the global list of IDM target services
98430e7468fSPeter Dunlap * and check for existing iSER CM service.
98530e7468fSPeter Dunlap */
98630e7468fSPeter Dunlap mutex_enter(&idm.idm_global_mutex);
98730e7468fSPeter Dunlap for (idm_svc = list_head(&idm.idm_tgt_svc_list);
98830e7468fSPeter Dunlap idm_svc != NULL;
98930e7468fSPeter Dunlap idm_svc = list_next(&idm.idm_tgt_svc_list, idm_svc)) {
99030e7468fSPeter Dunlap
99130e7468fSPeter Dunlap
99230e7468fSPeter Dunlap if (idm_svc->is_iser_svc == NULL) {
99330e7468fSPeter Dunlap
99430e7468fSPeter Dunlap /* Establish a new CM service for iSER requests */
99530e7468fSPeter Dunlap status = iser_tgt_svc_create(
99630e7468fSPeter Dunlap &idm_svc->is_svc_req, idm_svc);
99730e7468fSPeter Dunlap
99830e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
99930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portup_event "
100030e7468fSPeter Dunlap "status(0x%x): iser_tgt_svc_create failed: "
100130e7468fSPeter Dunlap "HCA(0x%llx) port(%d)", status,
100230e7468fSPeter Dunlap (longlong_t)event->ev_hca_guid,
100330e7468fSPeter Dunlap event->ev_port);
100430e7468fSPeter Dunlap
100530e7468fSPeter Dunlap continue;
100630e7468fSPeter Dunlap }
100730e7468fSPeter Dunlap }
100830e7468fSPeter Dunlap
100930e7468fSPeter Dunlap status = iser_ib_activate_port(
101030e7468fSPeter Dunlap idm_svc, event->ev_hca_guid, gid);
101130e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
101230e7468fSPeter Dunlap
101330e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portup_event "
101430e7468fSPeter Dunlap "status(0x%x): Bind service on port "
101530e7468fSPeter Dunlap "(%llx:%llx) failed",
101630e7468fSPeter Dunlap status, (longlong_t)gid.gid_prefix,
101730e7468fSPeter Dunlap (longlong_t)gid.gid_guid);
101830e7468fSPeter Dunlap
101930e7468fSPeter Dunlap continue;
102030e7468fSPeter Dunlap }
102130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portup_event: service bound "
102230e7468fSPeter Dunlap "HCA(0x%llx) port(%d)", (longlong_t)event->ev_hca_guid,
102330e7468fSPeter Dunlap event->ev_port);
102430e7468fSPeter Dunlap }
102530e7468fSPeter Dunlap mutex_exit(&idm.idm_global_mutex);
102630e7468fSPeter Dunlap
102730e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portup_event success: "
102830e7468fSPeter Dunlap "HCA(0x%llx) port(%d)", (longlong_t)event->ev_hca_guid,
102930e7468fSPeter Dunlap event->ev_port);
103030e7468fSPeter Dunlap }
103130e7468fSPeter Dunlap
103230e7468fSPeter Dunlap /*
103330e7468fSPeter Dunlap * iser_ib_handle_portdown_event()
103430e7468fSPeter Dunlap * This handles the IBT_EVENT_PORT_DOWN unaffiliated asynchronous error.
103530e7468fSPeter Dunlap *
103630e7468fSPeter Dunlap * Unconfigure the CM service on the deactivated port and teardown the
103730e7468fSPeter Dunlap * connections that are using the CM service.
103830e7468fSPeter Dunlap */
103930e7468fSPeter Dunlap /* ARGSUSED */
104030e7468fSPeter Dunlap static void
iser_ib_handle_portdown_event(ibt_hca_hdl_t hdl,ibt_async_event_t * event)104130e7468fSPeter Dunlap iser_ib_handle_portdown_event(ibt_hca_hdl_t hdl, ibt_async_event_t *event)
104230e7468fSPeter Dunlap {
104330e7468fSPeter Dunlap iser_hca_t *hca;
104430e7468fSPeter Dunlap ib_gid_t gid;
104530e7468fSPeter Dunlap int status;
104630e7468fSPeter Dunlap
104730e7468fSPeter Dunlap /*
104830e7468fSPeter Dunlap * Query all ports on the HCA and update the port information
104930e7468fSPeter Dunlap * maintainted in the iser_hca_t structure
105030e7468fSPeter Dunlap */
105130e7468fSPeter Dunlap hca = iser_ib_guid2hca(event->ev_hca_guid);
105230e7468fSPeter Dunlap ASSERT(hca != NULL);
105330e7468fSPeter Dunlap
105430e7468fSPeter Dunlap status = iser_ib_update_hcaports(hca);
105530e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
105630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portdown_event status(0x%x): "
105730e7468fSPeter Dunlap "ibt_ib_update_hcaports failed: HCA(0x%llx) port(%d)",
105830e7468fSPeter Dunlap status, (longlong_t)event->ev_hca_guid, event->ev_port);
105930e7468fSPeter Dunlap return;
106030e7468fSPeter Dunlap }
106130e7468fSPeter Dunlap
106230e7468fSPeter Dunlap /* get the gid of the new port */
106330e7468fSPeter Dunlap gid = hca->hca_port_info[event->ev_port - 1].p_sgid_tbl[0];
106430e7468fSPeter Dunlap iser_ib_deactivate_port(event->ev_hca_guid, gid);
106530e7468fSPeter Dunlap
106630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_portdown_event success: "
106730e7468fSPeter Dunlap "HCA(0x%llx) port(%d)", (longlong_t)event->ev_hca_guid,
106830e7468fSPeter Dunlap event->ev_port);
106930e7468fSPeter Dunlap }
107030e7468fSPeter Dunlap
107130e7468fSPeter Dunlap /*
107230e7468fSPeter Dunlap * iser_ib_handle_hca_detach_event()
107330e7468fSPeter Dunlap * Quiesce all activity bound for the port, teardown the connection, unbind
107430e7468fSPeter Dunlap * iSER services on all ports and release the HCA handle.
107530e7468fSPeter Dunlap */
107630e7468fSPeter Dunlap /* ARGSUSED */
107730e7468fSPeter Dunlap static void
iser_ib_handle_hca_detach_event(ibt_hca_hdl_t hdl,ibt_async_event_t * event)107830e7468fSPeter Dunlap iser_ib_handle_hca_detach_event(ibt_hca_hdl_t hdl, ibt_async_event_t *event)
107930e7468fSPeter Dunlap {
108030e7468fSPeter Dunlap iser_hca_t *nexthca, *hca;
108130e7468fSPeter Dunlap int i, status;
108230e7468fSPeter Dunlap
108330e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_handle_hca_detach_event: HCA(0x%llx)",
108430e7468fSPeter Dunlap (longlong_t)event->ev_hca_guid);
108530e7468fSPeter Dunlap
108630e7468fSPeter Dunlap hca = iser_ib_guid2hca(event->ev_hca_guid);
108730e7468fSPeter Dunlap for (i = 0; i < hca->hca_num_ports; i++) {
108830e7468fSPeter Dunlap iser_ib_deactivate_port(hca->hca_guid,
108930e7468fSPeter Dunlap hca->hca_port_info[i].p_sgid_tbl[0]);
109030e7468fSPeter Dunlap }
109130e7468fSPeter Dunlap
109230e7468fSPeter Dunlap /*
109330e7468fSPeter Dunlap * Update the HCA list maintained in the iser_state. Free the
109430e7468fSPeter Dunlap * resources allocated to the HCA, i.e. caches, protection domain
109530e7468fSPeter Dunlap */
109630e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
109730e7468fSPeter Dunlap
109830e7468fSPeter Dunlap for (hca = list_head(&iser_state->is_hcalist);
109930e7468fSPeter Dunlap hca != NULL;
110030e7468fSPeter Dunlap hca = nexthca) {
110130e7468fSPeter Dunlap
110230e7468fSPeter Dunlap nexthca = list_next(&iser_state->is_hcalist, hca);
110330e7468fSPeter Dunlap
110430e7468fSPeter Dunlap if (hca->hca_guid == event->ev_hca_guid) {
110530e7468fSPeter Dunlap
110630e7468fSPeter Dunlap list_remove(&iser_state->is_hcalist, hca);
110730e7468fSPeter Dunlap iser_state->is_num_hcas--;
110830e7468fSPeter Dunlap
110930e7468fSPeter Dunlap status = iser_ib_free_hca(hca);
111030e7468fSPeter Dunlap if (status != DDI_SUCCESS) {
111130e7468fSPeter Dunlap ISER_LOG(CE_WARN, "iser_ib_handle_hca_detach: "
111230e7468fSPeter Dunlap "Failed to free hca(%p)", (void *)hca);
111330e7468fSPeter Dunlap list_insert_tail(&iser_state->is_hcalist, hca);
111430e7468fSPeter Dunlap iser_state->is_num_hcas++;
111530e7468fSPeter Dunlap }
111630e7468fSPeter Dunlap /* No way to return status to IBT if this fails */
111730e7468fSPeter Dunlap }
111830e7468fSPeter Dunlap }
111930e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
112030e7468fSPeter Dunlap
112130e7468fSPeter Dunlap }
112230e7468fSPeter Dunlap
112330e7468fSPeter Dunlap /*
112430e7468fSPeter Dunlap * iser_ib_async_handler
112530e7468fSPeter Dunlap * An IBT Asynchronous Event handler is registered it with the framework and
112630e7468fSPeter Dunlap * passed via the ibt_attach() routine. This function handles the following
112730e7468fSPeter Dunlap * asynchronous events.
112830e7468fSPeter Dunlap * IBT_EVENT_PORT_UP
112930e7468fSPeter Dunlap * IBT_ERROR_PORT_DOWN
113030e7468fSPeter Dunlap * IBT_HCA_ATTACH_EVENT
113130e7468fSPeter Dunlap * IBT_HCA_DETACH_EVENT
113230e7468fSPeter Dunlap */
113330e7468fSPeter Dunlap /* ARGSUSED */
113430e7468fSPeter Dunlap void
iser_ib_async_handler(void * clntp,ibt_hca_hdl_t hdl,ibt_async_code_t code,ibt_async_event_t * event)113530e7468fSPeter Dunlap iser_ib_async_handler(void *clntp, ibt_hca_hdl_t hdl, ibt_async_code_t code,
113630e7468fSPeter Dunlap ibt_async_event_t *event)
113730e7468fSPeter Dunlap {
113830e7468fSPeter Dunlap switch (code) {
113930e7468fSPeter Dunlap case IBT_EVENT_PORT_UP:
114030e7468fSPeter Dunlap iser_ib_handle_portup_event(hdl, event);
114130e7468fSPeter Dunlap break;
114230e7468fSPeter Dunlap
114330e7468fSPeter Dunlap case IBT_ERROR_PORT_DOWN:
114430e7468fSPeter Dunlap iser_ib_handle_portdown_event(hdl, event);
114530e7468fSPeter Dunlap break;
114630e7468fSPeter Dunlap
114730e7468fSPeter Dunlap case IBT_HCA_ATTACH_EVENT:
114830e7468fSPeter Dunlap /*
114930e7468fSPeter Dunlap * A new HCA device is available for use, ignore this
115030e7468fSPeter Dunlap * event because the corresponding IBT_EVENT_PORT_UP
115130e7468fSPeter Dunlap * events will get triggered and handled accordingly.
115230e7468fSPeter Dunlap */
115330e7468fSPeter Dunlap break;
115430e7468fSPeter Dunlap
115530e7468fSPeter Dunlap case IBT_HCA_DETACH_EVENT:
115630e7468fSPeter Dunlap iser_ib_handle_hca_detach_event(hdl, event);
115730e7468fSPeter Dunlap break;
115830e7468fSPeter Dunlap
115930e7468fSPeter Dunlap default:
116030e7468fSPeter Dunlap break;
116130e7468fSPeter Dunlap }
116230e7468fSPeter Dunlap }
116330e7468fSPeter Dunlap
116430e7468fSPeter Dunlap /*
116530e7468fSPeter Dunlap * iser_ib_init_hcas
116630e7468fSPeter Dunlap *
116730e7468fSPeter Dunlap * This function opens all the HCA devices, gathers the HCA state information
116830e7468fSPeter Dunlap * and adds the HCA handle for each HCA found in the iser_soft_state.
116930e7468fSPeter Dunlap */
117030e7468fSPeter Dunlap static int
iser_ib_init_hcas(void)117130e7468fSPeter Dunlap iser_ib_init_hcas(void)
117230e7468fSPeter Dunlap {
117330e7468fSPeter Dunlap ib_guid_t *guid;
117430e7468fSPeter Dunlap int num_hcas;
117530e7468fSPeter Dunlap int i;
117630e7468fSPeter Dunlap iser_hca_t *hca;
117730e7468fSPeter Dunlap
117830e7468fSPeter Dunlap /* Retrieve the HCA list */
117930e7468fSPeter Dunlap num_hcas = ibt_get_hca_list(&guid);
118030e7468fSPeter Dunlap if (num_hcas == 0) {
118130e7468fSPeter Dunlap /*
118230e7468fSPeter Dunlap * This shouldn't happen, but might if we have all HCAs
118330e7468fSPeter Dunlap * detach prior to initialization.
118430e7468fSPeter Dunlap */
118530e7468fSPeter Dunlap return (DDI_FAILURE);
118630e7468fSPeter Dunlap }
118730e7468fSPeter Dunlap
118830e7468fSPeter Dunlap /* Initialize the hcalist lock */
118930e7468fSPeter Dunlap mutex_init(&iser_state->is_hcalist_lock, NULL, MUTEX_DRIVER, NULL);
119030e7468fSPeter Dunlap
119130e7468fSPeter Dunlap /* Create the HCA list */
119230e7468fSPeter Dunlap list_create(&iser_state->is_hcalist, sizeof (iser_hca_t),
119330e7468fSPeter Dunlap offsetof(iser_hca_t, hca_node));
119430e7468fSPeter Dunlap
119530e7468fSPeter Dunlap for (i = 0; i < num_hcas; i++) {
119630e7468fSPeter Dunlap
119730e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_init_hcas: initializing HCA "
119830e7468fSPeter Dunlap "(0x%llx)", (longlong_t)guid[i]);
119930e7468fSPeter Dunlap
120030e7468fSPeter Dunlap hca = iser_ib_alloc_hca(guid[i]);
120130e7468fSPeter Dunlap if (hca == NULL) {
120230e7468fSPeter Dunlap /* This shouldn't happen, teardown and fail */
120330e7468fSPeter Dunlap (void) iser_ib_fini_hcas();
120430e7468fSPeter Dunlap (void) ibt_free_hca_list(guid, num_hcas);
120530e7468fSPeter Dunlap return (DDI_FAILURE);
120630e7468fSPeter Dunlap }
120730e7468fSPeter Dunlap
120830e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
120930e7468fSPeter Dunlap list_insert_tail(&iser_state->is_hcalist, hca);
121030e7468fSPeter Dunlap iser_state->is_num_hcas++;
121130e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
121230e7468fSPeter Dunlap
121330e7468fSPeter Dunlap }
121430e7468fSPeter Dunlap
121530e7468fSPeter Dunlap /* Free the IBT HCA list */
121630e7468fSPeter Dunlap (void) ibt_free_hca_list(guid, num_hcas);
121730e7468fSPeter Dunlap
121830e7468fSPeter Dunlap /* Check that we've initialized at least one HCA */
121930e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
122030e7468fSPeter Dunlap if (list_is_empty(&iser_state->is_hcalist)) {
122130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_init_hcas: failed to initialize "
122230e7468fSPeter Dunlap "any HCAs");
122330e7468fSPeter Dunlap
122430e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
122530e7468fSPeter Dunlap (void) iser_ib_fini_hcas();
122630e7468fSPeter Dunlap return (DDI_FAILURE);
122730e7468fSPeter Dunlap }
122830e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
122930e7468fSPeter Dunlap
123030e7468fSPeter Dunlap return (DDI_SUCCESS);
123130e7468fSPeter Dunlap }
123230e7468fSPeter Dunlap
123330e7468fSPeter Dunlap /*
123430e7468fSPeter Dunlap * iser_ib_fini_hcas
123530e7468fSPeter Dunlap *
123630e7468fSPeter Dunlap * Teardown the iSER HCA list initialized above.
123730e7468fSPeter Dunlap */
123830e7468fSPeter Dunlap static int
iser_ib_fini_hcas(void)123930e7468fSPeter Dunlap iser_ib_fini_hcas(void)
124030e7468fSPeter Dunlap {
124130e7468fSPeter Dunlap iser_hca_t *nexthca, *hca;
124230e7468fSPeter Dunlap int status;
124330e7468fSPeter Dunlap
124430e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
124530e7468fSPeter Dunlap for (hca = list_head(&iser_state->is_hcalist);
124630e7468fSPeter Dunlap hca != NULL;
124730e7468fSPeter Dunlap hca = nexthca) {
124830e7468fSPeter Dunlap
124930e7468fSPeter Dunlap nexthca = list_next(&iser_state->is_hcalist, hca);
125030e7468fSPeter Dunlap
125130e7468fSPeter Dunlap list_remove(&iser_state->is_hcalist, hca);
125230e7468fSPeter Dunlap
125330e7468fSPeter Dunlap status = iser_ib_free_hca(hca);
125430e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
125530e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_fini_hcas: failed to free "
125630e7468fSPeter Dunlap "HCA during fini");
125730e7468fSPeter Dunlap list_insert_tail(&iser_state->is_hcalist, hca);
125830e7468fSPeter Dunlap return (DDI_FAILURE);
125930e7468fSPeter Dunlap }
126030e7468fSPeter Dunlap
126130e7468fSPeter Dunlap iser_state->is_num_hcas--;
126230e7468fSPeter Dunlap
126330e7468fSPeter Dunlap }
126430e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
126530e7468fSPeter Dunlap list_destroy(&iser_state->is_hcalist);
126630e7468fSPeter Dunlap mutex_destroy(&iser_state->is_hcalist_lock);
126730e7468fSPeter Dunlap
126830e7468fSPeter Dunlap return (DDI_SUCCESS);
126930e7468fSPeter Dunlap }
127030e7468fSPeter Dunlap
127130e7468fSPeter Dunlap /*
127230e7468fSPeter Dunlap * iser_ib_alloc_hca
127330e7468fSPeter Dunlap *
127430e7468fSPeter Dunlap * This function opens the given HCA device, gathers the HCA state information
127530e7468fSPeter Dunlap * and adds the HCA handle
127630e7468fSPeter Dunlap */
127730e7468fSPeter Dunlap static iser_hca_t *
iser_ib_alloc_hca(ib_guid_t guid)127830e7468fSPeter Dunlap iser_ib_alloc_hca(ib_guid_t guid)
127930e7468fSPeter Dunlap {
128030e7468fSPeter Dunlap iser_hca_t *hca;
128130e7468fSPeter Dunlap int status;
128230e7468fSPeter Dunlap
128330e7468fSPeter Dunlap /* Allocate an iser_hca_t HCA handle */
128430e7468fSPeter Dunlap hca = (iser_hca_t *)kmem_zalloc(sizeof (iser_hca_t), KM_SLEEP);
128530e7468fSPeter Dunlap
128630e7468fSPeter Dunlap /* Open this HCA */
128730e7468fSPeter Dunlap status = ibt_open_hca(iser_state->is_ibhdl, guid, &hca->hca_hdl);
128830e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
128930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_alloc_hca: ibt_open_hca failed:"
129030e7468fSPeter Dunlap " guid (0x%llx) status (0x%x)", (longlong_t)guid, status);
129130e7468fSPeter Dunlap kmem_free(hca, sizeof (iser_hca_t));
129230e7468fSPeter Dunlap return (NULL);
129330e7468fSPeter Dunlap }
129430e7468fSPeter Dunlap
129530e7468fSPeter Dunlap hca->hca_guid = guid;
129630e7468fSPeter Dunlap hca->hca_clnt_hdl = iser_state->is_ibhdl;
129730e7468fSPeter Dunlap
129830e7468fSPeter Dunlap /* Query the HCA */
129930e7468fSPeter Dunlap status = ibt_query_hca(hca->hca_hdl, &hca->hca_attr);
130030e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
130130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_alloc_hca: ibt_query_hca "
130230e7468fSPeter Dunlap "failure: guid (0x%llx) status (0x%x)",
130330e7468fSPeter Dunlap (longlong_t)guid, status);
130430e7468fSPeter Dunlap (void) ibt_close_hca(hca->hca_hdl);
130530e7468fSPeter Dunlap kmem_free(hca, sizeof (iser_hca_t));
130630e7468fSPeter Dunlap return (NULL);
130730e7468fSPeter Dunlap }
130830e7468fSPeter Dunlap
130930e7468fSPeter Dunlap /* Query all ports on the HCA */
131030e7468fSPeter Dunlap status = ibt_query_hca_ports(hca->hca_hdl, 0,
131130e7468fSPeter Dunlap &hca->hca_port_info, &hca->hca_num_ports,
131230e7468fSPeter Dunlap &hca->hca_port_info_sz);
131330e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
131430e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_alloc_hca: "
131530e7468fSPeter Dunlap "ibt_query_hca_ports failure: guid (0x%llx) "
131630e7468fSPeter Dunlap "status (0x%x)", (longlong_t)guid, status);
131730e7468fSPeter Dunlap (void) ibt_close_hca(hca->hca_hdl);
131830e7468fSPeter Dunlap kmem_free(hca, sizeof (iser_hca_t));
131930e7468fSPeter Dunlap return (NULL);
132030e7468fSPeter Dunlap }
132130e7468fSPeter Dunlap
132230e7468fSPeter Dunlap /* Allocate a single PD on this HCA */
132330e7468fSPeter Dunlap status = ibt_alloc_pd(hca->hca_hdl, IBT_PD_NO_FLAGS,
132430e7468fSPeter Dunlap &hca->hca_pdhdl);
132530e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
132630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_alloc_hca: ibt_alloc_pd "
132730e7468fSPeter Dunlap "failure: guid (0x%llx) status (0x%x)",
132830e7468fSPeter Dunlap (longlong_t)guid, status);
132930e7468fSPeter Dunlap (void) ibt_close_hca(hca->hca_hdl);
133030e7468fSPeter Dunlap ibt_free_portinfo(hca->hca_port_info, hca->hca_port_info_sz);
133130e7468fSPeter Dunlap kmem_free(hca, sizeof (iser_hca_t));
133230e7468fSPeter Dunlap return (NULL);
133330e7468fSPeter Dunlap }
133430e7468fSPeter Dunlap
133530e7468fSPeter Dunlap /* Initialize the message and data MR caches for this HCA */
133630e7468fSPeter Dunlap iser_init_hca_caches(hca);
133730e7468fSPeter Dunlap
133830e7468fSPeter Dunlap return (hca);
133930e7468fSPeter Dunlap }
134030e7468fSPeter Dunlap
134130e7468fSPeter Dunlap static int
iser_ib_free_hca(iser_hca_t * hca)134230e7468fSPeter Dunlap iser_ib_free_hca(iser_hca_t *hca)
134330e7468fSPeter Dunlap {
134430e7468fSPeter Dunlap int status;
134530e7468fSPeter Dunlap ibt_hca_portinfo_t *hca_port_info;
134630e7468fSPeter Dunlap uint_t hca_port_info_sz;
134730e7468fSPeter Dunlap
134830e7468fSPeter Dunlap ASSERT(hca != NULL);
134930e7468fSPeter Dunlap if (hca->hca_failed)
135030e7468fSPeter Dunlap return (DDI_FAILURE);
135130e7468fSPeter Dunlap
135230e7468fSPeter Dunlap hca_port_info = hca->hca_port_info;
135330e7468fSPeter Dunlap hca_port_info_sz = hca->hca_port_info_sz;
135430e7468fSPeter Dunlap
135530e7468fSPeter Dunlap /*
135630e7468fSPeter Dunlap * Free the memory regions before freeing
135730e7468fSPeter Dunlap * the associated protection domain
135830e7468fSPeter Dunlap */
135930e7468fSPeter Dunlap iser_fini_hca_caches(hca);
136030e7468fSPeter Dunlap
136130e7468fSPeter Dunlap status = ibt_free_pd(hca->hca_hdl, hca->hca_pdhdl);
136230e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
136330e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_free_hca: failed to free PD "
136430e7468fSPeter Dunlap "status=0x%x", status);
136530e7468fSPeter Dunlap goto out_caches;
136630e7468fSPeter Dunlap }
136730e7468fSPeter Dunlap
136830e7468fSPeter Dunlap status = ibt_close_hca(hca->hca_hdl);
136930e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
137030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_fini_hcas: failed to close HCA "
137130e7468fSPeter Dunlap "status=0x%x", status);
137230e7468fSPeter Dunlap goto out_pd;
137330e7468fSPeter Dunlap }
137430e7468fSPeter Dunlap
137530e7468fSPeter Dunlap ibt_free_portinfo(hca_port_info, hca_port_info_sz);
137630e7468fSPeter Dunlap
137730e7468fSPeter Dunlap kmem_free(hca, sizeof (iser_hca_t));
137830e7468fSPeter Dunlap return (DDI_SUCCESS);
137930e7468fSPeter Dunlap
138030e7468fSPeter Dunlap /*
138130e7468fSPeter Dunlap * We only managed to partially tear down the HCA, try to put it back
138230e7468fSPeter Dunlap * like it was before returning.
138330e7468fSPeter Dunlap */
138430e7468fSPeter Dunlap out_pd:
138530e7468fSPeter Dunlap status = ibt_alloc_pd(hca->hca_hdl, IBT_PD_NO_FLAGS, &hca->hca_pdhdl);
138630e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
138730e7468fSPeter Dunlap hca->hca_failed = B_TRUE;
138830e7468fSPeter Dunlap /* Report error and exit */
138930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_free_hca: could not re-alloc PD "
139030e7468fSPeter Dunlap "status=0x%x", status);
139130e7468fSPeter Dunlap return (DDI_FAILURE);
139230e7468fSPeter Dunlap }
139330e7468fSPeter Dunlap
139430e7468fSPeter Dunlap out_caches:
139530e7468fSPeter Dunlap iser_init_hca_caches(hca);
139630e7468fSPeter Dunlap
139730e7468fSPeter Dunlap return (DDI_FAILURE);
139830e7468fSPeter Dunlap }
139930e7468fSPeter Dunlap
140030e7468fSPeter Dunlap static int
iser_ib_update_hcaports(iser_hca_t * hca)140130e7468fSPeter Dunlap iser_ib_update_hcaports(iser_hca_t *hca)
140230e7468fSPeter Dunlap {
140330e7468fSPeter Dunlap ibt_hca_portinfo_t *pinfop, *oldpinfop;
140430e7468fSPeter Dunlap uint_t size, oldsize, nport;
140530e7468fSPeter Dunlap int status;
140630e7468fSPeter Dunlap
140730e7468fSPeter Dunlap ASSERT(hca != NULL);
140830e7468fSPeter Dunlap
140930e7468fSPeter Dunlap status = ibt_query_hca_ports(hca->hca_hdl, 0, &pinfop, &nport, &size);
141030e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
141130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "ibt_query_hca_ports failed: %d", status);
141230e7468fSPeter Dunlap return (status);
141330e7468fSPeter Dunlap }
141430e7468fSPeter Dunlap
141530e7468fSPeter Dunlap oldpinfop = hca->hca_port_info;
141630e7468fSPeter Dunlap oldsize = hca->hca_port_info_sz;
141730e7468fSPeter Dunlap hca->hca_port_info = pinfop;
141830e7468fSPeter Dunlap hca->hca_port_info_sz = size;
141930e7468fSPeter Dunlap
142030e7468fSPeter Dunlap (void) ibt_free_portinfo(oldpinfop, oldsize);
142130e7468fSPeter Dunlap
142230e7468fSPeter Dunlap return (IBT_SUCCESS);
142330e7468fSPeter Dunlap }
142430e7468fSPeter Dunlap
142530e7468fSPeter Dunlap /*
142630e7468fSPeter Dunlap * iser_ib_gid2hca
142730e7468fSPeter Dunlap * Given a gid, find the corresponding hca
142830e7468fSPeter Dunlap */
142930e7468fSPeter Dunlap iser_hca_t *
iser_ib_gid2hca(ib_gid_t gid)143030e7468fSPeter Dunlap iser_ib_gid2hca(ib_gid_t gid)
143130e7468fSPeter Dunlap {
143230e7468fSPeter Dunlap
143330e7468fSPeter Dunlap iser_hca_t *hca;
143430e7468fSPeter Dunlap int i;
143530e7468fSPeter Dunlap
143630e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
143730e7468fSPeter Dunlap for (hca = list_head(&iser_state->is_hcalist);
143830e7468fSPeter Dunlap hca != NULL;
143930e7468fSPeter Dunlap hca = list_next(&iser_state->is_hcalist, hca)) {
144030e7468fSPeter Dunlap
144130e7468fSPeter Dunlap for (i = 0; i < hca->hca_num_ports; i++) {
144230e7468fSPeter Dunlap if ((hca->hca_port_info[i].p_sgid_tbl[0].gid_prefix ==
144330e7468fSPeter Dunlap gid.gid_prefix) &&
144430e7468fSPeter Dunlap (hca->hca_port_info[i].p_sgid_tbl[0].gid_guid ==
144530e7468fSPeter Dunlap gid.gid_guid)) {
144630e7468fSPeter Dunlap
144730e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
144830e7468fSPeter Dunlap
144930e7468fSPeter Dunlap return (hca);
145030e7468fSPeter Dunlap }
145130e7468fSPeter Dunlap }
145230e7468fSPeter Dunlap }
145330e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
145430e7468fSPeter Dunlap return (NULL);
145530e7468fSPeter Dunlap }
145630e7468fSPeter Dunlap
145730e7468fSPeter Dunlap /*
145830e7468fSPeter Dunlap * iser_ib_guid2hca
145930e7468fSPeter Dunlap * Given a HCA guid, find the corresponding HCA
146030e7468fSPeter Dunlap */
146130e7468fSPeter Dunlap iser_hca_t *
iser_ib_guid2hca(ib_guid_t guid)146230e7468fSPeter Dunlap iser_ib_guid2hca(ib_guid_t guid)
146330e7468fSPeter Dunlap {
146430e7468fSPeter Dunlap
146530e7468fSPeter Dunlap iser_hca_t *hca;
146630e7468fSPeter Dunlap
146730e7468fSPeter Dunlap mutex_enter(&iser_state->is_hcalist_lock);
146830e7468fSPeter Dunlap for (hca = list_head(&iser_state->is_hcalist);
146930e7468fSPeter Dunlap hca != NULL;
147030e7468fSPeter Dunlap hca = list_next(&iser_state->is_hcalist, hca)) {
147130e7468fSPeter Dunlap
147230e7468fSPeter Dunlap if (hca->hca_guid == guid) {
147330e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
147430e7468fSPeter Dunlap return (hca);
147530e7468fSPeter Dunlap }
147630e7468fSPeter Dunlap }
147730e7468fSPeter Dunlap mutex_exit(&iser_state->is_hcalist_lock);
147830e7468fSPeter Dunlap return (NULL);
147930e7468fSPeter Dunlap }
148030e7468fSPeter Dunlap
148130e7468fSPeter Dunlap /*
148230e7468fSPeter Dunlap * iser_ib_conv_sockaddr2ibtaddr
148330e7468fSPeter Dunlap * This function converts a socket address into the IBT format
148430e7468fSPeter Dunlap */
iser_ib_conv_sockaddr2ibtaddr(idm_sockaddr_t * saddr,ibt_ip_addr_t * ibt_addr)148530e7468fSPeter Dunlap void iser_ib_conv_sockaddr2ibtaddr(
148630e7468fSPeter Dunlap idm_sockaddr_t *saddr, ibt_ip_addr_t *ibt_addr)
148730e7468fSPeter Dunlap {
148830e7468fSPeter Dunlap if (saddr == NULL) {
148930e7468fSPeter Dunlap ibt_addr->family = AF_UNSPEC;
149030e7468fSPeter Dunlap ibt_addr->un.ip4addr = 0;
149130e7468fSPeter Dunlap } else {
149230e7468fSPeter Dunlap switch (saddr->sin.sa_family) {
149330e7468fSPeter Dunlap case AF_INET:
149430e7468fSPeter Dunlap
149530e7468fSPeter Dunlap ibt_addr->family = saddr->sin4.sin_family;
149630e7468fSPeter Dunlap ibt_addr->un.ip4addr = saddr->sin4.sin_addr.s_addr;
149730e7468fSPeter Dunlap break;
149830e7468fSPeter Dunlap
149930e7468fSPeter Dunlap case AF_INET6:
150030e7468fSPeter Dunlap
150130e7468fSPeter Dunlap ibt_addr->family = saddr->sin6.sin6_family;
150230e7468fSPeter Dunlap ibt_addr->un.ip6addr = saddr->sin6.sin6_addr;
150330e7468fSPeter Dunlap break;
150430e7468fSPeter Dunlap
150530e7468fSPeter Dunlap default:
150630e7468fSPeter Dunlap ibt_addr->family = AF_UNSPEC;
150730e7468fSPeter Dunlap }
150830e7468fSPeter Dunlap
150930e7468fSPeter Dunlap }
151030e7468fSPeter Dunlap }
151130e7468fSPeter Dunlap
151230e7468fSPeter Dunlap /*
151330e7468fSPeter Dunlap * iser_ib_conv_ibtaddr2sockaddr
151430e7468fSPeter Dunlap * This function converts an IBT ip address handle to a sockaddr
151530e7468fSPeter Dunlap */
iser_ib_conv_ibtaddr2sockaddr(struct sockaddr_storage * ss,ibt_ip_addr_t * ibt_addr,in_port_t port)151630e7468fSPeter Dunlap void iser_ib_conv_ibtaddr2sockaddr(struct sockaddr_storage *ss,
151730e7468fSPeter Dunlap ibt_ip_addr_t *ibt_addr, in_port_t port)
151830e7468fSPeter Dunlap {
151930e7468fSPeter Dunlap struct sockaddr_in *sin;
152030e7468fSPeter Dunlap struct sockaddr_in6 *sin6;
152130e7468fSPeter Dunlap
152230e7468fSPeter Dunlap switch (ibt_addr->family) {
152330e7468fSPeter Dunlap case AF_INET:
152430e7468fSPeter Dunlap case AF_UNSPEC:
152530e7468fSPeter Dunlap
152630e7468fSPeter Dunlap sin = (struct sockaddr_in *)ibt_addr;
152730e7468fSPeter Dunlap sin->sin_port = ntohs(port);
152830e7468fSPeter Dunlap bcopy(sin, ss, sizeof (struct sockaddr_in));
152930e7468fSPeter Dunlap break;
153030e7468fSPeter Dunlap
153130e7468fSPeter Dunlap case AF_INET6:
153230e7468fSPeter Dunlap
153330e7468fSPeter Dunlap sin6 = (struct sockaddr_in6 *)ibt_addr;
153430e7468fSPeter Dunlap sin6->sin6_port = ntohs(port);
153530e7468fSPeter Dunlap bcopy(sin6, ss, sizeof (struct sockaddr_in6));
153630e7468fSPeter Dunlap break;
153730e7468fSPeter Dunlap
153830e7468fSPeter Dunlap default:
153930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_conv_ibtaddr2sockaddr: "
154030e7468fSPeter Dunlap "unknown family type: 0x%x", ibt_addr->family);
154130e7468fSPeter Dunlap }
154230e7468fSPeter Dunlap }
154330e7468fSPeter Dunlap
154430e7468fSPeter Dunlap /*
154530e7468fSPeter Dunlap * iser_ib_setup_cq
154630e7468fSPeter Dunlap * This function sets up the Completion Queue size and allocates the specified
154730e7468fSPeter Dunlap * Completion Queue
154830e7468fSPeter Dunlap */
154930e7468fSPeter Dunlap static int
iser_ib_setup_cq(ibt_hca_hdl_t hca_hdl,uint_t cq_size,ibt_cq_hdl_t * cq_hdl)155030e7468fSPeter Dunlap iser_ib_setup_cq(ibt_hca_hdl_t hca_hdl, uint_t cq_size, ibt_cq_hdl_t *cq_hdl)
155130e7468fSPeter Dunlap {
155230e7468fSPeter Dunlap
155330e7468fSPeter Dunlap ibt_cq_attr_t cq_attr;
155430e7468fSPeter Dunlap int status;
155530e7468fSPeter Dunlap
155630e7468fSPeter Dunlap cq_attr.cq_size = cq_size;
155730e7468fSPeter Dunlap cq_attr.cq_sched = 0;
155830e7468fSPeter Dunlap cq_attr.cq_flags = IBT_CQ_NO_FLAGS;
155930e7468fSPeter Dunlap
156030e7468fSPeter Dunlap /* Allocate a Completion Queue */
156130e7468fSPeter Dunlap status = ibt_alloc_cq(hca_hdl, &cq_attr, cq_hdl, NULL);
156230e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
156330e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_setup_cq: ibt_alloc_cq failure (%d)",
156430e7468fSPeter Dunlap status);
156530e7468fSPeter Dunlap return (status);
156630e7468fSPeter Dunlap }
156730e7468fSPeter Dunlap
156830e7468fSPeter Dunlap return (ISER_STATUS_SUCCESS);
156930e7468fSPeter Dunlap }
157030e7468fSPeter Dunlap
157130e7468fSPeter Dunlap /*
157230e7468fSPeter Dunlap * iser_ib_setup_chanargs
157330e7468fSPeter Dunlap *
157430e7468fSPeter Dunlap */
157530e7468fSPeter Dunlap static void
iser_ib_setup_chanargs(uint8_t hca_port,ibt_cq_hdl_t scq_hdl,ibt_cq_hdl_t rcq_hdl,uint_t sq_size,uint_t rq_size,ibt_pd_hdl_t hca_pdhdl,ibt_rc_chan_alloc_args_t * cargs)157630e7468fSPeter Dunlap iser_ib_setup_chanargs(uint8_t hca_port, ibt_cq_hdl_t scq_hdl,
157730e7468fSPeter Dunlap ibt_cq_hdl_t rcq_hdl, uint_t sq_size, uint_t rq_size,
157830e7468fSPeter Dunlap ibt_pd_hdl_t hca_pdhdl, ibt_rc_chan_alloc_args_t *cargs)
157930e7468fSPeter Dunlap {
158030e7468fSPeter Dunlap
158130e7468fSPeter Dunlap bzero(cargs, sizeof (ibt_rc_chan_alloc_args_t));
158230e7468fSPeter Dunlap
158330e7468fSPeter Dunlap /*
158430e7468fSPeter Dunlap * Set up the size of the channels send queue, receive queue and the
158530e7468fSPeter Dunlap * maximum number of elements in a scatter gather list of work requests
158630e7468fSPeter Dunlap * posted to the send and receive queues.
158730e7468fSPeter Dunlap */
158830e7468fSPeter Dunlap cargs->rc_sizes.cs_sq = sq_size;
158930e7468fSPeter Dunlap cargs->rc_sizes.cs_rq = rq_size;
159030e7468fSPeter Dunlap cargs->rc_sizes.cs_sq_sgl = ISER_IB_SGLIST_SIZE;
159130e7468fSPeter Dunlap cargs->rc_sizes.cs_rq_sgl = ISER_IB_SGLIST_SIZE;
159230e7468fSPeter Dunlap
159330e7468fSPeter Dunlap /*
159430e7468fSPeter Dunlap * All Work requests signaled on a WR basis will receive a send
159530e7468fSPeter Dunlap * request completion.
159630e7468fSPeter Dunlap */
159730e7468fSPeter Dunlap cargs->rc_flags = IBT_ALL_SIGNALED;
159830e7468fSPeter Dunlap
159930e7468fSPeter Dunlap /* Enable RDMA read and RDMA write on the channel end points */
160030e7468fSPeter Dunlap cargs->rc_control = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR;
160130e7468fSPeter Dunlap
160230e7468fSPeter Dunlap /* Set the local hca port on which the channel is allocated */
160330e7468fSPeter Dunlap cargs->rc_hca_port_num = hca_port;
160430e7468fSPeter Dunlap
160530e7468fSPeter Dunlap /* Set the Send and Receive Completion Queue handles */
160630e7468fSPeter Dunlap cargs->rc_scq = scq_hdl;
160730e7468fSPeter Dunlap cargs->rc_rcq = rcq_hdl;
160830e7468fSPeter Dunlap
160930e7468fSPeter Dunlap /* Set the protection domain associated with the channel */
161030e7468fSPeter Dunlap cargs->rc_pd = hca_pdhdl;
161130e7468fSPeter Dunlap
161230e7468fSPeter Dunlap /* No SRQ usage */
161330e7468fSPeter Dunlap cargs->rc_srq = NULL;
161430e7468fSPeter Dunlap }
161530e7468fSPeter Dunlap
161630e7468fSPeter Dunlap /*
161730e7468fSPeter Dunlap * iser_ib_init_qp
161830e7468fSPeter Dunlap * Initialize the QP handle
161930e7468fSPeter Dunlap */
162030e7468fSPeter Dunlap void
iser_ib_init_qp(iser_chan_t * chan,uint_t sq_size,uint_t rq_size)162130e7468fSPeter Dunlap iser_ib_init_qp(iser_chan_t *chan, uint_t sq_size, uint_t rq_size)
162230e7468fSPeter Dunlap {
162330e7468fSPeter Dunlap /* Initialize the handle lock */
162430e7468fSPeter Dunlap mutex_init(&chan->ic_qp.qp_lock, NULL, MUTEX_DRIVER, NULL);
162530e7468fSPeter Dunlap
162630e7468fSPeter Dunlap /* Record queue sizes */
162730e7468fSPeter Dunlap chan->ic_qp.sq_size = sq_size;
162830e7468fSPeter Dunlap chan->ic_qp.rq_size = rq_size;
162930e7468fSPeter Dunlap
163030e7468fSPeter Dunlap /* Initialize the RQ monitoring data */
163130e7468fSPeter Dunlap chan->ic_qp.rq_depth = rq_size;
163230e7468fSPeter Dunlap chan->ic_qp.rq_level = 0;
163330e7468fSPeter Dunlap chan->ic_qp.rq_lwm = (chan->ic_recvcq_sz * ISER_IB_RQ_LWM_PCT) / 100;
163430e7468fSPeter Dunlap
163530e7468fSPeter Dunlap /* Initialize the taskq flag */
163630e7468fSPeter Dunlap chan->ic_qp.rq_taskqpending = B_FALSE;
163730e7468fSPeter Dunlap }
163830e7468fSPeter Dunlap
163930e7468fSPeter Dunlap /*
164030e7468fSPeter Dunlap * iser_ib_fini_qp
164130e7468fSPeter Dunlap * Teardown the QP handle
164230e7468fSPeter Dunlap */
164330e7468fSPeter Dunlap void
iser_ib_fini_qp(iser_qp_t * qp)164430e7468fSPeter Dunlap iser_ib_fini_qp(iser_qp_t *qp)
164530e7468fSPeter Dunlap {
164630e7468fSPeter Dunlap /* Destroy the handle lock */
164730e7468fSPeter Dunlap mutex_destroy(&qp->qp_lock);
164830e7468fSPeter Dunlap }
164930e7468fSPeter Dunlap
165030e7468fSPeter Dunlap static int
iser_ib_activate_port(idm_svc_t * idm_svc,ib_guid_t guid,ib_gid_t gid)165130e7468fSPeter Dunlap iser_ib_activate_port(idm_svc_t *idm_svc, ib_guid_t guid, ib_gid_t gid)
165230e7468fSPeter Dunlap {
165330e7468fSPeter Dunlap iser_svc_t *iser_svc;
165430e7468fSPeter Dunlap iser_sbind_t *is_sbind;
165530e7468fSPeter Dunlap int status;
165630e7468fSPeter Dunlap
165730e7468fSPeter Dunlap iser_svc = idm_svc->is_iser_svc;
165830e7468fSPeter Dunlap
165930e7468fSPeter Dunlap /*
166030e7468fSPeter Dunlap * Save the address of the service bind handle in the
166130e7468fSPeter Dunlap * iser_svc_t to undo the service binding at a later time
166230e7468fSPeter Dunlap */
166330e7468fSPeter Dunlap is_sbind = kmem_zalloc(sizeof (iser_sbind_t), KM_SLEEP);
166430e7468fSPeter Dunlap is_sbind->is_gid = gid;
166530e7468fSPeter Dunlap is_sbind->is_guid = guid;
166630e7468fSPeter Dunlap
166730e7468fSPeter Dunlap status = ibt_bind_service(iser_svc->is_srvhdl, gid, NULL,
166830e7468fSPeter Dunlap idm_svc, &is_sbind->is_sbindhdl);
166930e7468fSPeter Dunlap
167030e7468fSPeter Dunlap if (status != IBT_SUCCESS) {
167130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_activate_port: status(0x%x): "
167230e7468fSPeter Dunlap "Bind service(%llx) on port(%llx:%llx) failed",
167330e7468fSPeter Dunlap status, (longlong_t)iser_svc->is_svcid,
167430e7468fSPeter Dunlap (longlong_t)gid.gid_prefix, (longlong_t)gid.gid_guid);
167530e7468fSPeter Dunlap
167630e7468fSPeter Dunlap kmem_free(is_sbind, sizeof (iser_sbind_t));
167730e7468fSPeter Dunlap
167830e7468fSPeter Dunlap return (status);
167930e7468fSPeter Dunlap }
168030e7468fSPeter Dunlap
168130e7468fSPeter Dunlap list_insert_tail(&iser_svc->is_sbindlist, is_sbind);
168230e7468fSPeter Dunlap
168330e7468fSPeter Dunlap return (IBT_SUCCESS);
168430e7468fSPeter Dunlap }
168530e7468fSPeter Dunlap
168630e7468fSPeter Dunlap static void
iser_ib_deactivate_port(ib_guid_t hca_guid,ib_gid_t gid)168730e7468fSPeter Dunlap iser_ib_deactivate_port(ib_guid_t hca_guid, ib_gid_t gid)
168830e7468fSPeter Dunlap {
168930e7468fSPeter Dunlap iser_svc_t *iser_svc;
169030e7468fSPeter Dunlap iser_conn_t *iser_conn;
169130e7468fSPeter Dunlap iser_sbind_t *is_sbind;
169230e7468fSPeter Dunlap idm_conn_t *idm_conn;
169330e7468fSPeter Dunlap
169430e7468fSPeter Dunlap /*
169530e7468fSPeter Dunlap * Iterate through the global list of IDM target connections.
169630e7468fSPeter Dunlap * Issue a TRANSPORT_FAIL for any connections on this port, and
169730e7468fSPeter Dunlap * if there is a bound service running on the port, tear it down.
169830e7468fSPeter Dunlap */
169930e7468fSPeter Dunlap mutex_enter(&idm.idm_global_mutex);
170030e7468fSPeter Dunlap for (idm_conn = list_head(&idm.idm_tgt_conn_list);
170130e7468fSPeter Dunlap idm_conn != NULL;
170230e7468fSPeter Dunlap idm_conn = list_next(&idm.idm_tgt_conn_list, idm_conn)) {
170330e7468fSPeter Dunlap
170430e7468fSPeter Dunlap if (idm_conn->ic_transport_type != IDM_TRANSPORT_TYPE_ISER) {
170530e7468fSPeter Dunlap /* this is not an iSER connection, skip it */
170630e7468fSPeter Dunlap continue;
170730e7468fSPeter Dunlap }
170830e7468fSPeter Dunlap
170930e7468fSPeter Dunlap iser_conn = idm_conn->ic_transport_private;
171030e7468fSPeter Dunlap if (iser_conn->ic_chan->ic_ibt_path.pi_hca_guid != hca_guid) {
171130e7468fSPeter Dunlap /* this iSER connection is on a different port */
171230e7468fSPeter Dunlap continue;
171330e7468fSPeter Dunlap }
171430e7468fSPeter Dunlap
171530e7468fSPeter Dunlap /* Fail the transport for this connection */
171630e7468fSPeter Dunlap idm_conn_event(idm_conn, CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
171730e7468fSPeter Dunlap
171830e7468fSPeter Dunlap if (idm_conn->ic_conn_type == CONN_TYPE_INI) {
171930e7468fSPeter Dunlap /* initiator connection, nothing else to do */
172030e7468fSPeter Dunlap continue;
172130e7468fSPeter Dunlap }
172230e7468fSPeter Dunlap
172330e7468fSPeter Dunlap /* Check for a service binding */
172430e7468fSPeter Dunlap iser_svc = idm_conn->ic_svc_binding->is_iser_svc;
172530e7468fSPeter Dunlap is_sbind = iser_ib_get_bind(iser_svc, hca_guid, gid);
172630e7468fSPeter Dunlap if (is_sbind != NULL) {
172730e7468fSPeter Dunlap /* This service is still bound, tear it down */
1728aedf2b3bSsrivijitha dugganapalli (void) ibt_unbind_service(iser_svc->is_srvhdl,
172930e7468fSPeter Dunlap is_sbind->is_sbindhdl);
173030e7468fSPeter Dunlap list_remove(&iser_svc->is_sbindlist, is_sbind);
173130e7468fSPeter Dunlap kmem_free(is_sbind, sizeof (iser_sbind_t));
173230e7468fSPeter Dunlap }
173330e7468fSPeter Dunlap }
173430e7468fSPeter Dunlap mutex_exit(&idm.idm_global_mutex);
173530e7468fSPeter Dunlap }
173630e7468fSPeter Dunlap
173730e7468fSPeter Dunlap static iser_sbind_t *
iser_ib_get_bind(iser_svc_t * iser_svc,ib_guid_t hca_guid,ib_gid_t gid)173830e7468fSPeter Dunlap iser_ib_get_bind(iser_svc_t *iser_svc, ib_guid_t hca_guid, ib_gid_t gid)
173930e7468fSPeter Dunlap {
174030e7468fSPeter Dunlap iser_sbind_t *is_sbind;
174130e7468fSPeter Dunlap
174230e7468fSPeter Dunlap for (is_sbind = list_head(&iser_svc->is_sbindlist);
174330e7468fSPeter Dunlap is_sbind != NULL;
174430e7468fSPeter Dunlap is_sbind = list_next(&iser_svc->is_sbindlist, is_sbind)) {
174530e7468fSPeter Dunlap
174630e7468fSPeter Dunlap if ((is_sbind->is_guid == hca_guid) &&
174730e7468fSPeter Dunlap (is_sbind->is_gid.gid_prefix == gid.gid_prefix) &&
174830e7468fSPeter Dunlap (is_sbind->is_gid.gid_guid == gid.gid_guid)) {
174930e7468fSPeter Dunlap return (is_sbind);
175030e7468fSPeter Dunlap }
175130e7468fSPeter Dunlap }
175230e7468fSPeter Dunlap return (NULL);
175330e7468fSPeter Dunlap }
1754