1b86efd96Sagiri /* 2b86efd96Sagiri * CDDL HEADER START 3b86efd96Sagiri * 4b86efd96Sagiri * The contents of this file are subject to the terms of the 5b86efd96Sagiri * Common Development and Distribution License (the "License"). 6b86efd96Sagiri * You may not use this file except in compliance with the License. 7b86efd96Sagiri * 8b86efd96Sagiri * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9b86efd96Sagiri * or http://www.opensolaris.org/os/licensing. 10b86efd96Sagiri * See the License for the specific language governing permissions 11b86efd96Sagiri * and limitations under the License. 12b86efd96Sagiri * 13b86efd96Sagiri * When distributing Covered Code, include this CDDL HEADER in each 14b86efd96Sagiri * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15b86efd96Sagiri * If applicable, add the following below this CDDL HEADER, with the 16b86efd96Sagiri * fields enclosed by brackets "[]" replaced with your own identifying 17b86efd96Sagiri * information: Portions Copyright [yyyy] [name of copyright owner] 18b86efd96Sagiri * 19b86efd96Sagiri * CDDL HEADER END 20b86efd96Sagiri */ 21b86efd96Sagiri /* 22*c1f8b08eSagiri * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23b86efd96Sagiri * Use is subject to license terms. 24b86efd96Sagiri */ 25b86efd96Sagiri /* 26b86efd96Sagiri * Copyright (c) 2005 SilverStorm Technologies, Inc. All rights reserved. 27b86efd96Sagiri * 28b86efd96Sagiri * This software is available to you under a choice of one of two 29b86efd96Sagiri * licenses. You may choose to be licensed under the terms of the GNU 30b86efd96Sagiri * General Public License (GPL) Version 2, available from the file 31b86efd96Sagiri * COPYING in the main directory of this source tree, or the 32b86efd96Sagiri * OpenIB.org BSD license below: 33b86efd96Sagiri * 34b86efd96Sagiri * Redistribution and use in source and binary forms, with or 35b86efd96Sagiri * without modification, are permitted provided that the following 36b86efd96Sagiri * conditions are met: 37b86efd96Sagiri * 38b86efd96Sagiri * - Redistributions of source code must retain the above 39b86efd96Sagiri * copyright notice, this list of conditions and the following 40b86efd96Sagiri * disclaimer. 41b86efd96Sagiri * 42b86efd96Sagiri * - Redistributions in binary form must reproduce the above 43b86efd96Sagiri * copyright notice, this list of conditions and the following 44b86efd96Sagiri * disclaimer in the documentation and/or other materials 45b86efd96Sagiri * provided with the distribution. 46b86efd96Sagiri * 47b86efd96Sagiri * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 48b86efd96Sagiri * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 49b86efd96Sagiri * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 50b86efd96Sagiri * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 51b86efd96Sagiri * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 52b86efd96Sagiri * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 53b86efd96Sagiri * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 54b86efd96Sagiri * SOFTWARE. 55b86efd96Sagiri * 56b86efd96Sagiri */ 57b86efd96Sagiri /* 58b86efd96Sagiri * Sun elects to include this software in Sun product 59b86efd96Sagiri * under the OpenIB BSD license. 60b86efd96Sagiri * 61b86efd96Sagiri * 62b86efd96Sagiri * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 63b86efd96Sagiri * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 64b86efd96Sagiri * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 65b86efd96Sagiri * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 66b86efd96Sagiri * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 67b86efd96Sagiri * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 68b86efd96Sagiri * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 69b86efd96Sagiri * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 70b86efd96Sagiri * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 71b86efd96Sagiri * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 72b86efd96Sagiri * POSSIBILITY OF SUCH DAMAGE. 73b86efd96Sagiri */ 74b86efd96Sagiri 75b86efd96Sagiri #pragma ident "%Z%%M% %I% %E% SMI" 76b86efd96Sagiri 77b86efd96Sagiri #include <sys/types.h> 78b86efd96Sagiri #include <sys/ddi.h> 79b86efd96Sagiri #include <sys/sunddi.h> 80b86efd96Sagiri #include <sys/ib/ibtl/ibti.h> 81b86efd96Sagiri #include <sys/ib/ibtl/ibtl_types.h> 82b86efd96Sagiri #include <sys/ib/clients/rds/rdsib_cm.h> 83b86efd96Sagiri #include <sys/ib/clients/rds/rdsib_ib.h> 84b86efd96Sagiri #include <sys/ib/clients/rds/rdsib_buf.h> 85b86efd96Sagiri #include <sys/ib/clients/rds/rdsib_ep.h> 86b86efd96Sagiri #include <sys/ib/clients/rds/rds_kstat.h> 87b86efd96Sagiri 88b86efd96Sagiri static void rds_async_handler(void *clntp, ibt_hca_hdl_t hdl, 89b86efd96Sagiri ibt_async_code_t code, ibt_async_event_t *event); 90b86efd96Sagiri 91b86efd96Sagiri static struct ibt_clnt_modinfo_s rds_ib_modinfo = { 92b86efd96Sagiri IBTI_V2, 93b86efd96Sagiri IBT_NETWORK, 94b86efd96Sagiri rds_async_handler, 95b86efd96Sagiri NULL, 96b86efd96Sagiri "RDS" 97b86efd96Sagiri }; 98b86efd96Sagiri 99b86efd96Sagiri /* performance tunables */ 100b86efd96Sagiri uint_t rds_no_interrupts = 0; 101b86efd96Sagiri uint_t rds_poll_percent_full = 25; 102b86efd96Sagiri uint_t rds_wc_signal = IBT_NEXT_SOLICITED; 103b86efd96Sagiri uint_t rds_waittime_ms = 100; /* ms */ 104b86efd96Sagiri 105b86efd96Sagiri extern dev_info_t *rdsib_dev_info; 106b86efd96Sagiri extern void rds_close_sessions(); 107b86efd96Sagiri 108b86efd96Sagiri static void 109b86efd96Sagiri rdsib_validate_chan_sizes(ibt_hca_attr_t *hattrp) 110b86efd96Sagiri { 111b86efd96Sagiri /* The SQ size should not be more than that supported by the HCA */ 112b86efd96Sagiri if (((MaxDataSendBuffers + RDS_NUM_ACKS) > hattrp->hca_max_chan_sz) || 113b86efd96Sagiri ((MaxDataSendBuffers + RDS_NUM_ACKS) > hattrp->hca_max_cq_sz)) { 114b86efd96Sagiri RDS_DPRINTF0("RDSIB", "MaxDataSendBuffers + %d is greater " 115b86efd96Sagiri "than that supported by the HCA driver " 116b86efd96Sagiri "(%d + %d > %d or %d), lowering it to a supported value.", 117b86efd96Sagiri RDS_NUM_ACKS, MaxDataSendBuffers, RDS_NUM_ACKS, 118b86efd96Sagiri hattrp->hca_max_chan_sz, hattrp->hca_max_cq_sz); 119b86efd96Sagiri 120b86efd96Sagiri MaxDataSendBuffers = (hattrp->hca_max_chan_sz > 121b86efd96Sagiri hattrp->hca_max_cq_sz) ? 122b86efd96Sagiri hattrp->hca_max_cq_sz - RDS_NUM_ACKS : 123b86efd96Sagiri hattrp->hca_max_chan_sz - RDS_NUM_ACKS; 124b86efd96Sagiri } 125b86efd96Sagiri 126b86efd96Sagiri /* The RQ size should not be more than that supported by the HCA */ 127b86efd96Sagiri if ((MaxDataRecvBuffers > hattrp->hca_max_chan_sz) || 128b86efd96Sagiri (MaxDataRecvBuffers > hattrp->hca_max_cq_sz)) { 129b86efd96Sagiri RDS_DPRINTF0("RDSIB", "MaxDataRecvBuffers is greater than that " 130b86efd96Sagiri "supported by the HCA driver (%d > %d or %d), lowering it " 131b86efd96Sagiri "to a supported value.", MaxDataRecvBuffers, 132b86efd96Sagiri hattrp->hca_max_chan_sz, hattrp->hca_max_cq_sz); 133b86efd96Sagiri 134b86efd96Sagiri MaxDataRecvBuffers = (hattrp->hca_max_chan_sz > 135b86efd96Sagiri hattrp->hca_max_cq_sz) ? hattrp->hca_max_cq_sz : 136b86efd96Sagiri hattrp->hca_max_chan_sz; 137b86efd96Sagiri } 138b86efd96Sagiri 139b86efd96Sagiri /* The SQ size should not be more than that supported by the HCA */ 140b86efd96Sagiri if ((MaxCtrlSendBuffers > hattrp->hca_max_chan_sz) || 141b86efd96Sagiri (MaxCtrlSendBuffers > hattrp->hca_max_cq_sz)) { 142b86efd96Sagiri RDS_DPRINTF0("RDSIB", "MaxCtrlSendBuffers is greater than that " 143b86efd96Sagiri "supported by the HCA driver (%d > %d or %d), lowering it " 144b86efd96Sagiri "to a supported value.", MaxCtrlSendBuffers, 145b86efd96Sagiri hattrp->hca_max_chan_sz, hattrp->hca_max_cq_sz); 146b86efd96Sagiri 147b86efd96Sagiri MaxCtrlSendBuffers = (hattrp->hca_max_chan_sz > 148b86efd96Sagiri hattrp->hca_max_cq_sz) ? hattrp->hca_max_cq_sz : 149b86efd96Sagiri hattrp->hca_max_chan_sz; 150b86efd96Sagiri } 151b86efd96Sagiri 152b86efd96Sagiri /* The RQ size should not be more than that supported by the HCA */ 153b86efd96Sagiri if ((MaxCtrlRecvBuffers > hattrp->hca_max_chan_sz) || 154b86efd96Sagiri (MaxCtrlRecvBuffers > hattrp->hca_max_cq_sz)) { 155b86efd96Sagiri RDS_DPRINTF0("RDSIB", "MaxCtrlRecvBuffers is greater than that " 156b86efd96Sagiri "supported by the HCA driver (%d > %d or %d), lowering it " 157b86efd96Sagiri "to a supported value.", MaxCtrlRecvBuffers, 158b86efd96Sagiri hattrp->hca_max_chan_sz, hattrp->hca_max_cq_sz); 159b86efd96Sagiri 160b86efd96Sagiri MaxCtrlRecvBuffers = (hattrp->hca_max_chan_sz > 161b86efd96Sagiri hattrp->hca_max_cq_sz) ? hattrp->hca_max_cq_sz : 162b86efd96Sagiri hattrp->hca_max_chan_sz; 163b86efd96Sagiri } 164b86efd96Sagiri 165b86efd96Sagiri /* The MaxRecvMemory should be less than that supported by the HCA */ 166b86efd96Sagiri if ((MaxRecvMemory * 1024) > hattrp->hca_max_memr_len) { 167b86efd96Sagiri RDS_DPRINTF0("RDSIB", "MaxRecvMemory is greater than that " 168b86efd96Sagiri "supported by the HCA driver (%d > %d), lowering it to %d", 169b86efd96Sagiri MaxRecvMemory, hattrp->hca_max_memr_len, 170b86efd96Sagiri hattrp->hca_max_memr_len); 171b86efd96Sagiri 172b86efd96Sagiri MaxRecvMemory = hattrp->hca_max_memr_len; 173b86efd96Sagiri } 174b86efd96Sagiri } 175b86efd96Sagiri 176b86efd96Sagiri /* 177b86efd96Sagiri * Called on open of first RDS socket 178b86efd96Sagiri */ 179b86efd96Sagiri int 180b86efd96Sagiri rdsib_open_ib() 181b86efd96Sagiri { 182b86efd96Sagiri ib_guid_t *guidp; 183b86efd96Sagiri rds_hca_t *hcap, *hcap1; 184b86efd96Sagiri uint_t ix, hcaix, nhcas; 185b86efd96Sagiri int ret; 186b86efd96Sagiri 187*c1f8b08eSagiri RDS_DPRINTF4("rdsib_open_ib", "enter: statep %p", rdsib_statep); 188b86efd96Sagiri 189b86efd96Sagiri ASSERT(rdsib_statep != NULL); 190b86efd96Sagiri if (rdsib_statep == NULL) { 191b86efd96Sagiri RDS_DPRINTF1("rdsib_open_ib", "RDS Statep not initialized"); 192b86efd96Sagiri return (-1); 193b86efd96Sagiri } 194b86efd96Sagiri 195b86efd96Sagiri /* How many hcas are there? */ 196b86efd96Sagiri nhcas = ibt_get_hca_list(&guidp); 197b86efd96Sagiri if (nhcas == 0) { 198b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", "No IB HCAs Available"); 199b86efd96Sagiri return (-1); 200b86efd96Sagiri } 201b86efd96Sagiri 202b86efd96Sagiri RDS_DPRINTF3("rdsib_open_ib", "Number of HCAs: %d", nhcas); 203b86efd96Sagiri 204b86efd96Sagiri /* Register with IBTF */ 205b86efd96Sagiri ret = ibt_attach(&rds_ib_modinfo, rdsib_dev_info, rdsib_statep, 206b86efd96Sagiri &rdsib_statep->rds_ibhdl); 207b86efd96Sagiri if (ret != IBT_SUCCESS) { 208b86efd96Sagiri RDS_DPRINTF2(LABEL, "ibt_attach failed: %d", ret); 209b86efd96Sagiri (void) ibt_free_hca_list(guidp, nhcas); 210b86efd96Sagiri return (-1); 211b86efd96Sagiri } 212b86efd96Sagiri 213b86efd96Sagiri /* 214b86efd96Sagiri * Open each HCA and gather its information. Don't care about HCAs 215b86efd96Sagiri * that cannot be opened. It is OK as long as atleast one HCA can be 216b86efd96Sagiri * opened. 217b86efd96Sagiri * Initialize a HCA only if all the information is available. 218b86efd96Sagiri */ 219b86efd96Sagiri hcap1 = NULL; 220b86efd96Sagiri for (ix = 0, hcaix = 0; ix < nhcas; ix++) { 221b86efd96Sagiri RDS_DPRINTF3(LABEL, "Open HCA: 0x%llx", guidp[ix]); 222b86efd96Sagiri 223b86efd96Sagiri hcap = (rds_hca_t *)kmem_zalloc(sizeof (rds_hca_t), KM_SLEEP); 224b86efd96Sagiri 225b86efd96Sagiri ret = ibt_open_hca(rdsib_statep->rds_ibhdl, guidp[ix], 226b86efd96Sagiri &hcap->hca_hdl); 227b86efd96Sagiri if (ret != IBT_SUCCESS) { 228b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", 229b86efd96Sagiri "ibt_open_hca: 0x%llx failed: %d", guidp[ix], ret); 230b86efd96Sagiri kmem_free(hcap, sizeof (rds_hca_t)); 231b86efd96Sagiri continue; 232b86efd96Sagiri } 233b86efd96Sagiri 234b86efd96Sagiri hcap->hca_guid = guidp[ix]; 235b86efd96Sagiri 236b86efd96Sagiri ret = ibt_query_hca(hcap->hca_hdl, &hcap->hca_attr); 237b86efd96Sagiri if (ret != IBT_SUCCESS) { 238b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", 239b86efd96Sagiri "Query HCA: 0x%llx failed: %d", guidp[ix], ret); 240b86efd96Sagiri ret = ibt_close_hca(hcap->hca_hdl); 241b86efd96Sagiri ASSERT(ret == IBT_SUCCESS); 242b86efd96Sagiri kmem_free(hcap, sizeof (rds_hca_t)); 243b86efd96Sagiri continue; 244b86efd96Sagiri } 245b86efd96Sagiri 246b86efd96Sagiri ret = ibt_query_hca_ports(hcap->hca_hdl, 0, 247b86efd96Sagiri &hcap->hca_pinfop, &hcap->hca_nports, &hcap->hca_pinfo_sz); 248b86efd96Sagiri if (ret != IBT_SUCCESS) { 249b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", 250b86efd96Sagiri "Query HCA 0x%llx ports failed: %d", guidp[ix], 251b86efd96Sagiri ret); 252b86efd96Sagiri ret = ibt_close_hca(hcap->hca_hdl); 253b86efd96Sagiri ASSERT(ret == IBT_SUCCESS); 254b86efd96Sagiri kmem_free(hcap, sizeof (rds_hca_t)); 255b86efd96Sagiri continue; 256b86efd96Sagiri } 257b86efd96Sagiri 258b86efd96Sagiri /* Only one PD per HCA is allocated, so do it here */ 259b86efd96Sagiri ret = ibt_alloc_pd(hcap->hca_hdl, IBT_PD_NO_FLAGS, 260b86efd96Sagiri &hcap->hca_pdhdl); 261b86efd96Sagiri if (ret != IBT_SUCCESS) { 262b86efd96Sagiri RDS_DPRINTF2(LABEL, "ibt_alloc_pd 0x%llx failed: %d", 263b86efd96Sagiri guidp[ix], ret); 264b86efd96Sagiri (void) ibt_free_portinfo(hcap->hca_pinfop, 265b86efd96Sagiri hcap->hca_pinfo_sz); 266b86efd96Sagiri ret = ibt_close_hca(hcap->hca_hdl); 267b86efd96Sagiri ASSERT(ret == IBT_SUCCESS); 268b86efd96Sagiri kmem_free(hcap, sizeof (rds_hca_t)); 269b86efd96Sagiri continue; 270b86efd96Sagiri } 271b86efd96Sagiri 272b86efd96Sagiri rdsib_validate_chan_sizes(&hcap->hca_attr); 273b86efd96Sagiri 274b86efd96Sagiri /* this HCA is fully initialized, go to the next one */ 275b86efd96Sagiri hcaix++; 276b86efd96Sagiri hcap->hca_nextp = hcap1; 277b86efd96Sagiri hcap1 = hcap; 278b86efd96Sagiri } 279b86efd96Sagiri 280b86efd96Sagiri /* free the HCA list, we are done with it */ 281b86efd96Sagiri (void) ibt_free_hca_list(guidp, nhcas); 282b86efd96Sagiri 283b86efd96Sagiri if (hcaix == 0) { 284b86efd96Sagiri /* Failed to Initialize even one HCA */ 285b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", "No HCAs are initialized"); 286b86efd96Sagiri (void) ibt_detach(rdsib_statep->rds_ibhdl); 287b86efd96Sagiri rdsib_statep->rds_ibhdl = NULL; 288b86efd96Sagiri return (-1); 289b86efd96Sagiri } 290b86efd96Sagiri 291b86efd96Sagiri if (hcaix < nhcas) { 292b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", "HCAs %d/%d failed to initialize", 293b86efd96Sagiri (nhcas - hcaix), nhcas); 294b86efd96Sagiri } 295b86efd96Sagiri 296b86efd96Sagiri rdsib_statep->rds_hcalistp = hcap1; 297b86efd96Sagiri rdsib_statep->rds_nhcas = hcaix; 298b86efd96Sagiri 299b86efd96Sagiri /* register the RDS service */ 300b86efd96Sagiri rdsib_statep->rds_srvhdl = 301b86efd96Sagiri rds_register_service(rdsib_statep->rds_ibhdl); 302b86efd96Sagiri if (rdsib_statep->rds_srvhdl == NULL) { 303b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", "Service registration failed"); 304b86efd96Sagiri } else { 305b86efd96Sagiri /* bind the service on all available ports */ 306b86efd96Sagiri ret = rds_bind_service(rdsib_statep); 307b86efd96Sagiri if (ret != 0) { 308b86efd96Sagiri RDS_DPRINTF2("rdsib_open_ib", "Bind service failed"); 309b86efd96Sagiri } 310b86efd96Sagiri } 311b86efd96Sagiri 312*c1f8b08eSagiri RDS_DPRINTF4("rdsib_open_ib", "return: statep %p", rdsib_statep); 313b86efd96Sagiri 314b86efd96Sagiri return (0); 315b86efd96Sagiri } 316b86efd96Sagiri 317b86efd96Sagiri /* 318b86efd96Sagiri * Called when all ports are closed. 319b86efd96Sagiri */ 320b86efd96Sagiri void 321b86efd96Sagiri rdsib_close_ib() 322b86efd96Sagiri { 323*c1f8b08eSagiri rds_hca_t *hcap, *nextp; 324b86efd96Sagiri int ret; 325b86efd96Sagiri 326*c1f8b08eSagiri RDS_DPRINTF2("rds_close_ib", "enter: statep %p", rdsib_statep); 327b86efd96Sagiri 328b86efd96Sagiri if (rdsib_statep->rds_srvhdl != NULL) { 329b86efd96Sagiri (void) ibt_unbind_all_services(rdsib_statep->rds_srvhdl); 330b86efd96Sagiri (void) ibt_deregister_service(rdsib_statep->rds_ibhdl, 331b86efd96Sagiri rdsib_statep->rds_srvhdl); 332b86efd96Sagiri } 333b86efd96Sagiri 334b86efd96Sagiri /* close and destroy all the sessions */ 335b86efd96Sagiri rds_close_sessions(NULL); 336b86efd96Sagiri 337*c1f8b08eSagiri /* Release all HCA resources */ 338*c1f8b08eSagiri rw_enter(&rdsib_statep->rds_hca_lock, RW_WRITER); 339b86efd96Sagiri hcap = rdsib_statep->rds_hcalistp; 340*c1f8b08eSagiri rdsib_statep->rds_hcalistp = NULL; 341*c1f8b08eSagiri rdsib_statep->rds_nhcas = 0; 342*c1f8b08eSagiri rw_exit(&rdsib_statep->rds_hca_lock); 343*c1f8b08eSagiri 344b86efd96Sagiri while (hcap != NULL) { 345*c1f8b08eSagiri nextp = hcap->hca_nextp; 346b86efd96Sagiri 347b86efd96Sagiri ret = ibt_free_pd(hcap->hca_hdl, hcap->hca_pdhdl); 348b86efd96Sagiri ASSERT(ret == IBT_SUCCESS); 349b86efd96Sagiri 350b86efd96Sagiri (void) ibt_free_portinfo(hcap->hca_pinfop, hcap->hca_pinfo_sz); 351b86efd96Sagiri 352b86efd96Sagiri ret = ibt_close_hca(hcap->hca_hdl); 353b86efd96Sagiri ASSERT(ret == IBT_SUCCESS); 354b86efd96Sagiri 355b86efd96Sagiri kmem_free(hcap, sizeof (rds_hca_t)); 356*c1f8b08eSagiri hcap = nextp; 357b86efd96Sagiri } 358b86efd96Sagiri 359b86efd96Sagiri /* Deregister with IBTF */ 360b86efd96Sagiri if (rdsib_statep->rds_ibhdl != NULL) { 361b86efd96Sagiri (void) ibt_detach(rdsib_statep->rds_ibhdl); 362b86efd96Sagiri rdsib_statep->rds_ibhdl = NULL; 363b86efd96Sagiri } 364b86efd96Sagiri 365*c1f8b08eSagiri RDS_DPRINTF2("rds_close_ib", "return: statep %p", rdsib_statep); 366b86efd96Sagiri } 367b86efd96Sagiri 368b86efd96Sagiri /* Return hcap, given the hca guid */ 369b86efd96Sagiri rds_hca_t * 370b86efd96Sagiri rds_get_hcap(rds_state_t *statep, ib_guid_t hca_guid) 371b86efd96Sagiri { 372b86efd96Sagiri rds_hca_t *hcap; 373b86efd96Sagiri 374b86efd96Sagiri RDS_DPRINTF4("rds_get_hcap", "rds_get_hcap: Enter: statep: 0x%p " 375b86efd96Sagiri "guid: %llx", statep, hca_guid); 376b86efd96Sagiri 377b86efd96Sagiri rw_enter(&statep->rds_hca_lock, RW_READER); 378b86efd96Sagiri 379b86efd96Sagiri hcap = statep->rds_hcalistp; 380b86efd96Sagiri while ((hcap != NULL) && (hcap->hca_guid != hca_guid)) { 381b86efd96Sagiri hcap = hcap->hca_nextp; 382b86efd96Sagiri } 383b86efd96Sagiri 384b86efd96Sagiri rw_exit(&statep->rds_hca_lock); 385b86efd96Sagiri 386b86efd96Sagiri RDS_DPRINTF4("rds_get_hcap", "rds_get_hcap: return"); 387b86efd96Sagiri 388b86efd96Sagiri return (hcap); 389b86efd96Sagiri } 390b86efd96Sagiri 391b86efd96Sagiri /* Return hcap, given a gid */ 392b86efd96Sagiri rds_hca_t * 393b86efd96Sagiri rds_gid_to_hcap(rds_state_t *statep, ib_gid_t gid) 394b86efd96Sagiri { 395*c1f8b08eSagiri rds_hca_t *hcap; 396*c1f8b08eSagiri uint_t ix; 397b86efd96Sagiri 398b86efd96Sagiri RDS_DPRINTF4("rds_gid_to_hcap", "Enter: statep: 0x%p gid: %llx:%llx", 399b86efd96Sagiri statep, gid.gid_prefix, gid.gid_guid); 400b86efd96Sagiri 401*c1f8b08eSagiri rw_enter(&statep->rds_hca_lock, RW_READER); 402*c1f8b08eSagiri 403*c1f8b08eSagiri hcap = statep->rds_hcalistp; 404*c1f8b08eSagiri while (hcap != NULL) { 405*c1f8b08eSagiri for (ix = 0; ix < hcap->hca_nports; ix++) { 406*c1f8b08eSagiri if ((hcap->hca_pinfop[ix].p_sgid_tbl[0].gid_prefix == 407*c1f8b08eSagiri gid.gid_prefix) && 408*c1f8b08eSagiri (hcap->hca_pinfop[ix].p_sgid_tbl[0].gid_guid == 409*c1f8b08eSagiri gid.gid_guid)) { 410*c1f8b08eSagiri RDS_DPRINTF4("rds_gid_to_hcap", 411*c1f8b08eSagiri "gid found in hcap: 0x%p", hcap); 412*c1f8b08eSagiri rw_exit(&statep->rds_hca_lock); 413*c1f8b08eSagiri return (hcap); 414*c1f8b08eSagiri } 415*c1f8b08eSagiri } 416*c1f8b08eSagiri hcap = hcap->hca_nextp; 417b86efd96Sagiri } 418b86efd96Sagiri 419*c1f8b08eSagiri rw_exit(&statep->rds_hca_lock); 420*c1f8b08eSagiri 421*c1f8b08eSagiri return (NULL); 422b86efd96Sagiri } 423b86efd96Sagiri 424b86efd96Sagiri /* This is called from the send CQ handler */ 425b86efd96Sagiri void 426b86efd96Sagiri rds_send_acknowledgement(rds_ep_t *ep) 427b86efd96Sagiri { 428b86efd96Sagiri int ret; 429b86efd96Sagiri uint_t ix; 430b86efd96Sagiri 431b86efd96Sagiri RDS_DPRINTF4("rds_send_acknowledgement", "Enter EP(%p)", ep); 432b86efd96Sagiri 433b86efd96Sagiri mutex_enter(&ep->ep_lock); 434b86efd96Sagiri 435b86efd96Sagiri ASSERT(ep->ep_rdmacnt != 0); 436b86efd96Sagiri 437b86efd96Sagiri /* 438b86efd96Sagiri * The previous ACK completed successfully, send the next one 439b86efd96Sagiri * if more messages were received after sending the last ACK 440b86efd96Sagiri */ 441b86efd96Sagiri if (ep->ep_rbufid != *(uintptr_t *)(uintptr_t)ep->ep_ackds.ds_va) { 442b86efd96Sagiri *(uintptr_t *)(uintptr_t)ep->ep_ackds.ds_va = ep->ep_rbufid; 443b86efd96Sagiri mutex_exit(&ep->ep_lock); 444b86efd96Sagiri 445b86efd96Sagiri /* send acknowledgement */ 446b86efd96Sagiri RDS_INCR_TXACKS(); 447b86efd96Sagiri ret = ibt_post_send(ep->ep_chanhdl, &ep->ep_ackwr, 1, &ix); 448b86efd96Sagiri if (ret != IBT_SUCCESS) { 449b86efd96Sagiri RDS_DPRINTF1("rds_send_acknowledgement", 450b86efd96Sagiri "EP(%p): ibt_post_send for acknowledgement " 451b86efd96Sagiri "failed: %d, SQ depth: %d", 452b86efd96Sagiri ep, ret, ep->ep_sndpool.pool_nbusy); 453b86efd96Sagiri mutex_enter(&ep->ep_lock); 454b86efd96Sagiri ep->ep_rdmacnt--; 455b86efd96Sagiri mutex_exit(&ep->ep_lock); 456b86efd96Sagiri } 457b86efd96Sagiri } else { 458b86efd96Sagiri /* ACKed all messages, no more to ACK */ 459b86efd96Sagiri ep->ep_rdmacnt--; 460b86efd96Sagiri mutex_exit(&ep->ep_lock); 461b86efd96Sagiri return; 462b86efd96Sagiri } 463b86efd96Sagiri 464b86efd96Sagiri RDS_DPRINTF4("rds_send_acknowledgement", "Return EP(%p)", ep); 465b86efd96Sagiri } 466b86efd96Sagiri 467b86efd96Sagiri static int 468b86efd96Sagiri rds_poll_ctrl_completions(ibt_cq_hdl_t cq, rds_ep_t *ep) 469b86efd96Sagiri { 470b86efd96Sagiri ibt_wc_t wc; 471b86efd96Sagiri uint_t npolled; 472b86efd96Sagiri rds_buf_t *bp; 473b86efd96Sagiri rds_ctrl_pkt_t *cpkt; 474b86efd96Sagiri rds_qp_t *recvqp; 475b86efd96Sagiri int ret = IBT_SUCCESS; 476b86efd96Sagiri 477b86efd96Sagiri RDS_DPRINTF4("rds_poll_ctrl_completions", "Enter: EP(%p)", ep); 478b86efd96Sagiri 479b86efd96Sagiri bzero(&wc, sizeof (ibt_wc_t)); 480b86efd96Sagiri ret = ibt_poll_cq(cq, &wc, 1, &npolled); 481b86efd96Sagiri if (ret != IBT_SUCCESS) { 482b86efd96Sagiri if (ret != IBT_CQ_EMPTY) { 483b86efd96Sagiri RDS_DPRINTF2(LABEL, "EP(%p) CQ(%p): ibt_poll_cq " 484b86efd96Sagiri "returned: %d", ep, cq, ret); 485b86efd96Sagiri } else { 486b86efd96Sagiri RDS_DPRINTF5(LABEL, "EP(%p) CQ(%p): ibt_poll_cq " 487b86efd96Sagiri "returned: IBT_CQ_EMPTY", ep, cq); 488b86efd96Sagiri } 489b86efd96Sagiri return (ret); 490b86efd96Sagiri } 491b86efd96Sagiri 492b86efd96Sagiri bp = (rds_buf_t *)(uintptr_t)wc.wc_id; 493b86efd96Sagiri 494b86efd96Sagiri if (wc.wc_status != IBT_WC_SUCCESS) { 495b86efd96Sagiri mutex_enter(&ep->ep_recvqp.qp_lock); 496b86efd96Sagiri ep->ep_recvqp.qp_level--; 497b86efd96Sagiri mutex_exit(&ep->ep_recvqp.qp_lock); 498b86efd96Sagiri 499b86efd96Sagiri /* Free the buffer */ 500b86efd96Sagiri bp->buf_state = RDS_RCVBUF_FREE; 501b86efd96Sagiri rds_free_recv_buf(bp, 1); 502b86efd96Sagiri 503b86efd96Sagiri /* Receive completion failure */ 504b86efd96Sagiri if (wc.wc_status != IBT_WC_WR_FLUSHED_ERR) { 505b86efd96Sagiri RDS_DPRINTF2("rds_poll_ctrl_completions", 506b86efd96Sagiri "EP(%p) CQ(%p) BP(%p): WC Error Status: %d", 507b86efd96Sagiri ep, cq, wc.wc_id, wc.wc_status); 508b86efd96Sagiri } 509b86efd96Sagiri return (ret); 510b86efd96Sagiri } 511b86efd96Sagiri 512b86efd96Sagiri /* there is one less in the RQ */ 513b86efd96Sagiri recvqp = &ep->ep_recvqp; 514b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 515b86efd96Sagiri recvqp->qp_level--; 516b86efd96Sagiri if ((recvqp->qp_taskqpending == B_FALSE) && 517b86efd96Sagiri (recvqp->qp_level <= recvqp->qp_lwm)) { 518b86efd96Sagiri /* Time to post more buffers into the RQ */ 519b86efd96Sagiri recvqp->qp_taskqpending = B_TRUE; 520b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 521b86efd96Sagiri 522b86efd96Sagiri ret = ddi_taskq_dispatch(rds_taskq, 523b86efd96Sagiri rds_post_recv_buf, (void *)ep->ep_chanhdl, DDI_NOSLEEP); 524b86efd96Sagiri if (ret != DDI_SUCCESS) { 525b86efd96Sagiri RDS_DPRINTF1(LABEL, "ddi_taskq_dispatch failed: %d", 526b86efd96Sagiri ret); 527b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 528b86efd96Sagiri recvqp->qp_taskqpending = B_FALSE; 529b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 530b86efd96Sagiri } 531b86efd96Sagiri } else { 532b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 533b86efd96Sagiri } 534b86efd96Sagiri 535b86efd96Sagiri cpkt = (rds_ctrl_pkt_t *)(uintptr_t)bp->buf_ds.ds_va; 536b86efd96Sagiri rds_handle_control_message(ep->ep_sp, cpkt); 537b86efd96Sagiri 538b86efd96Sagiri bp->buf_state = RDS_RCVBUF_FREE; 539b86efd96Sagiri rds_free_recv_buf(bp, 1); 540b86efd96Sagiri 541b86efd96Sagiri RDS_DPRINTF4("rds_poll_ctrl_completions", "Return: EP(%p)", ep); 542b86efd96Sagiri 543b86efd96Sagiri return (ret); 544b86efd96Sagiri } 545b86efd96Sagiri 546b86efd96Sagiri #define RDS_POST_FEW_ATATIME 100 547b86efd96Sagiri /* Post recv WRs into the RQ. Assumes the ep->refcnt is already incremented */ 548b86efd96Sagiri void 549b86efd96Sagiri rds_post_recv_buf(void *arg) 550b86efd96Sagiri { 551b86efd96Sagiri ibt_channel_hdl_t chanhdl; 552b86efd96Sagiri rds_ep_t *ep; 553b86efd96Sagiri rds_session_t *sp; 554b86efd96Sagiri rds_qp_t *recvqp; 555b86efd96Sagiri rds_bufpool_t *gp; 556b86efd96Sagiri rds_buf_t *bp, *bp1; 557b86efd96Sagiri ibt_recv_wr_t *wrp, wr[RDS_POST_FEW_ATATIME]; 558b86efd96Sagiri rds_hca_t *hcap; 559b86efd96Sagiri uint_t npost, nspace, rcv_len; 560b86efd96Sagiri uint_t ix, jx, kx; 561b86efd96Sagiri int ret; 562b86efd96Sagiri 563b86efd96Sagiri chanhdl = (ibt_channel_hdl_t)arg; 564b86efd96Sagiri RDS_DPRINTF4("rds_post_recv_buf", "Enter: CHAN(%p)", chanhdl); 565b86efd96Sagiri RDS_INCR_POST_RCV_BUF_CALLS(); 566b86efd96Sagiri 567b86efd96Sagiri ep = (rds_ep_t *)ibt_get_chan_private(chanhdl); 568b86efd96Sagiri ASSERT(ep != NULL); 569b86efd96Sagiri sp = ep->ep_sp; 570b86efd96Sagiri recvqp = &ep->ep_recvqp; 571b86efd96Sagiri 572b86efd96Sagiri RDS_DPRINTF5("rds_post_recv_buf", "EP(%p)", ep); 573b86efd96Sagiri 574b86efd96Sagiri /* get the hcap for the HCA hosting this channel */ 575b86efd96Sagiri hcap = rds_get_hcap(rdsib_statep, ep->ep_hca_guid); 576b86efd96Sagiri if (hcap == NULL) { 577b86efd96Sagiri RDS_DPRINTF2("rds_post_recv_buf", "HCA (0x%llx) not found", 578b86efd96Sagiri ep->ep_hca_guid); 579b86efd96Sagiri return; 580b86efd96Sagiri } 581b86efd96Sagiri 582b86efd96Sagiri /* Make sure the session is still connected */ 583b86efd96Sagiri rw_enter(&sp->session_lock, RW_READER); 584b86efd96Sagiri if ((sp->session_state != RDS_SESSION_STATE_INIT) && 585b86efd96Sagiri (sp->session_state != RDS_SESSION_STATE_CONNECTED)) { 586b86efd96Sagiri RDS_DPRINTF2("rds_post_recv_buf", "EP(%p): Session is not " 587b86efd96Sagiri "in active state (%d)", ep, sp->session_state); 588b86efd96Sagiri rw_exit(&sp->session_lock); 589b86efd96Sagiri return; 590b86efd96Sagiri } 591b86efd96Sagiri rw_exit(&sp->session_lock); 592b86efd96Sagiri 593b86efd96Sagiri /* how many can be posted */ 594b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 595b86efd96Sagiri nspace = recvqp->qp_depth - recvqp->qp_level; 596b86efd96Sagiri if (nspace == 0) { 597b86efd96Sagiri RDS_DPRINTF2("rds_post_recv_buf", "RQ is FULL"); 598b86efd96Sagiri recvqp->qp_taskqpending = B_FALSE; 599b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 600b86efd96Sagiri return; 601b86efd96Sagiri } 602b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 603b86efd96Sagiri 604b86efd96Sagiri if (ep->ep_type == RDS_EP_TYPE_DATA) { 605b86efd96Sagiri gp = &rds_dpool; 606b86efd96Sagiri rcv_len = RdsPktSize; 607b86efd96Sagiri } else { 608b86efd96Sagiri gp = &rds_cpool; 609b86efd96Sagiri rcv_len = RDS_CTRLPKT_SIZE; 610b86efd96Sagiri } 611b86efd96Sagiri 612b86efd96Sagiri bp = rds_get_buf(gp, nspace, &jx); 613b86efd96Sagiri if (bp == NULL) { 614b86efd96Sagiri RDS_DPRINTF2(LABEL, "EP(%p): No Recv buffers available", ep); 615b86efd96Sagiri /* try again later */ 616b86efd96Sagiri ret = ddi_taskq_dispatch(rds_taskq, rds_post_recv_buf, 617b86efd96Sagiri (void *)ep->ep_chanhdl, DDI_NOSLEEP); 618b86efd96Sagiri if (ret != DDI_SUCCESS) { 619b86efd96Sagiri RDS_DPRINTF1(LABEL, "ddi_taskq_dispatch failed: %d", 620b86efd96Sagiri ret); 621b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 622b86efd96Sagiri recvqp->qp_taskqpending = B_FALSE; 623b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 624b86efd96Sagiri } 625b86efd96Sagiri return; 626b86efd96Sagiri } 627b86efd96Sagiri 628b86efd96Sagiri if (jx != nspace) { 629b86efd96Sagiri RDS_DPRINTF2(LABEL, "EP(%p): Recv buffers " 630b86efd96Sagiri "needed: %d available: %d", ep, nspace, jx); 631b86efd96Sagiri nspace = jx; 632b86efd96Sagiri } 633b86efd96Sagiri 634b86efd96Sagiri bp1 = bp; 635b86efd96Sagiri for (ix = 0; ix < nspace; ix++) { 636b86efd96Sagiri bp1->buf_ep = ep; 637b86efd96Sagiri ASSERT(bp1->buf_state == RDS_RCVBUF_FREE); 638b86efd96Sagiri bp1->buf_state = RDS_RCVBUF_POSTED; 639b86efd96Sagiri bp1->buf_ds.ds_key = hcap->hca_lkey; 640b86efd96Sagiri bp1->buf_ds.ds_len = rcv_len; 641b86efd96Sagiri bp1 = bp1->buf_nextp; 642b86efd96Sagiri } 643b86efd96Sagiri 644b86efd96Sagiri #if 0 645b86efd96Sagiri wrp = kmem_zalloc(RDS_POST_FEW_ATATIME * sizeof (ibt_recv_wr_t), 646b86efd96Sagiri KM_SLEEP); 647b86efd96Sagiri #else 648b86efd96Sagiri wrp = &wr[0]; 649b86efd96Sagiri #endif 650b86efd96Sagiri 651b86efd96Sagiri npost = nspace; 652b86efd96Sagiri while (npost) { 653b86efd96Sagiri jx = (npost > RDS_POST_FEW_ATATIME) ? 654b86efd96Sagiri RDS_POST_FEW_ATATIME : npost; 655b86efd96Sagiri for (ix = 0; ix < jx; ix++) { 656b86efd96Sagiri wrp[ix].wr_id = (uintptr_t)bp; 657b86efd96Sagiri wrp[ix].wr_nds = 1; 658b86efd96Sagiri wrp[ix].wr_sgl = &bp->buf_ds; 659b86efd96Sagiri bp = bp->buf_nextp; 660b86efd96Sagiri } 661b86efd96Sagiri 662b86efd96Sagiri ret = ibt_post_recv(chanhdl, wrp, jx, &kx); 663b86efd96Sagiri if ((ret != IBT_SUCCESS) || (kx != jx)) { 664b86efd96Sagiri RDS_DPRINTF1(LABEL, "ibt_post_recv for %d WRs failed: " 665b86efd96Sagiri "%d", npost, ret); 666b86efd96Sagiri npost -= kx; 667b86efd96Sagiri break; 668b86efd96Sagiri } 669b86efd96Sagiri 670b86efd96Sagiri npost -= jx; 671b86efd96Sagiri } 672b86efd96Sagiri 673b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 674b86efd96Sagiri if (npost != 0) { 675b86efd96Sagiri RDS_DPRINTF2("rds_post_recv_buf", 676b86efd96Sagiri "EP(%p) Failed to post %d WRs", ep, npost); 677b86efd96Sagiri recvqp->qp_level += (nspace - npost); 678b86efd96Sagiri } else { 679b86efd96Sagiri recvqp->qp_level += nspace; 680b86efd96Sagiri } 681b86efd96Sagiri 682b86efd96Sagiri /* 683b86efd96Sagiri * sometimes, the recv WRs can get consumed as soon as they are 684b86efd96Sagiri * posted. In that case, taskq thread to post more WRs to the RQ will 685b86efd96Sagiri * not be scheduled as the taskqpending flag is still set. 686b86efd96Sagiri */ 687b86efd96Sagiri if (recvqp->qp_level == 0) { 688b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 689b86efd96Sagiri ret = ddi_taskq_dispatch(rds_taskq, 690b86efd96Sagiri rds_post_recv_buf, (void *)ep->ep_chanhdl, DDI_NOSLEEP); 691b86efd96Sagiri if (ret != DDI_SUCCESS) { 692b86efd96Sagiri RDS_DPRINTF1("rds_post_recv_buf", 693b86efd96Sagiri "ddi_taskq_dispatch failed: %d", ret); 694b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 695b86efd96Sagiri recvqp->qp_taskqpending = B_FALSE; 696b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 697b86efd96Sagiri } 698b86efd96Sagiri } else { 699b86efd96Sagiri recvqp->qp_taskqpending = B_FALSE; 700b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 701b86efd96Sagiri } 702b86efd96Sagiri 703b86efd96Sagiri #if 0 704b86efd96Sagiri kmem_free(wrp, RDS_POST_FEW_ATATIME * sizeof (ibt_recv_wr_t)); 705b86efd96Sagiri #endif 706b86efd96Sagiri 707b86efd96Sagiri RDS_DPRINTF4("rds_post_recv_buf", "Return: EP(%p)", ep); 708b86efd96Sagiri } 709b86efd96Sagiri 710b86efd96Sagiri static int 711b86efd96Sagiri rds_poll_data_completions(ibt_cq_hdl_t cq, rds_ep_t *ep) 712b86efd96Sagiri { 713b86efd96Sagiri ibt_wc_t wc; 714b86efd96Sagiri rds_buf_t *bp; 715b86efd96Sagiri rds_data_hdr_t *pktp; 716b86efd96Sagiri rds_qp_t *recvqp; 717b86efd96Sagiri uint_t npolled; 718b86efd96Sagiri int ret = IBT_SUCCESS; 719b86efd96Sagiri 720b86efd96Sagiri 721b86efd96Sagiri RDS_DPRINTF4("rds_poll_data_completions", "Enter: EP(%p)", ep); 722b86efd96Sagiri 723b86efd96Sagiri bzero(&wc, sizeof (ibt_wc_t)); 724b86efd96Sagiri ret = ibt_poll_cq(cq, &wc, 1, &npolled); 725b86efd96Sagiri if (ret != IBT_SUCCESS) { 726b86efd96Sagiri if (ret != IBT_CQ_EMPTY) { 727b86efd96Sagiri RDS_DPRINTF2(LABEL, "EP(%p) CQ(%p): ibt_poll_cq " 728b86efd96Sagiri "returned: %d", ep, cq, ret); 729b86efd96Sagiri } else { 730b86efd96Sagiri RDS_DPRINTF5(LABEL, "EP(%p) CQ(%p): ibt_poll_cq " 731b86efd96Sagiri "returned: IBT_CQ_EMPTY", ep, cq); 732b86efd96Sagiri } 733b86efd96Sagiri return (ret); 734b86efd96Sagiri } 735b86efd96Sagiri 736b86efd96Sagiri bp = (rds_buf_t *)(uintptr_t)wc.wc_id; 737b86efd96Sagiri ASSERT(bp->buf_state == RDS_RCVBUF_POSTED); 738b86efd96Sagiri bp->buf_state = RDS_RCVBUF_ONSOCKQ; 739b86efd96Sagiri bp->buf_nextp = NULL; 740b86efd96Sagiri 741b86efd96Sagiri if (wc.wc_status != IBT_WC_SUCCESS) { 742b86efd96Sagiri mutex_enter(&ep->ep_recvqp.qp_lock); 743b86efd96Sagiri ep->ep_recvqp.qp_level--; 744b86efd96Sagiri mutex_exit(&ep->ep_recvqp.qp_lock); 745b86efd96Sagiri 746b86efd96Sagiri /* free the buffer */ 747b86efd96Sagiri bp->buf_state = RDS_RCVBUF_FREE; 748b86efd96Sagiri rds_free_recv_buf(bp, 1); 749b86efd96Sagiri 750b86efd96Sagiri /* Receive completion failure */ 751b86efd96Sagiri if (wc.wc_status != IBT_WC_WR_FLUSHED_ERR) { 752b86efd96Sagiri RDS_DPRINTF2("rds_poll_data_completions", 753b86efd96Sagiri "EP(%p) CQ(%p) BP(%p): WC Error Status: %d", 754b86efd96Sagiri ep, cq, wc.wc_id, wc.wc_status); 755b86efd96Sagiri RDS_INCR_RXERRS(); 756b86efd96Sagiri } 757b86efd96Sagiri return (ret); 758b86efd96Sagiri } 759b86efd96Sagiri 760b86efd96Sagiri /* there is one less in the RQ */ 761b86efd96Sagiri recvqp = &ep->ep_recvqp; 762b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 763b86efd96Sagiri recvqp->qp_level--; 764b86efd96Sagiri if ((recvqp->qp_taskqpending == B_FALSE) && 765b86efd96Sagiri (recvqp->qp_level <= recvqp->qp_lwm)) { 766b86efd96Sagiri /* Time to post more buffers into the RQ */ 767b86efd96Sagiri recvqp->qp_taskqpending = B_TRUE; 768b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 769b86efd96Sagiri 770b86efd96Sagiri ret = ddi_taskq_dispatch(rds_taskq, 771b86efd96Sagiri rds_post_recv_buf, (void *)ep->ep_chanhdl, DDI_NOSLEEP); 772b86efd96Sagiri if (ret != DDI_SUCCESS) { 773b86efd96Sagiri RDS_DPRINTF1(LABEL, "ddi_taskq_dispatch failed: %d", 774b86efd96Sagiri ret); 775b86efd96Sagiri mutex_enter(&recvqp->qp_lock); 776b86efd96Sagiri recvqp->qp_taskqpending = B_FALSE; 777b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 778b86efd96Sagiri } 779b86efd96Sagiri } else { 780b86efd96Sagiri mutex_exit(&recvqp->qp_lock); 781b86efd96Sagiri } 782b86efd96Sagiri 783b86efd96Sagiri pktp = (rds_data_hdr_t *)(uintptr_t)bp->buf_ds.ds_va; 784b86efd96Sagiri ASSERT(pktp->dh_datalen != 0); 785b86efd96Sagiri 786b86efd96Sagiri RDS_DPRINTF5(LABEL, "Message Received: sendIP: 0x%x recvIP: 0x%x " 787b86efd96Sagiri "sendport: %d recvport: %d npkts: %d pktno: %d", ep->ep_remip, 788b86efd96Sagiri ep->ep_myip, pktp->dh_sendport, pktp->dh_recvport, 789b86efd96Sagiri pktp->dh_npkts, pktp->dh_psn); 790b86efd96Sagiri 791b86efd96Sagiri RDS_DPRINTF3(LABEL, "BP(%p): npkts: %d psn: %d", bp, 792b86efd96Sagiri pktp->dh_npkts, pktp->dh_psn); 793b86efd96Sagiri 794b86efd96Sagiri if (pktp->dh_npkts == 1) { 795b86efd96Sagiri /* single pkt or last packet */ 796b86efd96Sagiri if (pktp->dh_psn != 0) { 797b86efd96Sagiri /* last packet of a segmented message */ 798b86efd96Sagiri ASSERT(ep->ep_seglbp != NULL); 799b86efd96Sagiri ep->ep_seglbp->buf_nextp = bp; 800b86efd96Sagiri ep->ep_seglbp = bp; 801b86efd96Sagiri rds_received_msg(ep, ep->ep_segfbp); 802b86efd96Sagiri ep->ep_segfbp = NULL; 803b86efd96Sagiri ep->ep_seglbp = NULL; 804b86efd96Sagiri } else { 805b86efd96Sagiri /* single packet */ 806b86efd96Sagiri rds_received_msg(ep, bp); 807b86efd96Sagiri } 808b86efd96Sagiri } else { 809b86efd96Sagiri /* multi-pkt msg */ 810b86efd96Sagiri if (pktp->dh_psn == 0) { 811b86efd96Sagiri /* first packet */ 812b86efd96Sagiri ASSERT(ep->ep_segfbp == NULL); 813b86efd96Sagiri ep->ep_segfbp = bp; 814b86efd96Sagiri ep->ep_seglbp = bp; 815b86efd96Sagiri } else { 816b86efd96Sagiri /* intermediate packet */ 817b86efd96Sagiri ASSERT(ep->ep_segfbp != NULL); 818b86efd96Sagiri ep->ep_seglbp->buf_nextp = bp; 819b86efd96Sagiri ep->ep_seglbp = bp; 820b86efd96Sagiri } 821b86efd96Sagiri } 822b86efd96Sagiri 823b86efd96Sagiri RDS_DPRINTF4("rds_poll_data_completions", "Return: EP(%p)", ep); 824b86efd96Sagiri 825b86efd96Sagiri return (ret); 826b86efd96Sagiri } 827b86efd96Sagiri 828b86efd96Sagiri void 829b86efd96Sagiri rds_recvcq_handler(ibt_cq_hdl_t cq, void *arg) 830b86efd96Sagiri { 831b86efd96Sagiri rds_ep_t *ep; 832b86efd96Sagiri int ret = IBT_SUCCESS; 833b86efd96Sagiri int (*func)(ibt_cq_hdl_t, rds_ep_t *); 834b86efd96Sagiri 835b86efd96Sagiri ep = (rds_ep_t *)arg; 836b86efd96Sagiri 837b86efd96Sagiri RDS_DPRINTF4("rds_recvcq_handler", "enter: EP(%p)", ep); 838b86efd96Sagiri 839b86efd96Sagiri if (ep->ep_type == RDS_EP_TYPE_DATA) { 840b86efd96Sagiri func = rds_poll_data_completions; 841b86efd96Sagiri } else { 842b86efd96Sagiri func = rds_poll_ctrl_completions; 843b86efd96Sagiri } 844b86efd96Sagiri 845b86efd96Sagiri do { 846b86efd96Sagiri ret = func(cq, ep); 847b86efd96Sagiri } while (ret != IBT_CQ_EMPTY); 848b86efd96Sagiri 849b86efd96Sagiri /* enable the CQ */ 850b86efd96Sagiri ret = ibt_enable_cq_notify(cq, rds_wc_signal); 851b86efd96Sagiri if (ret != IBT_SUCCESS) { 852b86efd96Sagiri RDS_DPRINTF2(LABEL, "EP(%p) CQ(%p): ibt_enable_cq_notify " 853b86efd96Sagiri "failed: %d", ep, cq, ret); 854b86efd96Sagiri return; 855b86efd96Sagiri } 856b86efd96Sagiri 857b86efd96Sagiri do { 858b86efd96Sagiri ret = func(cq, ep); 859b86efd96Sagiri } while (ret != IBT_CQ_EMPTY); 860b86efd96Sagiri 861b86efd96Sagiri RDS_DPRINTF4("rds_recvcq_handler", "Return: EP(%p)", ep); 862b86efd96Sagiri } 863b86efd96Sagiri 864b86efd96Sagiri void 865b86efd96Sagiri rds_poll_send_completions(ibt_cq_hdl_t cq, rds_ep_t *ep, boolean_t lock) 866b86efd96Sagiri { 867b86efd96Sagiri ibt_wc_t wc[RDS_NUM_DATA_SEND_WCS]; 868b86efd96Sagiri uint_t npolled, nret, send_error = 0; 869b86efd96Sagiri rds_buf_t *headp, *tailp, *bp; 870b86efd96Sagiri int ret, ix; 871b86efd96Sagiri 872b86efd96Sagiri RDS_DPRINTF4("rds_poll_send_completions", "Enter EP(%p)", ep); 873b86efd96Sagiri 874b86efd96Sagiri headp = NULL; 875b86efd96Sagiri tailp = NULL; 876b86efd96Sagiri npolled = 0; 877b86efd96Sagiri do { 878b86efd96Sagiri ret = ibt_poll_cq(cq, wc, RDS_NUM_DATA_SEND_WCS, &nret); 879b86efd96Sagiri if (ret != IBT_SUCCESS) { 880b86efd96Sagiri if (ret != IBT_CQ_EMPTY) { 881b86efd96Sagiri RDS_DPRINTF2(LABEL, "EP(%p) CQ(%p): " 882b86efd96Sagiri "ibt_poll_cq returned: %d", ep, cq, ret); 883b86efd96Sagiri } else { 884b86efd96Sagiri RDS_DPRINTF5(LABEL, "EP(%p) CQ(%p): " 885b86efd96Sagiri "ibt_poll_cq returned: IBT_CQ_EMPTY", 886b86efd96Sagiri ep, cq); 887b86efd96Sagiri } 888b86efd96Sagiri 889b86efd96Sagiri break; 890b86efd96Sagiri } 891b86efd96Sagiri 892b86efd96Sagiri for (ix = 0; ix < nret; ix++) { 893b86efd96Sagiri if (wc[ix].wc_status == IBT_WC_SUCCESS) { 894b86efd96Sagiri if (wc[ix].wc_type == IBT_WRC_RDMAW) { 895b86efd96Sagiri rds_send_acknowledgement(ep); 896b86efd96Sagiri continue; 897b86efd96Sagiri } 898b86efd96Sagiri 899b86efd96Sagiri bp = (rds_buf_t *)(uintptr_t)wc[ix].wc_id; 900b86efd96Sagiri ASSERT(bp->buf_state == RDS_SNDBUF_PENDING); 901b86efd96Sagiri bp->buf_state = RDS_SNDBUF_FREE; 902b86efd96Sagiri } else if (wc[ix].wc_status == IBT_WC_WR_FLUSHED_ERR) { 903b86efd96Sagiri RDS_INCR_TXERRS(); 904b86efd96Sagiri RDS_DPRINTF5("rds_poll_send_completions", 905b86efd96Sagiri "EP(%p): WC ID: %p ERROR: %d", ep, 906b86efd96Sagiri wc[ix].wc_id, wc[ix].wc_status); 907b86efd96Sagiri 908b86efd96Sagiri if (wc[ix].wc_id == RDS_RDMAW_WRID) { 909b86efd96Sagiri mutex_enter(&ep->ep_lock); 910b86efd96Sagiri ep->ep_rdmacnt--; 911b86efd96Sagiri mutex_exit(&ep->ep_lock); 912b86efd96Sagiri continue; 913b86efd96Sagiri } 914b86efd96Sagiri 915b86efd96Sagiri bp = (rds_buf_t *)(uintptr_t)wc[ix].wc_id; 916b86efd96Sagiri bp->buf_state = RDS_SNDBUF_ERROR; 917b86efd96Sagiri } else { 918b86efd96Sagiri RDS_INCR_TXERRS(); 919b86efd96Sagiri RDS_DPRINTF2("rds_poll_send_completions", 920b86efd96Sagiri "EP(%p): WC ID: %p ERROR: %d", ep, 921b86efd96Sagiri wc[ix].wc_id, wc[ix].wc_status); 922b86efd96Sagiri if (send_error == 0) { 923b86efd96Sagiri rds_session_t *sp = ep->ep_sp; 924b86efd96Sagiri 925b86efd96Sagiri /* don't let anyone send anymore */ 926b86efd96Sagiri rw_enter(&sp->session_lock, RW_WRITER); 927b86efd96Sagiri if (sp->session_state != 928b86efd96Sagiri RDS_SESSION_STATE_ERROR) { 929b86efd96Sagiri sp->session_state = 930b86efd96Sagiri RDS_SESSION_STATE_ERROR; 931b86efd96Sagiri /* Make this the active end */ 932b86efd96Sagiri sp->session_type = 933b86efd96Sagiri RDS_SESSION_ACTIVE; 934b86efd96Sagiri } 935b86efd96Sagiri rw_exit(&sp->session_lock); 936b86efd96Sagiri } 937b86efd96Sagiri 938b86efd96Sagiri send_error++; 939b86efd96Sagiri 940b86efd96Sagiri if (wc[ix].wc_id == RDS_RDMAW_WRID) { 941b86efd96Sagiri mutex_enter(&ep->ep_lock); 942b86efd96Sagiri ep->ep_rdmacnt--; 943b86efd96Sagiri mutex_exit(&ep->ep_lock); 944b86efd96Sagiri continue; 945b86efd96Sagiri } 946b86efd96Sagiri 947b86efd96Sagiri bp = (rds_buf_t *)(uintptr_t)wc[ix].wc_id; 948b86efd96Sagiri bp->buf_state = RDS_SNDBUF_ERROR; 949b86efd96Sagiri } 950b86efd96Sagiri 951b86efd96Sagiri bp->buf_nextp = NULL; 952b86efd96Sagiri if (headp) { 953b86efd96Sagiri tailp->buf_nextp = bp; 954b86efd96Sagiri tailp = bp; 955b86efd96Sagiri } else { 956b86efd96Sagiri headp = bp; 957b86efd96Sagiri tailp = bp; 958b86efd96Sagiri } 959b86efd96Sagiri 960b86efd96Sagiri npolled++; 961b86efd96Sagiri } 962b86efd96Sagiri 963b86efd96Sagiri if (rds_no_interrupts && (npolled > 100)) { 964b86efd96Sagiri break; 965b86efd96Sagiri } 966b86efd96Sagiri 967b86efd96Sagiri if (rds_no_interrupts == 1) { 968b86efd96Sagiri break; 969b86efd96Sagiri } 970b86efd96Sagiri } while (ret != IBT_CQ_EMPTY); 971b86efd96Sagiri 972b86efd96Sagiri RDS_DPRINTF5("rds_poll_send_completions", "Npolled: %d send_error: %d", 973b86efd96Sagiri npolled, send_error); 974b86efd96Sagiri 975b86efd96Sagiri /* put the buffers to the pool */ 976b86efd96Sagiri if (npolled != 0) { 977b86efd96Sagiri rds_free_send_buf(ep, headp, tailp, npolled, lock); 978b86efd96Sagiri } 979b86efd96Sagiri 980b86efd96Sagiri if (send_error != 0) { 981b86efd96Sagiri rds_handle_send_error(ep); 982b86efd96Sagiri } 983b86efd96Sagiri 984b86efd96Sagiri RDS_DPRINTF4("rds_poll_send_completions", "Return EP(%p)", ep); 985b86efd96Sagiri } 986b86efd96Sagiri 987b86efd96Sagiri void 988b86efd96Sagiri rds_sendcq_handler(ibt_cq_hdl_t cq, void *arg) 989b86efd96Sagiri { 990b86efd96Sagiri rds_ep_t *ep; 991b86efd96Sagiri int ret; 992b86efd96Sagiri 993b86efd96Sagiri ep = (rds_ep_t *)arg; 994b86efd96Sagiri 995b86efd96Sagiri RDS_DPRINTF4("rds_sendcq_handler", "Enter: EP(%p)", ep); 996b86efd96Sagiri 997b86efd96Sagiri /* enable the CQ */ 998b86efd96Sagiri ret = ibt_enable_cq_notify(cq, IBT_NEXT_COMPLETION); 999b86efd96Sagiri if (ret != IBT_SUCCESS) { 1000b86efd96Sagiri RDS_DPRINTF2(LABEL, "EP(%p) CQ(%p): ibt_enable_cq_notify " 1001b86efd96Sagiri "failed: %d", ep, cq, ret); 1002b86efd96Sagiri return; 1003b86efd96Sagiri } 1004b86efd96Sagiri 1005b86efd96Sagiri rds_poll_send_completions(cq, ep, B_FALSE); 1006b86efd96Sagiri 1007b86efd96Sagiri RDS_DPRINTF4("rds_sendcq_handler", "Return: EP(%p)", ep); 1008b86efd96Sagiri } 1009b86efd96Sagiri 1010b86efd96Sagiri void 1011b86efd96Sagiri rds_ep_free_rc_channel(rds_ep_t *ep) 1012b86efd96Sagiri { 1013b86efd96Sagiri int ret; 1014b86efd96Sagiri 1015b86efd96Sagiri RDS_DPRINTF2("rds_ep_free_rc_channel", "EP(%p) - Enter", ep); 1016b86efd96Sagiri 1017b86efd96Sagiri ASSERT(mutex_owned(&ep->ep_lock)); 1018b86efd96Sagiri 1019b86efd96Sagiri /* free the QP */ 1020b86efd96Sagiri if (ep->ep_chanhdl != NULL) { 1021b86efd96Sagiri /* wait until the RQ is empty */ 1022b86efd96Sagiri (void) ibt_flush_channel(ep->ep_chanhdl); 1023b86efd96Sagiri (void) rds_is_recvq_empty(ep, B_TRUE); 1024b86efd96Sagiri ret = ibt_free_channel(ep->ep_chanhdl); 1025b86efd96Sagiri if (ret != IBT_SUCCESS) { 1026b86efd96Sagiri RDS_DPRINTF1("rds_ep_free_rc_channel", "EP(%p) " 1027b86efd96Sagiri "ibt_free_channel returned: %d", ep, ret); 1028b86efd96Sagiri } 1029b86efd96Sagiri ep->ep_chanhdl = NULL; 1030b86efd96Sagiri } else { 1031b86efd96Sagiri RDS_DPRINTF2("rds_ep_free_rc_channel", 1032b86efd96Sagiri "EP(%p) Channel is ALREADY FREE", ep); 1033b86efd96Sagiri } 1034b86efd96Sagiri 1035b86efd96Sagiri /* free the Send CQ */ 1036b86efd96Sagiri if (ep->ep_sendcq != NULL) { 1037b86efd96Sagiri ret = ibt_free_cq(ep->ep_sendcq); 1038b86efd96Sagiri if (ret != IBT_SUCCESS) { 1039b86efd96Sagiri RDS_DPRINTF1("rds_ep_free_rc_channel", 1040b86efd96Sagiri "EP(%p) - for sendcq, ibt_free_cq returned %d", 1041b86efd96Sagiri ep, ret); 1042b86efd96Sagiri } 1043b86efd96Sagiri ep->ep_sendcq = NULL; 1044b86efd96Sagiri } else { 1045b86efd96Sagiri RDS_DPRINTF2("rds_ep_free_rc_channel", 1046b86efd96Sagiri "EP(%p) SendCQ is ALREADY FREE", ep); 1047b86efd96Sagiri } 1048b86efd96Sagiri 1049b86efd96Sagiri /* free the Recv CQ */ 1050b86efd96Sagiri if (ep->ep_recvcq != NULL) { 1051b86efd96Sagiri ret = ibt_free_cq(ep->ep_recvcq); 1052b86efd96Sagiri if (ret != IBT_SUCCESS) { 1053b86efd96Sagiri RDS_DPRINTF1("rds_ep_free_rc_channel", 1054b86efd96Sagiri "EP(%p) - for recvcq, ibt_free_cq returned %d", 1055b86efd96Sagiri ep, ret); 1056b86efd96Sagiri } 1057b86efd96Sagiri ep->ep_recvcq = NULL; 1058b86efd96Sagiri } else { 1059b86efd96Sagiri RDS_DPRINTF2("rds_ep_free_rc_channel", 1060b86efd96Sagiri "EP(%p) RecvCQ is ALREADY FREE", ep); 1061b86efd96Sagiri } 1062b86efd96Sagiri 1063b86efd96Sagiri RDS_DPRINTF2("rds_ep_free_rc_channel", "EP(%p) - Return", ep); 1064b86efd96Sagiri } 1065b86efd96Sagiri 1066b86efd96Sagiri /* Allocate resources for RC channel */ 1067b86efd96Sagiri ibt_channel_hdl_t 1068b86efd96Sagiri rds_ep_alloc_rc_channel(rds_ep_t *ep, uint8_t hca_port) 1069b86efd96Sagiri { 1070b86efd96Sagiri int ret = IBT_SUCCESS; 1071b86efd96Sagiri ibt_cq_attr_t scqattr, rcqattr; 1072b86efd96Sagiri ibt_rc_chan_alloc_args_t chanargs; 1073b86efd96Sagiri ibt_channel_hdl_t chanhdl; 1074*c1f8b08eSagiri rds_session_t *sp; 1075b86efd96Sagiri rds_hca_t *hcap; 1076b86efd96Sagiri 1077b86efd96Sagiri RDS_DPRINTF4("rds_ep_alloc_rc_channel", "Enter: 0x%p port: %d", 1078b86efd96Sagiri ep, hca_port); 1079b86efd96Sagiri 1080*c1f8b08eSagiri /* Update the EP with the right IP address and HCA guid */ 1081*c1f8b08eSagiri sp = ep->ep_sp; 1082*c1f8b08eSagiri ASSERT(sp != NULL); 1083*c1f8b08eSagiri rw_enter(&sp->session_lock, RW_READER); 1084*c1f8b08eSagiri mutex_enter(&ep->ep_lock); 1085*c1f8b08eSagiri ep->ep_myip = sp->session_myip; 1086*c1f8b08eSagiri ep->ep_remip = sp->session_remip; 1087*c1f8b08eSagiri hcap = rds_gid_to_hcap(rdsib_statep, sp->session_lgid); 1088*c1f8b08eSagiri ep->ep_hca_guid = hcap->hca_guid; 1089*c1f8b08eSagiri mutex_exit(&ep->ep_lock); 1090*c1f8b08eSagiri rw_exit(&sp->session_lock); 1091b86efd96Sagiri 1092b86efd96Sagiri /* reset taskqpending flag here */ 1093b86efd96Sagiri ep->ep_recvqp.qp_taskqpending = B_FALSE; 1094b86efd96Sagiri 1095b86efd96Sagiri if (ep->ep_type == RDS_EP_TYPE_CTRL) { 1096b86efd96Sagiri scqattr.cq_size = MaxCtrlSendBuffers; 1097b86efd96Sagiri scqattr.cq_sched = NULL; 1098b86efd96Sagiri scqattr.cq_flags = IBT_CQ_NO_FLAGS; 1099b86efd96Sagiri 1100b86efd96Sagiri rcqattr.cq_size = MaxCtrlRecvBuffers; 1101b86efd96Sagiri rcqattr.cq_sched = NULL; 1102b86efd96Sagiri rcqattr.cq_flags = IBT_CQ_NO_FLAGS; 1103b86efd96Sagiri 1104b86efd96Sagiri chanargs.rc_sizes.cs_sq = MaxCtrlSendBuffers; 1105b86efd96Sagiri chanargs.rc_sizes.cs_rq = MaxCtrlRecvBuffers; 1106b86efd96Sagiri chanargs.rc_sizes.cs_sq_sgl = 1; 1107b86efd96Sagiri chanargs.rc_sizes.cs_rq_sgl = 1; 1108b86efd96Sagiri } else { 1109b86efd96Sagiri scqattr.cq_size = MaxDataSendBuffers + RDS_NUM_ACKS; 1110b86efd96Sagiri scqattr.cq_sched = NULL; 1111b86efd96Sagiri scqattr.cq_flags = IBT_CQ_NO_FLAGS; 1112b86efd96Sagiri 1113b86efd96Sagiri rcqattr.cq_size = MaxDataRecvBuffers; 1114b86efd96Sagiri rcqattr.cq_sched = NULL; 1115b86efd96Sagiri rcqattr.cq_flags = IBT_CQ_NO_FLAGS; 1116b86efd96Sagiri 1117b86efd96Sagiri chanargs.rc_sizes.cs_sq = MaxDataSendBuffers + RDS_NUM_ACKS; 1118b86efd96Sagiri chanargs.rc_sizes.cs_rq = MaxDataRecvBuffers; 1119b86efd96Sagiri chanargs.rc_sizes.cs_sq_sgl = 1; 1120b86efd96Sagiri chanargs.rc_sizes.cs_rq_sgl = 1; 1121b86efd96Sagiri } 1122b86efd96Sagiri 1123b86efd96Sagiri if (ep->ep_sendcq == NULL) { 1124b86efd96Sagiri /* returned size is always greater than the requested size */ 1125b86efd96Sagiri ret = ibt_alloc_cq(hcap->hca_hdl, &scqattr, 1126b86efd96Sagiri &ep->ep_sendcq, NULL); 1127b86efd96Sagiri if (ret != IBT_SUCCESS) { 1128b86efd96Sagiri RDS_DPRINTF2(LABEL, "ibt_alloc_cq for sendCQ " 1129b86efd96Sagiri "failed, size = %d: %d", scqattr.cq_size, ret); 1130b86efd96Sagiri return (NULL); 1131b86efd96Sagiri } 1132b86efd96Sagiri 1133b86efd96Sagiri (void) ibt_set_cq_handler(ep->ep_sendcq, rds_sendcq_handler, 1134b86efd96Sagiri ep); 1135b86efd96Sagiri 1136b86efd96Sagiri if (rds_no_interrupts == 0) { 1137b86efd96Sagiri ret = ibt_enable_cq_notify(ep->ep_sendcq, 1138b86efd96Sagiri IBT_NEXT_COMPLETION); 1139b86efd96Sagiri if (ret != IBT_SUCCESS) { 1140b86efd96Sagiri RDS_DPRINTF2(LABEL, 1141b86efd96Sagiri "ibt_enable_cq_notify failed: %d", ret); 1142b86efd96Sagiri (void) ibt_free_cq(ep->ep_sendcq); 1143b86efd96Sagiri ep->ep_sendcq = NULL; 1144b86efd96Sagiri return (NULL); 1145b86efd96Sagiri } 1146b86efd96Sagiri } 1147b86efd96Sagiri } 1148b86efd96Sagiri 1149b86efd96Sagiri if (ep->ep_recvcq == NULL) { 1150b86efd96Sagiri /* returned size is always greater than the requested size */ 1151b86efd96Sagiri ret = ibt_alloc_cq(hcap->hca_hdl, &rcqattr, 1152b86efd96Sagiri &ep->ep_recvcq, NULL); 1153b86efd96Sagiri if (ret != IBT_SUCCESS) { 1154b86efd96Sagiri RDS_DPRINTF2(LABEL, "ibt_alloc_cq for recvCQ " 1155b86efd96Sagiri "failed, size = %d: %d", rcqattr.cq_size, ret); 1156b86efd96Sagiri (void) ibt_free_cq(ep->ep_sendcq); 1157b86efd96Sagiri ep->ep_sendcq = NULL; 1158b86efd96Sagiri return (NULL); 1159b86efd96Sagiri } 1160b86efd96Sagiri 1161b86efd96Sagiri (void) ibt_set_cq_handler(ep->ep_recvcq, rds_recvcq_handler, 1162b86efd96Sagiri ep); 1163b86efd96Sagiri 1164b86efd96Sagiri ret = ibt_enable_cq_notify(ep->ep_recvcq, rds_wc_signal); 1165b86efd96Sagiri if (ret != IBT_SUCCESS) { 1166b86efd96Sagiri RDS_DPRINTF2(LABEL, 1167b86efd96Sagiri "ibt_enable_cq_notify failed: %d", ret); 1168b86efd96Sagiri (void) ibt_free_cq(ep->ep_recvcq); 1169b86efd96Sagiri ep->ep_recvcq = NULL; 1170b86efd96Sagiri (void) ibt_free_cq(ep->ep_sendcq); 1171b86efd96Sagiri ep->ep_sendcq = NULL; 1172b86efd96Sagiri return (NULL); 1173b86efd96Sagiri } 1174b86efd96Sagiri } 1175b86efd96Sagiri 1176b86efd96Sagiri chanargs.rc_flags = IBT_ALL_SIGNALED; 1177b86efd96Sagiri chanargs.rc_control = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR | 1178b86efd96Sagiri IBT_CEP_ATOMIC; 1179b86efd96Sagiri chanargs.rc_hca_port_num = hca_port; 1180b86efd96Sagiri chanargs.rc_scq = ep->ep_sendcq; 1181b86efd96Sagiri chanargs.rc_rcq = ep->ep_recvcq; 1182b86efd96Sagiri chanargs.rc_pd = hcap->hca_pdhdl; 1183b86efd96Sagiri chanargs.rc_srq = NULL; 1184b86efd96Sagiri 1185b86efd96Sagiri ret = ibt_alloc_rc_channel(hcap->hca_hdl, 1186b86efd96Sagiri IBT_ACHAN_NO_FLAGS, &chanargs, &chanhdl, NULL); 1187b86efd96Sagiri if (ret != IBT_SUCCESS) { 1188b86efd96Sagiri RDS_DPRINTF2(LABEL, "ibt_alloc_rc_channel fail: %d", 1189b86efd96Sagiri ret); 1190b86efd96Sagiri (void) ibt_free_cq(ep->ep_recvcq); 1191b86efd96Sagiri ep->ep_recvcq = NULL; 1192b86efd96Sagiri (void) ibt_free_cq(ep->ep_sendcq); 1193b86efd96Sagiri ep->ep_sendcq = NULL; 1194b86efd96Sagiri return (NULL); 1195b86efd96Sagiri } 1196b86efd96Sagiri 1197b86efd96Sagiri /* Chan private should contain the ep */ 1198b86efd96Sagiri (void) ibt_set_chan_private(chanhdl, ep); 1199b86efd96Sagiri 1200b86efd96Sagiri RDS_DPRINTF4("rds_ep_alloc_rc_channel", "Return: 0x%p", chanhdl); 1201b86efd96Sagiri 1202b86efd96Sagiri return (chanhdl); 1203b86efd96Sagiri } 1204b86efd96Sagiri 1205b86efd96Sagiri 1206b86efd96Sagiri #if 0 1207b86efd96Sagiri 1208b86efd96Sagiri /* Return node guid given a port gid */ 1209b86efd96Sagiri ib_guid_t 1210b86efd96Sagiri rds_gid_to_node_guid(ib_gid_t gid) 1211b86efd96Sagiri { 1212b86efd96Sagiri ibt_node_info_t nodeinfo; 1213b86efd96Sagiri int ret; 1214b86efd96Sagiri 1215b86efd96Sagiri RDS_DPRINTF4("rds_gid_to_node_guid", "Enter: gid: %llx:%llx", 1216b86efd96Sagiri gid.gid_prefix, gid.gid_guid); 1217b86efd96Sagiri 1218b86efd96Sagiri ret = ibt_gid_to_node_info(gid, &nodeinfo); 1219b86efd96Sagiri if (ret != IBT_SUCCESS) { 1220b86efd96Sagiri RDS_DPRINTF2(LABEL, "ibt_gid_node_info for gid: %llx:%llx " 1221b86efd96Sagiri "failed", gid.gid_prefix, gid.gid_guid); 1222b86efd96Sagiri return (0LL); 1223b86efd96Sagiri } 1224b86efd96Sagiri 1225b86efd96Sagiri RDS_DPRINTF4("rds_gid_to_node_guid", "Return: Node guid: %llx", 1226b86efd96Sagiri nodeinfo.n_node_guid); 1227b86efd96Sagiri 1228b86efd96Sagiri return (nodeinfo.n_node_guid); 1229b86efd96Sagiri } 1230b86efd96Sagiri 1231b86efd96Sagiri #endif 1232b86efd96Sagiri 1233b86efd96Sagiri static void 1234b86efd96Sagiri rds_handle_portup_event(rds_state_t *statep, ibt_hca_hdl_t hdl, 1235b86efd96Sagiri ibt_async_event_t *event) 1236b86efd96Sagiri { 1237b86efd96Sagiri rds_hca_t *hcap; 1238b86efd96Sagiri ibt_hca_portinfo_t *newpinfop, *oldpinfop; 1239b86efd96Sagiri uint_t newsize, oldsize, nport; 1240b86efd96Sagiri ib_gid_t gid; 1241b86efd96Sagiri int ret; 1242b86efd96Sagiri 1243*c1f8b08eSagiri RDS_DPRINTF2("rds_handle_portup_event", 1244*c1f8b08eSagiri "Enter: GUID: 0x%llx Statep: %p", event->ev_hca_guid, statep); 1245b86efd96Sagiri 1246b86efd96Sagiri hcap = rds_get_hcap(statep, event->ev_hca_guid); 1247*c1f8b08eSagiri if (hcap == NULL) { 1248*c1f8b08eSagiri RDS_DPRINTF2("rds_handle_portup_event", "HCA: 0x%llx is " 1249*c1f8b08eSagiri "not in our list", event->ev_hca_guid); 1250*c1f8b08eSagiri return; 1251*c1f8b08eSagiri } 1252b86efd96Sagiri 1253b86efd96Sagiri ret = ibt_query_hca_ports(hdl, 0, &newpinfop, &nport, &newsize); 1254b86efd96Sagiri if (ret != IBT_SUCCESS) { 1255b86efd96Sagiri RDS_DPRINTF2(LABEL, "ibt_query_hca_ports failed: %d", ret); 1256b86efd96Sagiri return; 1257b86efd96Sagiri } 1258b86efd96Sagiri 1259b86efd96Sagiri oldpinfop = hcap->hca_pinfop; 1260b86efd96Sagiri oldsize = hcap->hca_pinfo_sz; 1261b86efd96Sagiri hcap->hca_pinfop = newpinfop; 1262b86efd96Sagiri hcap->hca_pinfo_sz = newsize; 1263b86efd96Sagiri 1264b86efd96Sagiri /* structure copy */ 1265b86efd96Sagiri gid = newpinfop[event->ev_port - 1].p_sgid_tbl[0]; 1266b86efd96Sagiri 1267b86efd96Sagiri /* bind RDS service on the port, pass statep as cm_private */ 1268b86efd96Sagiri ret = ibt_bind_service(statep->rds_srvhdl, gid, NULL, statep, NULL); 1269b86efd96Sagiri if (ret != IBT_SUCCESS) { 1270b86efd96Sagiri RDS_DPRINTF2(LABEL, "Bind service for HCA: 0x%llx Port: %d " 1271b86efd96Sagiri "gid %llx:%llx returned: %d", event->ev_hca_guid, 1272b86efd96Sagiri event->ev_port, gid.gid_prefix, gid.gid_guid, ret); 1273b86efd96Sagiri } 1274b86efd96Sagiri 1275b86efd96Sagiri (void) ibt_free_portinfo(oldpinfop, oldsize); 1276b86efd96Sagiri 1277b86efd96Sagiri RDS_DPRINTF2("rds_handle_portup_event", "Return: GUID: 0x%llx", 1278b86efd96Sagiri event->ev_hca_guid); 1279b86efd96Sagiri } 1280b86efd96Sagiri 1281b86efd96Sagiri static void 1282b86efd96Sagiri rds_async_handler(void *clntp, ibt_hca_hdl_t hdl, ibt_async_code_t code, 1283b86efd96Sagiri ibt_async_event_t *event) 1284b86efd96Sagiri { 1285b86efd96Sagiri rds_state_t *statep; 1286b86efd96Sagiri 1287b86efd96Sagiri RDS_DPRINTF2("rds_async_handler", "Async code: %d", code); 1288b86efd96Sagiri 1289b86efd96Sagiri switch (code) { 1290b86efd96Sagiri case IBT_EVENT_PORT_UP: 1291b86efd96Sagiri statep = (rds_state_t *)clntp; 1292b86efd96Sagiri rds_handle_portup_event(statep, hdl, event); 1293b86efd96Sagiri break; 1294b86efd96Sagiri 1295b86efd96Sagiri default: 1296b86efd96Sagiri RDS_DPRINTF2(LABEL, "Async event: %d not handled", code); 1297b86efd96Sagiri } 1298b86efd96Sagiri 1299b86efd96Sagiri RDS_DPRINTF2("rds_async_handler", "Return: code: %d", code); 1300b86efd96Sagiri } 1301