1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * Copyright (c) 2009 Oracle. All rights reserved. 27 * 28 * This software is available to you under a choice of one of two 29 * licenses. You may choose to be licensed under the terms of the GNU 30 * General Public License (GPL) Version 2, available from the file 31 * COPYING in the main directory of this source tree, or the 32 * OpenIB.org BSD license below: 33 * 34 * Redistribution and use in source and binary forms, with or 35 * without modification, are permitted provided that the following 36 * conditions are met: 37 * 38 * - Redistributions of source code must retain the above 39 * copyright notice, this list of conditions and the following 40 * disclaimer. 41 * 42 * - Redistributions in binary form must reproduce the above 43 * copyright notice, this list of conditions and the following 44 * disclaimer in the documentation and/or other materials 45 * provided with the distribution. 46 * 47 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 48 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 49 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 50 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 51 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 52 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 53 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 54 * SOFTWARE. 55 * 56 */ 57 #include <sys/ib/clients/of/rdma/ib_verbs.h> 58 #include <sys/ib/clients/of/rdma/ib_addr.h> 59 #include <sys/ib/clients/of/rdma/rdma_cm.h> 60 61 #include <sys/ib/clients/rdsv3/ib.h> 62 #include <sys/ib/clients/rdsv3/rdma_transport.h> 63 #include <sys/ib/clients/rdsv3/rdsv3_debug.h> 64 65 kmutex_t rdsv3_rdma_listen_id_lock; 66 struct rdma_cm_id *rdsv3_rdma_listen_id = NULL; 67 68 int 69 rdsv3_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 70 struct rdma_cm_event *event) 71 { 72 /* this can be null in the listening path */ 73 struct rdsv3_connection *conn = cm_id->context; 74 struct rdsv3_transport *trans; 75 int ret = 0; 76 77 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 78 "conn %p id %p handling event %u", conn, cm_id, event->event); 79 80 trans = &rdsv3_ib_transport; 81 82 /* 83 * Prevent shutdown from tearing down the connection 84 * while we're executing. 85 */ 86 if (conn) { 87 mutex_enter(&conn->c_cm_lock); 88 89 /* 90 * If the connection is being shut down, bail out 91 * right away. We return 0 so cm_id doesn't get 92 * destroyed prematurely 93 */ 94 if (rdsv3_conn_state(conn) == RDSV3_CONN_DISCONNECTING) { 95 /* 96 * Reject incoming connections while we're tearing 97 * down an existing one. 98 */ 99 if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) 100 ret = 1; 101 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 102 "conn %p id %p incoming event %u when " 103 "disconnecting", conn, cm_id, event->event); 104 goto out; 105 } 106 } 107 108 switch (event->event) { 109 case RDMA_CM_EVENT_CONNECT_REQUEST: 110 ret = trans->cm_handle_connect(cm_id, event); 111 break; 112 113 case RDMA_CM_EVENT_ADDR_RESOLVED: 114 /* XXX do we need to clean up if this fails? */ 115 ret = rdma_resolve_route(cm_id, 116 RDSV3_RDMA_RESOLVE_TIMEOUT_MS); 117 break; 118 119 case RDMA_CM_EVENT_ROUTE_RESOLVED: 120 /* XXX worry about racing with listen acceptance */ 121 ret = trans->cm_initiate_connect(cm_id); 122 break; 123 124 case RDMA_CM_EVENT_ESTABLISHED: 125 trans->cm_connect_complete(conn, event); 126 break; 127 128 case RDMA_CM_EVENT_ADDR_ERROR: 129 case RDMA_CM_EVENT_ROUTE_ERROR: 130 case RDMA_CM_EVENT_CONNECT_ERROR: 131 case RDMA_CM_EVENT_UNREACHABLE: 132 case RDMA_CM_EVENT_REJECTED: 133 case RDMA_CM_EVENT_DEVICE_REMOVAL: 134 case RDMA_CM_EVENT_ADDR_CHANGE: 135 if (conn) 136 rdsv3_conn_drop(conn); 137 break; 138 139 case RDMA_CM_EVENT_DISCONNECTED: 140 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 141 "RDS/RDMA: DISCONNECT event - dropping connection " 142 "cm_id: %p", cm_id); 143 if (conn) { 144 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 145 "RDS/RDMA: DISCONNECT event - dropping connection " 146 "%u.%u.%u.%u->%u.%u.%u.%u", NIPQUAD(conn->c_laddr), 147 NIPQUAD(conn->c_faddr)); 148 rdsv3_conn_drop(conn); 149 } 150 break; 151 152 default: 153 /* things like device disconnect? */ 154 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 155 "unknown event %u!\n", event->event); 156 RDSV3_PANIC(); 157 break; 158 } 159 160 out: 161 if (conn) 162 mutex_exit(&conn->c_cm_lock); 163 164 RDSV3_DPRINTF2("rdsv3_rdma_cm_event_handler", 165 "id %p event %u handling ret %d", cm_id, event->event, ret); 166 167 return (ret); 168 } 169 170 static int 171 rdsv3_rdma_listen_init(void) 172 { 173 struct sockaddr_in sin; 174 struct rdma_cm_id *cm_id; 175 int ret; 176 177 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", "Enter"); 178 179 cm_id = rdma_create_id(rdsv3_rdma_cm_event_handler, NULL, RDMA_PS_TCP); 180 if (IS_ERR(cm_id)) { 181 ret = PTR_ERR(cm_id); 182 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 183 "RDS/RDMA: failed to setup listener, " 184 "rdma_create_id() returned %d", ret); 185 goto out; 186 } 187 188 sin.sin_family = PF_INET; 189 sin.sin_addr.s_addr = (uint32_t)htonl(INADDR_ANY); 190 sin.sin_port = (uint16_t)htons(RDSV3_PORT); 191 192 /* 193 * XXX I bet this binds the cm_id to a device. If we want to support 194 * fail-over we'll have to take this into consideration. 195 */ 196 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin); 197 if (ret) { 198 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 199 "RDS/RDMA: failed to setup listener, " 200 "rdma_bind_addr() returned %d", ret); 201 goto out; 202 } 203 204 ret = rdma_listen(cm_id, 128); 205 if (ret) { 206 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 207 "RDS/RDMA: failed to setup listener, " 208 "rdma_listen() returned %d", ret); 209 goto out; 210 } 211 212 RDSV3_DPRINTF5("rdsv3_rdma_listen_init", 213 "cm %p listening on port %u", cm_id, RDSV3_PORT); 214 215 rdsv3_rdma_listen_id = cm_id; 216 cm_id = NULL; 217 218 RDSV3_DPRINTF2("rdsv3_rdma_listen_init", 219 "Return: rdsv3_rdma_listen_id: %p", rdsv3_rdma_listen_id); 220 out: 221 if (cm_id) 222 rdma_destroy_id(cm_id); 223 return (ret); 224 } 225 226 static void rdsv3_rdma_listen_stop(void) 227 { 228 RDSV3_DPRINTF2("rdsv3_rdma_listen_stop", "cm %p", rdsv3_rdma_listen_id); 229 rdma_destroy_id(rdsv3_rdma_listen_id); 230 RDSV3_DPRINTF2("rdsv3_rdma_listen_stop", "Return"); 231 } 232 233 /* 234 * This function can be called via two routes. 235 * 1. During attach on a worker thread. 236 * 2. From rdsv3_create() for 1st socket. 237 */ 238 void 239 rdsv3_rdma_init() 240 { 241 int ret; 242 243 RDSV3_DPRINTF2("rdsv3_rdma_init", "Enter"); 244 245 mutex_enter(&rdsv3_rdma_listen_id_lock); 246 if (rdsv3_rdma_listen_id != NULL) { 247 RDSV3_DPRINTF2("rdsv3_rdma_init", 248 "rdsv3_rdma_listen_id is already initialized: %p", 249 rdsv3_rdma_listen_id); 250 mutex_exit(&rdsv3_rdma_listen_id_lock); 251 return; 252 } 253 254 ret = rdsv3_rdma_listen_init(); 255 if (ret) { 256 mutex_exit(&rdsv3_rdma_listen_id_lock); 257 return; 258 } 259 260 ret = rdsv3_ib_init(); 261 if (ret) { 262 rdsv3_rdma_listen_stop(); 263 } 264 mutex_exit(&rdsv3_rdma_listen_id_lock); 265 266 RDSV3_DPRINTF2("rdsv3_rdma_init", "Return"); 267 } 268 269 /*ARGSUSED*/ 270 void 271 rdsv3_rdma_exit(void *arg) 272 { 273 RDSV3_DPRINTF2("rdsv3_rdma_exit", "Enter"); 274 275 /* stop listening first to ensure no new connections are attempted */ 276 if (rdsv3_rdma_listen_id) { 277 rdsv3_rdma_listen_stop(); 278 rdsv3_ib_exit(); 279 rdsv3_rdma_listen_id = NULL; 280 } 281 282 RDSV3_DPRINTF2("rdsv3_rdma_exit", "Return"); 283 } 284