1 /* 2 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 3 */ 4 5 /* 6 * This file contains code imported from the OFED rds source file ib_cm.c 7 * Oracle elects to have and use the contents of ib_cm.c under and governed 8 * by the OpenIB.org BSD license (see below for full license text). However, 9 * the following notice accompanied the original version of this file: 10 */ 11 12 /* 13 * Copyright (c) 2006 Oracle. All rights reserved. 14 * 15 * This software is available to you under a choice of one of two 16 * licenses. You may choose to be licensed under the terms of the GNU 17 * General Public License (GPL) Version 2, available from the file 18 * COPYING in the main directory of this source tree, or the 19 * OpenIB.org BSD license below: 20 * 21 * Redistribution and use in source and binary forms, with or 22 * without modification, are permitted provided that the following 23 * conditions are met: 24 * 25 * - Redistributions of source code must retain the above 26 * copyright notice, this list of conditions and the following 27 * disclaimer. 28 * 29 * - Redistributions in binary form must reproduce the above 30 * copyright notice, this list of conditions and the following 31 * disclaimer in the documentation and/or other materials 32 * provided with the distribution. 33 * 34 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 35 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 36 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 37 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 38 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 39 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 40 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 41 * SOFTWARE. 42 * 43 */ 44 #include <sys/rds.h> 45 46 #include <sys/ib/clients/of/ofed_kernel.h> 47 #include <sys/ib/clients/of/rdma/ib_addr.h> 48 #include <sys/ib/clients/of/rdma/rdma_cm.h> 49 50 #include <sys/ib/clients/rdsv3/rdsv3.h> 51 #include <sys/ib/clients/rdsv3/ib.h> 52 #include <sys/ib/clients/rdsv3/rdsv3_debug.h> 53 54 extern int rdsv3_enable_snd_cq; 55 56 /* 57 * Set the selected protocol version 58 */ 59 static void 60 rdsv3_ib_set_protocol(struct rdsv3_connection *conn, unsigned int version) 61 { 62 RDSV3_DPRINTF4("rdsv3_ib_set_protocol", "conn: %p version: %d", 63 conn, version); 64 conn->c_version = version; 65 } 66 67 /* 68 * Set up flow control 69 */ 70 static void 71 rdsv3_ib_set_flow_control(struct rdsv3_connection *conn, uint32_t credits) 72 { 73 struct rdsv3_ib_connection *ic = conn->c_transport_data; 74 75 RDSV3_DPRINTF2("rdsv3_ib_set_flow_control", 76 "Enter: conn: %p credits: %d", conn, credits); 77 78 if (rdsv3_ib_sysctl_flow_control && credits != 0) { 79 /* We're doing flow control */ 80 ic->i_flowctl = 1; 81 rdsv3_ib_send_add_credits(conn, credits); 82 } else { 83 ic->i_flowctl = 0; 84 } 85 86 RDSV3_DPRINTF2("rdsv3_ib_set_flow_control", 87 "Return: conn: %p credits: %d", 88 conn, credits); 89 } 90 91 /* 92 * Tune RNR behavior. Without flow control, we use a rather 93 * low timeout, but not the absolute minimum - this should 94 * be tunable. 95 * 96 * We already set the RNR retry count to 7 (which is the 97 * smallest infinite number :-) above. 98 * If flow control is off, we want to change this back to 0 99 * so that we learn quickly when our credit accounting is 100 * buggy. 101 * 102 * Caller passes in a qp_attr pointer - don't waste stack spacv 103 * by allocation this twice. 104 */ 105 static void 106 rdsv3_ib_tune_rnr(struct rdsv3_ib_connection *ic, struct ib_qp_attr *attr) 107 { 108 int ret; 109 110 RDSV3_DPRINTF2("rdsv3_ib_tune_rnr", "Enter ic: %p attr: %p", 111 ic, attr); 112 113 attr->min_rnr_timer = IB_RNR_TIMER_000_32; 114 ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER); 115 if (ret) 116 RDSV3_DPRINTF2("rdsv3_ib_tune_rnr", 117 "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d", -ret); 118 } 119 120 /* 121 * Connection established. 122 * We get here for both outgoing and incoming connection. 123 */ 124 void 125 rdsv3_ib_cm_connect_complete(struct rdsv3_connection *conn, 126 struct rdma_cm_event *event) 127 { 128 const struct rdsv3_ib_connect_private *dp = NULL; 129 struct rdsv3_ib_connection *ic = conn->c_transport_data; 130 struct rdsv3_ib_device *rds_ibdev = 131 ib_get_client_data(ic->i_cm_id->device, &rdsv3_ib_client); 132 struct ib_qp_attr qp_attr; 133 int err; 134 135 RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete", 136 "Enter conn: %p event: %p", conn, event); 137 138 if (event->param.conn.private_data_len >= sizeof (*dp)) { 139 dp = event->param.conn.private_data; 140 141 /* make sure it isn't empty data */ 142 if (dp->dp_protocol_major) { 143 rdsv3_ib_set_protocol(conn, 144 RDS_PROTOCOL(dp->dp_protocol_major, 145 dp->dp_protocol_minor)); 146 rdsv3_ib_set_flow_control(conn, 147 ntohl(dp->dp_credit)); 148 } 149 } 150 151 if (conn->c_version < RDS_PROTOCOL(3, 1)) { 152 RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete", 153 "RDS/IB: Connection to %u.%u.%u.%u version %u.%u failed", 154 NIPQUAD(conn->c_faddr), 155 RDS_PROTOCOL_MAJOR(conn->c_version), 156 RDS_PROTOCOL_MINOR(conn->c_version)); 157 rdsv3_conn_destroy(conn); 158 return; 159 } else { 160 RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete", 161 "RDS/IB: connected to %u.%u.%u.%u version %u.%u%s", 162 NIPQUAD(conn->c_faddr), 163 RDS_PROTOCOL_MAJOR(conn->c_version), 164 RDS_PROTOCOL_MINOR(conn->c_version), 165 ic->i_flowctl ? ", flow control" : ""); 166 } 167 168 ASSERT(ic->i_soft_cq == NULL); 169 ic->i_soft_cq = rdsv3_af_intr_thr_create(rdsv3_ib_tasklet_fn, 170 (void *)ic, SCQ_INTR_BIND_CPU, rds_ibdev->aft_hcagp, 171 ic->i_cq->ibt_cq); 172 if (rdsv3_enable_snd_cq) { 173 ic->i_snd_soft_cq = rdsv3_af_intr_thr_create( 174 rdsv3_ib_snd_tasklet_fn, 175 (void *)ic, SCQ_INTR_BIND_CPU, rds_ibdev->aft_hcagp, 176 ic->i_snd_cq->ibt_cq); 177 } 178 ic->i_refill_rq = rdsv3_af_thr_create(rdsv3_ib_refill_fn, (void *)conn, 179 SCQ_WRK_BIND_CPU, rds_ibdev->aft_hcagp); 180 rdsv3_af_grp_draw(rds_ibdev->aft_hcagp); 181 182 (void) ib_req_notify_cq(ic->i_cq, IB_CQ_SOLICITED); 183 if (rdsv3_enable_snd_cq) { 184 (void) ib_req_notify_cq(ic->i_snd_cq, IB_CQ_NEXT_COMP); 185 } 186 187 /* 188 * Init rings and fill recv. this needs to wait until protocol 189 * negotiation 190 * is complete, since ring layout is different from 3.0 to 3.1. 191 */ 192 rdsv3_ib_send_init_ring(ic); 193 rdsv3_ib_recv_init_ring(ic); 194 /* 195 * Post receive buffers - as a side effect, this will update 196 * the posted credit count. 197 */ 198 (void) rdsv3_ib_recv_refill(conn, 1); 199 200 /* Tune RNR behavior */ 201 rdsv3_ib_tune_rnr(ic, &qp_attr); 202 203 qp_attr.qp_state = IB_QPS_RTS; 204 err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE); 205 if (err) 206 RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete", 207 "ib_modify_qp(IB_QP_STATE, RTS): err=%d", err); 208 209 /* update ib_device with this local ipaddr & conn */ 210 err = rdsv3_ib_update_ipaddr(rds_ibdev, conn->c_laddr); 211 if (err) 212 RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete", 213 "rdsv3_ib_update_ipaddr failed (%d)", err); 214 rdsv3_ib_add_conn(rds_ibdev, conn); 215 216 /* 217 * If the peer gave us the last packet it saw, process this as if 218 * we had received a regular ACK. 219 */ 220 if (dp && dp->dp_ack_seq) 221 rdsv3_send_drop_acked(conn, ntohll(dp->dp_ack_seq), NULL); 222 223 rdsv3_connect_complete(conn); 224 225 RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete", 226 "Return conn: %p event: %p", 227 conn, event); 228 } 229 230 static void 231 rdsv3_ib_cm_fill_conn_param(struct rdsv3_connection *conn, 232 struct rdma_conn_param *conn_param, 233 struct rdsv3_ib_connect_private *dp, 234 uint32_t protocol_version, 235 uint32_t max_responder_resources, 236 uint32_t max_initiator_depth) 237 { 238 struct rdsv3_ib_connection *ic = conn->c_transport_data; 239 struct rdsv3_ib_device *rds_ibdev; 240 241 RDSV3_DPRINTF2("rdsv3_ib_cm_fill_conn_param", 242 "Enter conn: %p conn_param: %p private: %p version: %d", 243 conn, conn_param, dp, protocol_version); 244 245 (void) memset(conn_param, 0, sizeof (struct rdma_conn_param)); 246 247 rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rdsv3_ib_client); 248 249 conn_param->responder_resources = 250 MIN(rds_ibdev->max_responder_resources, max_responder_resources); 251 conn_param->initiator_depth = 252 MIN(rds_ibdev->max_initiator_depth, max_initiator_depth); 253 conn_param->retry_count = min(rdsv3_ib_retry_count, 7); 254 conn_param->rnr_retry_count = 7; 255 256 if (dp) { 257 (void) memset(dp, 0, sizeof (*dp)); 258 dp->dp_saddr = conn->c_laddr; 259 dp->dp_daddr = conn->c_faddr; 260 dp->dp_protocol_major = RDS_PROTOCOL_MAJOR(protocol_version); 261 dp->dp_protocol_minor = RDS_PROTOCOL_MINOR(protocol_version); 262 dp->dp_protocol_minor_mask = 263 htons(RDSV3_IB_SUPPORTED_PROTOCOLS); 264 dp->dp_ack_seq = rdsv3_ib_piggyb_ack(ic); 265 266 /* Advertise flow control */ 267 if (ic->i_flowctl) { 268 unsigned int credits; 269 270 credits = IB_GET_POST_CREDITS( 271 atomic_get(&ic->i_credits)); 272 dp->dp_credit = htonl(credits); 273 atomic_add_32(&ic->i_credits, 274 -IB_SET_POST_CREDITS(credits)); 275 } 276 277 conn_param->private_data = dp; 278 conn_param->private_data_len = sizeof (*dp); 279 } 280 281 RDSV3_DPRINTF2("rdsv3_ib_cm_fill_conn_param", 282 "Return conn: %p conn_param: %p private: %p version: %d", 283 conn, conn_param, dp, protocol_version); 284 } 285 286 static void 287 rdsv3_ib_cq_event_handler(struct ib_event *event, void *data) 288 { 289 RDSV3_DPRINTF3("rdsv3_ib_cq_event_handler", "event %u data %p", 290 event->event, data); 291 } 292 293 static void 294 rdsv3_ib_snd_cq_comp_handler(struct ib_cq *cq, void *context) 295 { 296 struct rdsv3_connection *conn = context; 297 struct rdsv3_ib_connection *ic = conn->c_transport_data; 298 299 RDSV3_DPRINTF4("rdsv3_ib_snd_cq_comp_handler", 300 "Enter(conn: %p ic: %p cq: %p)", conn, ic, cq); 301 302 rdsv3_af_thr_fire(ic->i_snd_soft_cq); 303 } 304 305 void 306 rdsv3_ib_snd_tasklet_fn(void *data) 307 { 308 struct rdsv3_ib_connection *ic = (struct rdsv3_ib_connection *)data; 309 struct rdsv3_connection *conn = ic->conn; 310 struct rdsv3_ib_ack_state ack_state = { 0, }; 311 ibt_wc_t wc; 312 uint_t polled; 313 314 RDSV3_DPRINTF4("rdsv3_ib_snd_tasklet_fn", 315 "Enter(conn: %p ic: %p)", conn, ic); 316 317 /* 318 * Poll in a loop before and after enabling the next event 319 */ 320 while (ibt_poll_cq(RDSV3_CQ2CQHDL(ic->i_snd_cq), &wc, 1, &polled) == 321 IBT_SUCCESS) { 322 RDSV3_DPRINTF4("rdsv3_ib_snd_tasklet_fn", 323 "wc_id 0x%llx type %d status %u byte_len %u imm_data %u\n", 324 (unsigned long long)wc.wc_id, wc.wc_type, wc.wc_status, 325 wc.wc_bytes_xfer, ntohl(wc.wc_immed_data)); 326 327 ASSERT(wc.wc_id & RDSV3_IB_SEND_OP); 328 rdsv3_ib_send_cqe_handler(ic, &wc); 329 } 330 (void) ibt_enable_cq_notify(RDSV3_CQ2CQHDL(ic->i_snd_cq), 331 IBT_NEXT_COMPLETION); 332 while (ibt_poll_cq(RDSV3_CQ2CQHDL(ic->i_snd_cq), &wc, 1, &polled) == 333 IBT_SUCCESS) { 334 RDSV3_DPRINTF4("rdsv3_ib_snd_tasklet_fn", 335 "wc_id 0x%llx type %d status %u byte_len %u imm_data %u\n", 336 (unsigned long long)wc.wc_id, wc.wc_type, wc.wc_status, 337 wc.wc_bytes_xfer, ntohl(wc.wc_immed_data)); 338 339 ASSERT(wc.wc_id & RDSV3_IB_SEND_OP); 340 rdsv3_ib_send_cqe_handler(ic, &wc); 341 } 342 } 343 344 static void 345 rdsv3_ib_cq_comp_handler(struct ib_cq *cq, void *context) 346 { 347 struct rdsv3_connection *conn = context; 348 struct rdsv3_ib_connection *ic = conn->c_transport_data; 349 350 RDSV3_DPRINTF4("rdsv3_ib_cq_comp_handler", 351 "Enter(conn: %p cq: %p)", conn, cq); 352 353 rdsv3_ib_stats_inc(s_ib_evt_handler_call); 354 355 rdsv3_af_thr_fire(ic->i_soft_cq); 356 } 357 358 void 359 rdsv3_ib_refill_fn(void *data) 360 { 361 struct rdsv3_connection *conn = (struct rdsv3_connection *)data; 362 363 (void) rdsv3_ib_recv_refill(conn, 0); 364 } 365 366 void 367 rdsv3_ib_tasklet_fn(void *data) 368 { 369 struct rdsv3_ib_connection *ic = (struct rdsv3_ib_connection *)data; 370 struct rdsv3_connection *conn = ic->conn; 371 struct rdsv3_ib_ack_state ack_state = { 0, }; 372 ibt_wc_t wc; 373 uint_t polled; 374 375 RDSV3_DPRINTF4("rdsv3_ib_tasklet_fn", 376 "Enter(conn: %p ic: %p)", conn, ic); 377 378 rdsv3_ib_stats_inc(s_ib_tasklet_call); 379 380 /* 381 * Poll in a loop before and after enabling the next event 382 */ 383 while (ibt_poll_cq(RDSV3_CQ2CQHDL(ic->i_cq), &wc, 1, &polled) == 384 IBT_SUCCESS) { 385 RDSV3_DPRINTF4("rdsv3_ib_tasklet_fn", 386 "wc_id 0x%llx type %d status %u byte_len %u imm_data %u\n", 387 (unsigned long long)wc.wc_id, wc.wc_type, wc.wc_status, 388 wc.wc_bytes_xfer, ntohl(wc.wc_immed_data)); 389 390 if (wc.wc_id & RDSV3_IB_SEND_OP) { 391 rdsv3_ib_send_cqe_handler(ic, &wc); 392 } else { 393 rdsv3_ib_recv_cqe_handler(ic, &wc, &ack_state); 394 } 395 } 396 (void) ibt_enable_cq_notify(RDSV3_CQ2CQHDL(ic->i_cq), 397 IBT_NEXT_SOLICITED); 398 399 if (ack_state.ack_next_valid) { 400 rdsv3_ib_set_ack(ic, ack_state.ack_next, 401 ack_state.ack_required); 402 } 403 if (ack_state.ack_recv_valid && ack_state.ack_recv > ic->i_ack_recv) { 404 rdsv3_send_drop_acked(conn, ack_state.ack_recv, NULL); 405 ic->i_ack_recv = ack_state.ack_recv; 406 } 407 if (rdsv3_conn_up(conn)) { 408 if (!test_bit(RDSV3_LL_SEND_FULL, &conn->c_flags)) 409 (void) rdsv3_send_xmit(ic->conn); 410 rdsv3_ib_attempt_ack(ic); 411 } 412 } 413 414 static void 415 rdsv3_ib_qp_event_handler(struct ib_event *event, void *data) 416 { 417 struct rdsv3_connection *conn = data; 418 struct rdsv3_ib_connection *ic = conn->c_transport_data; 419 420 RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler", "conn %p ic %p event %u", 421 conn, ic, event->event); 422 423 switch (event->event) { 424 case IB_EVENT_COMM_EST: 425 (void) rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST); 426 break; 427 default: 428 if (conn) { 429 RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler", 430 "RDS/IB: Fatal QP Event %u - " 431 "connection %u.%u.%u.%u ->%u.%u.%u.%u " 432 "...reconnecting", 433 event->event, NIPQUAD(conn->c_laddr), 434 NIPQUAD(conn->c_faddr)); 435 rdsv3_conn_drop(conn); 436 } else { 437 RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler", 438 "RDS/IB: Fatal QP Event %u - connection" 439 "...reconnecting", event->event); 440 } 441 break; 442 } 443 444 RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler", "Return conn: %p event: %p", 445 conn, event); 446 } 447 448 extern int rdsv3_ib_alloc_hdrs(ib_device_t *dev, 449 struct rdsv3_ib_connection *ic); 450 extern void rdsv3_ib_free_hdrs(ib_device_t *dev, 451 struct rdsv3_ib_connection *ic); 452 453 /* 454 * This needs to be very careful to not leave IS_ERR pointers around for 455 * cleanup to trip over. 456 */ 457 static int 458 rdsv3_ib_setup_qp(struct rdsv3_connection *conn) 459 { 460 struct rdsv3_ib_connection *ic = conn->c_transport_data; 461 struct ib_device *dev = ic->i_cm_id->device; 462 struct ib_qp_init_attr attr; 463 struct rdsv3_ib_device *rds_ibdev; 464 ibt_send_wr_t *wrp; 465 ibt_wr_ds_t *sgl; 466 int ret, i; 467 468 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", "Enter conn: %p", conn); 469 470 /* 471 * rdsv3_ib_add_one creates a rdsv3_ib_device object per IB device, 472 * and allocates a protection domain, memory range and FMR pool 473 * for each. If that fails for any reason, it will not register 474 * the rds_ibdev at all. 475 */ 476 rds_ibdev = ib_get_client_data(dev, &rdsv3_ib_client); 477 if (!rds_ibdev) { 478 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 479 "RDS/IB: No client_data for device %s", dev->name); 480 return (-EOPNOTSUPP); 481 } 482 ic->rds_ibdev = rds_ibdev; 483 484 if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1) 485 rdsv3_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1); 486 if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1) 487 rdsv3_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1); 488 489 /* Protection domain and memory range */ 490 ic->i_pd = rds_ibdev->pd; 491 492 /* 493 * IB_CQ_VECTOR_LEAST_ATTACHED and/or the corresponding feature is 494 * not implmeneted in Hermon yet, but we can pass it to ib_create_cq() 495 * anyway. 496 */ 497 ic->i_cq = ib_create_cq(dev, rdsv3_ib_cq_comp_handler, 498 rdsv3_ib_cq_event_handler, conn, 499 ic->i_recv_ring.w_nr + ic->i_send_ring.w_nr + 1, 500 (intptr_t)rdsv3_af_grp_get_sched(ic->rds_ibdev->aft_hcagp)); 501 if (IS_ERR(ic->i_cq)) { 502 ret = PTR_ERR(ic->i_cq); 503 ic->i_cq = NULL; 504 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 505 "ib_create_cq failed: %d", ret); 506 goto out; 507 } 508 if (rdsv3_enable_snd_cq) { 509 ic->i_snd_cq = ib_create_cq(dev, rdsv3_ib_snd_cq_comp_handler, 510 rdsv3_ib_cq_event_handler, conn, ic->i_send_ring.w_nr + 1, 511 (intptr_t)rdsv3_af_grp_get_sched(ic->rds_ibdev->aft_hcagp)); 512 if (IS_ERR(ic->i_snd_cq)) { 513 ret = PTR_ERR(ic->i_snd_cq); 514 (void) ib_destroy_cq(ic->i_cq); 515 ic->i_cq = NULL; 516 ic->i_snd_cq = NULL; 517 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 518 "ib_create_cq send cq failed: %d", ret); 519 goto out; 520 } 521 } 522 523 /* XXX negotiate max send/recv with remote? */ 524 (void) memset(&attr, 0, sizeof (attr)); 525 attr.event_handler = rdsv3_ib_qp_event_handler; 526 attr.qp_context = conn; 527 /* + 1 to allow for the single ack message */ 528 attr.cap.max_send_wr = ic->i_send_ring.w_nr + 1; 529 attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1; 530 attr.cap.max_send_sge = rds_ibdev->max_sge; 531 attr.cap.max_recv_sge = RDSV3_IB_RECV_SGE; 532 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 533 attr.qp_type = IB_QPT_RC; 534 if (rdsv3_enable_snd_cq) { 535 attr.send_cq = ic->i_snd_cq; 536 } else { 537 attr.send_cq = ic->i_cq; 538 } 539 attr.recv_cq = ic->i_cq; 540 541 /* 542 * XXX this can fail if max_*_wr is too large? Are we supposed 543 * to back off until we get a value that the hardware can support? 544 */ 545 ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr); 546 if (ret) { 547 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 548 "rdma_create_qp failed: %d", ret); 549 goto out; 550 } 551 552 ret = rdsv3_ib_alloc_hdrs(dev, ic); 553 if (ret != 0) { 554 ret = -ENOMEM; 555 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 556 "rdsv3_ib_alloc_hdrs failed: %d", ret); 557 goto out; 558 } 559 560 ic->i_sends = kmem_alloc(ic->i_send_ring.w_nr * 561 sizeof (struct rdsv3_ib_send_work), KM_NOSLEEP); 562 if (ic->i_sends == NULL) { 563 ret = -ENOMEM; 564 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 565 "send allocation failed: %d", ret); 566 goto out; 567 } 568 (void) memset(ic->i_sends, 0, ic->i_send_ring.w_nr * 569 sizeof (struct rdsv3_ib_send_work)); 570 571 ic->i_send_wrs = 572 kmem_alloc(ic->i_send_ring.w_nr * (sizeof (ibt_send_wr_t) + 573 RDSV3_IB_MAX_SGE * sizeof (ibt_wr_ds_t)), KM_NOSLEEP); 574 if (ic->i_send_wrs == NULL) { 575 ret = -ENOMEM; 576 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 577 "Send WR allocation failed: %d", ret); 578 goto out; 579 } 580 sgl = (ibt_wr_ds_t *)((uint8_t *)ic->i_send_wrs + 581 (ic->i_send_ring.w_nr * sizeof (ibt_send_wr_t))); 582 for (i = 0; i < ic->i_send_ring.w_nr; i++) { 583 wrp = &ic->i_send_wrs[i]; 584 wrp->wr_sgl = &sgl[i * RDSV3_IB_MAX_SGE]; 585 } 586 587 ic->i_recvs = kmem_alloc(ic->i_recv_ring.w_nr * 588 sizeof (struct rdsv3_ib_recv_work), KM_NOSLEEP); 589 if (ic->i_recvs == NULL) { 590 ret = -ENOMEM; 591 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 592 "recv allocation failed: %d", ret); 593 goto out; 594 } 595 (void) memset(ic->i_recvs, 0, ic->i_recv_ring.w_nr * 596 sizeof (struct rdsv3_ib_recv_work)); 597 598 ic->i_recv_wrs = 599 kmem_alloc(ic->i_recv_ring.w_nr * sizeof (ibt_recv_wr_t), 600 KM_NOSLEEP); 601 if (ic->i_recv_wrs == NULL) { 602 ret = -ENOMEM; 603 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", 604 "Recv WR allocation failed: %d", ret); 605 goto out; 606 } 607 608 rdsv3_ib_recv_init_ack(ic); 609 610 RDSV3_DPRINTF2("rdsv3_ib_setup_qp", "conn %p pd %p mr %p cq %p", 611 conn, ic->i_pd, ic->i_mr, ic->i_cq); 612 613 out: 614 return (ret); 615 } 616 617 static uint32_t 618 rdsv3_ib_protocol_compatible(struct rdma_cm_event *event) 619 { 620 const struct rdsv3_ib_connect_private *dp = 621 event->param.conn.private_data; 622 uint16_t common; 623 uint32_t version = 0; 624 625 RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible", "Enter event: %p", 626 event); 627 628 /* 629 * rdma_cm private data is odd - when there is any private data in the 630 * request, we will be given a pretty large buffer without telling us 631 * the 632 * original size. The only way to tell the difference is by looking at 633 * the contents, which are initialized to zero. 634 * If the protocol version fields aren't set, 635 * this is a connection attempt 636 * from an older version. This could could be 3.0 or 2.0 - 637 * we can't tell. 638 * We really should have changed this for OFED 1.3 :-( 639 */ 640 641 /* Be paranoid. RDS always has privdata */ 642 if (!event->param.conn.private_data_len) { 643 RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible", 644 "RDS incoming connection has no private data, rejecting"); 645 return (0); 646 } 647 648 /* Even if len is crap *now* I still want to check it. -ASG */ 649 if (event->param.conn.private_data_len < sizeof (*dp) || 650 dp->dp_protocol_major == 0) 651 return (RDS_PROTOCOL_3_0); 652 653 common = ntohs(dp->dp_protocol_minor_mask) & 654 RDSV3_IB_SUPPORTED_PROTOCOLS; 655 if (dp->dp_protocol_major == 3 && common) { 656 version = RDS_PROTOCOL_3_0; 657 while ((common >>= 1) != 0) 658 version++; 659 } else { 660 RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible", 661 "RDS: Connection from %u.%u.%u.%u using " 662 "incompatible protocol version %u.%u\n", 663 NIPQUAD(dp->dp_saddr), 664 dp->dp_protocol_major, 665 dp->dp_protocol_minor); 666 } 667 668 RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible", "Return event: %p", 669 event); 670 671 return (version); 672 } 673 674 int 675 rdsv3_ib_cm_handle_connect(struct rdma_cm_id *cm_id, 676 struct rdma_cm_event *event) 677 { 678 uint64_be_t lguid = cm_id->route.path_rec->sgid.global.interface_id; 679 uint64_be_t fguid = cm_id->route.path_rec->dgid.global.interface_id; 680 const struct rdsv3_ib_connect_private *dp = 681 event->param.conn.private_data; 682 struct rdsv3_ib_connect_private dp_rep; 683 struct rdsv3_connection *conn = NULL; 684 struct rdsv3_ib_connection *ic = NULL; 685 struct rdma_conn_param conn_param; 686 uint32_t version; 687 int err, destroy = 1; 688 boolean_t conn_created = B_FALSE; 689 690 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 691 "Enter cm_id: %p event: %p", cm_id, event); 692 693 /* Check whether the remote protocol version matches ours. */ 694 version = rdsv3_ib_protocol_compatible(event); 695 if (!version) { 696 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 697 "version mismatch"); 698 goto out; 699 } 700 701 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 702 "saddr %u.%u.%u.%u daddr %u.%u.%u.%u RDSv%d.%d lguid 0x%llx fguid " 703 "0x%llx", NIPQUAD(dp->dp_saddr), NIPQUAD(dp->dp_daddr), 704 RDS_PROTOCOL_MAJOR(version), RDS_PROTOCOL_MINOR(version), 705 (unsigned long long)ntohll(lguid), 706 (unsigned long long)ntohll(fguid)); 707 708 conn = rdsv3_conn_create(dp->dp_daddr, dp->dp_saddr, 709 &rdsv3_ib_transport, KM_NOSLEEP); 710 if (IS_ERR(conn)) { 711 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 712 "rdsv3_conn_create failed (%ld)", PTR_ERR(conn)); 713 conn = NULL; 714 goto out; 715 } 716 717 /* 718 * The connection request may occur while the 719 * previous connection exist, e.g. in case of failover. 720 * But as connections may be initiated simultaneously 721 * by both hosts, we have a random backoff mechanism - 722 * see the comment above rdsv3_queue_reconnect() 723 */ 724 mutex_enter(&conn->c_cm_lock); 725 if (!rdsv3_conn_transition(conn, RDSV3_CONN_DOWN, 726 RDSV3_CONN_CONNECTING)) { 727 if (rdsv3_conn_state(conn) == RDSV3_CONN_UP) { 728 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 729 "incoming connect when connected: %p", 730 conn); 731 rdsv3_conn_drop(conn); 732 rdsv3_ib_stats_inc(s_ib_listen_closed_stale); 733 mutex_exit(&conn->c_cm_lock); 734 goto out; 735 } else if (rdsv3_conn_state(conn) == RDSV3_CONN_CONNECTING) { 736 /* Wait and see - our connect may still be succeeding */ 737 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 738 "peer-to-peer connection request: %p, " 739 "lguid: 0x%llx fguid: 0x%llx", 740 conn, lguid, fguid); 741 rdsv3_ib_stats_inc(s_ib_connect_raced); 742 } 743 mutex_exit(&conn->c_cm_lock); 744 goto out; 745 } 746 747 ic = conn->c_transport_data; 748 749 rdsv3_ib_set_protocol(conn, version); 750 rdsv3_ib_set_flow_control(conn, ntohl(dp->dp_credit)); 751 752 /* 753 * If the peer gave us the last packet it saw, process this as if 754 * we had received a regular ACK. 755 */ 756 if (dp->dp_ack_seq) 757 rdsv3_send_drop_acked(conn, ntohll(dp->dp_ack_seq), NULL); 758 759 ASSERT(!cm_id->context); 760 ASSERT(!ic->i_cm_id); 761 762 if (ic->i_cm_id != NULL) 763 RDSV3_PANIC(); 764 765 ic->i_cm_id = cm_id; 766 cm_id->context = conn; 767 768 /* 769 * We got halfway through setting up the ib_connection, if we 770 * fail now, we have to take the long route out of this mess. 771 */ 772 destroy = 0; 773 774 err = rdsv3_ib_setup_qp(conn); 775 if (err) { 776 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 777 "rdsv3_ib_setup_qp failed (%d)", err); 778 mutex_exit(&conn->c_cm_lock); 779 rdsv3_conn_drop(conn); 780 goto out; 781 } 782 783 rdsv3_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version, 784 event->param.conn.responder_resources, 785 event->param.conn.initiator_depth); 786 787 /* rdma_accept() calls rdma_reject() internally if it fails */ 788 err = rdma_accept(cm_id, &conn_param); 789 mutex_exit(&conn->c_cm_lock); 790 if (err) { 791 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 792 "rdma_accept failed (%d)", err); 793 rdsv3_conn_drop(conn); 794 goto out; 795 } 796 797 RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect", 798 "Return cm_id: %p event: %p", cm_id, event); 799 800 return (0); 801 802 out: 803 (void) rdma_reject(cm_id, NULL, 0); 804 return (destroy); 805 } 806 807 808 int 809 rdsv3_ib_cm_initiate_connect(struct rdma_cm_id *cm_id) 810 { 811 struct rdsv3_connection *conn = cm_id->context; 812 struct rdsv3_ib_connection *ic = conn->c_transport_data; 813 struct rdma_conn_param conn_param; 814 struct rdsv3_ib_connect_private dp; 815 int ret; 816 817 RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect", "Enter: cm_id: %p", 818 cm_id); 819 820 /* 821 * If the peer doesn't do protocol negotiation, we must 822 * default to RDSv3.0 823 */ 824 rdsv3_ib_set_protocol(conn, RDS_PROTOCOL_3_0); 825 ic->i_flowctl = 826 rdsv3_ib_sysctl_flow_control; /* advertise flow control */ 827 828 ret = rdsv3_ib_setup_qp(conn); 829 if (ret) { 830 RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect", 831 "rdsv3_ib_setup_qp failed (%d)", ret); 832 rdsv3_conn_drop(conn); 833 goto out; 834 } 835 836 rdsv3_ib_cm_fill_conn_param(conn, &conn_param, &dp, 837 RDS_PROTOCOL_VERSION, UINT_MAX, UINT_MAX); 838 839 ret = rdma_connect(cm_id, &conn_param); 840 if (ret) { 841 RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect", 842 "rdma_connect failed (%d)", ret); 843 rdsv3_conn_drop(conn); 844 } 845 846 RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect", 847 "Return: cm_id: %p", cm_id); 848 849 out: 850 /* 851 * Beware - returning non-zero tells the rdma_cm to destroy 852 * the cm_id. We should certainly not do it as long as we still 853 * "own" the cm_id. 854 */ 855 if (ret) { 856 if (ic->i_cm_id == cm_id) 857 ret = 0; 858 } 859 return (ret); 860 } 861 862 int 863 rdsv3_ib_conn_connect(struct rdsv3_connection *conn) 864 { 865 struct rdsv3_ib_connection *ic = conn->c_transport_data; 866 struct sockaddr_in src, dest; 867 ipaddr_t laddr, faddr; 868 int ret; 869 870 RDSV3_DPRINTF2("rdsv3_ib_conn_connect", "Enter: conn: %p", conn); 871 872 /* 873 * XXX I wonder what affect the port space has 874 */ 875 /* delegate cm event handler to rdma_transport */ 876 ic->i_cm_id = rdma_create_id(rdsv3_rdma_cm_event_handler, conn, 877 RDMA_PS_TCP); 878 if (IS_ERR(ic->i_cm_id)) { 879 ret = PTR_ERR(ic->i_cm_id); 880 ic->i_cm_id = NULL; 881 RDSV3_DPRINTF2("rdsv3_ib_conn_connect", 882 "rdma_create_id() failed: %d", ret); 883 goto out; 884 } 885 886 RDSV3_DPRINTF3("rdsv3_ib_conn_connect", 887 "created cm id %p for conn %p", ic->i_cm_id, conn); 888 889 /* The ipaddr should be in the network order */ 890 laddr = conn->c_laddr; 891 faddr = conn->c_faddr; 892 ret = rdsv3_sc_path_lookup(&laddr, &faddr); 893 if (ret == 0) { 894 RDSV3_DPRINTF2(LABEL, "Path not found (0x%x 0x%x)", 895 ntohl(laddr), ntohl(faddr)); 896 } 897 898 src.sin_family = AF_INET; 899 src.sin_addr.s_addr = (uint32_t)laddr; 900 src.sin_port = (uint16_t)htons(0); 901 902 dest.sin_family = AF_INET; 903 dest.sin_addr.s_addr = (uint32_t)faddr; 904 dest.sin_port = (uint16_t)htons(RDSV3_PORT); 905 906 ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src, 907 (struct sockaddr *)&dest, 908 RDSV3_RDMA_RESOLVE_TIMEOUT_MS); 909 if (ret) { 910 RDSV3_DPRINTF2("rdsv3_ib_conn_connect", 911 "addr resolve failed for cm id %p: %d", ic->i_cm_id, ret); 912 rdma_destroy_id(ic->i_cm_id); 913 ic->i_cm_id = NULL; 914 } 915 916 RDSV3_DPRINTF2("rdsv3_ib_conn_connect", "Return: conn: %p", conn); 917 918 out: 919 return (ret); 920 } 921 922 /* 923 * This is so careful about only cleaning up resources that were built up 924 * so that it can be called at any point during startup. In fact it 925 * can be called multiple times for a given connection. 926 */ 927 void 928 rdsv3_ib_conn_shutdown(struct rdsv3_connection *conn) 929 { 930 struct rdsv3_ib_connection *ic = conn->c_transport_data; 931 int err = 0; 932 933 RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown", 934 "cm %p pd %p cq %p qp %p", ic->i_cm_id, 935 ic->i_pd, ic->i_cq, ic->i_cm_id ? ic->i_cm_id->qp : NULL); 936 937 if (ic->i_cm_id) { 938 struct ib_device *dev = ic->i_cm_id->device; 939 940 RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown", 941 "disconnecting cm %p", ic->i_cm_id); 942 err = rdma_disconnect(ic->i_cm_id); 943 if (err) { 944 /* 945 * Actually this may happen quite frequently, when 946 * an outgoing connect raced with an incoming connect. 947 */ 948 RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown", 949 "failed to disconnect, cm: %p err %d", 950 ic->i_cm_id, err); 951 } 952 953 if (ic->i_cm_id->qp) { 954 (void) ibt_flush_qp( 955 ib_get_ibt_channel_hdl(ic->i_cm_id)); 956 /* 957 * Don't wait for the send ring to be empty -- there 958 * may be completed non-signaled entries sitting on 959 * there. We unmap these below. 960 */ 961 rdsv3_wait_event(&ic->i_recv_ring.w_empty_wait, 962 rdsv3_ib_ring_empty(&ic->i_recv_ring)); 963 /* 964 * Note that Linux original code calls 965 * rdma_destroy_qp() after rdsv3_ib_recv_clear_ring(ic). 966 */ 967 rdma_destroy_qp(ic->i_cm_id); 968 } 969 970 if (rdsv3_enable_snd_cq) { 971 if (ic->i_snd_soft_cq) { 972 rdsv3_af_thr_destroy(ic->i_snd_soft_cq); 973 ic->i_snd_soft_cq = NULL; 974 } 975 if (ic->i_snd_cq) 976 (void) ib_destroy_cq(ic->i_snd_cq); 977 } 978 if (ic->i_soft_cq) { 979 rdsv3_af_thr_destroy(ic->i_soft_cq); 980 ic->i_soft_cq = NULL; 981 } 982 if (ic->i_refill_rq) { 983 rdsv3_af_thr_destroy(ic->i_refill_rq); 984 ic->i_refill_rq = NULL; 985 } 986 if (ic->i_cq) 987 (void) ib_destroy_cq(ic->i_cq); 988 989 if (ic->i_mr) 990 rdsv3_ib_free_hdrs(dev, ic); 991 992 if (ic->i_sends) 993 rdsv3_ib_send_clear_ring(ic); 994 if (ic->i_recvs) 995 rdsv3_ib_recv_clear_ring(ic); 996 997 rdma_destroy_id(ic->i_cm_id); 998 999 /* 1000 * Move connection back to the nodev list. 1001 */ 1002 if (ic->i_on_dev_list) 1003 rdsv3_ib_remove_conn(ic->rds_ibdev, conn); 1004 1005 ic->i_cm_id = NULL; 1006 ic->i_pd = NULL; 1007 ic->i_mr = NULL; 1008 ic->i_cq = NULL; 1009 ic->i_snd_cq = NULL; 1010 ic->i_send_hdrs = NULL; 1011 ic->i_recv_hdrs = NULL; 1012 ic->i_ack = NULL; 1013 } 1014 ASSERT(!ic->i_on_dev_list); 1015 1016 /* Clear pending transmit */ 1017 if (ic->i_rm) { 1018 rdsv3_message_put(ic->i_rm); 1019 ic->i_rm = NULL; 1020 } 1021 1022 /* Clear the ACK state */ 1023 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); 1024 ic->i_ack_next = 0; 1025 ic->i_ack_recv = 0; 1026 1027 /* Clear flow control state */ 1028 ic->i_flowctl = 0; 1029 ic->i_credits = 0; 1030 1031 rdsv3_ib_ring_init(&ic->i_send_ring, rdsv3_ib_sysctl_max_send_wr); 1032 rdsv3_ib_ring_init(&ic->i_recv_ring, rdsv3_ib_sysctl_max_recv_wr); 1033 1034 if (ic->i_ibinc) { 1035 rdsv3_inc_put(&ic->i_ibinc->ii_inc); 1036 ic->i_ibinc = NULL; 1037 } 1038 1039 if (ic->i_sends) { 1040 kmem_free(ic->i_sends, 1041 ic->i_send_ring.w_nr * sizeof (struct rdsv3_ib_send_work)); 1042 ic->i_sends = NULL; 1043 } 1044 if (ic->i_send_wrs) { 1045 kmem_free(ic->i_send_wrs, ic->i_send_ring.w_nr * 1046 (sizeof (ibt_send_wr_t) + 1047 RDSV3_IB_MAX_SGE * sizeof (ibt_wr_ds_t))); 1048 ic->i_send_wrs = NULL; 1049 } 1050 if (ic->i_recvs) { 1051 kmem_free(ic->i_recvs, 1052 ic->i_recv_ring.w_nr * sizeof (struct rdsv3_ib_recv_work)); 1053 ic->i_recvs = NULL; 1054 } 1055 if (ic->i_recv_wrs) { 1056 kmem_free(ic->i_recv_wrs, ic->i_recv_ring.w_nr * 1057 (sizeof (ibt_recv_wr_t))); 1058 ic->i_recv_wrs = NULL; 1059 } 1060 1061 RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown", "Return conn: %p", conn); 1062 } 1063 1064 /* 1065 * the connection can be allocated from either rdsv3_conn_create_outgoing() 1066 * or rdsv3_conn_create(), so ddi_taskq_create() can be called with the 1067 * same string. This can print the kstat warning on the console. To prevent 1068 * it, this counter value is used. 1069 * Note that requests from rdsv3_conn_create_outgoing() refers to the cached 1070 * value with the mutex lock before it allocates the connection, so that 1071 * the warning cannot be produced in the case. (only between 1072 * rdsv3_conn_create() and rdsv3_conn_create_outgoing(). 1073 */ 1074 static int conn_cnt; 1075 1076 /* ARGSUSED */ 1077 int 1078 rdsv3_ib_conn_alloc(struct rdsv3_connection *conn, int gfp) 1079 { 1080 struct rdsv3_ib_connection *ic; 1081 1082 RDSV3_DPRINTF2("rdsv3_ib_conn_alloc", "conn: %p", conn); 1083 1084 /* XXX too lazy? */ 1085 ic = kmem_zalloc(sizeof (struct rdsv3_ib_connection), gfp); 1086 if (!ic) 1087 return (-ENOMEM); 1088 1089 list_link_init(&ic->ib_node); 1090 1091 mutex_init(&ic->i_recv_mutex, NULL, MUTEX_DRIVER, NULL); 1092 mutex_init(&ic->i_ack_lock, NULL, MUTEX_DRIVER, NULL); 1093 1094 /* 1095 * rdsv3_ib_conn_shutdown() waits for these to be emptied so they 1096 * must be initialized before it can be called. 1097 */ 1098 rdsv3_ib_ring_init(&ic->i_send_ring, rdsv3_ib_sysctl_max_send_wr); 1099 rdsv3_ib_ring_init(&ic->i_recv_ring, rdsv3_ib_sysctl_max_recv_wr); 1100 1101 ic->conn = conn; 1102 conn->c_transport_data = ic; 1103 1104 mutex_enter(&ib_nodev_conns_lock); 1105 list_insert_tail(&ib_nodev_conns, ic); 1106 mutex_exit(&ib_nodev_conns_lock); 1107 1108 RDSV3_DPRINTF2("rdsv3_ib_conn_alloc", "conn %p conn ic %p", 1109 conn, conn->c_transport_data); 1110 return (0); 1111 } 1112 1113 /* 1114 * Free a connection. Connection must be shut down and not set for reconnect. 1115 */ 1116 void 1117 rdsv3_ib_conn_free(void *arg) 1118 { 1119 struct rdsv3_ib_connection *ic = arg; 1120 kmutex_t *lock_ptr; 1121 1122 RDSV3_DPRINTF2("rdsv3_ib_conn_free", "ic %p\n", ic); 1123 1124 #ifndef __lock_lint 1125 /* 1126 * Conn is either on a dev's list or on the nodev list. 1127 * A race with shutdown() or connect() would cause problems 1128 * (since rds_ibdev would change) but that should never happen. 1129 */ 1130 lock_ptr = ic->i_on_dev_list ? 1131 &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock; 1132 1133 mutex_enter(lock_ptr); 1134 list_remove_node(&ic->ib_node); 1135 mutex_exit(lock_ptr); 1136 #endif 1137 kmem_free(ic, sizeof (*ic)); 1138 } 1139 1140 /* 1141 * An error occurred on the connection 1142 */ 1143 void 1144 __rdsv3_ib_conn_error(struct rdsv3_connection *conn) 1145 { 1146 rdsv3_conn_drop(conn); 1147 } 1148