1c0dd49bdSEiji Ota /*
216e76cddSagiri * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
3c0dd49bdSEiji Ota */
416e76cddSagiri
5c0dd49bdSEiji Ota /*
616e76cddSagiri * This file contains code imported from the OFED rds source file ib_cm.c
716e76cddSagiri * Oracle elects to have and use the contents of ib_cm.c under and governed
816e76cddSagiri * by the OpenIB.org BSD license (see below for full license text). However,
916e76cddSagiri * the following notice accompanied the original version of this file:
10c0dd49bdSEiji Ota */
11c0dd49bdSEiji Ota
12c0dd49bdSEiji Ota /*
13c0dd49bdSEiji Ota * Copyright (c) 2006 Oracle. All rights reserved.
14c0dd49bdSEiji Ota *
15c0dd49bdSEiji Ota * This software is available to you under a choice of one of two
16c0dd49bdSEiji Ota * licenses. You may choose to be licensed under the terms of the GNU
17c0dd49bdSEiji Ota * General Public License (GPL) Version 2, available from the file
18c0dd49bdSEiji Ota * COPYING in the main directory of this source tree, or the
19c0dd49bdSEiji Ota * OpenIB.org BSD license below:
20c0dd49bdSEiji Ota *
21c0dd49bdSEiji Ota * Redistribution and use in source and binary forms, with or
22c0dd49bdSEiji Ota * without modification, are permitted provided that the following
23c0dd49bdSEiji Ota * conditions are met:
24c0dd49bdSEiji Ota *
25c0dd49bdSEiji Ota * - Redistributions of source code must retain the above
26c0dd49bdSEiji Ota * copyright notice, this list of conditions and the following
27c0dd49bdSEiji Ota * disclaimer.
28c0dd49bdSEiji Ota *
29c0dd49bdSEiji Ota * - Redistributions in binary form must reproduce the above
30c0dd49bdSEiji Ota * copyright notice, this list of conditions and the following
31c0dd49bdSEiji Ota * disclaimer in the documentation and/or other materials
32c0dd49bdSEiji Ota * provided with the distribution.
33c0dd49bdSEiji Ota *
34c0dd49bdSEiji Ota * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35c0dd49bdSEiji Ota * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36c0dd49bdSEiji Ota * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37c0dd49bdSEiji Ota * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38c0dd49bdSEiji Ota * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39c0dd49bdSEiji Ota * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
40c0dd49bdSEiji Ota * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41c0dd49bdSEiji Ota * SOFTWARE.
42c0dd49bdSEiji Ota *
43c0dd49bdSEiji Ota */
44c0dd49bdSEiji Ota #include <sys/rds.h>
45c0dd49bdSEiji Ota
46c0dd49bdSEiji Ota #include <sys/ib/clients/of/ofed_kernel.h>
47c0dd49bdSEiji Ota #include <sys/ib/clients/of/rdma/ib_addr.h>
48c0dd49bdSEiji Ota #include <sys/ib/clients/of/rdma/rdma_cm.h>
49c0dd49bdSEiji Ota
50c0dd49bdSEiji Ota #include <sys/ib/clients/rdsv3/rdsv3.h>
51c0dd49bdSEiji Ota #include <sys/ib/clients/rdsv3/ib.h>
52c0dd49bdSEiji Ota #include <sys/ib/clients/rdsv3/rdsv3_debug.h>
53c0dd49bdSEiji Ota
545d5562f5SEiji Ota extern int rdsv3_enable_snd_cq;
55c0dd49bdSEiji Ota
56c0dd49bdSEiji Ota /*
57c0dd49bdSEiji Ota * Set the selected protocol version
58c0dd49bdSEiji Ota */
59c0dd49bdSEiji Ota static void
rdsv3_ib_set_protocol(struct rdsv3_connection * conn,unsigned int version)60c0dd49bdSEiji Ota rdsv3_ib_set_protocol(struct rdsv3_connection *conn, unsigned int version)
61c0dd49bdSEiji Ota {
62c0dd49bdSEiji Ota RDSV3_DPRINTF4("rdsv3_ib_set_protocol", "conn: %p version: %d",
63c0dd49bdSEiji Ota conn, version);
64c0dd49bdSEiji Ota conn->c_version = version;
65c0dd49bdSEiji Ota }
66c0dd49bdSEiji Ota
67c0dd49bdSEiji Ota /*
68c0dd49bdSEiji Ota * Set up flow control
69c0dd49bdSEiji Ota */
70c0dd49bdSEiji Ota static void
rdsv3_ib_set_flow_control(struct rdsv3_connection * conn,uint32_t credits)71c0dd49bdSEiji Ota rdsv3_ib_set_flow_control(struct rdsv3_connection *conn, uint32_t credits)
72c0dd49bdSEiji Ota {
73c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
74c0dd49bdSEiji Ota
75c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_set_flow_control",
76c0dd49bdSEiji Ota "Enter: conn: %p credits: %d", conn, credits);
77c0dd49bdSEiji Ota
78c0dd49bdSEiji Ota if (rdsv3_ib_sysctl_flow_control && credits != 0) {
79c0dd49bdSEiji Ota /* We're doing flow control */
80c0dd49bdSEiji Ota ic->i_flowctl = 1;
81c0dd49bdSEiji Ota rdsv3_ib_send_add_credits(conn, credits);
82c0dd49bdSEiji Ota } else {
83c0dd49bdSEiji Ota ic->i_flowctl = 0;
84c0dd49bdSEiji Ota }
85c0dd49bdSEiji Ota
86c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_set_flow_control",
87c0dd49bdSEiji Ota "Return: conn: %p credits: %d",
88c0dd49bdSEiji Ota conn, credits);
89c0dd49bdSEiji Ota }
90c0dd49bdSEiji Ota
91c0dd49bdSEiji Ota /*
92c0dd49bdSEiji Ota * Tune RNR behavior. Without flow control, we use a rather
93c0dd49bdSEiji Ota * low timeout, but not the absolute minimum - this should
94c0dd49bdSEiji Ota * be tunable.
95c0dd49bdSEiji Ota *
96c0dd49bdSEiji Ota * We already set the RNR retry count to 7 (which is the
97c0dd49bdSEiji Ota * smallest infinite number :-) above.
98c0dd49bdSEiji Ota * If flow control is off, we want to change this back to 0
99c0dd49bdSEiji Ota * so that we learn quickly when our credit accounting is
100c0dd49bdSEiji Ota * buggy.
101c0dd49bdSEiji Ota *
102c0dd49bdSEiji Ota * Caller passes in a qp_attr pointer - don't waste stack spacv
103c0dd49bdSEiji Ota * by allocation this twice.
104c0dd49bdSEiji Ota */
105c0dd49bdSEiji Ota static void
rdsv3_ib_tune_rnr(struct rdsv3_ib_connection * ic,struct ib_qp_attr * attr)106c0dd49bdSEiji Ota rdsv3_ib_tune_rnr(struct rdsv3_ib_connection *ic, struct ib_qp_attr *attr)
107c0dd49bdSEiji Ota {
108c0dd49bdSEiji Ota int ret;
109c0dd49bdSEiji Ota
110c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_tune_rnr", "Enter ic: %p attr: %p",
111c0dd49bdSEiji Ota ic, attr);
112c0dd49bdSEiji Ota
113c0dd49bdSEiji Ota attr->min_rnr_timer = IB_RNR_TIMER_000_32;
114c0dd49bdSEiji Ota ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);
115c0dd49bdSEiji Ota if (ret)
1166e18d381Sagiri RDSV3_DPRINTF2("rdsv3_ib_tune_rnr",
117c0dd49bdSEiji Ota "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d", -ret);
118c0dd49bdSEiji Ota }
119c0dd49bdSEiji Ota
120c0dd49bdSEiji Ota /*
121c0dd49bdSEiji Ota * Connection established.
122c0dd49bdSEiji Ota * We get here for both outgoing and incoming connection.
123c0dd49bdSEiji Ota */
124c0dd49bdSEiji Ota void
rdsv3_ib_cm_connect_complete(struct rdsv3_connection * conn,struct rdma_cm_event * event)125c0dd49bdSEiji Ota rdsv3_ib_cm_connect_complete(struct rdsv3_connection *conn,
126c0dd49bdSEiji Ota struct rdma_cm_event *event)
127c0dd49bdSEiji Ota {
128c0dd49bdSEiji Ota const struct rdsv3_ib_connect_private *dp = NULL;
129c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
1305d5562f5SEiji Ota struct rdsv3_ib_device *rds_ibdev =
1315d5562f5SEiji Ota ib_get_client_data(ic->i_cm_id->device, &rdsv3_ib_client);
132c0dd49bdSEiji Ota struct ib_qp_attr qp_attr;
133c0dd49bdSEiji Ota int err;
134c0dd49bdSEiji Ota
135c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
136c0dd49bdSEiji Ota "Enter conn: %p event: %p", conn, event);
137c0dd49bdSEiji Ota
138c0dd49bdSEiji Ota if (event->param.conn.private_data_len >= sizeof (*dp)) {
139c0dd49bdSEiji Ota dp = event->param.conn.private_data;
140c0dd49bdSEiji Ota
141c0dd49bdSEiji Ota /* make sure it isn't empty data */
142c0dd49bdSEiji Ota if (dp->dp_protocol_major) {
143c0dd49bdSEiji Ota rdsv3_ib_set_protocol(conn,
144c0dd49bdSEiji Ota RDS_PROTOCOL(dp->dp_protocol_major,
145c0dd49bdSEiji Ota dp->dp_protocol_minor));
146c0dd49bdSEiji Ota rdsv3_ib_set_flow_control(conn,
147c0dd49bdSEiji Ota ntohl(dp->dp_credit));
148c0dd49bdSEiji Ota }
149c0dd49bdSEiji Ota }
150c0dd49bdSEiji Ota
1515d5562f5SEiji Ota if (conn->c_version < RDS_PROTOCOL(3, 1)) {
1525d5562f5SEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
1535d5562f5SEiji Ota "RDS/IB: Connection to %u.%u.%u.%u version %u.%u failed",
1545d5562f5SEiji Ota NIPQUAD(conn->c_faddr),
1555d5562f5SEiji Ota RDS_PROTOCOL_MAJOR(conn->c_version),
1565d5562f5SEiji Ota RDS_PROTOCOL_MINOR(conn->c_version));
1575d5562f5SEiji Ota rdsv3_conn_destroy(conn);
1585d5562f5SEiji Ota return;
1595d5562f5SEiji Ota } else {
1605d5562f5SEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
1615d5562f5SEiji Ota "RDS/IB: connected to %u.%u.%u.%u version %u.%u%s",
1625d5562f5SEiji Ota NIPQUAD(conn->c_faddr),
1635d5562f5SEiji Ota RDS_PROTOCOL_MAJOR(conn->c_version),
1645d5562f5SEiji Ota RDS_PROTOCOL_MINOR(conn->c_version),
1655d5562f5SEiji Ota ic->i_flowctl ? ", flow control" : "");
1665d5562f5SEiji Ota }
1675d5562f5SEiji Ota
1685d5562f5SEiji Ota ASSERT(ic->i_soft_cq == NULL);
1695d5562f5SEiji Ota ic->i_soft_cq = rdsv3_af_intr_thr_create(rdsv3_ib_tasklet_fn,
1705d5562f5SEiji Ota (void *)ic, SCQ_INTR_BIND_CPU, rds_ibdev->aft_hcagp,
1715d5562f5SEiji Ota ic->i_cq->ibt_cq);
1725d5562f5SEiji Ota if (rdsv3_enable_snd_cq) {
1735d5562f5SEiji Ota ic->i_snd_soft_cq = rdsv3_af_intr_thr_create(
1745d5562f5SEiji Ota rdsv3_ib_snd_tasklet_fn,
1755d5562f5SEiji Ota (void *)ic, SCQ_INTR_BIND_CPU, rds_ibdev->aft_hcagp,
1765d5562f5SEiji Ota ic->i_snd_cq->ibt_cq);
1775d5562f5SEiji Ota }
178*5e12ddadSEiji Ota /* rdsv3_ib_refill_fn is expecting i_max_recv_alloc set */
179*5e12ddadSEiji Ota ic->i_max_recv_alloc = rdsv3_ib_sysctl_max_recv_allocation;
1805d5562f5SEiji Ota ic->i_refill_rq = rdsv3_af_thr_create(rdsv3_ib_refill_fn, (void *)conn,
1815d5562f5SEiji Ota SCQ_WRK_BIND_CPU, rds_ibdev->aft_hcagp);
1825d5562f5SEiji Ota rdsv3_af_grp_draw(rds_ibdev->aft_hcagp);
1835d5562f5SEiji Ota
1845d5562f5SEiji Ota (void) ib_req_notify_cq(ic->i_cq, IB_CQ_SOLICITED);
1855d5562f5SEiji Ota if (rdsv3_enable_snd_cq) {
1865d5562f5SEiji Ota (void) ib_req_notify_cq(ic->i_snd_cq, IB_CQ_NEXT_COMP);
1875d5562f5SEiji Ota }
188c0dd49bdSEiji Ota
189c0dd49bdSEiji Ota /*
190c0dd49bdSEiji Ota * Init rings and fill recv. this needs to wait until protocol
191c0dd49bdSEiji Ota * negotiation
192c0dd49bdSEiji Ota * is complete, since ring layout is different from 3.0 to 3.1.
193c0dd49bdSEiji Ota */
194c0dd49bdSEiji Ota rdsv3_ib_send_init_ring(ic);
195c0dd49bdSEiji Ota rdsv3_ib_recv_init_ring(ic);
196c0dd49bdSEiji Ota /*
197c0dd49bdSEiji Ota * Post receive buffers - as a side effect, this will update
198c0dd49bdSEiji Ota * the posted credit count.
199c0dd49bdSEiji Ota */
2005d5562f5SEiji Ota (void) rdsv3_ib_recv_refill(conn, 1);
201c0dd49bdSEiji Ota
202c0dd49bdSEiji Ota /* Tune RNR behavior */
203c0dd49bdSEiji Ota rdsv3_ib_tune_rnr(ic, &qp_attr);
204c0dd49bdSEiji Ota
205c0dd49bdSEiji Ota qp_attr.qp_state = IB_QPS_RTS;
206c0dd49bdSEiji Ota err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
207c0dd49bdSEiji Ota if (err)
2086e18d381Sagiri RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
209c0dd49bdSEiji Ota "ib_modify_qp(IB_QP_STATE, RTS): err=%d", err);
210c0dd49bdSEiji Ota
211c0dd49bdSEiji Ota /* update ib_device with this local ipaddr & conn */
212c0dd49bdSEiji Ota err = rdsv3_ib_update_ipaddr(rds_ibdev, conn->c_laddr);
213c0dd49bdSEiji Ota if (err)
2146e18d381Sagiri RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
215c0dd49bdSEiji Ota "rdsv3_ib_update_ipaddr failed (%d)", err);
216c0dd49bdSEiji Ota rdsv3_ib_add_conn(rds_ibdev, conn);
217c0dd49bdSEiji Ota
218c0dd49bdSEiji Ota /*
219c0dd49bdSEiji Ota * If the peer gave us the last packet it saw, process this as if
220c0dd49bdSEiji Ota * we had received a regular ACK.
221c0dd49bdSEiji Ota */
222c0dd49bdSEiji Ota if (dp && dp->dp_ack_seq)
223c0dd49bdSEiji Ota rdsv3_send_drop_acked(conn, ntohll(dp->dp_ack_seq), NULL);
224c0dd49bdSEiji Ota
225c0dd49bdSEiji Ota rdsv3_connect_complete(conn);
226c0dd49bdSEiji Ota
227c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_connect_complete",
228c0dd49bdSEiji Ota "Return conn: %p event: %p",
229c0dd49bdSEiji Ota conn, event);
230c0dd49bdSEiji Ota }
231c0dd49bdSEiji Ota
232c0dd49bdSEiji Ota static void
rdsv3_ib_cm_fill_conn_param(struct rdsv3_connection * conn,struct rdma_conn_param * conn_param,struct rdsv3_ib_connect_private * dp,uint32_t protocol_version,uint32_t max_responder_resources,uint32_t max_initiator_depth)233c0dd49bdSEiji Ota rdsv3_ib_cm_fill_conn_param(struct rdsv3_connection *conn,
234c0dd49bdSEiji Ota struct rdma_conn_param *conn_param,
235c0dd49bdSEiji Ota struct rdsv3_ib_connect_private *dp,
2365d5562f5SEiji Ota uint32_t protocol_version,
2375d5562f5SEiji Ota uint32_t max_responder_resources,
2385d5562f5SEiji Ota uint32_t max_initiator_depth)
239c0dd49bdSEiji Ota {
2405d5562f5SEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
2415d5562f5SEiji Ota struct rdsv3_ib_device *rds_ibdev;
2425d5562f5SEiji Ota
243c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_fill_conn_param",
244c0dd49bdSEiji Ota "Enter conn: %p conn_param: %p private: %p version: %d",
245c0dd49bdSEiji Ota conn, conn_param, dp, protocol_version);
246c0dd49bdSEiji Ota
247c0dd49bdSEiji Ota (void) memset(conn_param, 0, sizeof (struct rdma_conn_param));
2485d5562f5SEiji Ota
2495d5562f5SEiji Ota rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rdsv3_ib_client);
2505d5562f5SEiji Ota
2515d5562f5SEiji Ota conn_param->responder_resources =
2525d5562f5SEiji Ota MIN(rds_ibdev->max_responder_resources, max_responder_resources);
2535d5562f5SEiji Ota conn_param->initiator_depth =
2545d5562f5SEiji Ota MIN(rds_ibdev->max_initiator_depth, max_initiator_depth);
255c0dd49bdSEiji Ota conn_param->retry_count = min(rdsv3_ib_retry_count, 7);
256c0dd49bdSEiji Ota conn_param->rnr_retry_count = 7;
257c0dd49bdSEiji Ota
258c0dd49bdSEiji Ota if (dp) {
259c0dd49bdSEiji Ota (void) memset(dp, 0, sizeof (*dp));
260c0dd49bdSEiji Ota dp->dp_saddr = conn->c_laddr;
261c0dd49bdSEiji Ota dp->dp_daddr = conn->c_faddr;
262c0dd49bdSEiji Ota dp->dp_protocol_major = RDS_PROTOCOL_MAJOR(protocol_version);
263c0dd49bdSEiji Ota dp->dp_protocol_minor = RDS_PROTOCOL_MINOR(protocol_version);
264c0dd49bdSEiji Ota dp->dp_protocol_minor_mask =
265c0dd49bdSEiji Ota htons(RDSV3_IB_SUPPORTED_PROTOCOLS);
266c0dd49bdSEiji Ota dp->dp_ack_seq = rdsv3_ib_piggyb_ack(ic);
267c0dd49bdSEiji Ota
268c0dd49bdSEiji Ota /* Advertise flow control */
269c0dd49bdSEiji Ota if (ic->i_flowctl) {
270c0dd49bdSEiji Ota unsigned int credits;
271c0dd49bdSEiji Ota
272c0dd49bdSEiji Ota credits = IB_GET_POST_CREDITS(
273c0dd49bdSEiji Ota atomic_get(&ic->i_credits));
274c0dd49bdSEiji Ota dp->dp_credit = htonl(credits);
275c0dd49bdSEiji Ota atomic_add_32(&ic->i_credits,
276c0dd49bdSEiji Ota -IB_SET_POST_CREDITS(credits));
277c0dd49bdSEiji Ota }
278c0dd49bdSEiji Ota
279c0dd49bdSEiji Ota conn_param->private_data = dp;
280c0dd49bdSEiji Ota conn_param->private_data_len = sizeof (*dp);
281c0dd49bdSEiji Ota }
282c0dd49bdSEiji Ota
283c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_fill_conn_param",
284c0dd49bdSEiji Ota "Return conn: %p conn_param: %p private: %p version: %d",
285c0dd49bdSEiji Ota conn, conn_param, dp, protocol_version);
286c0dd49bdSEiji Ota }
287c0dd49bdSEiji Ota
288c0dd49bdSEiji Ota static void
rdsv3_ib_cq_event_handler(struct ib_event * event,void * data)289c0dd49bdSEiji Ota rdsv3_ib_cq_event_handler(struct ib_event *event, void *data)
290c0dd49bdSEiji Ota {
291c0dd49bdSEiji Ota RDSV3_DPRINTF3("rdsv3_ib_cq_event_handler", "event %u data %p",
292c0dd49bdSEiji Ota event->event, data);
293c0dd49bdSEiji Ota }
294c0dd49bdSEiji Ota
2955d5562f5SEiji Ota static void
rdsv3_ib_snd_cq_comp_handler(struct ib_cq * cq,void * context)2965d5562f5SEiji Ota rdsv3_ib_snd_cq_comp_handler(struct ib_cq *cq, void *context)
2975d5562f5SEiji Ota {
2985d5562f5SEiji Ota struct rdsv3_connection *conn = context;
2995d5562f5SEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
3005d5562f5SEiji Ota
3015d5562f5SEiji Ota RDSV3_DPRINTF4("rdsv3_ib_snd_cq_comp_handler",
3025d5562f5SEiji Ota "Enter(conn: %p ic: %p cq: %p)", conn, ic, cq);
3035d5562f5SEiji Ota
3045d5562f5SEiji Ota rdsv3_af_thr_fire(ic->i_snd_soft_cq);
3055d5562f5SEiji Ota }
3065d5562f5SEiji Ota
3075d5562f5SEiji Ota void
rdsv3_ib_snd_tasklet_fn(void * data)3085d5562f5SEiji Ota rdsv3_ib_snd_tasklet_fn(void *data)
3095d5562f5SEiji Ota {
3105d5562f5SEiji Ota struct rdsv3_ib_connection *ic = (struct rdsv3_ib_connection *)data;
3115d5562f5SEiji Ota struct rdsv3_connection *conn = ic->conn;
3125d5562f5SEiji Ota struct rdsv3_ib_ack_state ack_state = { 0, };
3135d5562f5SEiji Ota ibt_wc_t wc;
3145d5562f5SEiji Ota uint_t polled;
3155d5562f5SEiji Ota
3165d5562f5SEiji Ota RDSV3_DPRINTF4("rdsv3_ib_snd_tasklet_fn",
3175d5562f5SEiji Ota "Enter(conn: %p ic: %p)", conn, ic);
3185d5562f5SEiji Ota
3195d5562f5SEiji Ota /*
3205d5562f5SEiji Ota * Poll in a loop before and after enabling the next event
3215d5562f5SEiji Ota */
3225d5562f5SEiji Ota while (ibt_poll_cq(RDSV3_CQ2CQHDL(ic->i_snd_cq), &wc, 1, &polled) ==
3235d5562f5SEiji Ota IBT_SUCCESS) {
3243f756f37Sagiri RDSV3_DPRINTF4("rdsv3_ib_snd_tasklet_fn",
3255d5562f5SEiji Ota "wc_id 0x%llx type %d status %u byte_len %u imm_data %u\n",
3265d5562f5SEiji Ota (unsigned long long)wc.wc_id, wc.wc_type, wc.wc_status,
3275d5562f5SEiji Ota wc.wc_bytes_xfer, ntohl(wc.wc_immed_data));
3285d5562f5SEiji Ota
3295d5562f5SEiji Ota ASSERT(wc.wc_id & RDSV3_IB_SEND_OP);
3305d5562f5SEiji Ota rdsv3_ib_send_cqe_handler(ic, &wc);
3315d5562f5SEiji Ota }
3325d5562f5SEiji Ota (void) ibt_enable_cq_notify(RDSV3_CQ2CQHDL(ic->i_snd_cq),
3335d5562f5SEiji Ota IBT_NEXT_COMPLETION);
3343f756f37Sagiri while (ibt_poll_cq(RDSV3_CQ2CQHDL(ic->i_snd_cq), &wc, 1, &polled) ==
3355d5562f5SEiji Ota IBT_SUCCESS) {
3363f756f37Sagiri RDSV3_DPRINTF4("rdsv3_ib_snd_tasklet_fn",
3373f756f37Sagiri "wc_id 0x%llx type %d status %u byte_len %u imm_data %u\n",
3383f756f37Sagiri (unsigned long long)wc.wc_id, wc.wc_type, wc.wc_status,
3393f756f37Sagiri wc.wc_bytes_xfer, ntohl(wc.wc_immed_data));
3403f756f37Sagiri
3415d5562f5SEiji Ota ASSERT(wc.wc_id & RDSV3_IB_SEND_OP);
3425d5562f5SEiji Ota rdsv3_ib_send_cqe_handler(ic, &wc);
3435d5562f5SEiji Ota }
3445d5562f5SEiji Ota }
3455d5562f5SEiji Ota
3465d5562f5SEiji Ota static void
rdsv3_ib_cq_comp_handler(struct ib_cq * cq,void * context)3475d5562f5SEiji Ota rdsv3_ib_cq_comp_handler(struct ib_cq *cq, void *context)
3485d5562f5SEiji Ota {
3495d5562f5SEiji Ota struct rdsv3_connection *conn = context;
3505d5562f5SEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
3515d5562f5SEiji Ota
3525d5562f5SEiji Ota RDSV3_DPRINTF4("rdsv3_ib_cq_comp_handler",
3535d5562f5SEiji Ota "Enter(conn: %p cq: %p)", conn, cq);
3545d5562f5SEiji Ota
3555d5562f5SEiji Ota rdsv3_ib_stats_inc(s_ib_evt_handler_call);
3565d5562f5SEiji Ota
3575d5562f5SEiji Ota rdsv3_af_thr_fire(ic->i_soft_cq);
3585d5562f5SEiji Ota }
3595d5562f5SEiji Ota
3605d5562f5SEiji Ota void
rdsv3_ib_refill_fn(void * data)3615d5562f5SEiji Ota rdsv3_ib_refill_fn(void *data)
3625d5562f5SEiji Ota {
3635d5562f5SEiji Ota struct rdsv3_connection *conn = (struct rdsv3_connection *)data;
3645d5562f5SEiji Ota
3655d5562f5SEiji Ota (void) rdsv3_ib_recv_refill(conn, 0);
3665d5562f5SEiji Ota }
3675d5562f5SEiji Ota
3685d5562f5SEiji Ota void
rdsv3_ib_tasklet_fn(void * data)3695d5562f5SEiji Ota rdsv3_ib_tasklet_fn(void *data)
3705d5562f5SEiji Ota {
3715d5562f5SEiji Ota struct rdsv3_ib_connection *ic = (struct rdsv3_ib_connection *)data;
3725d5562f5SEiji Ota struct rdsv3_connection *conn = ic->conn;
3735d5562f5SEiji Ota struct rdsv3_ib_ack_state ack_state = { 0, };
374*5e12ddadSEiji Ota ibt_wc_t wc[RDSV3_IB_WC_POLL_SIZE];
3755d5562f5SEiji Ota uint_t polled;
376*5e12ddadSEiji Ota int i;
3775d5562f5SEiji Ota
3785d5562f5SEiji Ota RDSV3_DPRINTF4("rdsv3_ib_tasklet_fn",
3795d5562f5SEiji Ota "Enter(conn: %p ic: %p)", conn, ic);
3805d5562f5SEiji Ota
3815d5562f5SEiji Ota rdsv3_ib_stats_inc(s_ib_tasklet_call);
3825d5562f5SEiji Ota
3835d5562f5SEiji Ota /*
3845d5562f5SEiji Ota * Poll in a loop before and after enabling the next event
3855d5562f5SEiji Ota */
386*5e12ddadSEiji Ota while (ibt_poll_cq(RDSV3_CQ2CQHDL(ic->i_cq), &wc[0],
387*5e12ddadSEiji Ota RDSV3_IB_WC_POLL_SIZE, &polled) == IBT_SUCCESS) {
388*5e12ddadSEiji Ota for (i = 0; i < polled; i++) {
389*5e12ddadSEiji Ota RDSV3_DPRINTF4("rdsv3_ib_tasklet_fn",
390*5e12ddadSEiji Ota "wc_id 0x%llx type %d status %u byte_len %u \
391*5e12ddadSEiji Ota imm_data %u\n",
392*5e12ddadSEiji Ota (unsigned long long)wc[i].wc_id, wc[i].wc_type,
393*5e12ddadSEiji Ota wc[i].wc_status, wc[i].wc_bytes_xfer,
394*5e12ddadSEiji Ota ntohl(wc[i].wc_immed_data));
395*5e12ddadSEiji Ota
396*5e12ddadSEiji Ota if (wc[i].wc_id & RDSV3_IB_SEND_OP) {
397*5e12ddadSEiji Ota rdsv3_ib_send_cqe_handler(ic, &wc[i]);
398*5e12ddadSEiji Ota } else {
399*5e12ddadSEiji Ota rdsv3_ib_recv_cqe_handler(ic, &wc[i],
400*5e12ddadSEiji Ota &ack_state);
401*5e12ddadSEiji Ota }
4025d5562f5SEiji Ota }
4035d5562f5SEiji Ota }
4045d5562f5SEiji Ota (void) ibt_enable_cq_notify(RDSV3_CQ2CQHDL(ic->i_cq),
4055d5562f5SEiji Ota IBT_NEXT_SOLICITED);
406*5e12ddadSEiji Ota while (ibt_poll_cq(RDSV3_CQ2CQHDL(ic->i_cq), &wc[0],
407*5e12ddadSEiji Ota RDSV3_IB_WC_POLL_SIZE, &polled) == IBT_SUCCESS) {
408*5e12ddadSEiji Ota for (i = 0; i < polled; i++) {
409*5e12ddadSEiji Ota RDSV3_DPRINTF4("rdsv3_ib_tasklet_fn",
410*5e12ddadSEiji Ota "wc_id 0x%llx type %d status %u byte_len %u \
411*5e12ddadSEiji Ota imm_data %u\n",
412*5e12ddadSEiji Ota (unsigned long long)wc[i].wc_id, wc[i].wc_type,
413*5e12ddadSEiji Ota wc[i].wc_status, wc[i].wc_bytes_xfer,
414*5e12ddadSEiji Ota ntohl(wc[i].wc_immed_data));
415*5e12ddadSEiji Ota
416*5e12ddadSEiji Ota if (wc[i].wc_id & RDSV3_IB_SEND_OP) {
417*5e12ddadSEiji Ota rdsv3_ib_send_cqe_handler(ic, &wc[i]);
418*5e12ddadSEiji Ota } else {
419*5e12ddadSEiji Ota rdsv3_ib_recv_cqe_handler(ic, &wc[i],
420*5e12ddadSEiji Ota &ack_state);
421*5e12ddadSEiji Ota }
422*5e12ddadSEiji Ota }
423*5e12ddadSEiji Ota }
4245d5562f5SEiji Ota
4255d5562f5SEiji Ota if (ack_state.ack_next_valid) {
4265d5562f5SEiji Ota rdsv3_ib_set_ack(ic, ack_state.ack_next,
4275d5562f5SEiji Ota ack_state.ack_required);
4285d5562f5SEiji Ota }
4295d5562f5SEiji Ota if (ack_state.ack_recv_valid && ack_state.ack_recv > ic->i_ack_recv) {
4305d5562f5SEiji Ota rdsv3_send_drop_acked(conn, ack_state.ack_recv, NULL);
4315d5562f5SEiji Ota ic->i_ack_recv = ack_state.ack_recv;
4325d5562f5SEiji Ota }
4335d5562f5SEiji Ota if (rdsv3_conn_up(conn)) {
4345d5562f5SEiji Ota if (!test_bit(RDSV3_LL_SEND_FULL, &conn->c_flags))
4355d5562f5SEiji Ota (void) rdsv3_send_xmit(ic->conn);
4365d5562f5SEiji Ota rdsv3_ib_attempt_ack(ic);
4375d5562f5SEiji Ota }
4385d5562f5SEiji Ota }
4395d5562f5SEiji Ota
440c0dd49bdSEiji Ota static void
rdsv3_ib_qp_event_handler(struct ib_event * event,void * data)441c0dd49bdSEiji Ota rdsv3_ib_qp_event_handler(struct ib_event *event, void *data)
442c0dd49bdSEiji Ota {
443c0dd49bdSEiji Ota struct rdsv3_connection *conn = data;
444c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
445c0dd49bdSEiji Ota
446c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler", "conn %p ic %p event %u",
447c0dd49bdSEiji Ota conn, ic, event->event);
448c0dd49bdSEiji Ota
449c0dd49bdSEiji Ota switch (event->event) {
450c0dd49bdSEiji Ota case IB_EVENT_COMM_EST:
451c0dd49bdSEiji Ota (void) rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
452c0dd49bdSEiji Ota break;
453c0dd49bdSEiji Ota default:
454c0dd49bdSEiji Ota if (conn) {
455c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler",
456c0dd49bdSEiji Ota "RDS/IB: Fatal QP Event %u - "
457c0dd49bdSEiji Ota "connection %u.%u.%u.%u ->%u.%u.%u.%u "
458c0dd49bdSEiji Ota "...reconnecting",
459c0dd49bdSEiji Ota event->event, NIPQUAD(conn->c_laddr),
460c0dd49bdSEiji Ota NIPQUAD(conn->c_faddr));
461c0dd49bdSEiji Ota rdsv3_conn_drop(conn);
462c0dd49bdSEiji Ota } else {
463c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler",
464c0dd49bdSEiji Ota "RDS/IB: Fatal QP Event %u - connection"
465c0dd49bdSEiji Ota "...reconnecting", event->event);
466c0dd49bdSEiji Ota }
467c0dd49bdSEiji Ota break;
468c0dd49bdSEiji Ota }
469c0dd49bdSEiji Ota
470c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_qp_event_handler", "Return conn: %p event: %p",
471c0dd49bdSEiji Ota conn, event);
472c0dd49bdSEiji Ota }
473c0dd49bdSEiji Ota
474c0dd49bdSEiji Ota extern int rdsv3_ib_alloc_hdrs(ib_device_t *dev,
475c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic);
476c0dd49bdSEiji Ota extern void rdsv3_ib_free_hdrs(ib_device_t *dev,
477c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic);
478c0dd49bdSEiji Ota
479c0dd49bdSEiji Ota /*
480c0dd49bdSEiji Ota * This needs to be very careful to not leave IS_ERR pointers around for
481c0dd49bdSEiji Ota * cleanup to trip over.
482c0dd49bdSEiji Ota */
483c0dd49bdSEiji Ota static int
rdsv3_ib_setup_qp(struct rdsv3_connection * conn)484c0dd49bdSEiji Ota rdsv3_ib_setup_qp(struct rdsv3_connection *conn)
485c0dd49bdSEiji Ota {
486c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
487c0dd49bdSEiji Ota struct ib_device *dev = ic->i_cm_id->device;
488c0dd49bdSEiji Ota struct ib_qp_init_attr attr;
489c0dd49bdSEiji Ota struct rdsv3_ib_device *rds_ibdev;
490c0dd49bdSEiji Ota ibt_send_wr_t *wrp;
491c0dd49bdSEiji Ota ibt_wr_ds_t *sgl;
492c0dd49bdSEiji Ota int ret, i;
493c0dd49bdSEiji Ota
494c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp", "Enter conn: %p", conn);
495c0dd49bdSEiji Ota
496c0dd49bdSEiji Ota /*
497c0dd49bdSEiji Ota * rdsv3_ib_add_one creates a rdsv3_ib_device object per IB device,
498c0dd49bdSEiji Ota * and allocates a protection domain, memory range and FMR pool
499c0dd49bdSEiji Ota * for each. If that fails for any reason, it will not register
500c0dd49bdSEiji Ota * the rds_ibdev at all.
501c0dd49bdSEiji Ota */
502c0dd49bdSEiji Ota rds_ibdev = ib_get_client_data(dev, &rdsv3_ib_client);
5035d5562f5SEiji Ota if (!rds_ibdev) {
5046e18d381Sagiri RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
505c0dd49bdSEiji Ota "RDS/IB: No client_data for device %s", dev->name);
506c0dd49bdSEiji Ota return (-EOPNOTSUPP);
507c0dd49bdSEiji Ota }
508d2b539e7Sagiri ic->rds_ibdev = rds_ibdev;
509c0dd49bdSEiji Ota
510c0dd49bdSEiji Ota if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1)
511c0dd49bdSEiji Ota rdsv3_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1);
512c0dd49bdSEiji Ota if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1)
513c0dd49bdSEiji Ota rdsv3_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1);
514c0dd49bdSEiji Ota
515c0dd49bdSEiji Ota /* Protection domain and memory range */
516c0dd49bdSEiji Ota ic->i_pd = rds_ibdev->pd;
517c0dd49bdSEiji Ota
518cadbfdc3SEiji Ota /*
519cadbfdc3SEiji Ota * IB_CQ_VECTOR_LEAST_ATTACHED and/or the corresponding feature is
520cadbfdc3SEiji Ota * not implmeneted in Hermon yet, but we can pass it to ib_create_cq()
521cadbfdc3SEiji Ota * anyway.
522cadbfdc3SEiji Ota */
5235d5562f5SEiji Ota ic->i_cq = ib_create_cq(dev, rdsv3_ib_cq_comp_handler,
524c0dd49bdSEiji Ota rdsv3_ib_cq_event_handler, conn,
5255d5562f5SEiji Ota ic->i_recv_ring.w_nr + ic->i_send_ring.w_nr + 1,
52617a2b317SBill Taylor rdsv3_af_grp_get_sched(ic->rds_ibdev->aft_hcagp));
5275d5562f5SEiji Ota if (IS_ERR(ic->i_cq)) {
5285d5562f5SEiji Ota ret = PTR_ERR(ic->i_cq);
5295d5562f5SEiji Ota ic->i_cq = NULL;
530c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
5315d5562f5SEiji Ota "ib_create_cq failed: %d", ret);
532c0dd49bdSEiji Ota goto out;
533c0dd49bdSEiji Ota }
5345d5562f5SEiji Ota if (rdsv3_enable_snd_cq) {
5355d5562f5SEiji Ota ic->i_snd_cq = ib_create_cq(dev, rdsv3_ib_snd_cq_comp_handler,
5365d5562f5SEiji Ota rdsv3_ib_cq_event_handler, conn, ic->i_send_ring.w_nr + 1,
53717a2b317SBill Taylor rdsv3_af_grp_get_sched(ic->rds_ibdev->aft_hcagp));
5385d5562f5SEiji Ota if (IS_ERR(ic->i_snd_cq)) {
5395d5562f5SEiji Ota ret = PTR_ERR(ic->i_snd_cq);
5405d5562f5SEiji Ota (void) ib_destroy_cq(ic->i_cq);
5415d5562f5SEiji Ota ic->i_cq = NULL;
5425d5562f5SEiji Ota ic->i_snd_cq = NULL;
5435d5562f5SEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
5445d5562f5SEiji Ota "ib_create_cq send cq failed: %d", ret);
5455d5562f5SEiji Ota goto out;
5465d5562f5SEiji Ota }
547c0dd49bdSEiji Ota }
548c0dd49bdSEiji Ota
549c0dd49bdSEiji Ota /* XXX negotiate max send/recv with remote? */
550c0dd49bdSEiji Ota (void) memset(&attr, 0, sizeof (attr));
551c0dd49bdSEiji Ota attr.event_handler = rdsv3_ib_qp_event_handler;
552c0dd49bdSEiji Ota attr.qp_context = conn;
553c0dd49bdSEiji Ota /* + 1 to allow for the single ack message */
554c0dd49bdSEiji Ota attr.cap.max_send_wr = ic->i_send_ring.w_nr + 1;
555c0dd49bdSEiji Ota attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;
556c0dd49bdSEiji Ota attr.cap.max_send_sge = rds_ibdev->max_sge;
557c0dd49bdSEiji Ota attr.cap.max_recv_sge = RDSV3_IB_RECV_SGE;
558c0dd49bdSEiji Ota attr.sq_sig_type = IB_SIGNAL_REQ_WR;
559c0dd49bdSEiji Ota attr.qp_type = IB_QPT_RC;
5605d5562f5SEiji Ota if (rdsv3_enable_snd_cq) {
5615d5562f5SEiji Ota attr.send_cq = ic->i_snd_cq;
5625d5562f5SEiji Ota } else {
5635d5562f5SEiji Ota attr.send_cq = ic->i_cq;
5645d5562f5SEiji Ota }
5655d5562f5SEiji Ota attr.recv_cq = ic->i_cq;
566c0dd49bdSEiji Ota
567c0dd49bdSEiji Ota /*
568c0dd49bdSEiji Ota * XXX this can fail if max_*_wr is too large? Are we supposed
569c0dd49bdSEiji Ota * to back off until we get a value that the hardware can support?
570c0dd49bdSEiji Ota */
571c0dd49bdSEiji Ota ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
572c0dd49bdSEiji Ota if (ret) {
573c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
574c0dd49bdSEiji Ota "rdma_create_qp failed: %d", ret);
575c0dd49bdSEiji Ota goto out;
576c0dd49bdSEiji Ota }
577c0dd49bdSEiji Ota
578c0dd49bdSEiji Ota ret = rdsv3_ib_alloc_hdrs(dev, ic);
579c0dd49bdSEiji Ota if (ret != 0) {
580c0dd49bdSEiji Ota ret = -ENOMEM;
581c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
582c0dd49bdSEiji Ota "rdsv3_ib_alloc_hdrs failed: %d", ret);
583c0dd49bdSEiji Ota goto out;
584c0dd49bdSEiji Ota }
585c0dd49bdSEiji Ota
586c0dd49bdSEiji Ota ic->i_sends = kmem_alloc(ic->i_send_ring.w_nr *
587c0dd49bdSEiji Ota sizeof (struct rdsv3_ib_send_work), KM_NOSLEEP);
588c0dd49bdSEiji Ota if (ic->i_sends == NULL) {
589c0dd49bdSEiji Ota ret = -ENOMEM;
590c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
591c0dd49bdSEiji Ota "send allocation failed: %d", ret);
592c0dd49bdSEiji Ota goto out;
593c0dd49bdSEiji Ota }
594c0dd49bdSEiji Ota (void) memset(ic->i_sends, 0, ic->i_send_ring.w_nr *
595c0dd49bdSEiji Ota sizeof (struct rdsv3_ib_send_work));
596c0dd49bdSEiji Ota
597c0dd49bdSEiji Ota ic->i_send_wrs =
5983f756f37Sagiri kmem_alloc(ic->i_send_ring.w_nr * (sizeof (ibt_send_wr_t) +
599c0dd49bdSEiji Ota RDSV3_IB_MAX_SGE * sizeof (ibt_wr_ds_t)), KM_NOSLEEP);
600c0dd49bdSEiji Ota if (ic->i_send_wrs == NULL) {
601c0dd49bdSEiji Ota ret = -ENOMEM;
602c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
603d2b539e7Sagiri "Send WR allocation failed: %d", ret);
604c0dd49bdSEiji Ota goto out;
605c0dd49bdSEiji Ota }
606c0dd49bdSEiji Ota sgl = (ibt_wr_ds_t *)((uint8_t *)ic->i_send_wrs +
6073f756f37Sagiri (ic->i_send_ring.w_nr * sizeof (ibt_send_wr_t)));
6083f756f37Sagiri for (i = 0; i < ic->i_send_ring.w_nr; i++) {
609c0dd49bdSEiji Ota wrp = &ic->i_send_wrs[i];
610c0dd49bdSEiji Ota wrp->wr_sgl = &sgl[i * RDSV3_IB_MAX_SGE];
611c0dd49bdSEiji Ota }
612c0dd49bdSEiji Ota
613c0dd49bdSEiji Ota ic->i_recvs = kmem_alloc(ic->i_recv_ring.w_nr *
614c0dd49bdSEiji Ota sizeof (struct rdsv3_ib_recv_work), KM_NOSLEEP);
615c0dd49bdSEiji Ota if (ic->i_recvs == NULL) {
616c0dd49bdSEiji Ota ret = -ENOMEM;
617c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
618c0dd49bdSEiji Ota "recv allocation failed: %d", ret);
619c0dd49bdSEiji Ota goto out;
620c0dd49bdSEiji Ota }
621c0dd49bdSEiji Ota (void) memset(ic->i_recvs, 0, ic->i_recv_ring.w_nr *
622c0dd49bdSEiji Ota sizeof (struct rdsv3_ib_recv_work));
623c0dd49bdSEiji Ota
624d2b539e7Sagiri ic->i_recv_wrs =
625d2b539e7Sagiri kmem_alloc(ic->i_recv_ring.w_nr * sizeof (ibt_recv_wr_t),
626d2b539e7Sagiri KM_NOSLEEP);
627d2b539e7Sagiri if (ic->i_recv_wrs == NULL) {
628d2b539e7Sagiri ret = -ENOMEM;
629d2b539e7Sagiri RDSV3_DPRINTF2("rdsv3_ib_setup_qp",
630d2b539e7Sagiri "Recv WR allocation failed: %d", ret);
631d2b539e7Sagiri goto out;
632d2b539e7Sagiri }
633d2b539e7Sagiri
634c0dd49bdSEiji Ota rdsv3_ib_recv_init_ack(ic);
635c0dd49bdSEiji Ota
6365d5562f5SEiji Ota RDSV3_DPRINTF2("rdsv3_ib_setup_qp", "conn %p pd %p mr %p cq %p",
6375d5562f5SEiji Ota conn, ic->i_pd, ic->i_mr, ic->i_cq);
638c0dd49bdSEiji Ota
639c0dd49bdSEiji Ota out:
640c0dd49bdSEiji Ota return (ret);
641c0dd49bdSEiji Ota }
642c0dd49bdSEiji Ota
643c0dd49bdSEiji Ota static uint32_t
rdsv3_ib_protocol_compatible(struct rdma_cm_event * event)644c0dd49bdSEiji Ota rdsv3_ib_protocol_compatible(struct rdma_cm_event *event)
645c0dd49bdSEiji Ota {
646c0dd49bdSEiji Ota const struct rdsv3_ib_connect_private *dp =
647c0dd49bdSEiji Ota event->param.conn.private_data;
648c0dd49bdSEiji Ota uint16_t common;
649c0dd49bdSEiji Ota uint32_t version = 0;
650c0dd49bdSEiji Ota
651c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible", "Enter event: %p",
652c0dd49bdSEiji Ota event);
653c0dd49bdSEiji Ota
654c0dd49bdSEiji Ota /*
655c0dd49bdSEiji Ota * rdma_cm private data is odd - when there is any private data in the
656c0dd49bdSEiji Ota * request, we will be given a pretty large buffer without telling us
657c0dd49bdSEiji Ota * the
658c0dd49bdSEiji Ota * original size. The only way to tell the difference is by looking at
659c0dd49bdSEiji Ota * the contents, which are initialized to zero.
660c0dd49bdSEiji Ota * If the protocol version fields aren't set,
661c0dd49bdSEiji Ota * this is a connection attempt
662c0dd49bdSEiji Ota * from an older version. This could could be 3.0 or 2.0 -
663c0dd49bdSEiji Ota * we can't tell.
664c0dd49bdSEiji Ota * We really should have changed this for OFED 1.3 :-(
665c0dd49bdSEiji Ota */
666c0dd49bdSEiji Ota
667c0dd49bdSEiji Ota /* Be paranoid. RDS always has privdata */
668c0dd49bdSEiji Ota if (!event->param.conn.private_data_len) {
669c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible",
670c0dd49bdSEiji Ota "RDS incoming connection has no private data, rejecting");
671c0dd49bdSEiji Ota return (0);
672c0dd49bdSEiji Ota }
673c0dd49bdSEiji Ota
674c0dd49bdSEiji Ota /* Even if len is crap *now* I still want to check it. -ASG */
675c0dd49bdSEiji Ota if (event->param.conn.private_data_len < sizeof (*dp) ||
676c0dd49bdSEiji Ota dp->dp_protocol_major == 0)
677c0dd49bdSEiji Ota return (RDS_PROTOCOL_3_0);
678c0dd49bdSEiji Ota
679c0dd49bdSEiji Ota common = ntohs(dp->dp_protocol_minor_mask) &
680c0dd49bdSEiji Ota RDSV3_IB_SUPPORTED_PROTOCOLS;
681c0dd49bdSEiji Ota if (dp->dp_protocol_major == 3 && common) {
682c0dd49bdSEiji Ota version = RDS_PROTOCOL_3_0;
683c0dd49bdSEiji Ota while ((common >>= 1) != 0)
684c0dd49bdSEiji Ota version++;
685c0dd49bdSEiji Ota } else {
6866e18d381Sagiri RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible",
687c0dd49bdSEiji Ota "RDS: Connection from %u.%u.%u.%u using "
688c0dd49bdSEiji Ota "incompatible protocol version %u.%u\n",
689c0dd49bdSEiji Ota NIPQUAD(dp->dp_saddr),
690c0dd49bdSEiji Ota dp->dp_protocol_major,
691c0dd49bdSEiji Ota dp->dp_protocol_minor);
692c0dd49bdSEiji Ota }
693c0dd49bdSEiji Ota
694c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_protocol_compatible", "Return event: %p",
695c0dd49bdSEiji Ota event);
696c0dd49bdSEiji Ota
697c0dd49bdSEiji Ota return (version);
698c0dd49bdSEiji Ota }
699c0dd49bdSEiji Ota
700c0dd49bdSEiji Ota int
rdsv3_ib_cm_handle_connect(struct rdma_cm_id * cm_id,struct rdma_cm_event * event)701c0dd49bdSEiji Ota rdsv3_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
702c0dd49bdSEiji Ota struct rdma_cm_event *event)
703c0dd49bdSEiji Ota {
704c0dd49bdSEiji Ota uint64_be_t lguid = cm_id->route.path_rec->sgid.global.interface_id;
705c0dd49bdSEiji Ota uint64_be_t fguid = cm_id->route.path_rec->dgid.global.interface_id;
706c0dd49bdSEiji Ota const struct rdsv3_ib_connect_private *dp =
707c0dd49bdSEiji Ota event->param.conn.private_data;
708c0dd49bdSEiji Ota struct rdsv3_ib_connect_private dp_rep;
709c0dd49bdSEiji Ota struct rdsv3_connection *conn = NULL;
710c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = NULL;
711c0dd49bdSEiji Ota struct rdma_conn_param conn_param;
712c0dd49bdSEiji Ota uint32_t version;
713c0dd49bdSEiji Ota int err, destroy = 1;
714c0dd49bdSEiji Ota boolean_t conn_created = B_FALSE;
715c0dd49bdSEiji Ota
716c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
717c0dd49bdSEiji Ota "Enter cm_id: %p event: %p", cm_id, event);
718c0dd49bdSEiji Ota
719c0dd49bdSEiji Ota /* Check whether the remote protocol version matches ours. */
720c0dd49bdSEiji Ota version = rdsv3_ib_protocol_compatible(event);
721c0dd49bdSEiji Ota if (!version) {
722c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
723c0dd49bdSEiji Ota "version mismatch");
724c0dd49bdSEiji Ota goto out;
725c0dd49bdSEiji Ota }
726c0dd49bdSEiji Ota
727c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
728c0dd49bdSEiji Ota "saddr %u.%u.%u.%u daddr %u.%u.%u.%u RDSv%d.%d lguid 0x%llx fguid "
729c0dd49bdSEiji Ota "0x%llx", NIPQUAD(dp->dp_saddr), NIPQUAD(dp->dp_daddr),
730c0dd49bdSEiji Ota RDS_PROTOCOL_MAJOR(version), RDS_PROTOCOL_MINOR(version),
731c0dd49bdSEiji Ota (unsigned long long)ntohll(lguid),
732c0dd49bdSEiji Ota (unsigned long long)ntohll(fguid));
733c0dd49bdSEiji Ota
734c0dd49bdSEiji Ota conn = rdsv3_conn_create(dp->dp_daddr, dp->dp_saddr,
735c0dd49bdSEiji Ota &rdsv3_ib_transport, KM_NOSLEEP);
736c0dd49bdSEiji Ota if (IS_ERR(conn)) {
737c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
738c0dd49bdSEiji Ota "rdsv3_conn_create failed (%ld)", PTR_ERR(conn));
739c0dd49bdSEiji Ota conn = NULL;
740c0dd49bdSEiji Ota goto out;
741c0dd49bdSEiji Ota }
742c0dd49bdSEiji Ota
743c0dd49bdSEiji Ota /*
744c0dd49bdSEiji Ota * The connection request may occur while the
745c0dd49bdSEiji Ota * previous connection exist, e.g. in case of failover.
746c0dd49bdSEiji Ota * But as connections may be initiated simultaneously
747c0dd49bdSEiji Ota * by both hosts, we have a random backoff mechanism -
748c0dd49bdSEiji Ota * see the comment above rdsv3_queue_reconnect()
749c0dd49bdSEiji Ota */
750c0dd49bdSEiji Ota mutex_enter(&conn->c_cm_lock);
751c0dd49bdSEiji Ota if (!rdsv3_conn_transition(conn, RDSV3_CONN_DOWN,
752c0dd49bdSEiji Ota RDSV3_CONN_CONNECTING)) {
753c0dd49bdSEiji Ota if (rdsv3_conn_state(conn) == RDSV3_CONN_UP) {
754c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
755c0dd49bdSEiji Ota "incoming connect when connected: %p",
756c0dd49bdSEiji Ota conn);
757c0dd49bdSEiji Ota rdsv3_conn_drop(conn);
758c0dd49bdSEiji Ota rdsv3_ib_stats_inc(s_ib_listen_closed_stale);
759c0dd49bdSEiji Ota mutex_exit(&conn->c_cm_lock);
760c0dd49bdSEiji Ota goto out;
761c0dd49bdSEiji Ota } else if (rdsv3_conn_state(conn) == RDSV3_CONN_CONNECTING) {
762c0dd49bdSEiji Ota /* Wait and see - our connect may still be succeeding */
763c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
764c0dd49bdSEiji Ota "peer-to-peer connection request: %p, "
765c0dd49bdSEiji Ota "lguid: 0x%llx fguid: 0x%llx",
766c0dd49bdSEiji Ota conn, lguid, fguid);
767c0dd49bdSEiji Ota rdsv3_ib_stats_inc(s_ib_connect_raced);
768c0dd49bdSEiji Ota }
769c0dd49bdSEiji Ota mutex_exit(&conn->c_cm_lock);
770c0dd49bdSEiji Ota goto out;
771c0dd49bdSEiji Ota }
772c0dd49bdSEiji Ota
773c0dd49bdSEiji Ota ic = conn->c_transport_data;
774c0dd49bdSEiji Ota
775c0dd49bdSEiji Ota rdsv3_ib_set_protocol(conn, version);
776c0dd49bdSEiji Ota rdsv3_ib_set_flow_control(conn, ntohl(dp->dp_credit));
777c0dd49bdSEiji Ota
778c0dd49bdSEiji Ota /*
779c0dd49bdSEiji Ota * If the peer gave us the last packet it saw, process this as if
780c0dd49bdSEiji Ota * we had received a regular ACK.
781c0dd49bdSEiji Ota */
782c0dd49bdSEiji Ota if (dp->dp_ack_seq)
783c0dd49bdSEiji Ota rdsv3_send_drop_acked(conn, ntohll(dp->dp_ack_seq), NULL);
784c0dd49bdSEiji Ota
785c0dd49bdSEiji Ota ASSERT(!cm_id->context);
786c0dd49bdSEiji Ota ASSERT(!ic->i_cm_id);
787c0dd49bdSEiji Ota
788c0dd49bdSEiji Ota if (ic->i_cm_id != NULL)
789c0dd49bdSEiji Ota RDSV3_PANIC();
790c0dd49bdSEiji Ota
791c0dd49bdSEiji Ota ic->i_cm_id = cm_id;
792c0dd49bdSEiji Ota cm_id->context = conn;
793c0dd49bdSEiji Ota
794c0dd49bdSEiji Ota /*
795c0dd49bdSEiji Ota * We got halfway through setting up the ib_connection, if we
796c0dd49bdSEiji Ota * fail now, we have to take the long route out of this mess.
797c0dd49bdSEiji Ota */
798c0dd49bdSEiji Ota destroy = 0;
799c0dd49bdSEiji Ota
800c0dd49bdSEiji Ota err = rdsv3_ib_setup_qp(conn);
801c0dd49bdSEiji Ota if (err) {
802c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
803c0dd49bdSEiji Ota "rdsv3_ib_setup_qp failed (%d)", err);
8046e18d381Sagiri mutex_exit(&conn->c_cm_lock);
805c0dd49bdSEiji Ota rdsv3_conn_drop(conn);
806c0dd49bdSEiji Ota goto out;
807c0dd49bdSEiji Ota }
808c0dd49bdSEiji Ota
8095d5562f5SEiji Ota rdsv3_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
8105d5562f5SEiji Ota event->param.conn.responder_resources,
8115d5562f5SEiji Ota event->param.conn.initiator_depth);
812c0dd49bdSEiji Ota
813c0dd49bdSEiji Ota /* rdma_accept() calls rdma_reject() internally if it fails */
814c0dd49bdSEiji Ota err = rdma_accept(cm_id, &conn_param);
815c0dd49bdSEiji Ota mutex_exit(&conn->c_cm_lock);
816c0dd49bdSEiji Ota if (err) {
817c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
818c0dd49bdSEiji Ota "rdma_accept failed (%d)", err);
819c0dd49bdSEiji Ota rdsv3_conn_drop(conn);
820c0dd49bdSEiji Ota goto out;
821c0dd49bdSEiji Ota }
822c0dd49bdSEiji Ota
823c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_handle_connect",
824c0dd49bdSEiji Ota "Return cm_id: %p event: %p", cm_id, event);
825c0dd49bdSEiji Ota
826c0dd49bdSEiji Ota return (0);
827c0dd49bdSEiji Ota
828c0dd49bdSEiji Ota out:
829c0dd49bdSEiji Ota (void) rdma_reject(cm_id, NULL, 0);
830c0dd49bdSEiji Ota return (destroy);
831c0dd49bdSEiji Ota }
832c0dd49bdSEiji Ota
833c0dd49bdSEiji Ota
834c0dd49bdSEiji Ota int
rdsv3_ib_cm_initiate_connect(struct rdma_cm_id * cm_id)835c0dd49bdSEiji Ota rdsv3_ib_cm_initiate_connect(struct rdma_cm_id *cm_id)
836c0dd49bdSEiji Ota {
837c0dd49bdSEiji Ota struct rdsv3_connection *conn = cm_id->context;
838c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
839c0dd49bdSEiji Ota struct rdma_conn_param conn_param;
840c0dd49bdSEiji Ota struct rdsv3_ib_connect_private dp;
841c0dd49bdSEiji Ota int ret;
842c0dd49bdSEiji Ota
843c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect", "Enter: cm_id: %p",
844c0dd49bdSEiji Ota cm_id);
845c0dd49bdSEiji Ota
846c0dd49bdSEiji Ota /*
847c0dd49bdSEiji Ota * If the peer doesn't do protocol negotiation, we must
848c0dd49bdSEiji Ota * default to RDSv3.0
849c0dd49bdSEiji Ota */
850c0dd49bdSEiji Ota rdsv3_ib_set_protocol(conn, RDS_PROTOCOL_3_0);
851c0dd49bdSEiji Ota ic->i_flowctl =
852c0dd49bdSEiji Ota rdsv3_ib_sysctl_flow_control; /* advertise flow control */
853c0dd49bdSEiji Ota
854c0dd49bdSEiji Ota ret = rdsv3_ib_setup_qp(conn);
855c0dd49bdSEiji Ota if (ret) {
856c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect",
857c0dd49bdSEiji Ota "rdsv3_ib_setup_qp failed (%d)", ret);
858c0dd49bdSEiji Ota rdsv3_conn_drop(conn);
859c0dd49bdSEiji Ota goto out;
860c0dd49bdSEiji Ota }
861c0dd49bdSEiji Ota
8625d5562f5SEiji Ota rdsv3_ib_cm_fill_conn_param(conn, &conn_param, &dp,
8635d5562f5SEiji Ota RDS_PROTOCOL_VERSION, UINT_MAX, UINT_MAX);
864c0dd49bdSEiji Ota
865c0dd49bdSEiji Ota ret = rdma_connect(cm_id, &conn_param);
866c0dd49bdSEiji Ota if (ret) {
867c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect",
868c0dd49bdSEiji Ota "rdma_connect failed (%d)", ret);
869c0dd49bdSEiji Ota rdsv3_conn_drop(conn);
870c0dd49bdSEiji Ota }
871c0dd49bdSEiji Ota
872c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_cm_initiate_connect",
873c0dd49bdSEiji Ota "Return: cm_id: %p", cm_id);
874c0dd49bdSEiji Ota
875c0dd49bdSEiji Ota out:
876c0dd49bdSEiji Ota /*
877c0dd49bdSEiji Ota * Beware - returning non-zero tells the rdma_cm to destroy
878c0dd49bdSEiji Ota * the cm_id. We should certainly not do it as long as we still
879c0dd49bdSEiji Ota * "own" the cm_id.
880c0dd49bdSEiji Ota */
881c0dd49bdSEiji Ota if (ret) {
882c0dd49bdSEiji Ota if (ic->i_cm_id == cm_id)
883c0dd49bdSEiji Ota ret = 0;
884c0dd49bdSEiji Ota }
885c0dd49bdSEiji Ota return (ret);
886c0dd49bdSEiji Ota }
887c0dd49bdSEiji Ota
888c0dd49bdSEiji Ota int
rdsv3_ib_conn_connect(struct rdsv3_connection * conn)889c0dd49bdSEiji Ota rdsv3_ib_conn_connect(struct rdsv3_connection *conn)
890c0dd49bdSEiji Ota {
891c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
892c0dd49bdSEiji Ota struct sockaddr_in src, dest;
893c0dd49bdSEiji Ota ipaddr_t laddr, faddr;
894c0dd49bdSEiji Ota int ret;
895c0dd49bdSEiji Ota
896c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_connect", "Enter: conn: %p", conn);
897c0dd49bdSEiji Ota
898c0dd49bdSEiji Ota /*
899c0dd49bdSEiji Ota * XXX I wonder what affect the port space has
900c0dd49bdSEiji Ota */
901c0dd49bdSEiji Ota /* delegate cm event handler to rdma_transport */
902c0dd49bdSEiji Ota ic->i_cm_id = rdma_create_id(rdsv3_rdma_cm_event_handler, conn,
903c0dd49bdSEiji Ota RDMA_PS_TCP);
904c0dd49bdSEiji Ota if (IS_ERR(ic->i_cm_id)) {
905c0dd49bdSEiji Ota ret = PTR_ERR(ic->i_cm_id);
906c0dd49bdSEiji Ota ic->i_cm_id = NULL;
907c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_connect",
908c0dd49bdSEiji Ota "rdma_create_id() failed: %d", ret);
909c0dd49bdSEiji Ota goto out;
910c0dd49bdSEiji Ota }
911c0dd49bdSEiji Ota
912c0dd49bdSEiji Ota RDSV3_DPRINTF3("rdsv3_ib_conn_connect",
913c0dd49bdSEiji Ota "created cm id %p for conn %p", ic->i_cm_id, conn);
914c0dd49bdSEiji Ota
915c0dd49bdSEiji Ota /* The ipaddr should be in the network order */
916c0dd49bdSEiji Ota laddr = conn->c_laddr;
917c0dd49bdSEiji Ota faddr = conn->c_faddr;
918c0dd49bdSEiji Ota ret = rdsv3_sc_path_lookup(&laddr, &faddr);
919c0dd49bdSEiji Ota if (ret == 0) {
920c0dd49bdSEiji Ota RDSV3_DPRINTF2(LABEL, "Path not found (0x%x 0x%x)",
921c0dd49bdSEiji Ota ntohl(laddr), ntohl(faddr));
922c0dd49bdSEiji Ota }
923c0dd49bdSEiji Ota
924c0dd49bdSEiji Ota src.sin_family = AF_INET;
925c0dd49bdSEiji Ota src.sin_addr.s_addr = (uint32_t)laddr;
926c0dd49bdSEiji Ota src.sin_port = (uint16_t)htons(0);
927c0dd49bdSEiji Ota
928c0dd49bdSEiji Ota dest.sin_family = AF_INET;
929c0dd49bdSEiji Ota dest.sin_addr.s_addr = (uint32_t)faddr;
930c0dd49bdSEiji Ota dest.sin_port = (uint16_t)htons(RDSV3_PORT);
931c0dd49bdSEiji Ota
932c0dd49bdSEiji Ota ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
933c0dd49bdSEiji Ota (struct sockaddr *)&dest,
934c0dd49bdSEiji Ota RDSV3_RDMA_RESOLVE_TIMEOUT_MS);
935c0dd49bdSEiji Ota if (ret) {
936c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_connect",
937c0dd49bdSEiji Ota "addr resolve failed for cm id %p: %d", ic->i_cm_id, ret);
938c0dd49bdSEiji Ota rdma_destroy_id(ic->i_cm_id);
939c0dd49bdSEiji Ota ic->i_cm_id = NULL;
940c0dd49bdSEiji Ota }
941c0dd49bdSEiji Ota
942c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_connect", "Return: conn: %p", conn);
943c0dd49bdSEiji Ota
944c0dd49bdSEiji Ota out:
945c0dd49bdSEiji Ota return (ret);
946c0dd49bdSEiji Ota }
947c0dd49bdSEiji Ota
948c0dd49bdSEiji Ota /*
949c0dd49bdSEiji Ota * This is so careful about only cleaning up resources that were built up
950c0dd49bdSEiji Ota * so that it can be called at any point during startup. In fact it
951c0dd49bdSEiji Ota * can be called multiple times for a given connection.
952c0dd49bdSEiji Ota */
953c0dd49bdSEiji Ota void
rdsv3_ib_conn_shutdown(struct rdsv3_connection * conn)954c0dd49bdSEiji Ota rdsv3_ib_conn_shutdown(struct rdsv3_connection *conn)
955c0dd49bdSEiji Ota {
956c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
957c0dd49bdSEiji Ota int err = 0;
958c0dd49bdSEiji Ota
959c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown",
9605d5562f5SEiji Ota "cm %p pd %p cq %p qp %p", ic->i_cm_id,
9615d5562f5SEiji Ota ic->i_pd, ic->i_cq, ic->i_cm_id ? ic->i_cm_id->qp : NULL);
962c0dd49bdSEiji Ota
963c0dd49bdSEiji Ota if (ic->i_cm_id) {
964c0dd49bdSEiji Ota struct ib_device *dev = ic->i_cm_id->device;
965c0dd49bdSEiji Ota
966c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown",
967c0dd49bdSEiji Ota "disconnecting cm %p", ic->i_cm_id);
968c0dd49bdSEiji Ota err = rdma_disconnect(ic->i_cm_id);
969c0dd49bdSEiji Ota if (err) {
970c0dd49bdSEiji Ota /*
971c0dd49bdSEiji Ota * Actually this may happen quite frequently, when
972c0dd49bdSEiji Ota * an outgoing connect raced with an incoming connect.
973c0dd49bdSEiji Ota */
974c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown",
975c0dd49bdSEiji Ota "failed to disconnect, cm: %p err %d",
976c0dd49bdSEiji Ota ic->i_cm_id, err);
977c0dd49bdSEiji Ota }
978c0dd49bdSEiji Ota
979c0dd49bdSEiji Ota if (ic->i_cm_id->qp) {
980c0dd49bdSEiji Ota (void) ibt_flush_qp(
981c0dd49bdSEiji Ota ib_get_ibt_channel_hdl(ic->i_cm_id));
9825d5562f5SEiji Ota /*
9835d5562f5SEiji Ota * Don't wait for the send ring to be empty -- there
9845d5562f5SEiji Ota * may be completed non-signaled entries sitting on
9855d5562f5SEiji Ota * there. We unmap these below.
9865d5562f5SEiji Ota */
9875d5562f5SEiji Ota rdsv3_wait_event(&ic->i_recv_ring.w_empty_wait,
988c0dd49bdSEiji Ota rdsv3_ib_ring_empty(&ic->i_recv_ring));
9895d5562f5SEiji Ota /*
9905d5562f5SEiji Ota * Note that Linux original code calls
9915d5562f5SEiji Ota * rdma_destroy_qp() after rdsv3_ib_recv_clear_ring(ic).
9925d5562f5SEiji Ota */
993c0dd49bdSEiji Ota rdma_destroy_qp(ic->i_cm_id);
994c0dd49bdSEiji Ota }
995c0dd49bdSEiji Ota
9965d5562f5SEiji Ota if (rdsv3_enable_snd_cq) {
9975d5562f5SEiji Ota if (ic->i_snd_soft_cq) {
9985d5562f5SEiji Ota rdsv3_af_thr_destroy(ic->i_snd_soft_cq);
9995d5562f5SEiji Ota ic->i_snd_soft_cq = NULL;
10005d5562f5SEiji Ota }
10015d5562f5SEiji Ota if (ic->i_snd_cq)
10025d5562f5SEiji Ota (void) ib_destroy_cq(ic->i_snd_cq);
10035d5562f5SEiji Ota }
10045d5562f5SEiji Ota if (ic->i_soft_cq) {
10055d5562f5SEiji Ota rdsv3_af_thr_destroy(ic->i_soft_cq);
10065d5562f5SEiji Ota ic->i_soft_cq = NULL;
10075d5562f5SEiji Ota }
10085d5562f5SEiji Ota if (ic->i_refill_rq) {
10095d5562f5SEiji Ota rdsv3_af_thr_destroy(ic->i_refill_rq);
10105d5562f5SEiji Ota ic->i_refill_rq = NULL;
10115d5562f5SEiji Ota }
10125d5562f5SEiji Ota if (ic->i_cq)
10135d5562f5SEiji Ota (void) ib_destroy_cq(ic->i_cq);
1014c0dd49bdSEiji Ota
1015c0dd49bdSEiji Ota if (ic->i_mr)
1016c0dd49bdSEiji Ota rdsv3_ib_free_hdrs(dev, ic);
1017c0dd49bdSEiji Ota
1018c0dd49bdSEiji Ota if (ic->i_sends)
1019c0dd49bdSEiji Ota rdsv3_ib_send_clear_ring(ic);
1020c0dd49bdSEiji Ota if (ic->i_recvs)
1021c0dd49bdSEiji Ota rdsv3_ib_recv_clear_ring(ic);
1022c0dd49bdSEiji Ota
1023c0dd49bdSEiji Ota rdma_destroy_id(ic->i_cm_id);
1024c0dd49bdSEiji Ota
1025c0dd49bdSEiji Ota /*
1026c0dd49bdSEiji Ota * Move connection back to the nodev list.
1027c0dd49bdSEiji Ota */
1028d2b539e7Sagiri if (ic->i_on_dev_list)
1029c0dd49bdSEiji Ota rdsv3_ib_remove_conn(ic->rds_ibdev, conn);
1030c0dd49bdSEiji Ota
1031c0dd49bdSEiji Ota ic->i_cm_id = NULL;
1032c0dd49bdSEiji Ota ic->i_pd = NULL;
1033c0dd49bdSEiji Ota ic->i_mr = NULL;
10345d5562f5SEiji Ota ic->i_cq = NULL;
10355d5562f5SEiji Ota ic->i_snd_cq = NULL;
1036c0dd49bdSEiji Ota ic->i_send_hdrs = NULL;
1037c0dd49bdSEiji Ota ic->i_recv_hdrs = NULL;
1038c0dd49bdSEiji Ota ic->i_ack = NULL;
1039c0dd49bdSEiji Ota }
1040d2b539e7Sagiri ASSERT(!ic->i_on_dev_list);
1041c0dd49bdSEiji Ota
1042c0dd49bdSEiji Ota /* Clear pending transmit */
1043c0dd49bdSEiji Ota if (ic->i_rm) {
1044c0dd49bdSEiji Ota rdsv3_message_put(ic->i_rm);
1045c0dd49bdSEiji Ota ic->i_rm = NULL;
1046c0dd49bdSEiji Ota }
1047c0dd49bdSEiji Ota
1048c0dd49bdSEiji Ota /* Clear the ACK state */
1049c0dd49bdSEiji Ota clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
1050c0dd49bdSEiji Ota ic->i_ack_next = 0;
1051c0dd49bdSEiji Ota ic->i_ack_recv = 0;
1052c0dd49bdSEiji Ota
1053c0dd49bdSEiji Ota /* Clear flow control state */
1054c0dd49bdSEiji Ota ic->i_flowctl = 0;
1055c0dd49bdSEiji Ota ic->i_credits = 0;
1056c0dd49bdSEiji Ota
1057c0dd49bdSEiji Ota rdsv3_ib_ring_init(&ic->i_send_ring, rdsv3_ib_sysctl_max_send_wr);
1058c0dd49bdSEiji Ota rdsv3_ib_ring_init(&ic->i_recv_ring, rdsv3_ib_sysctl_max_recv_wr);
1059c0dd49bdSEiji Ota
1060c0dd49bdSEiji Ota if (ic->i_ibinc) {
1061c0dd49bdSEiji Ota rdsv3_inc_put(&ic->i_ibinc->ii_inc);
1062c0dd49bdSEiji Ota ic->i_ibinc = NULL;
1063c0dd49bdSEiji Ota }
1064c0dd49bdSEiji Ota
1065c0dd49bdSEiji Ota if (ic->i_sends) {
1066c0dd49bdSEiji Ota kmem_free(ic->i_sends,
1067c0dd49bdSEiji Ota ic->i_send_ring.w_nr * sizeof (struct rdsv3_ib_send_work));
1068c0dd49bdSEiji Ota ic->i_sends = NULL;
1069c0dd49bdSEiji Ota }
1070c0dd49bdSEiji Ota if (ic->i_send_wrs) {
10713f756f37Sagiri kmem_free(ic->i_send_wrs, ic->i_send_ring.w_nr *
1072c0dd49bdSEiji Ota (sizeof (ibt_send_wr_t) +
1073c0dd49bdSEiji Ota RDSV3_IB_MAX_SGE * sizeof (ibt_wr_ds_t)));
1074c0dd49bdSEiji Ota ic->i_send_wrs = NULL;
1075c0dd49bdSEiji Ota }
1076c0dd49bdSEiji Ota if (ic->i_recvs) {
1077c0dd49bdSEiji Ota kmem_free(ic->i_recvs,
1078c0dd49bdSEiji Ota ic->i_recv_ring.w_nr * sizeof (struct rdsv3_ib_recv_work));
1079c0dd49bdSEiji Ota ic->i_recvs = NULL;
1080c0dd49bdSEiji Ota }
10815d5562f5SEiji Ota if (ic->i_recv_wrs) {
10825d5562f5SEiji Ota kmem_free(ic->i_recv_wrs, ic->i_recv_ring.w_nr *
10835d5562f5SEiji Ota (sizeof (ibt_recv_wr_t)));
10845d5562f5SEiji Ota ic->i_recv_wrs = NULL;
10855d5562f5SEiji Ota }
1086c0dd49bdSEiji Ota
1087c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_shutdown", "Return conn: %p", conn);
1088c0dd49bdSEiji Ota }
1089c0dd49bdSEiji Ota
1090c0dd49bdSEiji Ota /* ARGSUSED */
1091c0dd49bdSEiji Ota int
rdsv3_ib_conn_alloc(struct rdsv3_connection * conn,int gfp)1092c0dd49bdSEiji Ota rdsv3_ib_conn_alloc(struct rdsv3_connection *conn, int gfp)
1093c0dd49bdSEiji Ota {
1094c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic;
1095c0dd49bdSEiji Ota
1096c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_alloc", "conn: %p", conn);
1097c0dd49bdSEiji Ota
1098c0dd49bdSEiji Ota /* XXX too lazy? */
1099c0dd49bdSEiji Ota ic = kmem_zalloc(sizeof (struct rdsv3_ib_connection), gfp);
11005d5562f5SEiji Ota if (!ic)
1101c0dd49bdSEiji Ota return (-ENOMEM);
1102c0dd49bdSEiji Ota
1103c0dd49bdSEiji Ota list_link_init(&ic->ib_node);
1104c0dd49bdSEiji Ota
1105c0dd49bdSEiji Ota mutex_init(&ic->i_recv_mutex, NULL, MUTEX_DRIVER, NULL);
1106c0dd49bdSEiji Ota mutex_init(&ic->i_ack_lock, NULL, MUTEX_DRIVER, NULL);
1107c0dd49bdSEiji Ota
1108c0dd49bdSEiji Ota /*
1109c0dd49bdSEiji Ota * rdsv3_ib_conn_shutdown() waits for these to be emptied so they
1110c0dd49bdSEiji Ota * must be initialized before it can be called.
1111c0dd49bdSEiji Ota */
1112c0dd49bdSEiji Ota rdsv3_ib_ring_init(&ic->i_send_ring, rdsv3_ib_sysctl_max_send_wr);
1113c0dd49bdSEiji Ota rdsv3_ib_ring_init(&ic->i_recv_ring, rdsv3_ib_sysctl_max_recv_wr);
1114c0dd49bdSEiji Ota
1115c0dd49bdSEiji Ota ic->conn = conn;
1116c0dd49bdSEiji Ota conn->c_transport_data = ic;
1117c0dd49bdSEiji Ota
1118c0dd49bdSEiji Ota mutex_enter(&ib_nodev_conns_lock);
1119c0dd49bdSEiji Ota list_insert_tail(&ib_nodev_conns, ic);
1120c0dd49bdSEiji Ota mutex_exit(&ib_nodev_conns_lock);
1121c0dd49bdSEiji Ota
1122c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_alloc", "conn %p conn ic %p",
1123c0dd49bdSEiji Ota conn, conn->c_transport_data);
1124c0dd49bdSEiji Ota return (0);
1125c0dd49bdSEiji Ota }
1126c0dd49bdSEiji Ota
1127c0dd49bdSEiji Ota /*
1128c0dd49bdSEiji Ota * Free a connection. Connection must be shut down and not set for reconnect.
1129c0dd49bdSEiji Ota */
1130c0dd49bdSEiji Ota void
rdsv3_ib_conn_free(void * arg)1131c0dd49bdSEiji Ota rdsv3_ib_conn_free(void *arg)
1132c0dd49bdSEiji Ota {
1133c0dd49bdSEiji Ota struct rdsv3_ib_connection *ic = arg;
1134c0dd49bdSEiji Ota kmutex_t *lock_ptr;
1135c0dd49bdSEiji Ota
1136c0dd49bdSEiji Ota RDSV3_DPRINTF2("rdsv3_ib_conn_free", "ic %p\n", ic);
1137c0dd49bdSEiji Ota
1138c0dd49bdSEiji Ota #ifndef __lock_lint
1139c0dd49bdSEiji Ota /*
1140c0dd49bdSEiji Ota * Conn is either on a dev's list or on the nodev list.
1141c0dd49bdSEiji Ota * A race with shutdown() or connect() would cause problems
1142c0dd49bdSEiji Ota * (since rds_ibdev would change) but that should never happen.
1143c0dd49bdSEiji Ota */
1144d2b539e7Sagiri lock_ptr = ic->i_on_dev_list ?
1145c0dd49bdSEiji Ota &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;
1146c0dd49bdSEiji Ota
1147c0dd49bdSEiji Ota mutex_enter(lock_ptr);
1148c0dd49bdSEiji Ota list_remove_node(&ic->ib_node);
1149c0dd49bdSEiji Ota mutex_exit(lock_ptr);
1150c0dd49bdSEiji Ota #endif
1151c0dd49bdSEiji Ota kmem_free(ic, sizeof (*ic));
1152c0dd49bdSEiji Ota }
1153c0dd49bdSEiji Ota
1154c0dd49bdSEiji Ota /*
1155c0dd49bdSEiji Ota * An error occurred on the connection
1156c0dd49bdSEiji Ota */
1157c0dd49bdSEiji Ota void
__rdsv3_ib_conn_error(struct rdsv3_connection * conn)1158c0dd49bdSEiji Ota __rdsv3_ib_conn_error(struct rdsv3_connection *conn)
1159c0dd49bdSEiji Ota {
1160c0dd49bdSEiji Ota rdsv3_conn_drop(conn);
1161c0dd49bdSEiji Ota }
1162