1a0fcf16hselasky/*-
2a0fcf16hselasky * SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
3a0fcf16hselasky *
45115240jeff * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
55115240jeff * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
65115240jeff * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
75115240jeff * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
85115240jeff *
95115240jeff * This software is available to you under a choice of one of two
105115240jeff * licenses.  You may choose to be licensed under the terms of the GNU
115115240jeff * General Public License (GPL) Version 2, available from the file
125115240jeff * COPYING in the main directory of this source tree, or the
135115240jeff * OpenIB.org BSD license below:
145115240jeff *
155115240jeff *     Redistribution and use in source and binary forms, with or
165115240jeff *     without modification, are permitted provided that the following
175115240jeff *     conditions are met:
185115240jeff *
195115240jeff *      - Redistributions of source code must retain the above
205115240jeff *        copyright notice, this list of conditions and the following
215115240jeff *        disclaimer.
225115240jeff *
235115240jeff *      - Redistributions in binary form must reproduce the above
245115240jeff *        copyright notice, this list of conditions and the following
255115240jeff *        disclaimer in the documentation and/or other materials
265115240jeff *        provided with the distribution.
275115240jeff *
285115240jeff * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
295115240jeff * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
305115240jeff * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
315115240jeff * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
325115240jeff * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
335115240jeff * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
345115240jeff * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
355115240jeff * SOFTWARE.
365115240jeff */
375115240jeff
382692d50hselasky#include <sys/cdefs.h>
392692d50hselasky__FBSDID("$FreeBSD$");
402692d50hselasky
410d8f1c2hselasky#define	LINUXKPI_PARAM_PREFIX ibcore_
420d8f1c2hselasky
435115240jeff#include <linux/completion.h>
445115240jeff#include <linux/in.h>
455115240jeff#include <linux/in6.h>
465115240jeff#include <linux/mutex.h>
475115240jeff#include <linux/random.h>
485115240jeff#include <linux/idr.h>
495115240jeff#include <linux/inetdevice.h>
505dbc43fhselasky#include <linux/slab.h>
515dbc43fhselasky#include <linux/module.h>
525dbc43fhselasky#include <net/route.h>
536004273melifaro#include <net/route/nhop.h>
545115240jeff
555115240jeff#include <net/tcp.h>
565115240jeff#include <net/ipv6.h>
575115240jeff
586004273melifaro#include <netinet/in_fib.h>
596004273melifaro
606004273melifaro#include <netinet6/in6_fib.h>
6179ee7efhselasky#include <netinet6/scope6_var.h>
6279ee7efhselasky#include <netinet6/ip6_var.h>
6379ee7efhselasky
645115240jeff#include <rdma/rdma_cm.h>
655115240jeff#include <rdma/rdma_cm_ib.h>
66f1b5552cem#include <rdma/rdma_sdp.h>
6779ee7efhselasky#include <rdma/ib.h>
6879ee7efhselasky#include <rdma/ib_addr.h>
695115240jeff#include <rdma/ib_cache.h>
705115240jeff#include <rdma/ib_cm.h>
715115240jeff#include <rdma/ib_sa.h>
725115240jeff#include <rdma/iw_cm.h>
735115240jeff
7479ee7efhselasky#include <sys/priv.h>
7579ee7efhselasky
7679ee7efhselasky#include "core_priv.h"
7779ee7efhselasky
785115240jeffMODULE_AUTHOR("Sean Hefty");
795115240jeffMODULE_DESCRIPTION("Generic RDMA CM Agent");
805115240jeffMODULE_LICENSE("Dual BSD/GPL");
815115240jeff
825115240jeff#define CMA_CM_RESPONSE_TIMEOUT 20
8379ee7efhselasky#define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000
845115240jeff#define CMA_MAX_CM_RETRIES 15
855115240jeff#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
865dbc43fhselasky#define CMA_IBOE_PACKET_LIFETIME 18
875115240jeff
8879ee7efhselaskystatic const char * const cma_events[] = {
8979ee7efhselasky	[RDMA_CM_EVENT_ADDR_RESOLVED]	 = "address resolved",
9079ee7efhselasky	[RDMA_CM_EVENT_ADDR_ERROR]	 = "address error",
9179ee7efhselasky	[RDMA_CM_EVENT_ROUTE_RESOLVED]	 = "route resolved ",
9279ee7efhselasky	[RDMA_CM_EVENT_ROUTE_ERROR]	 = "route error",
9379ee7efhselasky	[RDMA_CM_EVENT_CONNECT_REQUEST]	 = "connect request",
9479ee7efhselasky	[RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response",
9579ee7efhselasky	[RDMA_CM_EVENT_CONNECT_ERROR]	 = "connect error",
9679ee7efhselasky	[RDMA_CM_EVENT_UNREACHABLE]	 = "unreachable",
9779ee7efhselasky	[RDMA_CM_EVENT_REJECTED]	 = "rejected",
9879ee7efhselasky	[RDMA_CM_EVENT_ESTABLISHED]	 = "established",
9979ee7efhselasky	[RDMA_CM_EVENT_DISCONNECTED]	 = "disconnected",
10079ee7efhselasky	[RDMA_CM_EVENT_DEVICE_REMOVAL]	 = "device removal",
10179ee7efhselasky	[RDMA_CM_EVENT_MULTICAST_JOIN]	 = "multicast join",
10279ee7efhselasky	[RDMA_CM_EVENT_MULTICAST_ERROR]	 = "multicast error",
10379ee7efhselasky	[RDMA_CM_EVENT_ADDR_CHANGE]	 = "address change",
10479ee7efhselasky	[RDMA_CM_EVENT_TIMEWAIT_EXIT]	 = "timewait exit",
10579ee7efhselasky};
1065dbc43fhselasky
10779ee7efhselaskyconst char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
10879ee7efhselasky{
10979ee7efhselasky	size_t index = event;
1105dbc43fhselasky
11179ee7efhselasky	return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ?
11279ee7efhselasky			cma_events[index] : "unrecognized event";
11379ee7efhselasky}
11479ee7efhselaskyEXPORT_SYMBOL(rdma_event_msg);
1155dbc43fhselasky
1165d1e727hselaskystatic int cma_check_linklocal(struct rdma_dev_addr *, struct sockaddr *);
1175115240jeffstatic void cma_add_one(struct ib_device *device);
11879ee7efhselaskystatic void cma_remove_one(struct ib_device *device, void *client_data);
119f1b5552cemstatic enum rdma_port_space rdma_ps_from_service_id(__be64 service_id);
1205115240jeff
1215115240jeffstatic struct ib_client cma_client = {
1225115240jeff	.name   = "cma",
1235115240jeff	.add    = cma_add_one,
1245115240jeff	.remove = cma_remove_one
1255115240jeff};
1265115240jeff
1275115240jeffstatic struct ib_sa_client sa_client;
1285115240jeffstatic struct rdma_addr_client addr_client;
1295115240jeffstatic LIST_HEAD(dev_list);
1305115240jeffstatic LIST_HEAD(listen_any_list);
1315115240jeffstatic DEFINE_MUTEX(lock);
1325115240jeffstatic struct workqueue_struct *cma_wq;
13379ee7efhselasky
13479ee7efhselaskystruct cma_pernet {
13579ee7efhselasky	struct idr tcp_ps;
13679ee7efhselasky	struct idr udp_ps;
13779ee7efhselasky	struct idr ipoib_ps;
13879ee7efhselasky	struct idr ib_ps;
139f1b5552cem	struct idr sdp_ps;
14079ee7efhselasky};
14179ee7efhselasky
14279ee7efhselaskyVNET_DEFINE(struct cma_pernet, cma_pernet);
14379ee7efhselasky
14479ee7efhselaskystatic struct cma_pernet *cma_pernet_ptr(struct vnet *vnet)
14579ee7efhselasky{
14679ee7efhselasky	struct cma_pernet *retval;
14779ee7efhselasky
14879ee7efhselasky	CURVNET_SET_QUIET(vnet);
14979ee7efhselasky	retval = &VNET(cma_pernet);
15079ee7efhselasky	CURVNET_RESTORE();
15179ee7efhselasky
15279ee7efhselasky	return (retval);
15379ee7efhselasky}
15479ee7efhselasky
15579ee7efhselaskystatic struct idr *cma_pernet_idr(struct vnet *net, enum rdma_port_space ps)
15679ee7efhselasky{
15779ee7efhselasky	struct cma_pernet *pernet = cma_pernet_ptr(net);
15879ee7efhselasky
15979ee7efhselasky	switch (ps) {
16079ee7efhselasky	case RDMA_PS_TCP:
16179ee7efhselasky		return &pernet->tcp_ps;
16279ee7efhselasky	case RDMA_PS_UDP:
16379ee7efhselasky		return &pernet->udp_ps;
16479ee7efhselasky	case RDMA_PS_IPOIB:
16579ee7efhselasky		return &pernet->ipoib_ps;
16679ee7efhselasky	case RDMA_PS_IB:
16779ee7efhselasky		return &pernet->ib_ps;
168f1b5552cem	case RDMA_PS_SDP:
169f1b5552cem		return &pernet->sdp_ps;
17079ee7efhselasky	default:
17179ee7efhselasky		return NULL;
17279ee7efhselasky	}
17379ee7efhselasky}
1745115240jeff
1755115240jeffstruct cma_device {
1765115240jeff	struct list_head	list;
1775115240jeff	struct ib_device	*device;
1785115240jeff	struct completion	comp;
1795115240jeff	atomic_t		refcount;
1805115240jeff	struct list_head	id_list;
18179ee7efhselasky	struct sysctl_ctx_list	sysctl_ctx;
18279ee7efhselasky	enum ib_gid_type	*default_gid_type;
1835115240jeff};
1845115240jeff
1855115240jeffstruct rdma_bind_list {
18679ee7efhselasky	enum rdma_port_space	ps;
1875115240jeff	struct hlist_head	owners;
1885115240jeff	unsigned short		port;
1895115240jeff};
1905115240jeff
19179ee7efhselaskystruct class_port_info_context {
19279ee7efhselasky	struct ib_class_port_info	*class_port_info;
19379ee7efhselasky	struct ib_device		*device;
19479ee7efhselasky	struct completion		done;
19579ee7efhselasky	struct ib_sa_query		*sa_query;
19679ee7efhselasky	u8				port_num;
19779ee7efhselasky};
19879ee7efhselasky
19979ee7efhselaskystatic int cma_ps_alloc(struct vnet *vnet, enum rdma_port_space ps,
20079ee7efhselasky			struct rdma_bind_list *bind_list, int snum)
20179ee7efhselasky{
20279ee7efhselasky	struct idr *idr = cma_pernet_idr(vnet, ps);
20379ee7efhselasky
20479ee7efhselasky	return idr_alloc(idr, bind_list, snum, snum + 1, GFP_KERNEL);
20579ee7efhselasky}
20679ee7efhselasky
20779ee7efhselaskystatic struct rdma_bind_list *cma_ps_find(struct vnet *net,
20879ee7efhselasky					  enum rdma_port_space ps, int snum)
20979ee7efhselasky{
21079ee7efhselasky	struct idr *idr = cma_pernet_idr(net, ps);
21179ee7efhselasky
21279ee7efhselasky	return idr_find(idr, snum);
21379ee7efhselasky}
21479ee7efhselasky
21579ee7efhselaskystatic void cma_ps_remove(struct vnet *net, enum rdma_port_space ps, int snum)
21679ee7efhselasky{
21779ee7efhselasky	struct idr *idr = cma_pernet_idr(net, ps);
21879ee7efhselasky
21979ee7efhselasky	idr_remove(idr, snum);
22079ee7efhselasky}
22179ee7efhselasky
2225dbc43fhselaskyenum {
2235dbc43fhselasky	CMA_OPTION_AFONLY,
2245dbc43fhselasky};
2255dbc43fhselasky
22679ee7efhselaskyvoid cma_ref_dev(struct cma_device *cma_dev)
22779ee7efhselasky{
22879ee7efhselasky	atomic_inc(&cma_dev->refcount);
22979ee7efhselasky}
23079ee7efhselasky
23179ee7efhselaskystruct cma_device *cma_enum_devices_by_ibdev(cma_device_filter	filter,
23279ee7efhselasky					     void		*cookie)
23379ee7efhselasky{
23479ee7efhselasky	struct cma_device *cma_dev;
23579ee7efhselasky	struct cma_device *found_cma_dev = NULL;
23679ee7efhselasky
23779ee7efhselasky	mutex_lock(&lock);
23879ee7efhselasky
23979ee7efhselasky	list_for_each_entry(cma_dev, &dev_list, list)
24079ee7efhselasky		if (filter(cma_dev->device, cookie)) {
24179ee7efhselasky			found_cma_dev = cma_dev;
24279ee7efhselasky			break;
24379ee7efhselasky		}
24479ee7efhselasky
24579ee7efhselasky	if (found_cma_dev)
24679ee7efhselasky		cma_ref_dev(found_cma_dev);
24779ee7efhselasky	mutex_unlock(&lock);
24879ee7efhselasky	return found_cma_dev;
24979ee7efhselasky}
25079ee7efhselasky
25179ee7efhselaskyint cma_get_default_gid_type(struct cma_device *cma_dev,
25279ee7efhselasky			     unsigned int port)
25379ee7efhselasky{
25479ee7efhselasky	if (port < rdma_start_port(cma_dev->device) ||
25579ee7efhselasky	    port > rdma_end_port(cma_dev->device))
25679ee7efhselasky		return -EINVAL;
25779ee7efhselasky
25879ee7efhselasky	return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
25979ee7efhselasky}
26079ee7efhselasky
26179ee7efhselaskyint cma_set_default_gid_type(struct cma_device *cma_dev,
26279ee7efhselasky			     unsigned int port,
26379ee7efhselasky			     enum ib_gid_type default_gid_type)
26479ee7efhselasky{
26579ee7efhselasky	unsigned long supported_gids;
26679ee7efhselasky
26779ee7efhselasky	if (port < rdma_start_port(cma_dev->device) ||
26879ee7efhselasky	    port > rdma_end_port(cma_dev->device))
26979ee7efhselasky		return -EINVAL;
27079ee7efhselasky
27179ee7efhselasky	supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
27279ee7efhselasky
27379ee7efhselasky	if (!(supported_gids & 1 << default_gid_type))
27479ee7efhselasky		return -EINVAL;
27579ee7efhselasky
27679ee7efhselasky	cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
27779ee7efhselasky		default_gid_type;
27879ee7efhselasky
27979ee7efhselasky	return 0;
28079ee7efhselasky}
28179ee7efhselasky
28279ee7efhselaskystruct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
28379ee7efhselasky{
28479ee7efhselasky	return cma_dev->device;
28579ee7efhselasky}
28679ee7efhselasky
2875115240jeff/*
2885115240jeff * Device removal can occur at anytime, so we need extra handling to
2895115240jeff * serialize notifying the user of device removal with other callbacks.
2905115240jeff * We do this by disabling removal notification while a callback is in process,
2915115240jeff * and reporting it after the callback completes.
2925115240jeff */
2935115240jeffstruct rdma_id_private {
2945115240jeff	struct rdma_cm_id	id;
2955115240jeff
2965115240jeff	struct rdma_bind_list	*bind_list;
2975115240jeff	struct hlist_node	node;
2985115240jeff	struct list_head	list; /* listen_any_list or cma_device.list */
2995115240jeff	struct list_head	listen_list; /* per device listens */
3005115240jeff	struct cma_device	*cma_dev;
3015115240jeff	struct list_head	mc_list;
3025115240jeff
3035115240jeff	int			internal_id;
3045dbc43fhselasky	enum rdma_cm_state	state;
3055115240jeff	spinlock_t		lock;
3065115240jeff	struct mutex		qp_mutex;
3075115240jeff
3085115240jeff	struct completion	comp;
3095115240jeff	atomic_t		refcount;
3105115240jeff	struct mutex		handler_mutex;
3115115240jeff
3125115240jeff	int			backlog;
3135115240jeff	int			timeout_ms;
3145115240jeff	struct ib_sa_query	*query;
3155115240jeff	int			query_id;
3165115240jeff	union {
3175115240jeff		struct ib_cm_id	*ib;
3185115240jeff		struct iw_cm_id	*iw;
3195115240jeff	} cm_id;
3205115240jeff
3215115240jeff	u32			seq_num;
3225115240jeff	u32			qkey;
3235115240jeff	u32			qp_num;
3245dbc43fhselasky	pid_t			owner;
3255dbc43fhselasky	u32			options;
3265115240jeff	u8			srq;
3275115240jeff	u8			tos;
3285dbc43fhselasky	u8			reuseaddr;
3295dbc43fhselasky	u8			afonly;
33079ee7efhselasky	enum ib_gid_type	gid_type;
3315115240jeff};
3325115240jeff
3335115240jeffstruct cma_multicast {
3345115240jeff	struct rdma_id_private *id_priv;
3355115240jeff	union {
3365115240jeff		struct ib_sa_multicast *ib;
3375115240jeff	} multicast;
3385115240jeff	struct list_head	list;
3395115240jeff	void			*context;
3405115240jeff	struct sockaddr_storage	addr;
3415115240jeff	struct kref		mcref;
34279ee7efhselasky	bool			igmp_joined;
34379ee7efhselasky	u8			join_state;
3445115240jeff};
3455115240jeff
3465115240jeffstruct cma_work {
3475115240jeff	struct work_struct	work;
3485115240jeff	struct rdma_id_private	*id;
3495dbc43fhselasky	enum rdma_cm_state	old_state;
3505dbc43fhselasky	enum rdma_cm_state	new_state;
3515115240jeff	struct rdma_cm_event	event;
3525115240jeff};
3535115240jeff
3545115240jeffstruct cma_ndev_work {
3555115240jeff	struct work_struct	work;
3565115240jeff	struct rdma_id_private	*id;
3575115240jeff	struct rdma_cm_event	event;
3585115240jeff};
3595115240jeff
3605115240jeffstruct iboe_mcast_work {
3615115240jeff	struct work_struct	 work;
3625115240jeff	struct rdma_id_private	*id;
3635115240jeff	struct cma_multicast	*mc;
3645115240jeff};
3655115240jeff
3665115240jeffstruct cma_hdr {
3675115240jeff	u8 cma_version;
3685115240jeff	u8 ip_version;	/* IP version: 7:4 */
3695115240jeff	__be16 port;
3705115240jeff	union cma_ip_addr src_addr;
3715115240jeff	union cma_ip_addr dst_addr;
3725115240jeff};
3735115240jeff
37479ee7efhselasky#define CMA_VERSION 0x00
375f1b5552cem#define SDP_MAJ_VERSION 0x2
3765115240jeff
37779ee7efhselaskystruct cma_req_info {
37879ee7efhselasky	struct ib_device *device;
37979ee7efhselasky	int port;
38079ee7efhselasky	union ib_gid local_gid;
38179ee7efhselasky	__be64 service_id;
38279ee7efhselasky	u16 pkey;
38379ee7efhselasky	bool has_gid:1;
3845115240jeff};
3855115240jeff
3865dbc43fhselaskystatic int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
3875115240jeff{
3885115240jeff	unsigned long flags;
3895115240jeff	int ret;
3905115240jeff
3915115240jeff	spin_lock_irqsave(&id_priv->lock, flags);
3925115240jeff	ret = (id_priv->state == comp);
3935115240jeff	spin_unlock_irqrestore(&id_priv->lock, flags);
3945115240jeff	return ret;
3955115240jeff}
3965115240jeff
3975115240jeffstatic int cma_comp_exch(struct rdma_id_private *id_priv,
3985dbc43fhselasky			 enum rdma_cm_state comp, enum rdma_cm_state exch)
3995115240jeff{
4005115240jeff	unsigned long flags;
4015115240jeff	int ret;
4025115240jeff
4035115240jeff	spin_lock_irqsave(&id_priv->lock, flags);
4045115240jeff	if ((ret = (id_priv->state == comp)))
4055115240jeff		id_priv->state = exch;
4065115240jeff	spin_unlock_irqrestore(&id_priv->lock, flags);
4075115240jeff	return ret;
4085115240jeff}
4095115240jeff
4105dbc43fhselaskystatic enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
4115dbc43fhselasky				   enum rdma_cm_state exch)
4125115240jeff{
4135115240jeff	unsigned long flags;
4145dbc43fhselasky	enum rdma_cm_state old;
4155115240jeff
4165115240jeff	spin_lock_irqsave(&id_priv->lock, flags);
4175115240jeff	old = id_priv->state;
4185115240jeff	id_priv->state = exch;
4195115240jeff	spin_unlock_irqrestore(&id_priv->lock, flags);
4205115240jeff	return old;
4215115240jeff}
4225115240jeff
42379ee7efhselaskystatic inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
4245115240jeff{
4255115240jeff	return hdr->ip_version >> 4;
4265115240jeff}
4275115240jeff
4285115240jeffstatic inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
4295115240jeff{
4305115240jeff	hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
4315115240jeff}
4325115240jeff
433f1b5552cemstatic inline u8 sdp_get_majv(u8 sdp_version)
434f1b5552cem{
435f1b5552cem	return sdp_version >> 4;
436f1b5552cem}
437f1b5552cem
438f1b5552cemstatic inline u8 sdp_get_ip_ver(const struct sdp_hh *hh)
439f1b5552cem{
440f1b5552cem	return hh->ipv_cap >> 4;
441f1b5552cem}
442f1b5552cem
443f1b5552cemstatic inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
444f1b5552cem{
445f1b5552cem	hh->ipv_cap = (ip_ver << 4) | (hh->ipv_cap & 0xF);
446f1b5552cem}
447f1b5552cem
44888533a8hselaskystatic int cma_igmp_send(struct net_device *ndev, const union ib_gid *mgid, bool join)
44988533a8hselasky{
45088533a8hselasky	int retval;
45188533a8hselasky
45288533a8hselasky	if (ndev) {
4538b9d289vangyzen		union rdma_sockaddr addr;
45488533a8hselasky
4558b9d289vangyzen		rdma_gid2ip(&addr._sockaddr, mgid);
45688533a8hselasky
45788533a8hselasky		CURVNET_SET_QUIET(ndev->if_vnet);
45888533a8hselasky		if (join)
4598b9d289vangyzen			retval = -if_addmulti(ndev, &addr._sockaddr, NULL);
46088533a8hselasky		else
4618b9d289vangyzen			retval = -if_delmulti(ndev, &addr._sockaddr);
46288533a8hselasky		CURVNET_RESTORE();
46388533a8hselasky	} else {
46488533a8hselasky		retval = -ENODEV;
46588533a8hselasky	}
46688533a8hselasky	return retval;
46788533a8hselasky}
46888533a8hselasky
46979ee7efhselaskystatic void _cma_attach_to_dev(struct rdma_id_private *id_priv,
47079ee7efhselasky			       struct cma_device *cma_dev)
4715115240jeff{
47279ee7efhselasky	cma_ref_dev(cma_dev);
4735115240jeff	id_priv->cma_dev = cma_dev;
47479ee7efhselasky	id_priv->gid_type = 0;
4755115240jeff	id_priv->id.device = cma_dev->device;
4765115240jeff	id_priv->id.route.addr.dev_addr.transport =
4775115240jeff		rdma_node_get_transport(cma_dev->device->node_type);
4785115240jeff	list_add_tail(&id_priv->list, &cma_dev->id_list);
4795115240jeff}
4805115240jeff
48179ee7efhselaskystatic void cma_attach_to_dev(struct rdma_id_private *id_priv,
48279ee7efhselasky			      struct cma_device *cma_dev)
48379ee7efhselasky{
48479ee7efhselasky	_cma_attach_to_dev(id_priv, cma_dev);
48579ee7efhselasky	id_priv->gid_type =
48679ee7efhselasky		cma_dev->default_gid_type[id_priv->id.port_num -
48779ee7efhselasky					  rdma_start_port(cma_dev->device)];
48879ee7efhselasky}
48979ee7efhselasky
49079ee7efhselaskyvoid cma_deref_dev(struct cma_device *cma_dev)
4915115240jeff{
4925115240jeff	if (atomic_dec_and_test(&cma_dev->refcount))
4935115240jeff		complete(&cma_dev->comp);
4945115240jeff}
4955115240jeff
4965115240jeffstatic inline void release_mc(struct kref *kref)
4975115240jeff{
4985115240jeff	struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
4995115240jeff
5005115240jeff	kfree(mc->multicast.ib);
5015115240jeff	kfree(mc);
5025115240jeff}
5035115240jeff
5045dbc43fhselaskystatic void cma_release_dev(struct rdma_id_private *id_priv)
5055115240jeff{
5065dbc43fhselasky	mutex_lock(&lock);
5075115240jeff	list_del(&id_priv->list);
5085115240jeff	cma_deref_dev(id_priv->cma_dev);
5095115240jeff	id_priv->cma_dev = NULL;
5105dbc43fhselasky	mutex_unlock(&lock);
5115115240jeff}
5125115240jeff
51379ee7efhselaskystatic inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
51479ee7efhselasky{
51579ee7efhselasky	return (struct sockaddr *) &id_priv->id.route.addr.src_addr;
51679ee7efhselasky}
51779ee7efhselasky
51879ee7efhselaskystatic inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
51979ee7efhselasky{
52079ee7efhselasky	return (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
52179ee7efhselasky}
52279ee7efhselasky
52379ee7efhselaskystatic inline unsigned short cma_family(struct rdma_id_private *id_priv)
52479ee7efhselasky{
52579ee7efhselasky	return id_priv->id.route.addr.src_addr.ss_family;
52679ee7efhselasky}
52779ee7efhselasky
52879ee7efhselaskystatic int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
5295115240jeff{
5305115240jeff	struct ib_sa_mcmember_rec rec;
5315115240jeff	int ret = 0;
5325115240jeff
53379ee7efhselasky	if (id_priv->qkey) {
53479ee7efhselasky		if (qkey && id_priv->qkey != qkey)
53579ee7efhselasky			return -EINVAL;
5365115240jeff		return 0;
53779ee7efhselasky	}
53879ee7efhselasky
53979ee7efhselasky	if (qkey) {
54079ee7efhselasky		id_priv->qkey = qkey;
54179ee7efhselasky		return 0;
54279ee7efhselasky	}
5435115240jeff
5445115240jeff	switch (id_priv->id.ps) {
5455115240jeff	case RDMA_PS_UDP:
54679ee7efhselasky	case RDMA_PS_IB:
5475115240jeff		id_priv->qkey = RDMA_UDP_QKEY;
5485115240jeff		break;
5495115240jeff	case RDMA_PS_IPOIB:
5505115240jeff		ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
5515115240jeff		ret = ib_sa_get_mcmember_rec(id_priv->id.device,
5525115240jeff					     id_priv->id.port_num, &rec.mgid,
5535115240jeff					     &rec);
5545115240jeff		if (!ret)
5555115240jeff			id_priv->qkey = be32_to_cpu(rec.qkey);
5565115240jeff		break;
5575115240jeff	default:
5585115240jeff		break;
5595115240jeff	}
5605115240jeff	return ret;
5615115240jeff}
5625115240jeff
56379ee7efhselaskystatic void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
5645dbc43fhselasky{
56579ee7efhselasky	dev_addr->dev_type = ARPHRD_INFINIBAND;
56679ee7efhselasky	rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
56779ee7efhselasky	ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
56879ee7efhselasky}
5695dbc43fhselasky
57079ee7efhselaskystatic int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
57179ee7efhselasky{
57279ee7efhselasky	int ret;
5735dbc43fhselasky
57479ee7efhselasky	if (addr->sa_family != AF_IB) {
5751a0dfe6hselasky		ret = rdma_translate_ip(addr, dev_addr);
57679ee7efhselasky	} else {
57779ee7efhselasky		cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
57879ee7efhselasky		ret = 0;
5795dbc43fhselasky	}
5805dbc43fhselasky
58179ee7efhselasky	return ret;
5825dbc43fhselasky}
5835dbc43fhselasky
58479ee7efhselaskystatic inline int cma_validate_port(struct ib_device *device, u8 port,
58579ee7efhselasky				    enum ib_gid_type gid_type,
58622afd65hselasky				    union ib_gid *gid,
58722afd65hselasky				    const struct rdma_dev_addr *dev_addr)
588057d736np{
58922afd65hselasky	const int dev_type = dev_addr->dev_type;
59022afd65hselasky	struct net_device *ndev;
59179ee7efhselasky	int ret = -ENODEV;
592057d736np
59379ee7efhselasky	if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
59479ee7efhselasky		return ret;
595057d736np
59679ee7efhselasky	if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
59779ee7efhselasky		return ret;
598057d736np
59979ee7efhselasky	if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
60022afd65hselasky		ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
60179ee7efhselasky	} else {
60222afd65hselasky		ndev = NULL;
60379ee7efhselasky		gid_type = IB_GID_TYPE_IB;
60479ee7efhselasky	}
605057d736np
60679ee7efhselasky	ret = ib_find_cached_gid_by_port(device, gid, gid_type, port,
60779ee7efhselasky					 ndev, NULL);
608057d736np
60979ee7efhselasky	if (ndev)
61079ee7efhselasky		dev_put(ndev);
611057d736np
61279ee7efhselasky	return ret;
613057d736np}
614057d736np
615562f522markjstatic int cma_acquire_dev(struct rdma_id_private *id_priv,
616562f522markj			   struct rdma_id_private *listen_id_priv)
6175115240jeff{
6185115240jeff	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
6195115240jeff	struct cma_device *cma_dev;
62079ee7efhselasky	union ib_gid gid, iboe_gid, *gidp;
6215115240jeff	int ret = -ENODEV;
62279ee7efhselasky	u8 port;
6235115240jeff
62479ee7efhselasky	if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
6255dbc43fhselasky	    id_priv->id.ps == RDMA_PS_IPOIB)
6265dbc43fhselasky		return -EINVAL;
6275dbc43fhselasky
6285dbc43fhselasky	mutex_lock(&lock);
6295dbc43fhselasky	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
6305dbc43fhselasky		    &iboe_gid);
6315115240jeff
6325115240jeff	memcpy(&gid, dev_addr->src_dev_addr +
6335115240jeff	       rdma_addr_gid_offset(dev_addr), sizeof gid);
63479ee7efhselasky
63579ee7efhselasky	if (listen_id_priv) {
636562f522markj		cma_dev = listen_id_priv->cma_dev;
637562f522markj		port = listen_id_priv->id.port_num;
638c11489eslavash
639c11489eslavash		if (rdma_is_port_valid(cma_dev->device, port)) {
640c11489eslavash			gidp = rdma_protocol_roce(cma_dev->device, port) ?
641c11489eslavash			       &iboe_gid : &gid;
642c11489eslavash
643c11489eslavash			ret = cma_validate_port(cma_dev->device, port,
644c11489eslavash				rdma_protocol_ib(cma_dev->device, port) ?
645c11489eslavash				IB_GID_TYPE_IB :
646c11489eslavash				listen_id_priv->gid_type, gidp, dev_addr);
647c11489eslavash			if (!ret) {
648c11489eslavash				id_priv->id.port_num = port;
649c11489eslavash				goto out;
650c11489eslavash			}
651562f522markj		}
652562f522markj	}
65379ee7efhselasky
6545115240jeff	list_for_each_entry(cma_dev, &dev_list, list) {
655562f522markj		for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
656562f522markj			if (listen_id_priv &&
657562f522markj			    listen_id_priv->cma_dev == cma_dev &&
658562f522markj			    listen_id_priv->id.port_num == port)
659562f522markj				continue;
66079ee7efhselasky
66179ee7efhselasky			gidp = rdma_protocol_roce(cma_dev->device, port) ?
66279ee7efhselasky			       &iboe_gid : &gid;
66379ee7efhselasky
66479ee7efhselasky			ret = cma_validate_port(cma_dev->device, port,
66579ee7efhselasky						rdma_protocol_ib(cma_dev->device, port) ?
66679ee7efhselasky						IB_GID_TYPE_IB :
66779ee7efhselasky						cma_dev->default_gid_type[port - 1],
66822afd65hselasky						gidp, dev_addr);
66979ee7efhselasky			if (!ret) {
67079ee7efhselasky				id_priv->id.port_num = port;
67179ee7efhselasky				goto out;
672ad04104markj			}
673562f522markj		}
6745dbc43fhselasky	}
6755115240jeff
6765115240jeffout:
6775115240jeff	if (!ret)
6785115240jeff		cma_attach_to_dev(id_priv, cma_dev);
6795115240jeff
6805dbc43fhselasky	mutex_unlock(&lock);
6815115240jeff	return ret;
6825115240jeff}
6835115240jeff
68479ee7efhselasky/*
68579ee7efhselasky * Select the source IB device and address to reach the destination IB address.
68679ee7efhselasky */
68779ee7efhselaskystatic int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
6885115240jeff{
68979ee7efhselasky	struct cma_device *cma_dev, *cur_dev;
69079ee7efhselasky	struct sockaddr_ib *addr;
69179ee7efhselasky	union ib_gid gid, sgid, *dgid;
69279ee7efhselasky	u16 pkey, index;
69379ee7efhselasky	u8 p;
69479ee7efhselasky	int i;
6955115240jeff
69679ee7efhselasky	cma_dev = NULL;
69779ee7efhselasky	addr = (struct sockaddr_ib *) cma_dst_addr(id_priv);
69879ee7efhselasky	dgid = (union ib_gid *) &addr->sib_addr;
69979ee7efhselasky	pkey = ntohs(addr->sib_pkey);
70079ee7efhselasky
70179ee7efhselasky	list_for_each_entry(cur_dev, &dev_list, list) {
70279ee7efhselasky		for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
70379ee7efhselasky			if (!rdma_cap_af_ib(cur_dev->device, p))
70479ee7efhselasky				continue;
70579ee7efhselasky
70679ee7efhselasky			if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index))
70779ee7efhselasky				continue;
70879ee7efhselasky
70979ee7efhselasky			for (i = 0; !ib_get_cached_gid(cur_dev->device, p, i,
71079ee7efhselasky						       &gid, NULL);
71179ee7efhselasky			     i++) {
71279ee7efhselasky				if (!memcmp(&gid, dgid, sizeof(gid))) {
71379ee7efhselasky					cma_dev = cur_dev;
71479ee7efhselasky					sgid = gid;
71579ee7efhselasky					id_priv->id.port_num = p;
71679ee7efhselasky					goto found;
71779ee7efhselasky				}
71879ee7efhselasky
71979ee7efhselasky				if (!cma_dev && (gid.global.subnet_prefix ==
72079ee7efhselasky						 dgid->global.subnet_prefix)) {
72179ee7efhselasky					cma_dev = cur_dev;
72279ee7efhselasky					sgid = gid;
72379ee7efhselasky					id_priv->id.port_num = p;
72479ee7efhselasky				}
72579ee7efhselasky			}
72679ee7efhselasky		}
7275115240jeff	}
72879ee7efhselasky
72979ee7efhselasky	if (!cma_dev)
73079ee7efhselasky		return -ENODEV;
73179ee7efhselasky
73279ee7efhselaskyfound:
73379ee7efhselasky	cma_attach_to_dev(id_priv, cma_dev);
73479ee7efhselasky	addr = (struct sockaddr_ib *) cma_src_addr(id_priv);
73579ee7efhselasky	memcpy(&addr->sib_addr, &sgid, sizeof sgid);
73679ee7efhselasky	cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
7375115240jeff	return 0;
7385115240jeff}
7395115240jeff
74079ee7efhselaskystatic void cma_deref_id(struct rdma_id_private *id_priv)
74179ee7efhselasky{
74279ee7efhselasky	if (atomic_dec_and_test(&id_priv->refcount))
74379ee7efhselasky		complete(&id_priv->comp);
74479ee7efhselasky}
74579ee7efhselasky
74679ee7efhselaskystruct rdma_cm_id *rdma_create_id(struct vnet *net,
74779ee7efhselasky				  rdma_cm_event_handler event_handler,
7485dbc43fhselasky				  void *context, enum rdma_port_space ps,
7495dbc43fhselasky				  enum ib_qp_type qp_type)
7505115240jeff{
7515115240jeff	struct rdma_id_private *id_priv;
7525115240jeff
753c893379hselasky#ifdef VIMAGE
754c893379hselasky	if (net == NULL)
755c893379hselasky		return ERR_PTR(-EINVAL);
756c893379hselasky#endif
7575115240jeff	id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
7585115240jeff	if (!id_priv)
7595115240jeff		return ERR_PTR(-ENOMEM);
7605115240jeff
76179ee7efhselasky	id_priv->owner = task_pid_nr(current);
7625dbc43fhselasky	id_priv->state = RDMA_CM_IDLE;
7635115240jeff	id_priv->id.context = context;
7645115240jeff	id_priv->id.event_handler = event_handler;
7655115240jeff	id_priv->id.ps = ps;
7665dbc43fhselasky	id_priv->id.qp_type = qp_type;
7675115240jeff	spin_lock_init(&id_priv->lock);
7685115240jeff	mutex_init(&id_priv->qp_mutex);
7695115240jeff	init_completion(&id_priv->comp);
7705115240jeff	atomic_set(&id_priv->refcount, 1);
7715115240jeff	mutex_init(&id_priv->handler_mutex);
7725115240jeff	INIT_LIST_HEAD(&id_priv->listen_list);
7735115240jeff	INIT_LIST_HEAD(&id_priv->mc_list);
7745115240jeff	get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
775c893379hselasky	id_priv->id.route.addr.dev_addr.net = net;
7765115240jeff
7775115240jeff	return &id_priv->id;
7785115240jeff}
7795115240jeffEXPORT_SYMBOL(rdma_create_id);
7805115240jeff
7815115240jeffstatic int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
7825115240jeff{
7835115240jeff	struct ib_qp_attr qp_attr;
7845115240jeff	int qp_attr_mask, ret;
7855115240jeff
7865115240jeff	qp_attr.qp_state = IB_QPS_INIT;
7875115240jeff	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
7885115240jeff	if (ret)
7895115240jeff		return ret;
7905115240jeff
7915115240jeff	ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
7925115240jeff	if (ret)
7935115240jeff		return ret;
7945115240jeff
7955115240jeff	qp_attr.qp_state = IB_QPS_RTR;
7965115240jeff	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
7975115240jeff	if (ret)
7985115240jeff		return ret;
7995115240jeff
8005115240jeff	qp_attr.qp_state = IB_QPS_RTS;
8015115240jeff	qp_attr.sq_psn = 0;
8025115240jeff	ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
8035115240jeff
8045115240jeff	return ret;
8055115240jeff}
8065115240jeff
8075115240jeffstatic int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
8085115240jeff{
8095115240jeff	struct ib_qp_attr qp_attr;
8105115240jeff	int qp_attr_mask, ret;
8115115240jeff
8125115240jeff	qp_attr.qp_state = IB_QPS_INIT;
8135115240jeff	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
8145115240jeff	if (ret)
8155115240jeff		return ret;
8165115240jeff
8175115240jeff	return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
8185115240jeff}
8195115240jeff
8205115240jeffint rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
8215115240jeff		   struct ib_qp_init_attr *qp_init_attr)
8225115240jeff{
8235115240jeff	struct rdma_id_private *id_priv;
8245115240jeff	struct ib_qp *qp;
8255115240jeff	int ret;
8265115240jeff
8275115240jeff	id_priv = container_of(id, struct rdma_id_private, id);
8285115240jeff	if (id->device != pd->device)
8295115240jeff		return -EINVAL;
8305115240jeff
83179ee7efhselasky	qp_init_attr->port_num = id->port_num;
8325115240jeff	qp = ib_create_qp(pd, qp_init_attr);
8335115240jeff	if (IS_ERR(qp))
8345115240jeff		return PTR_ERR(qp);
8355115240jeff
8365dbc43fhselasky	if (id->qp_type == IB_QPT_UD)
8375115240jeff		ret = cma_init_ud_qp(id_priv, qp);
8385115240jeff	else
8395115240jeff		ret = cma_init_conn_qp(id_priv, qp);
8405115240jeff	if (ret)
8415115240jeff		goto err;
8425115240jeff
8435115240jeff	id->qp = qp;
8445115240jeff	id_priv->qp_num = qp->qp_num;
8455115240jeff	id_priv->srq = (qp->srq != NULL);
8465115240jeff	return 0;
8475115240jefferr:
8485115240jeff	ib_destroy_qp(qp);
8495115240jeff	return ret;
8505115240jeff}
8515115240jeffEXPORT_SYMBOL(rdma_create_qp);
8525115240jeff
8535115240jeffvoid rdma_destroy_qp(struct rdma_cm_id *id)
8545115240jeff{
8555115240jeff	struct rdma_id_private *id_priv;
8565115240jeff
8575115240jeff	id_priv = container_of(id, struct rdma_id_private, id);
8585115240jeff	mutex_lock(&id_priv->qp_mutex);
8595115240jeff	ib_destroy_qp(id_priv->id.qp);
8605115240jeff	id_priv->id.qp = NULL;
8615115240jeff	mutex_unlock(&id_priv->qp_mutex);
8625115240jeff}
8635115240jeffEXPORT_SYMBOL(rdma_destroy_qp);
8645115240jeff
8655115240jeffstatic int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
8665115240jeff			     struct rdma_conn_param *conn_param)
8675115240jeff{
8685115240jeff	struct ib_qp_attr qp_attr;
8695115240jeff	int qp_attr_mask, ret;
8705dbc43fhselasky	union ib_gid sgid;
8715115240jeff
8725115240jeff	mutex_lock(&id_priv->qp_mutex);
8735115240jeff	if (!id_priv->id.qp) {
8745115240jeff		ret = 0;
8755115240jeff		goto out;
8765115240jeff	}
8775115240jeff
8785115240jeff	/* Need to update QP attributes from default values. */
8795115240jeff	qp_attr.qp_state = IB_QPS_INIT;
8805115240jeff	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
8815115240jeff	if (ret)
8825115240jeff		goto out;
8835115240jeff
8845115240jeff	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
8855115240jeff	if (ret)
8865115240jeff		goto out;
8875115240jeff
8885115240jeff	qp_attr.qp_state = IB_QPS_RTR;
8895115240jeff	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
8905115240jeff	if (ret)
8915115240jeff		goto out;
89279ee7efhselasky
8935dbc43fhselasky	ret = ib_query_gid(id_priv->id.device, id_priv->id.port_num,
89479ee7efhselasky			   qp_attr.ah_attr.grh.sgid_index, &sgid, NULL);
8955dbc43fhselasky	if (ret)
8965dbc43fhselasky		goto out;
8975dbc43fhselasky
89879ee7efhselasky	BUG_ON(id_priv->cma_dev->device != id_priv->id.device);
8995115240jeff
9005115240jeff	if (conn_param)
9015115240jeff		qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
9025115240jeff	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
9035115240jeffout:
9045115240jeff	mutex_unlock(&id_priv->qp_mutex);
9055115240jeff	return ret;
9065115240jeff}
9075115240jeff
9085115240jeffstatic int cma_modify_qp_rts(struct rdma_id_private *id_priv,
9095115240jeff			     struct rdma_conn_param *conn_param)
9105115240jeff{
9115115240jeff	struct ib_qp_attr qp_attr;
9125115240jeff	int qp_attr_mask, ret;
9135115240jeff
9145115240jeff	mutex_lock(&id_priv->qp_mutex);
9155115240jeff	if (!id_priv->id.qp) {
9165115240jeff		ret = 0;
9175115240jeff		goto out;
9185115240jeff	}
9195115240jeff
9205115240jeff	qp_attr.qp_state = IB_QPS_RTS;
9215115240jeff	ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
9225115240jeff	if (ret)
9235115240jeff		goto out;
9245115240jeff
9255115240jeff	if (conn_param)
9265115240jeff		qp_attr.max_rd_atomic = conn_param->initiator_depth;
9275115240jeff	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
9285115240jeffout:
9295115240jeff	mutex_unlock(&id_priv->qp_mutex);
9305115240jeff	return ret;
9315115240jeff}
9325115240jeff
9335115240jeffstatic int cma_modify_qp_err(struct rdma_id_private *id_priv)
9345115240jeff{
9355115240jeff	struct ib_qp_attr qp_attr;
9365115240jeff	int ret;
9375115240jeff
9385115240jeff	mutex_lock(&id_priv->qp_mutex);
9395115240jeff	if (!id_priv->id.qp) {
9405115240jeff		ret = 0;
9415115240jeff		goto out;
9425115240jeff	}
9435115240jeff
9445115240jeff	qp_attr.qp_state = IB_QPS_ERR;
9455115240jeff	ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
9465115240jeffout:
9475115240jeff	mutex_unlock(&id_priv->qp_mutex);
9485115240jeff	return ret;
9495115240jeff}
9505115240jeff
9515115240jeffstatic int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
9525115240jeff			       struct ib_qp_attr *qp_attr, int *qp_attr_mask)
9535115240jeff{
9545115240jeff	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
9555115240jeff	int ret;
9565115240jeff	u16 pkey;
9575115240jeff
95879ee7efhselasky	if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num))
9595115240jeff		pkey = 0xffff;
96079ee7efhselasky	else
96179ee7efhselasky		pkey = ib_addr_get_pkey(dev_addr);
9625115240jeff
9635115240jeff	ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
9645115240jeff				  pkey, &qp_attr->pkey_index);
9655115240jeff	if (ret)
9665115240jeff		return ret;
9675115240jeff
9685115240jeff	qp_attr->port_num = id_priv->id.port_num;
9695115240jeff	*qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
9705115240jeff
9715dbc43fhselasky	if (id_priv->id.qp_type == IB_QPT_UD) {
97279ee7efhselasky		ret = cma_set_qkey(id_priv, 0);
9735115240jeff		if (ret)
9745115240jeff			return ret;
9755115240jeff
9765115240jeff		qp_attr->qkey = id_priv->qkey;
9775115240jeff		*qp_attr_mask |= IB_QP_QKEY;
9785115240jeff	} else {
9795115240jeff		qp_attr->qp_access_flags = 0;
9805115240jeff		*qp_attr_mask |= IB_QP_ACCESS_FLAGS;
9815115240jeff	}
9825115240jeff	return 0;
9835115240jeff}
9845115240jeff
9855115240jeffint rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
9865115240jeff		       int *qp_attr_mask)
9875115240jeff{
9885115240jeff	struct rdma_id_private *id_priv;
9895115240jeff	int ret = 0;
9905115240jeff
9915115240jeff	id_priv = container_of(id, struct rdma_id_private, id);
99279ee7efhselasky	if (rdma_cap_ib_cm(id->device, id->port_num)) {
9935dbc43fhselasky		if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
9945115240jeff			ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
9955115240jeff		else
9965115240jeff			ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
9975115240jeff						 qp_attr_mask);
99879ee7efhselasky
9995115240jeff		if (qp_attr->qp_state == IB_QPS_RTR)
10005115240jeff			qp_attr->rq_psn = id_priv->seq_num;
100179ee7efhselasky	} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
10025115240jeff		if (!id_priv->cm_id.iw) {
10035115240jeff			qp_attr->qp_access_flags = 0;
10045115240jeff			*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
10055115240jeff		} else
10065115240jeff			ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
10075115240jeff						 qp_attr_mask);
10089498114hselasky		qp_attr->port_num = id_priv->id.port_num;
10099498114hselasky		*qp_attr_mask |= IB_QP_PORT;
101079ee7efhselasky	} else
10115115240jeff		ret = -ENOSYS;
10125115240jeff
10135115240jeff	return ret;
10145115240jeff}
10155115240jeffEXPORT_SYMBOL(rdma_init_qp_attr);
10165115240jeff
10175115240jeffstatic inline int cma_zero_addr(struct sockaddr *addr)
10185115240jeff{
101979ee7efhselasky	switch (addr->sa_family) {
102079ee7efhselasky	case AF_INET:
102179ee7efhselasky		return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
102279ee7efhselasky	case AF_INET6:
102379ee7efhselasky		return ipv6_addr_any(&((struct sockaddr_in6 *) addr)->sin6_addr);
102479ee7efhselasky	case AF_IB:
102579ee7efhselasky		return ib_addr_any(&((struct sockaddr_ib *) addr)->sib_addr);
102679ee7efhselasky	default:
102779ee7efhselasky		return 0;
10285115240jeff	}
10295115240jeff}
10305115240jeff
10315115240jeffstatic inline int cma_loopback_addr(struct sockaddr *addr)
10325115240jeff{
103379ee7efhselasky	switch (addr->sa_family) {
103479ee7efhselasky	case AF_INET:
103579ee7efhselasky		return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
103679ee7efhselasky	case AF_INET6:
103779ee7efhselasky		return ipv6_addr_loopback(&((struct sockaddr_in6 *) addr)->sin6_addr);
103879ee7efhselasky	case AF_IB:
103979ee7efhselasky		return ib_addr_loopback(&((struct sockaddr_ib *) addr)->sib_addr);
104079ee7efhselasky	default:
104179ee7efhselasky		return 0;
104279ee7efhselasky	}
10435115240jeff}
10445115240jeff
10455115240jeffstatic inline int cma_any_addr(struct sockaddr *addr)
10465115240jeff{
10475115240jeff	return cma_zero_addr(addr) || cma_loopback_addr(addr);
10485115240jeff}
10495115240jeff
10505dbc43fhselaskystatic int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
10515dbc43fhselasky{
10525dbc43fhselasky	if (src->sa_family != dst->sa_family)
10535dbc43fhselasky		return -1;
10545dbc43fhselasky
10555dbc43fhselasky	switch (src->sa_family) {
10565dbc43fhselasky	case AF_INET:
10575dbc43fhselasky		return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
10585dbc43fhselasky		       ((struct sockaddr_in *) dst)->sin_addr.s_addr;
105979ee7efhselasky	case AF_INET6:
10605dbc43fhselasky		return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
10615dbc43fhselasky				     &((struct sockaddr_in6 *) dst)->sin6_addr);
106279ee7efhselasky	default:
106379ee7efhselasky		return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
106479ee7efhselasky				   &((struct sockaddr_ib *) dst)->sib_addr);
10655dbc43fhselasky	}
10665dbc43fhselasky}
10675dbc43fhselasky
106879ee7efhselaskystatic __be16 cma_port(struct sockaddr *addr)
10695115240jeff{
107079ee7efhselasky	struct sockaddr_ib *sib;
107179ee7efhselasky
107279ee7efhselasky	switch (addr->sa_family) {
107379ee7efhselasky	case AF_INET:
10745115240jeff		return ((struct sockaddr_in *) addr)->sin_port;
107579ee7efhselasky	case AF_INET6:
10765115240jeff		return ((struct sockaddr_in6 *) addr)->sin6_port;
107779ee7efhselasky	case AF_IB:
107879ee7efhselasky		sib = (struct sockaddr_ib *) addr;
107979ee7efhselasky		return htons((u16) (be64_to_cpu(sib->sib_sid) &
108079ee7efhselasky				    be64_to_cpu(sib->sib_sid_mask)));
108179ee7efhselasky	default:
108279ee7efhselasky		return 0;
108379ee7efhselasky	}
10845115240jeff}
10855115240jeff
10865115240jeffstatic inline int cma_any_port(struct sockaddr *addr)
10875115240jeff{
10885115240jeff	return !cma_port(addr);
10895115240jeff}
10905115240jeff
109179ee7efhselaskystatic void cma_save_ib_info(struct sockaddr *src_addr,
109279ee7efhselasky			     struct sockaddr *dst_addr,
109379ee7efhselasky			     struct rdma_cm_id *listen_id,
109479ee7efhselasky			     struct ib_sa_path_rec *path)
10955115240jeff{
109679ee7efhselasky	struct sockaddr_ib *listen_ib, *ib;
109779ee7efhselasky
109879ee7efhselasky	listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr;
109979ee7efhselasky	if (src_addr) {
110079ee7efhselasky		ib = (struct sockaddr_ib *)src_addr;
110179ee7efhselasky		ib->sib_family = AF_IB;
110279ee7efhselasky		if (path) {
110379ee7efhselasky			ib->sib_pkey = path->pkey;
110479ee7efhselasky			ib->sib_flowinfo = path->flow_label;
110579ee7efhselasky			memcpy(&ib->sib_addr, &path->sgid, 16);
110679ee7efhselasky			ib->sib_sid = path->service_id;
110779ee7efhselasky			ib->sib_scope_id = 0;
110879ee7efhselasky		} else {
110979ee7efhselasky			ib->sib_pkey = listen_ib->sib_pkey;
111079ee7efhselasky			ib->sib_flowinfo = listen_ib->sib_flowinfo;
111179ee7efhselasky			ib->sib_addr = listen_ib->sib_addr;
111279ee7efhselasky			ib->sib_sid = listen_ib->sib_sid;
111379ee7efhselasky			ib->sib_scope_id = listen_ib->sib_scope_id;
111479ee7efhselasky		}
111579ee7efhselasky		ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
111679ee7efhselasky	}
111779ee7efhselasky	if (dst_addr) {
111879ee7efhselasky		ib = (struct sockaddr_ib *)dst_addr;
111979ee7efhselasky		ib->sib_family = AF_IB;
112079ee7efhselasky		if (path) {
112179ee7efhselasky			ib->sib_pkey = path->pkey;
112279ee7efhselasky			ib->sib_flowinfo = path->flow_label;
112379ee7efhselasky			memcpy(&ib->sib_addr, &path->dgid, 16);
112479ee7efhselasky		}
112579ee7efhselasky	}
112679ee7efhselasky}
11275115240jeff
112879ee7efhselaskystatic void cma_save_ip4_info(struct sockaddr_in *src_addr,
112979ee7efhselasky			      struct sockaddr_in *dst_addr,
113079ee7efhselasky			      struct cma_hdr *hdr,
113179ee7efhselasky			      __be16 local_port)
113279ee7efhselasky{
113379ee7efhselasky	if (src_addr) {
113479ee7efhselasky		*src_addr = (struct sockaddr_in) {
1135b00a0b4hselasky			.sin_len = sizeof(struct sockaddr_in),
113679ee7efhselasky			.sin_family = AF_INET,
113779ee7efhselasky			.sin_addr.s_addr = hdr->dst_addr.ip4.addr,
113879ee7efhselasky			.sin_port = local_port,
113979ee7efhselasky		};
114079ee7efhselasky	}
11415115240jeff
114279ee7efhselasky	if (dst_addr) {
114379ee7efhselasky		*dst_addr = (struct sockaddr_in) {
1144b00a0b4hselasky			.sin_len = sizeof(struct sockaddr_in),
114579ee7efhselasky			.sin_family = AF_INET,
114679ee7efhselasky			.sin_addr.s_addr = hdr->src_addr.ip4.addr,
114779ee7efhselasky			.sin_port = hdr->port,
114879ee7efhselasky		};
11495115240jeff	}
115079ee7efhselasky}
11515115240jeff
11521f96ecchselaskystatic void cma_ip6_clear_scope_id(struct in6_addr *addr)
11531f96ecchselasky{
11541f96ecchselasky	/* make sure link local scope ID gets zeroed */
11551f96ecchselasky	if (IN6_IS_SCOPE_LINKLOCAL(addr) ||
11561f96ecchselasky	    IN6_IS_ADDR_MC_INTFACELOCAL(addr)) {
11571f96ecchselasky		/* use byte-access to be alignment safe */
11581f96ecchselasky		addr->s6_addr[2] = 0;
11591f96ecchselasky		addr->s6_addr[3] = 0;
11601f96ecchselasky	}
11611f96ecchselasky}
11621f96ecchselasky
116379ee7efhselaskystatic void cma_save_ip6_info(struct sockaddr_in6 *src_addr,
116479ee7efhselasky			      struct sockaddr_in6 *dst_addr,
116579ee7efhselasky			      struct cma_hdr *hdr,
116679ee7efhselasky			      __be16 local_port)
116779ee7efhselasky{
116879ee7efhselasky	if (src_addr) {
116979ee7efhselasky		*src_addr = (struct sockaddr_in6) {
11706c898cfhselasky			.sin6_len = sizeof(struct sockaddr_in6),
117179ee7efhselasky			.sin6_family = AF_INET6,
117279ee7efhselasky			.sin6_addr = hdr->dst_addr.ip6,
117379ee7efhselasky			.sin6_port = local_port,
117479ee7efhselasky		};
11751f96ecchselasky		cma_ip6_clear_scope_id(&src_addr->sin6_addr);
117679ee7efhselasky	}
117779ee7efhselasky
117879ee7efhselasky	if (dst_addr) {
117979ee7efhselasky		*dst_addr = (struct sockaddr_in6) {
1180b00a0b4hselasky			.sin6_len = sizeof(struct sockaddr_in6),
118179ee7efhselasky			.sin6_family = AF_INET6,
118279ee7efhselasky			.sin6_addr = hdr->src_addr.ip6,
118379ee7efhselasky			.sin6_port = hdr->port,
118479ee7efhselasky		};
11851f96ecchselasky		cma_ip6_clear_scope_id(&dst_addr->sin6_addr);
118679ee7efhselasky	}
118779ee7efhselasky}
118879ee7efhselasky
118979ee7efhselaskystatic u16 cma_port_from_service_id(__be64 service_id)
119079ee7efhselasky{
119179ee7efhselasky	return (u16)be64_to_cpu(service_id);
11925115240jeff}
11935115240jeff
1194f1b5552cemstatic int sdp_save_ip_info(struct sockaddr *src_addr,
1195f1b5552cem			    struct sockaddr *dst_addr,
1196f1b5552cem			    const struct sdp_hh *hdr,
1197f1b5552cem			    __be64 service_id)
1198f1b5552cem{
1199f1b5552cem	__be16 local_port;
1200f1b5552cem
1201f1b5552cem	BUG_ON(src_addr == NULL || dst_addr == NULL);
1202f1b5552cem
1203f1b5552cem	if (sdp_get_majv(hdr->majv_minv) != SDP_MAJ_VERSION)
1204f1b5552cem		return -EINVAL;
1205f1b5552cem
1206f1b5552cem	local_port = htons(cma_port_from_service_id(service_id));
1207f1b5552cem
1208f1b5552cem	switch (sdp_get_ip_ver(hdr)) {
1209f1b5552cem	case 4: {
1210f1b5552cem		struct sockaddr_in *s4, *d4;
1211f1b5552cem
1212f1b5552cem		s4 = (void *)src_addr;
1213f1b5552cem		d4 = (void *)dst_addr;
1214f1b5552cem
1215f1b5552cem		*s4 = (struct sockaddr_in) {
1216f1b5552cem			.sin_len = sizeof(*s4),
1217f1b5552cem			.sin_family = AF_INET,
1218f1b5552cem			.sin_addr.s_addr = hdr->dst_addr.ip4.addr,
1219f1b5552cem			.sin_port = local_port,
1220f1b5552cem		};
1221f1b5552cem		*d4 = (struct sockaddr_in) {
1222f1b5552cem			.sin_len = sizeof(*d4),
1223f1b5552cem			.sin_family = AF_INET,
1224f1b5552cem			.sin_addr.s_addr = hdr->src_addr.ip4.addr,
1225f1b5552cem			.sin_port = hdr->port,
1226f1b5552cem		};
1227f1b5552cem		break;
1228f1b5552cem	}
1229f1b5552cem	case 6: {
1230f1b5552cem		struct sockaddr_in6 *s6, *d6;
1231f1b5552cem
1232f1b5552cem		s6 = (void *)src_addr;
1233f1b5552cem		d6 = (void *)dst_addr;
1234f1b5552cem
1235f1b5552cem		*s6 = (struct sockaddr_in6) {
1236f1b5552cem			.sin6_len = sizeof(*s6),
1237f1b5552cem			.sin6_family = AF_INET6,
1238f1b5552cem			.sin6_addr = hdr->dst_addr.ip6,
1239f1b5552cem			.sin6_port = local_port,
1240f1b5552cem		};
1241f1b5552cem		*d6 = (struct sockaddr_in6) {
1242f1b5552cem			.sin6_len = sizeof(*d6),
1243f1b5552cem			.sin6_family = AF_INET6,
1244f1b5552cem			.sin6_addr = hdr->src_addr.ip6,
1245f1b5552cem			.sin6_port = hdr->port,
1246f1b5552cem		};
1247f1b5552cem		cma_ip6_clear_scope_id(&s6->sin6_addr);
1248f1b5552cem		cma_ip6_clear_scope_id(&d6->sin6_addr);
1249f1b5552cem		break;
1250f1b5552cem	}
1251f1b5552cem	default:
1252f1b5552cem		return -EAFNOSUPPORT;
1253f1b5552cem	}
1254f1b5552cem
1255f1b5552cem	return 0;
1256f1b5552cem}
1257f1b5552cem
125879ee7efhselaskystatic int cma_save_ip_info(struct sockaddr *src_addr,
125979ee7efhselasky			    struct sockaddr *dst_addr,
126079ee7efhselasky			    struct ib_cm_event *ib_event,
126179ee7efhselasky			    __be64 service_id)
12625115240jeff{
126379ee7efhselasky	struct cma_hdr *hdr;
126479ee7efhselasky	__be16 port;
126579ee7efhselasky
1266f1b5552cem	if (rdma_ps_from_service_id(service_id) == RDMA_PS_SDP)
1267f1b5552cem		return sdp_save_ip_info(src_addr, dst_addr,
1268f1b5552cem		    ib_event->private_data, service_id);
1269f1b5552cem
127079ee7efhselasky	hdr = ib_event->private_data;
127179ee7efhselasky	if (hdr->cma_version != CMA_VERSION)
127279ee7efhselasky		return -EINVAL;
12735115240jeff
127479ee7efhselasky	port = htons(cma_port_from_service_id(service_id));
127579ee7efhselasky
127679ee7efhselasky	switch (cma_get_ip_ver(hdr)) {
12775115240jeff	case 4:
127879ee7efhselasky		cma_save_ip4_info((struct sockaddr_in *)src_addr,
127979ee7efhselasky				  (struct sockaddr_in *)dst_addr, hdr, port);
12805115240jeff		break;
12815115240jeff	case 6:
128279ee7efhselasky		cma_save_ip6_info((struct sockaddr_in6 *)src_addr,
128379ee7efhselasky				  (struct sockaddr_in6 *)dst_addr, hdr, port);
12845115240jeff		break;
12855115240jeff	default:
128679ee7efhselasky		return -EAFNOSUPPORT;
12875115240jeff	}
128879ee7efhselasky
128979ee7efhselasky	return 0;
12905115240jeff}
12915115240jeff
129279ee7efhselaskystatic int cma_save_net_info(struct sockaddr *src_addr,
129379ee7efhselasky			     struct sockaddr *dst_addr,
129479ee7efhselasky			     struct rdma_cm_id *listen_id,
129579ee7efhselasky			     struct ib_cm_event *ib_event,
129679ee7efhselasky			     sa_family_t sa_family, __be64 service_id)
12975115240jeff{
129879ee7efhselasky	if (sa_family == AF_IB) {
129979ee7efhselasky		if (ib_event->event == IB_CM_REQ_RECEIVED)
130079ee7efhselasky			cma_save_ib_info(src_addr, dst_addr, listen_id,
130179ee7efhselasky					 ib_event->param.req_rcvd.primary_path);
130279ee7efhselasky		else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
130379ee7efhselasky			cma_save_ib_info(src_addr, dst_addr, listen_id, NULL);
13045115240jeff		return 0;
130579ee7efhselasky	}
130679ee7efhselasky
130779ee7efhselasky	return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id);
130879ee7efhselasky}
130979ee7efhselasky
131079ee7efhselaskystatic int cma_save_req_info(const struct ib_cm_event *ib_event,
131179ee7efhselasky			     struct cma_req_info *req)
131279ee7efhselasky{
131379ee7efhselasky	const struct ib_cm_req_event_param *req_param =
131479ee7efhselasky		&ib_event->param.req_rcvd;
131579ee7efhselasky	const struct ib_cm_sidr_req_event_param *sidr_param =
131679ee7efhselasky		&ib_event->param.sidr_req_rcvd;
131779ee7efhselasky
131879ee7efhselasky	switch (ib_event->event) {
131979ee7efhselasky	case IB_CM_REQ_RECEIVED:
132079ee7efhselasky		req->device	= req_param->listen_id->device;
132179ee7efhselasky		req->port	= req_param->port;
132279ee7efhselasky		memcpy(&req->local_gid, &req_param->primary_path->sgid,
132379ee7efhselasky		       sizeof(req->local_gid));
132479ee7efhselasky		req->has_gid	= true;
132579ee7efhselasky		req->service_id	= req_param->primary_path->service_id;
132679ee7efhselasky		req->pkey	= be16_to_cpu(req_param->primary_path->pkey);
132779ee7efhselasky		if (req->pkey != req_param->bth_pkey)
132879ee7efhselasky			pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
132979ee7efhselasky					    "RDMA CMA: in the future this may cause the request to be dropped\n",
133079ee7efhselasky					    req_param->bth_pkey, req->pkey);
133179ee7efhselasky		break;
133279ee7efhselasky	case IB_CM_SIDR_REQ_RECEIVED:
133379ee7efhselasky		req->device	= sidr_param->listen_id->device;
133479ee7efhselasky		req->port	= sidr_param->port;
133579ee7efhselasky		req->has_gid	= false;
133679ee7efhselasky		req->service_id	= sidr_param->service_id;
133779ee7efhselasky		req->pkey	= sidr_param->pkey;
133879ee7efhselasky		if (req->pkey != sidr_param->bth_pkey)
133979ee7efhselasky			pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
134079ee7efhselasky					    "RDMA CMA: in the future this may cause the request to be dropped\n",
134179ee7efhselasky					    sidr_param->bth_pkey, req->pkey);
134279ee7efhselasky		break;
134379ee7efhselasky	default:
134479ee7efhselasky		return -EINVAL;
134579ee7efhselasky	}
134679ee7efhselasky
134779ee7efhselasky	return 0;
134879ee7efhselasky}
134979ee7efhselasky
135079ee7efhselaskystatic bool validate_ipv4_net_dev(struct net_device *net_dev,
135179ee7efhselasky				  const struct sockaddr_in *dst_addr,
135279ee7efhselasky				  const struct sockaddr_in *src_addr)
135379ee7efhselasky{
135479ee7efhselasky#ifdef INET
135579ee7efhselasky	__be32 daddr = dst_addr->sin_addr.s_addr,
135679ee7efhselasky	       saddr = src_addr->sin_addr.s_addr;
135714eba6bhselasky	struct net_device *dst_dev;
13586004273melifaro	struct nhop_object *nh;
135979ee7efhselasky	bool ret;
136079ee7efhselasky
136179ee7efhselasky	if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
136279ee7efhselasky	    ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) ||
136379ee7efhselasky	    ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) ||
136479ee7efhselasky	    ipv4_is_loopback(saddr))
136579ee7efhselasky		return false;
136679ee7efhselasky
136714eba6bhselasky	dst_dev = ip_dev_find(net_dev->if_vnet, daddr);
136814eba6bhselasky	if (dst_dev != net_dev) {
136914eba6bhselasky		if (dst_dev != NULL)
137014eba6bhselasky			dev_put(dst_dev);
137179ee7efhselasky		return false;
1372bd785e4hselasky	}
137314eba6bhselasky	dev_put(dst_dev);
137479ee7efhselasky
137579ee7efhselasky	/*
137636bca24slavash	 * Check for loopback.
137736bca24slavash	 */
137836bca24slavash	if (saddr == daddr)
137936bca24slavash		return true;
138036bca24slavash
138179ee7efhselasky	CURVNET_SET(net_dev->if_vnet);
13826004273melifaro	nh = fib4_lookup(RT_DEFAULT_FIB, src_addr->sin_addr, 0, NHR_NONE, 0);
13836004273melifaro	if (nh != NULL)
13846004273melifaro		ret = (nh->nh_ifp == net_dev);
13856004273melifaro	else
138679ee7efhselasky		ret = false;
138714eba6bhselasky	CURVNET_RESTORE();
138879ee7efhselasky	return ret;
138979ee7efhselasky#else
139079ee7efhselasky	return false;
139179ee7efhselasky#endif
139279ee7efhselasky}
139379ee7efhselasky
139479ee7efhselaskystatic bool validate_ipv6_net_dev(struct net_device *net_dev,
139579ee7efhselasky				  const struct sockaddr_in6 *dst_addr,
139679ee7efhselasky				  const struct sockaddr_in6 *src_addr)
139779ee7efhselasky{
139879ee7efhselasky#ifdef INET6
139914eba6bhselasky	struct sockaddr_in6 src_tmp = *src_addr;
140036bca24slavash	struct sockaddr_in6 dst_tmp = *dst_addr;
140114eba6bhselasky	struct net_device *dst_dev;
14026004273melifaro	struct nhop_object *nh;
140379ee7efhselasky	bool ret;
140479ee7efhselasky
14055413daaslavash	dst_dev = ip6_dev_find(net_dev->if_vnet, dst_tmp.sin6_addr,
14065413daaslavash	    net_dev->if_index);
140714eba6bhselasky	if (dst_dev != net_dev) {
140814eba6bhselasky		if (dst_dev != NULL)
140914eba6bhselasky			dev_put(dst_dev);
141079ee7efhselasky		return false;
141114eba6bhselasky	}
1412cc6a289slavash	dev_put(dst_dev);
141314eba6bhselasky
141414eba6bhselasky	CURVNET_SET(net_dev->if_vnet);
141579ee7efhselasky
141679ee7efhselasky	/*
14176004273melifaro	 * Make sure the scope ID gets embedded.
141814eba6bhselasky	 */
141914eba6bhselasky	src_tmp.sin6_scope_id = net_dev->if_index;
142014eba6bhselasky	sa6_embedscope(&src_tmp, 0);
142114eba6bhselasky
142236bca24slavash	dst_tmp.sin6_scope_id = net_dev->if_index;
142336bca24slavash	sa6_embedscope(&dst_tmp, 0);
142436bca24slavash
142536bca24slavash	/*
142636bca24slavash	 * Check for loopback after scope ID
142736bca24slavash	 * has been embedded:
142836bca24slavash	 */
142936bca24slavash	if (memcmp(&src_tmp.sin6_addr, &dst_tmp.sin6_addr,
143036bca24slavash	    sizeof(dst_tmp.sin6_addr)) == 0) {
143136bca24slavash		ret = true;
143279ee7efhselasky	} else {
143336bca24slavash		/* non-loopback case */
14346004273melifaro		nh = fib6_lookup(RT_DEFAULT_FIB, &src_addr->sin6_addr,
14356004273melifaro		    net_dev->if_index, NHR_NONE, 0);
14366004273melifaro		if (nh != NULL)
14376004273melifaro			ret = (nh->nh_ifp == net_dev);
14386004273melifaro		else
143936bca24slavash			ret = false;
144079ee7efhselasky	}
144114eba6bhselasky	CURVNET_RESTORE();
144279ee7efhselasky	return ret;
144379ee7efhselasky#else
144479ee7efhselasky	return false;
144579ee7efhselasky#endif
144679ee7efhselasky}
144779ee7efhselasky
144879ee7efhselaskystatic bool validate_net_dev(struct net_device *net_dev,
144979ee7efhselasky			     const struct sockaddr *daddr,
145079ee7efhselasky			     const struct sockaddr *saddr)
145179ee7efhselasky{
145279ee7efhselasky	const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr;
145379ee7efhselasky	const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr;
145479ee7efhselasky	const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
145579ee7efhselasky	const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr;
145679ee7efhselasky
145779ee7efhselasky	switch (daddr->sa_family) {
145879ee7efhselasky	case AF_INET:
145979ee7efhselasky		return saddr->sa_family == AF_INET &&
146079ee7efhselasky		       validate_ipv4_net_dev(net_dev, daddr4, saddr4);
146179ee7efhselasky
146279ee7efhselasky	case AF_INET6:
146379ee7efhselasky		return saddr->sa_family == AF_INET6 &&
146479ee7efhselasky		       validate_ipv6_net_dev(net_dev, daddr6, saddr6);
146579ee7efhselasky
146679ee7efhselasky	default:
146779ee7efhselasky		return false;
146879ee7efhselasky	}
146979ee7efhselasky}
147079ee7efhselasky
1471c893379hselaskystatic struct net_device *
1472c893379hselaskyroce_get_net_dev_by_cm_event(struct ib_device *device, u8 port_num,
1473c893379hselasky    const struct ib_cm_event *ib_event)
1474c893379hselasky{
1475c893379hselasky	struct ib_gid_attr sgid_attr;
1476c893379hselasky	union ib_gid sgid;
1477c893379hselasky	int err = -EINVAL;
1478c893379hselasky
1479c893379hselasky	if (ib_event->event == IB_CM_REQ_RECEIVED) {
1480c893379hselasky		err = ib_get_cached_gid(device, port_num,
1481c893379hselasky		    ib_event->param.req_rcvd.ppath_sgid_index, &sgid, &sgid_attr);
1482c893379hselasky	} else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
1483c893379hselasky		err = ib_get_cached_gid(device, port_num,
1484c893379hselasky		    ib_event->param.sidr_req_rcvd.sgid_index, &sgid, &sgid_attr);
1485c893379hselasky	}
1486c893379hselasky	if (err)
1487c893379hselasky		return (NULL);
1488c893379hselasky	return (sgid_attr.ndev);
1489c893379hselasky}
1490c893379hselasky
149179ee7efhselaskystatic struct net_device *cma_get_net_dev(struct ib_cm_event *ib_event,
149279ee7efhselasky					  const struct cma_req_info *req)
149379ee7efhselasky{
149479ee7efhselasky	struct sockaddr_storage listen_addr_storage, src_addr_storage;
149579ee7efhselasky	struct sockaddr *listen_addr = (struct sockaddr *)&listen_addr_storage,
149679ee7efhselasky			*src_addr = (struct sockaddr *)&src_addr_storage;
149779ee7efhselasky	struct net_device *net_dev;
149879ee7efhselasky	const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL;
14996004273melifaro	struct epoch_tracker et;
150079ee7efhselasky	int err;
150179ee7efhselasky
150279ee7efhselasky	err = cma_save_ip_info(listen_addr, src_addr, ib_event,
150379ee7efhselasky			       req->service_id);
150479ee7efhselasky	if (err)
150579ee7efhselasky		return ERR_PTR(err);
150679ee7efhselasky
1507c893379hselasky	if (rdma_protocol_roce(req->device, req->port)) {
1508c893379hselasky		net_dev = roce_get_net_dev_by_cm_event(req->device, req->port,
1509c893379hselasky						       ib_event);
1510c893379hselasky	} else {
1511c893379hselasky		net_dev = ib_get_net_dev_by_params(req->device, req->port,
1512c893379hselasky						   req->pkey,
1513c893379hselasky						   gid, listen_addr);
1514c893379hselasky	}
151579ee7efhselasky	if (!net_dev)
151679ee7efhselasky		return ERR_PTR(-ENODEV);
151779ee7efhselasky
15186004273melifaro	NET_EPOCH_ENTER(et);
151979ee7efhselasky	if (!validate_net_dev(net_dev, listen_addr, src_addr)) {
15206004273melifaro		NET_EPOCH_EXIT(et);
152179ee7efhselasky		dev_put(net_dev);
152279ee7efhselasky		return ERR_PTR(-EHOSTUNREACH);
152379ee7efhselasky	}
15246004273melifaro	NET_EPOCH_EXIT(et);
152579ee7efhselasky
152679ee7efhselasky	return net_dev;
152779ee7efhselasky}
152879ee7efhselasky
152979ee7efhselaskystatic enum rdma_port_space rdma_ps_from_service_id(__be64 service_id)
153079ee7efhselasky{
153179ee7efhselasky	return (be64_to_cpu(service_id) >> 16) & 0xffff;
153279ee7efhselasky}
153379ee7efhselasky
1534f1b5552cemstatic bool sdp_match_private_data(struct rdma_id_private *id_priv,
1535f1b5552cem				   const struct sdp_hh *hdr,
1536f1b5552cem				   struct sockaddr *addr)
1537f1b5552cem{
1538f1b5552cem	__be32 ip4_addr;
1539f1b5552cem	struct in6_addr ip6_addr;
1540f1b5552cem
1541f1b5552cem	switch (addr->sa_family) {
1542f1b5552cem	case AF_INET:
1543f1b5552cem		ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
1544f1b5552cem		if (sdp_get_ip_ver(hdr) != 4)
1545f1b5552cem			return false;
1546f1b5552cem		if (!cma_any_addr(addr) &&
1547f1b5552cem		    hdr->dst_addr.ip4.addr != ip4_addr)
1548f1b5552cem			return false;
1549f1b5552cem		break;
1550f1b5552cem	case AF_INET6:
1551f1b5552cem		ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
1552f1b5552cem		if (sdp_get_ip_ver(hdr) != 6)
1553f1b5552cem			return false;
1554f1b5552cem		cma_ip6_clear_scope_id(&ip6_addr);
1555f1b5552cem		if (!cma_any_addr(addr) &&
1556f1b5552cem		    memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
1557f1b5552cem			return false;
1558f1b5552cem		break;
1559f1b5552cem	case AF_IB:
1560f1b5552cem		return true;
1561f1b5552cem	default:
1562f1b5552cem		return false;
1563f1b5552cem	}
1564f1b5552cem
1565f1b5552cem	return true;
1566f1b5552cem}
1567f1b5552cem
156879ee7efhselaskystatic bool cma_match_private_data(struct rdma_id_private *id_priv,
1569f1b5552cem				   const void *vhdr)
157079ee7efhselasky{
1571f1b5552cem	const struct cma_hdr *hdr = vhdr;
157279ee7efhselasky	struct sockaddr *addr = cma_src_addr(id_priv);
157379ee7efhselasky	__be32 ip4_addr;
157479ee7efhselasky	struct in6_addr ip6_addr;
157579ee7efhselasky
157679ee7efhselasky	if (cma_any_addr(addr) && !id_priv->afonly)
157779ee7efhselasky		return true;
157879ee7efhselasky
1579f1b5552cem	if (id_priv->id.ps == RDMA_PS_SDP)
1580f1b5552cem		return sdp_match_private_data(id_priv, vhdr, addr);
1581f1b5552cem
158279ee7efhselasky	switch (addr->sa_family) {
158379ee7efhselasky	case AF_INET:
158479ee7efhselasky		ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
158579ee7efhselasky		if (cma_get_ip_ver(hdr) != 4)
158679ee7efhselasky			return false;
158779ee7efhselasky		if (!cma_any_addr(addr) &&
158879ee7efhselasky		    hdr->dst_addr.ip4.addr != ip4_addr)
158979ee7efhselasky			return false;
159079ee7efhselasky		break;
159179ee7efhselasky	case AF_INET6:
159279ee7efhselasky		ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
159379ee7efhselasky		if (cma_get_ip_ver(hdr) != 6)
159479ee7efhselasky			return false;
15951f96ecchselasky		cma_ip6_clear_scope_id(&ip6_addr);
159679ee7efhselasky		if (!cma_any_addr(addr) &&
159779ee7efhselasky		    memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
159879ee7efhselasky			return false;
159979ee7efhselasky		break;
160079ee7efhselasky	case AF_IB:
160179ee7efhselasky		return true;
16025115240jeff	default:
160379ee7efhselasky		return false;
16045115240jeff	}
160579ee7efhselasky
160679ee7efhselasky	return true;
160779ee7efhselasky}
160879ee7efhselasky
160979ee7efhselaskystatic bool cma_protocol_roce_dev_port(struct ib_device *device, int port_num)
161079ee7efhselasky{
161179ee7efhselasky	enum rdma_link_layer ll = rdma_port_get_link_layer(device, port_num);
161279ee7efhselasky	enum rdma_transport_type transport =
161379ee7efhselasky		rdma_node_get_transport(device->node_type);
161479ee7efhselasky
161579ee7efhselasky	return ll == IB_LINK_LAYER_ETHERNET && transport == RDMA_TRANSPORT_IB;
161679ee7efhselasky}
161779ee7efhselasky
161879ee7efhselaskystatic bool cma_protocol_roce(const struct rdma_cm_id *id)
161979ee7efhselasky{
162079ee7efhselasky	struct ib_device *device = id->device;
162179ee7efhselasky	const int port_num = id->port_num ?: rdma_start_port(device);
162279ee7efhselasky
162379ee7efhselasky	return cma_protocol_roce_dev_port(device, port_num);
162479ee7efhselasky}
162579ee7efhselasky
162679ee7efhselaskystatic bool cma_match_net_dev(const struct rdma_cm_id *id,
162779ee7efhselasky			      const struct net_device *net_dev,
162879ee7efhselasky			      u8 port_num)
162979ee7efhselasky{
163079ee7efhselasky	const struct rdma_addr *addr = &id->route.addr;
163179ee7efhselasky
1632f1b5552cem	if (!net_dev) {
1633f1b5552cem		if (id->port_num && id->port_num != port_num)
1634f1b5552cem			return false;
1635f1b5552cem
1636f1b5552cem		if (id->ps == RDMA_PS_SDP) {
1637f1b5552cem			if (addr->src_addr.ss_family == AF_INET ||
1638f1b5552cem			    addr->src_addr.ss_family == AF_INET6)
1639f1b5552cem				return true;
1640f1b5552cem			return false;
1641f1b5552cem		}
164279ee7efhselasky		/* This request is an AF_IB request or a RoCE request */
1643f1b5552cem		return addr->src_addr.ss_family == AF_IB ||
1644f1b5552cem		    cma_protocol_roce_dev_port(id->device, port_num);
1645f1b5552cem	}
164679ee7efhselasky
164779ee7efhselasky	return !addr->dev_addr.bound_dev_if ||
164879ee7efhselasky	       (net_eq(dev_net(net_dev), addr->dev_addr.net) &&
164979ee7efhselasky		addr->dev_addr.bound_dev_if == net_dev->if_index);
165079ee7efhselasky}
165179ee7efhselasky
165279ee7efhselaskystatic struct rdma_id_private *cma_find_listener(
165379ee7efhselasky		const struct rdma_bind_list *bind_list,
165479ee7efhselasky		const struct ib_cm_id *cm_id,
165579ee7efhselasky		const struct ib_cm_event *ib_event,
165679ee7efhselasky		const struct cma_req_info *req,
165779ee7efhselasky		const struct net_device *net_dev)
165879ee7efhselasky{
165979ee7efhselasky	struct rdma_id_private *id_priv, *id_priv_dev;
166079ee7efhselasky
166179ee7efhselasky	if (!bind_list)
166279ee7efhselasky		return ERR_PTR(-EINVAL);
166379ee7efhselasky
166479ee7efhselasky	hlist_for_each_entry(id_priv, &bind_list->owners, node) {
166579ee7efhselasky		if (cma_match_private_data(id_priv, ib_event->private_data)) {
166679ee7efhselasky			if (id_priv->id.device == cm_id->device &&
166779ee7efhselasky			    cma_match_net_dev(&id_priv->id, net_dev, req->port))
166879ee7efhselasky				return id_priv;
166979ee7efhselasky			list_for_each_entry(id_priv_dev,
167079ee7efhselasky					    &id_priv->listen_list,
167179ee7efhselasky					    listen_list) {
167279ee7efhselasky				if (id_priv_dev->id.device == cm_id->device &&
167379ee7efhselasky				    cma_match_net_dev(&id_priv_dev->id, net_dev, req->port))
167479ee7efhselasky					return id_priv_dev;
167579ee7efhselasky			}
167679ee7efhselasky		}
167779ee7efhselasky	}
167879ee7efhselasky
167979ee7efhselasky	return ERR_PTR(-EINVAL);
168079ee7efhselasky}
168179ee7efhselasky
168279ee7efhselaskystatic struct rdma_id_private *cma_id_from_event(struct ib_cm_id *cm_id,
168379ee7efhselasky						 struct ib_cm_event *ib_event,
168479ee7efhselasky						 struct net_device **net_dev)
168579ee7efhselasky{
168679ee7efhselasky	struct cma_req_info req;
168779ee7efhselasky	struct rdma_bind_list *bind_list;
168879ee7efhselasky	struct rdma_id_private *id_priv;
168979ee7efhselasky	int err;
169079ee7efhselasky
169179ee7efhselasky	err = cma_save_req_info(ib_event, &req);
169279ee7efhselasky	if (err)
169379ee7efhselasky		return ERR_PTR(err);
169479ee7efhselasky
1695f1b5552cem	if (rdma_ps_from_service_id(cm_id->service_id) == RDMA_PS_SDP) {
1696f1b5552cem		*net_dev = NULL;
1697f1b5552cem		goto there_is_no_net_dev;
1698f1b5552cem	}
1699f1b5552cem
170079ee7efhselasky	*net_dev = cma_get_net_dev(ib_event, &req);
170179ee7efhselasky	if (IS_ERR(*net_dev)) {
170279ee7efhselasky		if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) {
170379ee7efhselasky			/* Assuming the protocol is AF_IB */
170479ee7efhselasky			*net_dev = NULL;
170579ee7efhselasky		} else {
170679ee7efhselasky			return ERR_CAST(*net_dev);
170779ee7efhselasky		}
170879ee7efhselasky	}
170979ee7efhselasky
1710f1b5552cemthere_is_no_net_dev:
171179ee7efhselasky	bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net,
171279ee7efhselasky				rdma_ps_from_service_id(req.service_id),
171379ee7efhselasky				cma_port_from_service_id(req.service_id));
171479ee7efhselasky	id_priv = cma_find_listener(bind_list, cm_id, ib_event, &req, *net_dev);
171579ee7efhselasky	if (IS_ERR(id_priv) && *net_dev) {
171679ee7efhselasky		dev_put(*net_dev);
171779ee7efhselasky		*net_dev = NULL;
171879ee7efhselasky	}
171979ee7efhselasky
172079ee7efhselasky	return id_priv;
172179ee7efhselasky}
172279ee7efhselasky
172379ee7efhselaskystatic inline int cma_user_data_offset(struct rdma_id_private *id_priv)
172479ee7efhselasky{
1725f1b5552cem	if (cma_family(id_priv) == AF_IB)
1726f1b5552cem		return 0;
1727f1b5552cem	if (id_priv->id.ps == RDMA_PS_SDP)
1728f1b5552cem		return 0;
1729f1b5552cem	return sizeof(struct cma_hdr);
17305115240jeff}
17315115240jeff
17325115240jeffstatic void cma_cancel_route(struct rdma_id_private *id_priv)
17335115240jeff{
173479ee7efhselasky	if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) {
17355115240jeff		if (id_priv->query)
17365115240jeff			ib_sa_cancel_query(id_priv->query_id, id_priv->query);
17375115240jeff	}
17385115240jeff}
17395115240jeff
17405115240jeffstatic void cma_cancel_listens(struct rdma_id_private *id_priv)
17415115240jeff{
17425115240jeff	struct rdma_id_private *dev_id_priv;
17435115240jeff
17445115240jeff	/*
17455115240jeff	 * Remove from listen_any_list to prevent added devices from spawning
17465115240jeff	 * additional listen requests.
17475115240jeff	 */
17485115240jeff	mutex_lock(&lock);
17495115240jeff	list_del(&id_priv->list);
17505115240jeff
17515115240jeff	while (!list_empty(&id_priv->listen_list)) {
17525115240jeff		dev_id_priv = list_entry(id_priv->listen_list.next,
17535115240jeff					 struct rdma_id_private, listen_list);
17545115240jeff		/* sync with device removal to avoid duplicate destruction */
17555115240jeff