1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 #include <sys/types.h>
25 #include <sys/stream.h>
26 #include <sys/dlpi.h>
27 #include <sys/stropts.h>
28 #include <sys/strsun.h>
29 #include <sys/sysmacros.h>
30 #include <sys/strlog.h>
31 #include <sys/ddi.h>
32 #include <sys/cmn_err.h>
33 #include <sys/socket.h>
34 #include <net/if.h>
35 #include <net/if_types.h>
36 #include <netinet/in.h>
37 #include <sys/ethernet.h>
38 #include <inet/arp.h>
39 #include <inet/ip.h>
40 #include <inet/ip6.h>
41 #include <inet/ip_ire.h>
42 #include <inet/ip_if.h>
43 #include <inet/ip_ftable.h>
44 
45 #include <sys/sunddi.h>
46 #include <sys/ksynch.h>
47 
48 #include <sys/rds.h>
49 #include <sys/socket.h>
50 #include <sys/socketvar.h>
51 #include <sys/sockio.h>
52 #include <sys/sysmacros.h>
53 #include <inet/common.h>
54 #include <inet/ip.h>
55 #include <net/if_types.h>
56 
57 #include <sys/ib/clients/rdsv3/rdsv3.h>
58 #include <sys/ib/clients/rdsv3/rdma.h>
59 #include <sys/ib/clients/rdsv3/ib.h>
60 #include <sys/ib/clients/rdsv3/rdsv3_impl.h>
61 #include <sys/ib/clients/rdsv3/rdsv3_debug.h>
62 
63 #include <sys/dls.h>
64 #include <sys/mac.h>
65 #include <sys/mac_client.h>
66 #include <sys/mac_provider.h>
67 #include <sys/mac_client_priv.h>
68 
69 uint_t			rdsv3_one_sec_in_hz;
70 ddi_taskq_t		*rdsv3_taskq = NULL;
71 extern kmem_cache_t	*rdsv3_alloc_cache;
72 
73 extern unsigned int	ip_ocsum(ushort_t *address, int halfword_count,
74     unsigned int sum);
75 
76 /*
77  * Check if the IP interface named by `lifrp' is RDS-capable.
78  */
79 boolean_t
rdsv3_capable_interface(struct lifreq * lifrp)80 rdsv3_capable_interface(struct lifreq *lifrp)
81 {
82 	char	ifname[LIFNAMSIZ];
83 	char	drv[MAXLINKNAMELEN];
84 	uint_t	ppa;
85 	char	*cp;
86 
87 	RDSV3_DPRINTF4("rdsv3_capable_interface", "Enter");
88 
89 	if (lifrp->lifr_type == IFT_IB)
90 		return (B_TRUE);
91 
92 	/*
93 	 * Strip off the logical interface portion before getting
94 	 * intimate with the name.
95 	 */
96 	(void) strlcpy(ifname, lifrp->lifr_name, LIFNAMSIZ);
97 	if ((cp = strchr(ifname, ':')) != NULL)
98 		*cp = '\0';
99 
100 	if (strcmp("lo0", ifname) == 0) {
101 		/*
102 		 * loopback is considered RDS-capable
103 		 */
104 		return (B_TRUE);
105 	}
106 
107 	return (ddi_parse(ifname, drv, &ppa) == DDI_SUCCESS &&
108 	    rdsv3_if_lookup_by_name(drv));
109 }
110 
111 int
rdsv3_do_ip_ioctl(ksocket_t so4,void ** ipaddrs,int * size,int * nifs)112 rdsv3_do_ip_ioctl(ksocket_t so4, void **ipaddrs, int *size, int *nifs)
113 {
114 	struct lifnum		lifn;
115 	struct lifconf		lifc;
116 	struct lifreq		*lp, *rlp, lifr;
117 	int			rval = 0;
118 	int			numifs;
119 	int			bufsize, rbufsize;
120 	void			*buf, *rbuf;
121 	int			i, j, n, rc;
122 
123 	*ipaddrs = NULL;
124 	*size = 0;
125 	*nifs = 0;
126 
127 	RDSV3_DPRINTF4("rdsv3_do_ip_ioctl", "Enter");
128 
129 retry_count:
130 	/* snapshot the current number of interfaces */
131 	lifn.lifn_family = PF_UNSPEC;
132 	lifn.lifn_flags = LIFC_NOXMIT | LIFC_TEMPORARY | LIFC_ALLZONES;
133 	lifn.lifn_count = 0;
134 	rval = ksocket_ioctl(so4, SIOCGLIFNUM, (intptr_t)&lifn, &rval,
135 	    CRED());
136 	if (rval != 0) {
137 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl",
138 		    "ksocket_ioctl returned: %d", rval);
139 		return (rval);
140 	}
141 
142 	numifs = lifn.lifn_count;
143 	if (numifs <= 0) {
144 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl", "No interfaces found");
145 		return (0);
146 	}
147 
148 	/* allocate extra room in case more interfaces appear */
149 	numifs += 10;
150 
151 	/* get the interface names and ip addresses */
152 	bufsize = numifs * sizeof (struct lifreq);
153 	buf = kmem_alloc(bufsize, KM_SLEEP);
154 
155 	lifc.lifc_family = AF_UNSPEC;
156 	lifc.lifc_flags = LIFC_NOXMIT | LIFC_TEMPORARY | LIFC_ALLZONES;
157 	lifc.lifc_len = bufsize;
158 	lifc.lifc_buf = buf;
159 	rc = ksocket_ioctl(so4, SIOCGLIFCONF, (intptr_t)&lifc, &rval, CRED());
160 	if (rc != 0) {
161 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl", "SIOCGLIFCONF failed");
162 		kmem_free(buf, bufsize);
163 		return (rc);
164 	}
165 	/* if our extra room is used up, try again */
166 	if (bufsize <= lifc.lifc_len) {
167 		kmem_free(buf, bufsize);
168 		buf = NULL;
169 		goto retry_count;
170 	}
171 	/* calc actual number of ifconfs */
172 	n = lifc.lifc_len / sizeof (struct lifreq);
173 
174 	/*
175 	 * Count the RDS interfaces
176 	 */
177 	for (i = 0, j = 0, lp = lifc.lifc_req; i < n; i++, lp++) {
178 
179 		/*
180 		 * Copy as the SIOCGLIFFLAGS ioctl is destructive
181 		 */
182 		bcopy(lp, &lifr, sizeof (struct lifreq));
183 		/*
184 		 * fetch the flags using the socket of the correct family
185 		 */
186 		switch (lifr.lifr_addr.ss_family) {
187 		case AF_INET:
188 			rc = ksocket_ioctl(so4, SIOCGLIFFLAGS, (intptr_t)&lifr,
189 			    &rval, CRED());
190 			break;
191 		default:
192 			continue;
193 		}
194 
195 		if (rc != 0) continue;
196 
197 		/*
198 		 * If we got the flags, skip uninteresting
199 		 * interfaces based on flags
200 		 */
201 		if ((lifr.lifr_flags & IFF_UP) != IFF_UP)
202 			continue;
203 		if (lifr.lifr_flags &
204 		    (IFF_ANYCAST|IFF_NOLOCAL|IFF_DEPRECATED))
205 			continue;
206 		if (!rdsv3_capable_interface(&lifr))
207 			continue;
208 		j++;
209 	}
210 
211 	if (j <= 0) {
212 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl", "No RDS interfaces");
213 		kmem_free(buf, bufsize);
214 		return (rval);
215 	}
216 
217 	numifs = j;
218 
219 	/* This is the buffer we pass back */
220 	rbufsize = numifs * sizeof (struct lifreq);
221 	rbuf = kmem_alloc(rbufsize, KM_SLEEP);
222 	rlp = (struct lifreq *)rbuf;
223 
224 	/*
225 	 * Examine the array of interfaces and filter uninteresting ones
226 	 */
227 	for (i = 0, lp = lifc.lifc_req; i < n; i++, lp++) {
228 
229 		/*
230 		 * Copy the address as the SIOCGLIFFLAGS ioctl is destructive
231 		 */
232 		bcopy(lp, &lifr, sizeof (struct lifreq));
233 		/*
234 		 * fetch the flags using the socket of the correct family
235 		 */
236 		switch (lifr.lifr_addr.ss_family) {
237 		case AF_INET:
238 			rc = ksocket_ioctl(so4, SIOCGLIFFLAGS, (intptr_t)&lifr,
239 			    &rval, CRED());
240 			break;
241 		default:
242 			continue;
243 		}
244 
245 
246 		if (rc != 0) {
247 			RDSV3_DPRINTF2("rdsv3_do_ip_ioctl",
248 			    "ksocket_ioctl failed" " for %s", lifr.lifr_name);
249 			continue;
250 		}
251 
252 		/*
253 		 * If we got the flags, skip uninteresting
254 		 * interfaces based on flags
255 		 */
256 		if ((lifr.lifr_flags & IFF_UP) != IFF_UP)
257 			continue;
258 		if (lifr.lifr_flags &
259 		    (IFF_ANYCAST|IFF_NOLOCAL|IFF_DEPRECATED))
260 			continue;
261 		if (!rdsv3_capable_interface(&lifr))
262 			continue;
263 
264 		/* save the record */
265 		bcopy(lp, rlp, sizeof (struct lifreq));
266 		rlp->lifr_addr.ss_family = AF_INET_OFFLOAD;
267 		rlp++;
268 	}
269 
270 	kmem_free(buf, bufsize);
271 
272 	*ipaddrs = rbuf;
273 	*size = rbufsize;
274 	*nifs = numifs;
275 
276 	RDSV3_DPRINTF4("rdsv3_do_ip_ioctl", "Return");
277 
278 	return (rval);
279 }
280 
281 /*
282  * Check if the IP interface named by `ifrp' is RDS-capable.
283  */
284 boolean_t
rdsv3_capable_interface_old(struct ifreq * ifrp)285 rdsv3_capable_interface_old(struct ifreq *ifrp)
286 {
287 	char	ifname[IFNAMSIZ];
288 	char	drv[MAXLINKNAMELEN];
289 	uint_t	ppa;
290 	char	*cp;
291 
292 	RDSV3_DPRINTF4("rdsv3_capable_interface_old", "Enter");
293 
294 	/*
295 	 * Strip off the logical interface portion before getting
296 	 * intimate with the name.
297 	 */
298 	(void) strlcpy(ifname, ifrp->ifr_name, IFNAMSIZ);
299 	if ((cp = strchr(ifname, ':')) != NULL)
300 		*cp = '\0';
301 
302 	RDSV3_DPRINTF4("rdsv3_capable_interface_old", "ifname: %s", ifname);
303 
304 	if ((strcmp("lo0", ifname) == 0) ||
305 	    (strncmp("ibd", ifname, 3) == 0)) {
306 		/*
307 		 * loopback and IB are considered RDS-capable
308 		 */
309 		return (B_TRUE);
310 	}
311 
312 	return (ddi_parse(ifname, drv, &ppa) == DDI_SUCCESS &&
313 	    rdsv3_if_lookup_by_name(drv));
314 }
315 
316 int
rdsv3_do_ip_ioctl_old(ksocket_t so4,void ** ipaddrs,int * size,int * nifs)317 rdsv3_do_ip_ioctl_old(ksocket_t so4, void **ipaddrs, int *size, int *nifs)
318 {
319 	uint_t			ifn;
320 	struct ifconf		ifc;
321 	struct ifreq		*lp, *rlp, ifr;
322 	int			rval = 0;
323 	int			numifs;
324 	int			bufsize, rbufsize;
325 	void			*buf, *rbuf;
326 	int			i, j, n, rc;
327 
328 	*ipaddrs = NULL;
329 	*size = 0;
330 	*nifs = 0;
331 
332 	RDSV3_DPRINTF4("rdsv3_do_ip_ioctl_old", "Enter");
333 
334 retry_count:
335 	rval = ksocket_ioctl(so4, SIOCGIFNUM, (intptr_t)&ifn, &rval,
336 	    CRED());
337 	if (rval != 0) {
338 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old",
339 		    "ksocket_ioctl(SIOCGIFNUM) returned: %d", rval);
340 		return (rval);
341 	}
342 
343 	numifs = ifn;
344 	if (numifs <= 0) {
345 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old", "No interfaces found");
346 		return (0);
347 	}
348 
349 	/* allocate extra room in case more interfaces appear */
350 	numifs += 10;
351 
352 	/* get the interface names and ip addresses */
353 	bufsize = numifs * sizeof (struct ifreq);
354 	buf = kmem_alloc(bufsize, KM_SLEEP);
355 
356 	ifc.ifc_len = bufsize;
357 	ifc.ifc_buf = buf;
358 	rc = ksocket_ioctl(so4, SIOCGIFCONF, (intptr_t)&ifc, &rval, CRED());
359 	if (rc != 0) {
360 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old",
361 		    "SIOCGLIFCONF failed: %d", rc);
362 		kmem_free(buf, bufsize);
363 		return (rc);
364 	}
365 	/* if our extra room is used up, try again */
366 	if (bufsize <= ifc.ifc_len) {
367 		kmem_free(buf, bufsize);
368 		buf = NULL;
369 		goto retry_count;
370 	}
371 	/* calc actual number of ifconfs */
372 	n = ifc.ifc_len / sizeof (struct ifreq);
373 
374 	/*
375 	 * Count the RDS interfaces
376 	 */
377 	for (i = 0, j = 0, lp = ifc.ifc_req; i < n; i++, lp++) {
378 
379 		/*
380 		 * Copy as the SIOCGIFFLAGS ioctl is destructive
381 		 */
382 		bcopy(lp, &ifr, sizeof (struct ifreq));
383 		/*
384 		 * fetch the flags using the socket of the correct family
385 		 */
386 		switch (ifr.ifr_addr.sa_family) {
387 		case AF_INET:
388 			rc = ksocket_ioctl(so4, SIOCGIFFLAGS, (intptr_t)&ifr,
389 			    &rval, CRED());
390 			break;
391 		default:
392 			continue;
393 		}
394 
395 		if (rc != 0) continue;
396 
397 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old",
398 		    "1. ifr_name: %s, flags: %d", ifr.ifr_name,
399 		    (ushort_t)ifr.ifr_flags);
400 
401 		/*
402 		 * If we got the flags, skip uninteresting
403 		 * interfaces based on flags
404 		 */
405 		if ((((ushort_t)ifr.ifr_flags) & IFF_UP) != IFF_UP)
406 			continue;
407 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old",
408 		    "2. ifr_name: %s, flags: %d", ifr.ifr_name,
409 		    (ushort_t)ifr.ifr_flags);
410 		if (((ushort_t)ifr.ifr_flags) &
411 		    (IFF_ANYCAST|IFF_NOLOCAL|IFF_DEPRECATED))
412 			continue;
413 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old",
414 		    "3. ifr_name: %s, flags: %d", ifr.ifr_name,
415 		    (ushort_t)ifr.ifr_flags);
416 		if (!rdsv3_capable_interface_old(&ifr))
417 			continue;
418 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old",
419 		    "4. ifr_name: %s, flags: %d", ifr.ifr_name,
420 		    (ushort_t)ifr.ifr_flags);
421 		j++;
422 	}
423 
424 	if (j <= 0) {
425 		RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old", "No RDS interfaces");
426 		kmem_free(buf, bufsize);
427 		return (rval);
428 	}
429 
430 	numifs = j;
431 
432 	/* This is the buffer we pass back */
433 	rbufsize = numifs * sizeof (struct ifreq);
434 	rbuf = kmem_alloc(rbufsize, KM_SLEEP);
435 	rlp = (struct ifreq *)rbuf;
436 
437 	/*
438 	 * Examine the array of interfaces and filter uninteresting ones
439 	 */
440 	for (i = 0, lp = ifc.ifc_req; i < n; i++, lp++) {
441 
442 		/*
443 		 * Copy the address as the SIOCGIFFLAGS ioctl is destructive
444 		 */
445 		bcopy(lp, &ifr, sizeof (struct ifreq));
446 		/*
447 		 * fetch the flags using the socket of the correct family
448 		 */
449 		switch (ifr.ifr_addr.sa_family) {
450 		case AF_INET:
451 			rc = ksocket_ioctl(so4, SIOCGIFFLAGS, (intptr_t)&ifr,
452 			    &rval, CRED());
453 			break;
454 		default:
455 			continue;
456 		}
457 
458 
459 		if (rc != 0) {
460 			RDSV3_DPRINTF2("rdsv3_do_ip_ioctl_old",
461 			    "ksocket_ioctl failed: %d for %s",
462 			    rc, ifr.ifr_name);
463 			continue;
464 		}
465 
466 		/*
467 		 * If we got the flags, skip uninteresting
468 		 * interfaces based on flags
469 		 */
470 		if ((((ushort_t)ifr.ifr_flags) & IFF_UP) != IFF_UP)
471 			continue;
472 		if (((ushort_t)ifr.ifr_flags) &
473 		    (IFF_ANYCAST|IFF_NOLOCAL|IFF_DEPRECATED))
474 			continue;
475 		if (!rdsv3_capable_interface_old(&ifr))
476 			continue;
477 
478 		/* save the record */
479 		bcopy(lp, rlp, sizeof (struct ifreq));
480 		rlp->ifr_addr.sa_family = AF_INET_OFFLOAD;
481 		rlp++;
482 	}
483 
484 	kmem_free(buf, bufsize);
485 
486 	*ipaddrs = rbuf;
487 	*size = rbufsize;
488 	*nifs = numifs;
489 
490 	RDSV3_DPRINTF4("rdsv3_do_ip_ioctl_old", "Return");
491 
492 	return (rval);
493 }
494 
495 boolean_t
rdsv3_isloopback(ipaddr_t addr)496 rdsv3_isloopback(ipaddr_t addr)
497 {
498 	ip_stack_t *ipst;
499 
500 	ipst = netstack_find_by_zoneid(GLOBAL_ZONEID)->netstack_ip;
501 	ASSERT(ipst != NULL);
502 	if (ip_type_v4(addr, ipst) != IRE_LOOPBACK) {
503 		netstack_rele(ipst->ips_netstack);
504 		return (B_FALSE);
505 	}
506 	netstack_rele(ipst->ips_netstack);
507 	return (B_TRUE);
508 }
509 
510 /*
511  * Work Queue Implementation
512  */
513 
514 #define	RDSV3_WQ_THREAD_IDLE		0
515 #define	RDSV3_WQ_THREAD_RUNNING		1
516 #define	RDSV3_WQ_THREAD_FLUSHING	2
517 #define	RDSV3_WQ_THREAD_EXITING		3
518 
519 /* worker thread */
520 void
rdsv3_worker_thread(void * arg)521 rdsv3_worker_thread(void *arg)
522 {
523 	rdsv3_workqueue_struct_t *wq = arg;
524 	rdsv3_work_t *work;
525 
526 	RDSV3_DPRINTF4("rdsv3_worker_thread", "Enter(wq: 0x%p)", wq);
527 
528 	mutex_enter(&wq->wq_lock);
529 	work = list_remove_head(&wq->wq_queue);
530 	while (work) {
531 		mutex_exit(&wq->wq_lock);
532 
533 		/* process work */
534 		work->func(work);
535 
536 		mutex_enter(&wq->wq_lock);
537 		work = list_remove_head(&wq->wq_queue);
538 	}
539 
540 	/* No more work, go home, until called again */
541 	if (wq->wq_state != RDSV3_WQ_THREAD_EXITING) {
542 		wq->wq_state = RDSV3_WQ_THREAD_IDLE;
543 	}
544 	mutex_exit(&wq->wq_lock);
545 
546 	RDSV3_DPRINTF4("rdsv3_worker_thread", "Return(wq: 0x%p)", wq);
547 }
548 
549 /* XXX */
550 void
rdsv3_flush_workqueue(rdsv3_workqueue_struct_t * wq)551 rdsv3_flush_workqueue(rdsv3_workqueue_struct_t *wq)
552 {
553 	RDSV3_DPRINTF4("rdsv3_flush_workqueue", "Enter(wq: %p)", wq);
554 
555 	mutex_enter(&wq->wq_lock);
556 	switch (wq->wq_state) {
557 	case RDSV3_WQ_THREAD_IDLE:
558 		/* nothing to do */
559 		ASSERT(list_is_empty(&wq->wq_queue));
560 		break;
561 
562 	case RDSV3_WQ_THREAD_RUNNING:
563 		wq->wq_state = RDSV3_WQ_THREAD_FLUSHING;
564 		/* FALLTHRU */
565 	case RDSV3_WQ_THREAD_FLUSHING:
566 		/* already flushing, wait until the flushing is complete */
567 		do {
568 			mutex_exit(&wq->wq_lock);
569 			delay(drv_usectohz(1000000));
570 			mutex_enter(&wq->wq_lock);
571 		} while (wq->wq_state == RDSV3_WQ_THREAD_FLUSHING);
572 		break;
573 	case RDSV3_WQ_THREAD_EXITING:
574 		mutex_exit(&wq->wq_lock);
575 		rdsv3_worker_thread(wq);
576 		return;
577 	}
578 	mutex_exit(&wq->wq_lock);
579 
580 	RDSV3_DPRINTF4("rdsv3_flush_workqueue", "Return(wq: %p)", wq);
581 }
582 
583 void
rdsv3_queue_work(rdsv3_workqueue_struct_t * wq,rdsv3_work_t * wp)584 rdsv3_queue_work(rdsv3_workqueue_struct_t *wq, rdsv3_work_t *wp)
585 {
586 	RDSV3_DPRINTF4("rdsv3_queue_work", "Enter(wq: %p, wp: %p)", wq, wp);
587 
588 	mutex_enter(&wq->wq_lock);
589 
590 	if (list_link_active(&wp->work_item)) {
591 		/* This is already in the queue, ignore this call */
592 		mutex_exit(&wq->wq_lock);
593 		RDSV3_DPRINTF3("rdsv3_queue_work", "already queued: %p", wp);
594 		return;
595 	}
596 
597 	switch (wq->wq_state) {
598 	case RDSV3_WQ_THREAD_RUNNING:
599 		list_insert_tail(&wq->wq_queue, wp);
600 		mutex_exit(&wq->wq_lock);
601 		break;
602 
603 	case RDSV3_WQ_THREAD_FLUSHING:
604 		do {
605 			mutex_exit(&wq->wq_lock);
606 			delay(drv_usectohz(1000000));
607 			mutex_enter(&wq->wq_lock);
608 		} while (wq->wq_state == RDSV3_WQ_THREAD_FLUSHING);
609 
610 		if (wq->wq_state == RDSV3_WQ_THREAD_RUNNING) {
611 			list_insert_tail(&wq->wq_queue, wp);
612 			mutex_exit(&wq->wq_lock);
613 			break;
614 		}
615 		/* FALLTHRU */
616 
617 	case RDSV3_WQ_THREAD_IDLE:
618 		list_insert_tail(&wq->wq_queue, wp);
619 		wq->wq_state = RDSV3_WQ_THREAD_RUNNING;
620 		mutex_exit(&wq->wq_lock);
621 
622 		(void) ddi_taskq_dispatch(rdsv3_taskq, rdsv3_worker_thread, wq,
623 		    DDI_SLEEP);
624 		break;
625 
626 	case RDSV3_WQ_THREAD_EXITING:
627 		mutex_exit(&wq->wq_lock);
628 		break;
629 	}
630 
631 	RDSV3_DPRINTF4("rdsv3_queue_work", "Return(wq: %p, wp: %p)", wq, wp);
632 }
633 
634 /* timeout handler for delayed work queuing */
635 void
rdsv3_work_timeout_handler(void * arg)636 rdsv3_work_timeout_handler(void *arg)
637 {
638 	rdsv3_delayed_work_t *dwp = (rdsv3_delayed_work_t *)arg;
639 
640 	RDSV3_DPRINTF4("rdsv3_work_timeout_handler",
641 	    "Enter(wq: %p, wp: %p)", dwp->wq, &dwp->work);
642 
643 	mutex_enter(&dwp->lock);
644 	dwp->timeid = 0;
645 	mutex_exit(&dwp->lock);
646 
647 	mutex_enter(&dwp->wq->wq_lock);
648 	dwp->wq->wq_pending--;
649 	if (dwp->wq->wq_state == RDSV3_WQ_THREAD_EXITING) {
650 		mutex_exit(&dwp->wq->wq_lock);
651 		return;
652 	}
653 	mutex_exit(&dwp->wq->wq_lock);
654 
655 	rdsv3_queue_work(dwp->wq, &dwp->work);
656 
657 	RDSV3_DPRINTF4("rdsv3_work_timeout_handler",
658 	    "Return(wq: %p, wp: %p)", dwp->wq, &dwp->work);
659 }
660 
661 void
rdsv3_queue_delayed_work(rdsv3_workqueue_struct_t * wq,rdsv3_delayed_work_t * dwp,uint_t delay)662 rdsv3_queue_delayed_work(rdsv3_workqueue_struct_t *wq,
663     rdsv3_delayed_work_t *dwp, uint_t delay)
664 {
665 	RDSV3_DPRINTF4("rdsv3_queue_delayed_work",
666 	    "Enter(wq: %p, wp: %p)", wq, dwp);
667 
668 	if (delay == 0) {
669 		rdsv3_queue_work(wq, &dwp->work);
670 		return;
671 	}
672 
673 	mutex_enter(&wq->wq_lock);
674 	if (wq->wq_state == RDSV3_WQ_THREAD_EXITING) {
675 		mutex_exit(&wq->wq_lock);
676 		RDSV3_DPRINTF4("rdsv3_queue_delayed_work",
677 		    "WQ exiting - don't queue (wq: %p, wp: %p)", wq, dwp);
678 		return;
679 	}
680 	wq->wq_pending++;
681 	mutex_exit(&wq->wq_lock);
682 
683 	mutex_enter(&dwp->lock);
684 	if (dwp->timeid == 0) {
685 		dwp->wq = wq;
686 		dwp->timeid = timeout(rdsv3_work_timeout_handler, dwp,
687 		    jiffies + (delay * rdsv3_one_sec_in_hz));
688 		mutex_exit(&dwp->lock);
689 	} else {
690 		mutex_exit(&dwp->lock);
691 		RDSV3_DPRINTF4("rdsv3_queue_delayed_work", "Already queued: %p",
692 		    dwp);
693 		mutex_enter(&wq->wq_lock);
694 		wq->wq_pending--;
695 		mutex_exit(&wq->wq_lock);
696 	}
697 
698 	RDSV3_DPRINTF4("rdsv3_queue_delayed_work",
699 	    "Return(wq: %p, wp: %p)", wq, dwp);
700 }
701 
702 void
rdsv3_cancel_delayed_work(rdsv3_delayed_work_t * dwp)703 rdsv3_cancel_delayed_work(rdsv3_delayed_work_t *dwp)
704 {
705 	RDSV3_DPRINTF4("rdsv3_cancel_delayed_work",
706 	    "Enter(wq: %p, dwp: %p)", dwp->wq, dwp);
707 
708 	mutex_enter(&dwp->lock);
709 	if (dwp->timeid != 0) {
710 		(void) untimeout(dwp->timeid);
711 		dwp->timeid = 0;
712 	} else {
713 		RDSV3_DPRINTF4("rdsv3_cancel_delayed_work",
714 		    "Nothing to cancel (wq: %p, dwp: %p)", dwp->wq, dwp);
715 		mutex_exit(&dwp->lock);
716 		return;
717 	}
718 	mutex_exit(&dwp->lock);
719 
720 	mutex_enter(&dwp->wq->wq_lock);
721 	dwp->wq->wq_pending--;
722 	mutex_exit(&dwp->wq->wq_lock);
723 
724 	RDSV3_DPRINTF4("rdsv3_cancel_delayed_work",
725 	    "Return(wq: %p, dwp: %p)", dwp->wq, dwp);
726 }
727 
728 void
rdsv3_destroy_task_workqueue(rdsv3_workqueue_struct_t * wq)729 rdsv3_destroy_task_workqueue(rdsv3_workqueue_struct_t *wq)
730 {
731 	RDSV3_DPRINTF2("rdsv3_destroy_workqueue", "Enter");
732 
733 	ASSERT(wq);
734 
735 	mutex_enter(&wq->wq_lock);
736 	wq->wq_state = RDSV3_WQ_THREAD_EXITING;
737 
738 	while (wq->wq_pending > 0) {
739 		mutex_exit(&wq->wq_lock);
740 		delay(drv_usectohz(1000000));
741 		mutex_enter(&wq->wq_lock);
742 	};
743 	mutex_exit(&wq->wq_lock);
744 
745 	rdsv3_flush_workqueue(wq);
746 
747 	list_destroy(&wq->wq_queue);
748 	mutex_destroy(&wq->wq_lock);
749 	kmem_free(wq, sizeof (rdsv3_workqueue_struct_t));
750 
751 	ASSERT(rdsv3_taskq);
752 	ddi_taskq_destroy(rdsv3_taskq);
753 
754 	wq = NULL;
755 	rdsv3_taskq = NULL;
756 
757 	RDSV3_DPRINTF2("rdsv3_destroy_workqueue", "Return");
758 }
759 
760 /* ARGSUSED */
761 void
rdsv3_rdma_init_worker(struct rdsv3_work_s * work)762 rdsv3_rdma_init_worker(struct rdsv3_work_s *work)
763 {
764 	rdsv3_rdma_init();
765 }
766 
767 #define	RDSV3_NUM_TASKQ_THREADS	1
768 rdsv3_workqueue_struct_t *
rdsv3_create_task_workqueue(char * name)769 rdsv3_create_task_workqueue(char *name)
770 {
771 	rdsv3_workqueue_struct_t	*wq;
772 
773 	RDSV3_DPRINTF2("create_singlethread_workqueue", "Enter (dip: %p)",
774 	    rdsv3_dev_info);
775 
776 	rdsv3_taskq = ddi_taskq_create(rdsv3_dev_info, name,
777 	    RDSV3_NUM_TASKQ_THREADS, TASKQ_DEFAULTPRI, 0);
778 	if (rdsv3_taskq == NULL) {
779 		RDSV3_DPRINTF2(__FILE__,
780 		    "ddi_taskq_create failed for rdsv3_taskq");
781 		return (NULL);
782 	}
783 
784 	wq = kmem_zalloc(sizeof (rdsv3_workqueue_struct_t), KM_NOSLEEP);
785 	if (wq == NULL) {
786 		RDSV3_DPRINTF2(__FILE__, "kmem_zalloc failed for wq");
787 		ddi_taskq_destroy(rdsv3_taskq);
788 		return (NULL);
789 	}
790 
791 	list_create(&wq->wq_queue, sizeof (struct rdsv3_work_s),
792 	    offsetof(struct rdsv3_work_s, work_item));
793 	mutex_init(&wq->wq_lock, NULL, MUTEX_DRIVER, NULL);
794 	wq->wq_state = RDSV3_WQ_THREAD_IDLE;
795 	wq->wq_pending = 0;
796 	rdsv3_one_sec_in_hz = drv_usectohz(1000000);
797 
798 	RDSV3_DPRINTF2("create_singlethread_workqueue", "Return");
799 
800 	return (wq);
801 }
802 
803 /*
804  * Implementation for struct sock
805  */
806 
807 void
rdsv3_sock_exit_data(struct rsock * sk)808 rdsv3_sock_exit_data(struct rsock *sk)
809 {
810 	struct rdsv3_sock *rs = sk->sk_protinfo;
811 
812 	RDSV3_DPRINTF4("rdsv3_sock_exit_data", "rs: %p sk: %p", rs, sk);
813 
814 	ASSERT(rs != NULL);
815 	ASSERT(rdsv3_sk_sock_flag(sk, SOCK_DEAD));
816 
817 	rs->rs_sk = NULL;
818 
819 	list_destroy(&rs->rs_send_queue);
820 	list_destroy(&rs->rs_notify_queue);
821 	list_destroy(&rs->rs_recv_queue);
822 
823 	rw_destroy(&rs->rs_recv_lock);
824 	mutex_destroy(&rs->rs_lock);
825 
826 	mutex_destroy(&rs->rs_rdma_lock);
827 	avl_destroy(&rs->rs_rdma_keys);
828 
829 	mutex_destroy(&rs->rs_conn_lock);
830 	mutex_destroy(&rs->rs_congested_lock);
831 	cv_destroy(&rs->rs_congested_cv);
832 
833 	rdsv3_exit_waitqueue(sk->sk_sleep);
834 	kmem_free(sk->sk_sleep, sizeof (rdsv3_wait_queue_t));
835 	mutex_destroy(&sk->sk_lock);
836 
837 	kmem_cache_free(rdsv3_alloc_cache, sk);
838 	RDSV3_DPRINTF4("rdsv3_sock_exit_data", "rs: %p sk: %p", rs, sk);
839 }
840 
841 /* XXX - figure out right values */
842 #define	RDSV3_RECV_HIWATER	(256 * 1024)
843 #define	RDSV3_RECV_LOWATER	128
844 #define	RDSV3_XMIT_HIWATER	(256 * 1024)
845 #define	RDSV3_XMIT_LOWATER	1024
846 
847 struct rsock *
rdsv3_sk_alloc()848 rdsv3_sk_alloc()
849 {
850 	struct rsock *sk;
851 
852 	sk = kmem_cache_alloc(rdsv3_alloc_cache, KM_SLEEP);
853 	if (sk == NULL) {
854 		RDSV3_DPRINTF2("rdsv3_create", "kmem_cache_alloc failed");
855 		return (NULL);
856 	}
857 
858 	bzero(sk, sizeof (struct rsock) + sizeof (struct rdsv3_sock));
859 	return (sk);
860 }
861 
862 void
rdsv3_sock_init_data(struct rsock * sk)863 rdsv3_sock_init_data(struct rsock *sk)
864 {
865 	sk->sk_sleep = kmem_zalloc(sizeof (rdsv3_wait_queue_t), KM_SLEEP);
866 	rdsv3_init_waitqueue(sk->sk_sleep);
867 
868 	mutex_init(&sk->sk_lock, NULL, MUTEX_DRIVER, NULL);
869 	sk->sk_refcount = 1;
870 	sk->sk_protinfo = (struct rdsv3_sock *)(sk + 1);
871 	sk->sk_sndbuf = RDSV3_XMIT_HIWATER;
872 	sk->sk_rcvbuf = RDSV3_RECV_HIWATER;
873 }
874 
875 /*
876  * Connection cache
877  */
878 /* ARGSUSED */
879 int
rdsv3_conn_constructor(void * buf,void * arg,int kmflags)880 rdsv3_conn_constructor(void *buf, void *arg, int kmflags)
881 {
882 	struct rdsv3_connection *conn = buf;
883 
884 	bzero(conn, sizeof (struct rdsv3_connection));
885 
886 	conn->c_next_tx_seq = 1;
887 	mutex_init(&conn->c_lock, NULL, MUTEX_DRIVER, NULL);
888 	mutex_init(&conn->c_send_lock, NULL, MUTEX_DRIVER, NULL);
889 	conn->c_send_generation = 1;
890 	conn->c_senders = 0;
891 
892 	list_create(&conn->c_send_queue, sizeof (struct rdsv3_message),
893 	    offsetof(struct rdsv3_message, m_conn_item));
894 	list_create(&conn->c_retrans, sizeof (struct rdsv3_message),
895 	    offsetof(struct rdsv3_message, m_conn_item));
896 	return (0);
897 }
898 
899 /* ARGSUSED */
900 void
rdsv3_conn_destructor(void * buf,void * arg)901 rdsv3_conn_destructor(void *buf, void *arg)
902 {
903 	struct rdsv3_connection *conn = buf;
904 
905 	ASSERT(list_is_empty(&conn->c_send_queue));
906 	ASSERT(list_is_empty(&conn->c_retrans));
907 	list_destroy(&conn->c_send_queue);
908 	list_destroy(&conn->c_retrans);
909 	mutex_destroy(&conn->c_send_lock);
910 	mutex_destroy(&conn->c_lock);
911 }
912 
913 int
rdsv3_conn_compare(const void * conn1,const void * conn2)914 rdsv3_conn_compare(const void *conn1, const void *conn2)
915 {
916 	uint32_be_t	laddr1, faddr1, laddr2, faddr2;
917 
918 	laddr1 = ((rdsv3_conn_info_t *)conn1)->c_laddr;
919 	laddr2 = ((struct rdsv3_connection *)conn2)->c_laddr;
920 
921 	if (laddr1 == laddr2) {
922 		faddr1 = ((rdsv3_conn_info_t *)conn1)->c_faddr;
923 		faddr2 = ((struct rdsv3_connection *)conn2)->c_faddr;
924 		if (faddr1 == faddr2)
925 			return (0);
926 		if (faddr1 < faddr2)
927 			return (-1);
928 		return (1);
929 	}
930 
931 	if (laddr1 < laddr2)
932 		return (-1);
933 
934 	return (1);
935 }
936 
937 /* rdsv3_ib_incoming cache */
938 /* ARGSUSED */
939 int
rdsv3_ib_inc_constructor(void * buf,void * arg,int kmflags)940 rdsv3_ib_inc_constructor(void *buf, void *arg, int kmflags)
941 {
942 	list_create(&((struct rdsv3_ib_incoming *)buf)->ii_frags,
943 	    sizeof (struct rdsv3_page_frag),
944 	    offsetof(struct rdsv3_page_frag, f_item));
945 
946 	return (0);
947 }
948 
949 /* ARGSUSED */
950 void
rdsv3_ib_inc_destructor(void * buf,void * arg)951 rdsv3_ib_inc_destructor(void *buf, void *arg)
952 {
953 	list_destroy(&((struct rdsv3_ib_incoming *)buf)->ii_frags);
954 }
955 
956 /* ib_frag_slab cache */
957 /* ARGSUSED */
958 int
rdsv3_ib_frag_constructor(void * buf,void * arg,int kmflags)959 rdsv3_ib_frag_constructor(void *buf, void *arg, int kmflags)
960 {
961 	struct rdsv3_page_frag *frag = (struct rdsv3_page_frag *)buf;
962 	struct rdsv3_ib_device *rds_ibdev = (struct rdsv3_ib_device *)arg;
963 	ibt_iov_attr_t iov_attr;
964 	ibt_iov_t iov_arr[1];
965 	ibt_all_wr_t wr;
966 
967 	bzero(frag, sizeof (struct rdsv3_page_frag));
968 	list_link_init(&frag->f_item);
969 
970 	frag->f_page = kmem_alloc(PAGE_SIZE, kmflags);
971 	if (frag->f_page == NULL) {
972 		RDSV3_DPRINTF2("rdsv3_ib_frag_constructor",
973 		    "kmem_alloc for %d failed", PAGE_SIZE);
974 		return (-1);
975 	}
976 	frag->f_offset = 0;
977 
978 	iov_attr.iov_as = NULL;
979 	iov_attr.iov = &iov_arr[0];
980 	iov_attr.iov_buf = NULL;
981 	iov_attr.iov_list_len = 1;
982 	iov_attr.iov_wr_nds = 1;
983 	iov_attr.iov_lso_hdr_sz = 0;
984 	iov_attr.iov_flags = IBT_IOV_SLEEP | IBT_IOV_RECV;
985 
986 	iov_arr[0].iov_addr = frag->f_page;
987 	iov_arr[0].iov_len = PAGE_SIZE;
988 
989 	wr.recv.wr_nds = 1;
990 	wr.recv.wr_sgl = &frag->f_sge;
991 
992 	if (ibt_map_mem_iov(ib_get_ibt_hca_hdl(rds_ibdev->dev),
993 	    &iov_attr, &wr, &frag->f_mapped) != IBT_SUCCESS) {
994 		RDSV3_DPRINTF2("rdsv3_ib_frag_constructor",
995 		    "ibt_map_mem_iov failed");
996 		kmem_free(frag->f_page, PAGE_SIZE);
997 		return (-1);
998 	}
999 
1000 	return (0);
1001 }
1002 
1003 /* ARGSUSED */
1004 void
rdsv3_ib_frag_destructor(void * buf,void * arg)1005 rdsv3_ib_frag_destructor(void *buf, void *arg)
1006 {
1007 	struct rdsv3_page_frag *frag = (struct rdsv3_page_frag *)buf;
1008 	struct rdsv3_ib_device *rds_ibdev = (struct rdsv3_ib_device *)arg;
1009 
1010 	/* unmap the page */
1011 	if (ibt_unmap_mem_iov(ib_get_ibt_hca_hdl(rds_ibdev->dev),
1012 	    frag->f_mapped) != IBT_SUCCESS)
1013 		RDSV3_DPRINTF2("rdsv3_ib_frag_destructor",
1014 		    "ibt_unmap_mem_iov failed");
1015 
1016 	/* free the page */
1017 	kmem_free(frag->f_page, PAGE_SIZE);
1018 }
1019 
1020 /* loop.c */
1021 extern kmutex_t loop_conns_lock;
1022 extern list_t loop_conns;
1023 
1024 struct rdsv3_loop_connection
1025 {
1026 	struct list_node loop_node;
1027 	struct rdsv3_connection *conn;
1028 };
1029 
1030 void
rdsv3_loop_init(void)1031 rdsv3_loop_init(void)
1032 {
1033 	list_create(&loop_conns, sizeof (struct rdsv3_loop_connection),
1034 	    offsetof(struct rdsv3_loop_connection, loop_node));
1035 	mutex_init(&loop_conns_lock, NULL, MUTEX_DRIVER, NULL);
1036 }
1037 
1038 /* rdma.c */
1039 /* IB Rkey is used here for comparison */
1040 int
rdsv3_mr_compare(const void * mr1,const void * mr2)1041 rdsv3_mr_compare(const void *mr1, const void *mr2)
1042 {
1043 	uint32_t key1 = *(uint32_t *)mr1;
1044 	uint32_t key2 = ((struct rdsv3_mr *)mr2)->r_key;
1045 
1046 	if (key1 < key2)
1047 		return (-1);
1048 	if (key1 > key2)
1049 		return (1);
1050 	return (0);
1051 }
1052 
1053 /* transport.c */
1054 extern struct rdsv3_transport *transports[];
1055 extern krwlock_t		trans_sem;
1056 
1057 void
rdsv3_trans_exit(void)1058 rdsv3_trans_exit(void)
1059 {
1060 	struct rdsv3_transport *trans;
1061 	int i;
1062 
1063 	RDSV3_DPRINTF2("rdsv3_trans_exit", "Enter");
1064 
1065 	/* currently, only IB transport */
1066 	rw_enter(&trans_sem, RW_READER);
1067 	trans = NULL;
1068 	for (i = 0; i < RDS_TRANS_COUNT; i++) {
1069 		if (transports[i]) {
1070 			trans = transports[i];
1071 			break;
1072 		}
1073 	}
1074 	rw_exit(&trans_sem);
1075 
1076 	/* trans->exit() will remove the trans from the list */
1077 	if (trans)
1078 		trans->exit();
1079 
1080 	rw_destroy(&trans_sem);
1081 
1082 	RDSV3_DPRINTF2("rdsv3_trans_exit", "Return");
1083 }
1084 
1085 void
rdsv3_trans_init()1086 rdsv3_trans_init()
1087 {
1088 	RDSV3_DPRINTF2("rdsv3_trans_init", "Enter");
1089 
1090 	rw_init(&trans_sem, NULL, RW_DRIVER, NULL);
1091 
1092 	RDSV3_DPRINTF2("rdsv3_trans_init", "Return");
1093 }
1094 
1095 int
rdsv3_put_cmsg(struct nmsghdr * msg,int level,int type,size_t size,void * payload)1096 rdsv3_put_cmsg(struct nmsghdr *msg, int level, int type, size_t size,
1097     void *payload)
1098 {
1099 	struct cmsghdr *cp;
1100 	char *bp;
1101 	size_t cmlen;
1102 	size_t cmspace;
1103 	size_t bufsz;
1104 
1105 	RDSV3_DPRINTF4("rdsv3_put_cmsg",
1106 	    "Enter(msg: %p level: %d type: %d sz: %d)",
1107 	    msg, level, type, size);
1108 
1109 	if (msg == NULL || msg->msg_controllen == 0) {
1110 		return (0);
1111 	}
1112 	/* check for first cmsg or this is another cmsg to be appended */
1113 	if (msg->msg_control == NULL)
1114 		msg->msg_controllen = 0;
1115 
1116 	cmlen = CMSG_LEN(size);
1117 	cmspace = CMSG_SPACE(size);
1118 	bufsz = msg->msg_controllen + cmspace;
1119 
1120 	/* extend the existing cmsg to append the next cmsg */
1121 	bp = kmem_alloc(bufsz, KM_SLEEP);
1122 	if (msg->msg_control) {
1123 		bcopy(msg->msg_control, bp, msg->msg_controllen);
1124 		kmem_free(msg->msg_control, (size_t)msg->msg_controllen);
1125 	}
1126 
1127 	/* assign payload the proper cmsg location */
1128 	cp = (struct cmsghdr *)(bp + msg->msg_controllen);
1129 	cp->cmsg_len = cmlen;
1130 	cp->cmsg_level = level;
1131 	cp->cmsg_type = type;
1132 
1133 	bcopy(payload, CMSG_DATA(cp), cmlen -
1134 	    (unsigned int)_CMSG_DATA_ALIGN(sizeof (struct cmsghdr)));
1135 
1136 	msg->msg_control = bp;
1137 	msg->msg_controllen = bufsz;
1138 
1139 	RDSV3_DPRINTF4("rdsv3_put_cmsg", "Return(cmsg_len: %d)", cp->cmsg_len);
1140 
1141 	return (0);
1142 }
1143 
1144 /* ARGSUSED */
1145 int
rdsv3_verify_bind_address(ipaddr_t addr)1146 rdsv3_verify_bind_address(ipaddr_t addr)
1147 {
1148 	return (1);
1149 }
1150 
1151 /* checksum */
1152 uint16_t
rdsv3_ip_fast_csum(void * hdr,size_t length)1153 rdsv3_ip_fast_csum(void *hdr, size_t length)
1154 {
1155 	return (0xffff &
1156 	    (uint16_t)(~ip_ocsum((ushort_t *)hdr, (int)length <<1, 0)));
1157 }
1158 
1159 /* scatterlist implementation */
1160 /* ARGSUSED */
1161 caddr_t
rdsv3_ib_sg_dma_address(ib_device_t * dev,struct rdsv3_scatterlist * scat,uint_t offset)1162 rdsv3_ib_sg_dma_address(ib_device_t *dev, struct rdsv3_scatterlist *scat,
1163     uint_t offset)
1164 {
1165 	return (0);
1166 }
1167 
1168 uint_t
rdsv3_ib_dma_map_sg(struct ib_device * dev,struct rdsv3_scatterlist * scat,uint_t num)1169 rdsv3_ib_dma_map_sg(struct ib_device *dev, struct rdsv3_scatterlist *scat,
1170     uint_t num)
1171 {
1172 	struct rdsv3_scatterlist *s, *first;
1173 	ibt_iov_t *iov;
1174 	ibt_wr_ds_t *sgl;
1175 	ibt_iov_attr_t iov_attr;
1176 	ibt_send_wr_t swr;
1177 	uint_t i;
1178 
1179 	RDSV3_DPRINTF4("rdsv3_ib_dma_map_sg", "scat %p, num: %d", scat, num);
1180 
1181 	s = first = &scat[0];
1182 	ASSERT(first->mihdl == NULL);
1183 
1184 	iov = kmem_alloc(num * sizeof (ibt_iov_t), KM_SLEEP);
1185 	sgl = kmem_zalloc((num * 2) *  sizeof (ibt_wr_ds_t), KM_SLEEP);
1186 
1187 	for (i = 0; i < num; i++, s++) {
1188 		iov[i].iov_addr = s->vaddr;
1189 		iov[i].iov_len = s->length;
1190 	}
1191 
1192 	iov_attr.iov_as = NULL;
1193 	iov_attr.iov = iov;
1194 	iov_attr.iov_buf = NULL;
1195 	iov_attr.iov_list_len = num;
1196 	iov_attr.iov_wr_nds = num * 2;
1197 	iov_attr.iov_lso_hdr_sz = 0;
1198 	iov_attr.iov_flags = IBT_IOV_SLEEP;
1199 
1200 	swr.wr_sgl = sgl;
1201 
1202 	i = ibt_map_mem_iov(ib_get_ibt_hca_hdl(dev),
1203 	    &iov_attr, (ibt_all_wr_t *)&swr, &first->mihdl);
1204 	kmem_free(iov, num * sizeof (ibt_iov_t));
1205 	if (i != IBT_SUCCESS) {
1206 		RDSV3_DPRINTF2("rdsv3_ib_dma_map_sg",
1207 		    "ibt_map_mem_iov returned: %d", i);
1208 		return (0);
1209 	}
1210 
1211 	s = first;
1212 	for (i = 0; i < num; i++, s++, sgl++) {
1213 		s->sgl = sgl;
1214 	}
1215 
1216 	return (num);
1217 }
1218 
1219 void
rdsv3_ib_dma_unmap_sg(ib_device_t * dev,struct rdsv3_scatterlist * scat,uint_t num)1220 rdsv3_ib_dma_unmap_sg(ib_device_t *dev, struct rdsv3_scatterlist *scat,
1221     uint_t num)
1222 {
1223 	/* Zero length messages have no scatter gather entries */
1224 	if (num != 0) {
1225 		ASSERT(scat->mihdl != NULL);
1226 		ASSERT(scat->sgl != NULL);
1227 
1228 		(void) ibt_unmap_mem_iov(ib_get_ibt_hca_hdl(dev), scat->mihdl);
1229 
1230 		kmem_free(scat->sgl, (num * 2)  * sizeof (ibt_wr_ds_t));
1231 		scat->sgl = NULL;
1232 		scat->mihdl = NULL;
1233 	}
1234 }
1235 
1236 int
rdsv3_ib_alloc_hdrs(ib_device_t * dev,struct rdsv3_ib_connection * ic)1237 rdsv3_ib_alloc_hdrs(ib_device_t *dev, struct rdsv3_ib_connection *ic)
1238 {
1239 	caddr_t addr;
1240 	size_t size;
1241 	ibt_mr_attr_t mr_attr;
1242 	ibt_mr_desc_t mr_desc;
1243 	ibt_mr_hdl_t mr_hdl;
1244 	int ret;
1245 
1246 	RDSV3_DPRINTF4("rdsv3_ib_alloc_hdrs", "Enter(dev: %p)", dev);
1247 
1248 	ASSERT(ic->i_mr == NULL);
1249 
1250 	size = (ic->i_send_ring.w_nr + ic->i_recv_ring.w_nr + 1) *
1251 	    sizeof (struct rdsv3_header);
1252 
1253 	addr = kmem_zalloc(size, KM_NOSLEEP);
1254 	if (addr == NULL)
1255 		return (-1);
1256 
1257 	mr_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)addr;
1258 	mr_attr.mr_len = size;
1259 	mr_attr.mr_as = NULL;
1260 	mr_attr.mr_flags = IBT_MR_ENABLE_LOCAL_WRITE;
1261 	ret = ibt_register_mr(ib_get_ibt_hca_hdl(dev), RDSV3_PD2PDHDL(ic->i_pd),
1262 	    &mr_attr, &mr_hdl, &mr_desc);
1263 	if (ret != IBT_SUCCESS) {
1264 		RDSV3_DPRINTF2("rdsv3_ib_alloc_hdrs",
1265 		    "ibt_register_mr returned: " "%d", ret);
1266 		return (-1);
1267 	}
1268 
1269 	ic->i_mr =
1270 	    (struct rdsv3_hdrs_mr *)kmem_alloc(sizeof (struct rdsv3_hdrs_mr),
1271 	    KM_SLEEP);
1272 	ic->i_mr->addr = addr;
1273 	ic->i_mr->size = size;
1274 	ic->i_mr->hdl =	mr_hdl;
1275 	ic->i_mr->lkey = mr_desc.md_lkey;
1276 
1277 	ic->i_send_hdrs = (struct rdsv3_header *)addr;
1278 	ic->i_send_hdrs_dma = (uint64_t)(uintptr_t)addr;
1279 
1280 	ic->i_recv_hdrs = (struct rdsv3_header *)(addr +
1281 	    (ic->i_send_ring.w_nr * sizeof (struct rdsv3_header)));
1282 	ic->i_recv_hdrs_dma = (uint64_t)(uintptr_t)(addr +
1283 	    (ic->i_send_ring.w_nr * sizeof (struct rdsv3_header)));
1284 
1285 	ic->i_ack = (struct rdsv3_header *)(addr +
1286 	    ((ic->i_send_ring.w_nr + ic->i_recv_ring.w_nr) *
1287 	    sizeof (struct rdsv3_header)));
1288 	ic->i_ack_dma = (uint64_t)(uintptr_t)(addr +
1289 	    ((ic->i_send_ring.w_nr + ic->i_recv_ring.w_nr) *
1290 	    sizeof (struct rdsv3_header)));
1291 
1292 	RDSV3_DPRINTF4("rdsv3_ib_alloc_hdrs", "Return(dev: %p)", dev);
1293 
1294 	return (0);
1295 }
1296 
1297 void
rdsv3_ib_free_hdrs(ib_device_t * dev,struct rdsv3_ib_connection * ic)1298 rdsv3_ib_free_hdrs(ib_device_t *dev, struct rdsv3_ib_connection *ic)
1299 {
1300 	RDSV3_DPRINTF4("rdsv3_ib_free_hdrs", "Enter(dev: %p)", dev);
1301 	ASSERT(ic->i_mr != NULL);
1302 
1303 	ic->i_send_hdrs = NULL;
1304 	ic->i_send_hdrs_dma = 0;
1305 
1306 	ic->i_recv_hdrs = NULL;
1307 	ic->i_recv_hdrs_dma = 0;
1308 
1309 	ic->i_ack = NULL;
1310 	ic->i_ack_dma = 0;
1311 
1312 	(void) ibt_deregister_mr(ib_get_ibt_hca_hdl(dev), ic->i_mr->hdl);
1313 
1314 	kmem_free(ic->i_mr->addr, ic->i_mr->size);
1315 	kmem_free(ic->i_mr, sizeof (struct rdsv3_hdrs_mr));
1316 
1317 	ic->i_mr = NULL;
1318 	RDSV3_DPRINTF4("rdsv3_ib_free_hdrs", "Return(dev: %p)", dev);
1319 }
1320 
1321 /*
1322  * atomic_add_unless - add unless the number is a given value
1323  * @v: pointer of type atomic_t
1324  * @a: the amount to add to v...
1325  * @u: ...unless v is equal to u.
1326  *
1327  * Atomically adds @a to @v, so long as it was not @u.
1328  * Returns non-zero if @v was not @u, and zero otherwise.
1329  */
1330 int
atomic_add_unless(atomic_t * v,uint_t a,ulong_t u)1331 atomic_add_unless(atomic_t *v, uint_t a, ulong_t u)
1332 {
1333 	uint_t c, old;
1334 
1335 	c = *v;
1336 	while (c != u && (old = atomic_cas_uint(v, c, c + a)) != c) {
1337 		c = old;
1338 	}
1339 	return ((ulong_t)c != u);
1340 }
1341