xref: /illumos-gate/usr/src/uts/common/io/ib/ibtl/ibtl_srq.c (revision f7327bbd)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/ib/ibtl/impl/ibtl.h>
26 
27 /*
28  * ibtl_srq.c
29  *	These routines implement (most of) the verbs related to
30  *	Shared Receive Queues.
31  */
32 
33 /*
34  * Globals
35  */
36 
37 static char ibtf_srq[] = "ibtl_srq";
38 
39 /*
40  * This file contains code for the TI SRQ calls
41  */
42 
43 /*
44  *
45  * ibt_alloc_srq() - Allocate a completion queue
46  */
47 ibt_status_t
ibt_alloc_srq(ibt_hca_hdl_t hca_hdl,ibt_srq_flags_t flags,ibt_pd_hdl_t pd,ibt_srq_sizes_t * srq_sizes,ibt_srq_hdl_t * ibt_srq_p,ibt_srq_sizes_t * real_sizes_p)48 ibt_alloc_srq(ibt_hca_hdl_t hca_hdl, ibt_srq_flags_t flags, ibt_pd_hdl_t pd,
49     ibt_srq_sizes_t *srq_sizes, ibt_srq_hdl_t *ibt_srq_p,
50     ibt_srq_sizes_t *real_sizes_p)
51 {
52 	ibt_status_t 		status;
53 	ibt_srq_hdl_t		ibt_srq;
54 
55 	IBTF_DPRINTF_L3(ibtf_srq, "ibt_alloc_srq(%p, %p)",
56 	    hca_hdl, srq_sizes);
57 
58 	ibt_srq = kmem_zalloc(sizeof (struct ibtl_srq_s), KM_SLEEP);
59 	*ibt_srq_p = ibt_srq;
60 
61 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibt_srq->srq_ibc_srq_hdl))
62 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(ibt_srq->srq_hca))
63 	/*
64 	 * Set the following values before creating CI SRQ, to avoid race
65 	 * conditions on async callback.
66 	 */
67 	ibt_srq->srq_hca = hca_hdl;
68 
69 	status = IBTL_HCA2CIHCAOPS_P(hca_hdl)->ibc_alloc_srq(
70 	    IBTL_HCA2CIHCA(hca_hdl), flags, ibt_srq, pd, srq_sizes,
71 	    &ibt_srq->srq_ibc_srq_hdl, real_sizes_p);
72 
73 	if (status != IBT_SUCCESS) {
74 		IBTF_DPRINTF_L2(ibtf_srq, "ibt_alloc_srq: "
75 		    "CI SRQ handle allocation failed: status = %d", status);
76 		kmem_free(ibt_srq, sizeof (struct ibtl_srq_s));
77 		*ibt_srq_p = NULL;
78 		return (status);
79 	}
80 
81 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibt_srq->srq_ibc_srq_hdl))
82 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibt_srq->srq_hca))
83 
84 	/* Update the srq resource count */
85 	atomic_inc_32(&hca_hdl->ha_srq_cnt);
86 
87 	return (IBT_SUCCESS);
88 }
89 
90 
91 /*
92  * ibt_free_srq() - Free a shared receive queue
93  *
94  */
95 ibt_status_t
ibt_free_srq(ibt_srq_hdl_t ibt_srq)96 ibt_free_srq(ibt_srq_hdl_t ibt_srq)
97 {
98 	ibt_status_t	status;
99 	ibtl_hca_t	*ibt_hca = ibt_srq->srq_hca;
100 
101 	IBTF_DPRINTF_L3(ibtf_srq, "ibt_free_srq(%p)", ibt_srq);
102 
103 	status = ((IBTL_SRQ2CIHCAOPS_P(ibt_srq))->ibc_free_srq)
104 	    (IBTL_SRQ2CIHCA(ibt_srq), ibt_srq->srq_ibc_srq_hdl);
105 
106 	if (status != IBT_SUCCESS) {
107 		IBTF_DPRINTF_L2(ibtf_srq, "ibt_free_srq: "
108 		    "CI SRQ handle de-allocation failed: status = %d", status);
109 		return (status);
110 	}
111 
112 	ibtl_free_srq_async_check(ibt_srq);
113 
114 	/* Update the srq resource count */
115 	atomic_dec_32(&ibt_hca->ha_srq_cnt);
116 
117 	return (status);
118 }
119 
120 
121 /*
122  * ibt_query_srq() - Returns the size of the srq
123  */
124 ibt_status_t
ibt_query_srq(ibt_srq_hdl_t ibt_srq,ibt_pd_hdl_t * pd_p,ibt_srq_sizes_t * sizes_p,uint_t * limit)125 ibt_query_srq(ibt_srq_hdl_t ibt_srq, ibt_pd_hdl_t *pd_p,
126     ibt_srq_sizes_t *sizes_p, uint_t *limit)
127 {
128 	IBTF_DPRINTF_L3(ibtf_srq, "ibt_query_srq(%p)", ibt_srq);
129 
130 	return (IBTL_SRQ2CIHCAOPS_P(ibt_srq)->ibc_query_srq(
131 	    IBTL_SRQ2CIHCA(ibt_srq), ibt_srq->srq_ibc_srq_hdl, pd_p,
132 	    sizes_p, limit));
133 }
134 
135 
136 /*
137  *  ibt_resize_srq() - Change the size of a srq.
138  */
139 ibt_status_t
ibt_modify_srq(ibt_srq_hdl_t ibt_srq,ibt_srq_modify_flags_t flags,uint_t size,uint_t limit,uint_t * real_size_p)140 ibt_modify_srq(ibt_srq_hdl_t ibt_srq, ibt_srq_modify_flags_t flags,
141     uint_t size, uint_t limit, uint_t *real_size_p)
142 {
143 	IBTF_DPRINTF_L3(ibtf_srq, "ibt_modify_srq(%p, %d, %d, %d)",
144 	    ibt_srq, flags, size, limit);
145 
146 	return (IBTL_SRQ2CIHCAOPS_P(ibt_srq)->ibc_modify_srq(
147 	    IBTL_SRQ2CIHCA(ibt_srq), ibt_srq->srq_ibc_srq_hdl,
148 	    flags, size, limit, real_size_p));
149 }
150 
151 
152 _NOTE(SCHEME_PROTECTS_DATA("client managed", ibtl_srq_s::srq_clnt_private))
153 
154 /*
155  * ibt_set_srq_private - Sets the private data on a given SRQ
156  *
157  *      ibt_srq          The ibt_srq_hdl_t of the allocated SRQ.
158  *      clnt_private    The client private data.
159  */
160 void
ibt_set_srq_private(ibt_srq_hdl_t ibt_srq,void * clnt_private)161 ibt_set_srq_private(ibt_srq_hdl_t ibt_srq, void *clnt_private)
162 {
163 	ibt_srq->srq_clnt_private = clnt_private;
164 }
165 
166 
167 /*
168  * ibt_get_srq_private - Retrieves the private data for a given SRQ
169  *
170  *      ibt_srq          The ibt_srq_hdl_t of the allocated SRQ.
171  */
172 void *
ibt_get_srq_private(ibt_srq_hdl_t ibt_srq)173 ibt_get_srq_private(ibt_srq_hdl_t ibt_srq)
174 {
175 	return (ibt_srq->srq_clnt_private);
176 }
177 
178 /*
179  * Function:
180  *	ibt_post_srq
181  * Input:
182  *	srq	- SRQ.
183  *	wr_list	- Address of array[size] of work requests.
184  *	size	- Number of work requests.
185  * Output:
186  *	posted	- Address to return the number of work requests
187  *		  successfully posted.  May be NULL.
188  * Description:
189  *	Post one or more receive work requests to the SRQ.
190  */
191 
192 ibt_status_t
ibt_post_srq(ibt_srq_hdl_t srq,ibt_recv_wr_t * wr_list,uint_t size,uint_t * posted)193 ibt_post_srq(ibt_srq_hdl_t srq, ibt_recv_wr_t *wr_list, uint_t size,
194     uint_t *posted)
195 {
196 	IBTF_DPRINTF_L4(ibtf_srq, "ibt_post_srq(%p, %p, %d)",
197 	    srq, wr_list, size);
198 
199 	return (IBTL_SRQ2CIHCAOPS_P(srq)->ibc_post_srq(IBTL_SRQ2CIHCA(srq),
200 	    srq->srq_ibc_srq_hdl, wr_list, size, posted));
201 }
202