1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * hermon_srq.c
28  *    Hermon Shared Receive Queue Processing Routines
29  *
30  *    Implements all the routines necessary for allocating, freeing, querying,
31  *    modifying and posting shared receive queues.
32  */
33 
34 #include <sys/sysmacros.h>
35 #include <sys/types.h>
36 #include <sys/conf.h>
37 #include <sys/ddi.h>
38 #include <sys/sunddi.h>
39 #include <sys/modctl.h>
40 #include <sys/bitmap.h>
41 
42 #include <sys/ib/adapters/hermon/hermon.h>
43 
44 static void hermon_srq_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
45     hermon_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl);
46 
47 /*
48  * hermon_srq_alloc()
49  *    Context: Can be called only from user or kernel context.
50  */
51 int
hermon_srq_alloc(hermon_state_t * state,hermon_srq_info_t * srqinfo,uint_t sleepflag)52 hermon_srq_alloc(hermon_state_t *state, hermon_srq_info_t *srqinfo,
53     uint_t sleepflag)
54 {
55 	ibt_srq_hdl_t		ibt_srqhdl;
56 	hermon_pdhdl_t		pd;
57 	ibt_srq_sizes_t		*sizes;
58 	ibt_srq_sizes_t		*real_sizes;
59 	hermon_srqhdl_t		*srqhdl;
60 	ibt_srq_flags_t		flags;
61 	hermon_rsrc_t		*srqc, *rsrc;
62 	hermon_hw_srqc_t	srqc_entry;
63 	uint32_t		*buf;
64 	hermon_srqhdl_t		srq;
65 	hermon_umap_db_entry_t	*umapdb;
66 	ibt_mr_attr_t		mr_attr;
67 	hermon_mr_options_t	mr_op;
68 	hermon_mrhdl_t		mr;
69 	uint64_t		value, srq_desc_off;
70 	uint32_t		log_srq_size;
71 	uint32_t		uarpg;
72 	uint_t			srq_is_umap;
73 	int			flag, status;
74 	uint_t			max_sgl;
75 	uint_t			wqesz;
76 	uint_t			srq_wr_sz;
77 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sizes))
78 
79 	/*
80 	 * options-->wq_location used to be for location, now explicitly
81 	 * LOCATION_NORMAL
82 	 */
83 
84 	/*
85 	 * Extract the necessary info from the hermon_srq_info_t structure
86 	 */
87 	real_sizes = srqinfo->srqi_real_sizes;
88 	sizes	   = srqinfo->srqi_sizes;
89 	pd	   = srqinfo->srqi_pd;
90 	ibt_srqhdl = srqinfo->srqi_ibt_srqhdl;
91 	flags	   = srqinfo->srqi_flags;
92 	srqhdl	   = srqinfo->srqi_srqhdl;
93 
94 	/*
95 	 * Determine whether SRQ is being allocated for userland access or
96 	 * whether it is being allocated for kernel access.  If the SRQ is
97 	 * being allocated for userland access, then lookup the UAR doorbell
98 	 * page number for the current process.  Note:  If this is not found
99 	 * (e.g. if the process has not previously open()'d the Hermon driver),
100 	 * then an error is returned.
101 	 */
102 	srq_is_umap = (flags & IBT_SRQ_USER_MAP) ? 1 : 0;
103 	if (srq_is_umap) {
104 		status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
105 		    MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
106 		if (status != DDI_SUCCESS) {
107 			status = IBT_INVALID_PARAM;
108 			goto srqalloc_fail3;
109 		}
110 		uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
111 	} else {
112 		uarpg = state->hs_kernel_uar_index;
113 	}
114 
115 	/* Increase PD refcnt */
116 	hermon_pd_refcnt_inc(pd);
117 
118 	/* Allocate an SRQ context entry */
119 	status = hermon_rsrc_alloc(state, HERMON_SRQC, 1, sleepflag, &srqc);
120 	if (status != DDI_SUCCESS) {
121 		status = IBT_INSUFF_RESOURCE;
122 		goto srqalloc_fail1;
123 	}
124 
125 	/* Allocate the SRQ Handle entry */
126 	status = hermon_rsrc_alloc(state, HERMON_SRQHDL, 1, sleepflag, &rsrc);
127 	if (status != DDI_SUCCESS) {
128 		status = IBT_INSUFF_RESOURCE;
129 		goto srqalloc_fail2;
130 	}
131 
132 	srq = (hermon_srqhdl_t)rsrc->hr_addr;
133 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq))
134 
135 	bzero(srq, sizeof (struct hermon_sw_srq_s));
136 	/* Calculate the SRQ number */
137 
138 	/* just use the index, implicit in Hermon */
139 	srq->srq_srqnum = srqc->hr_indx;
140 
141 	/*
142 	 * If this will be a user-mappable SRQ, then allocate an entry for
143 	 * the "userland resources database".  This will later be added to
144 	 * the database (after all further SRQ operations are successful).
145 	 * If we fail here, we must undo the reference counts and the
146 	 * previous resource allocation.
147 	 */
148 	if (srq_is_umap) {
149 		umapdb = hermon_umap_db_alloc(state->hs_instance,
150 		    srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC,
151 		    (uint64_t)(uintptr_t)rsrc);
152 		if (umapdb == NULL) {
153 			status = IBT_INSUFF_RESOURCE;
154 			goto srqalloc_fail3;
155 		}
156 	}
157 
158 	/*
159 	 * Allocate the doorbell record.  Hermon just needs one for the
160 	 * SRQ, and use uarpg (above) as the uar index
161 	 */
162 
163 	status = hermon_dbr_alloc(state, uarpg, &srq->srq_wq_dbr_acchdl,
164 	    &srq->srq_wq_vdbr, &srq->srq_wq_pdbr, &srq->srq_rdbr_mapoffset);
165 	if (status != DDI_SUCCESS) {
166 		status = IBT_INSUFF_RESOURCE;
167 		goto srqalloc_fail4;
168 	}
169 
170 	/*
171 	 * Calculate the appropriate size for the SRQ.
172 	 * Note:  All Hermon SRQs must be a power-of-2 in size.  Also
173 	 * they may not be any smaller than HERMON_SRQ_MIN_SIZE.  This step
174 	 * is to round the requested size up to the next highest power-of-2
175 	 */
176 	srq_wr_sz = max(sizes->srq_wr_sz + 1, HERMON_SRQ_MIN_SIZE);
177 	log_srq_size = highbit(srq_wr_sz);
178 	if (ISP2(srq_wr_sz)) {
179 		log_srq_size = log_srq_size - 1;
180 	}
181 
182 	/*
183 	 * Next we verify that the rounded-up size is valid (i.e. consistent
184 	 * with the device limits and/or software-configured limits).  If not,
185 	 * then obviously we have a lot of cleanup to do before returning.
186 	 */
187 	if (log_srq_size > state->hs_cfg_profile->cp_log_max_srq_sz) {
188 		status = IBT_HCA_WR_EXCEEDED;
189 		goto srqalloc_fail4a;
190 	}
191 
192 	/*
193 	 * Next we verify that the requested number of SGL is valid (i.e.
194 	 * consistent with the device limits and/or software-configured
195 	 * limits).  If not, then obviously the same cleanup needs to be done.
196 	 */
197 	max_sgl = state->hs_ibtfinfo.hca_attr->hca_max_srq_sgl;
198 	if (sizes->srq_sgl_sz > max_sgl) {
199 		status = IBT_HCA_SGL_EXCEEDED;
200 		goto srqalloc_fail4a;
201 	}
202 
203 	/*
204 	 * Determine the SRQ's WQE sizes.  This depends on the requested
205 	 * number of SGLs.  Note: This also has the side-effect of
206 	 * calculating the real number of SGLs (for the calculated WQE size)
207 	 */
208 	hermon_srq_sgl_to_logwqesz(state, sizes->srq_sgl_sz,
209 	    HERMON_QP_WQ_TYPE_RECVQ, &srq->srq_wq_log_wqesz,
210 	    &srq->srq_wq_sgl);
211 
212 	/*
213 	 * Allocate the memory for SRQ work queues.  Note:  The location from
214 	 * which we will allocate these work queues is always
215 	 * QUEUE_LOCATION_NORMAL.  Since Hermon work queues are not
216 	 * allowed to cross a 32-bit (4GB) boundary, the alignment of the work
217 	 * queue memory is very important.  We used to allocate work queues
218 	 * (the combined receive and send queues) so that they would be aligned
219 	 * on their combined size.  That alignment guaranteed that they would
220 	 * never cross the 4GB boundary (Hermon work queues are on the order of
221 	 * MBs at maximum).  Now we are able to relax this alignment constraint
222 	 * by ensuring that the IB address assigned to the queue memory (as a
223 	 * result of the hermon_mr_register() call) is offset from zero.
224 	 * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
225 	 * guarantee the alignment, but when attempting to use IOMMU bypass
226 	 * mode we found that we were not allowed to specify any alignment that
227 	 * was more restrictive than the system page size.  So we avoided this
228 	 * constraint by passing two alignment values, one for the memory
229 	 * allocation itself and the other for the DMA handle (for later bind).
230 	 * This used to cause more memory than necessary to be allocated (in
231 	 * order to guarantee the more restrictive alignment contraint).  But
232 	 * be guaranteeing the zero-based IB virtual address for the queue, we
233 	 * are able to conserve this memory.
234 	 *
235 	 * Note: If SRQ is not user-mappable, then it may come from either
236 	 * kernel system memory or from HCA-attached local DDR memory.
237 	 *
238 	 * Note2: We align this queue on a pagesize boundary.  This is required
239 	 * to make sure that all the resulting IB addresses will start at 0, for
240 	 * a zero-based queue.  By making sure we are aligned on at least a
241 	 * page, any offset we use into our queue will be the same as when we
242 	 * perform hermon_srq_modify() operations later.
243 	 */
244 	wqesz = (1 << srq->srq_wq_log_wqesz);
245 	srq->srq_wqinfo.qa_size = (1 << log_srq_size) * wqesz;
246 	srq->srq_wqinfo.qa_alloc_align = PAGESIZE;
247 	srq->srq_wqinfo.qa_bind_align = PAGESIZE;
248 	if (srq_is_umap) {
249 		srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
250 	} else {
251 		srq->srq_wqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
252 	}
253 	status = hermon_queue_alloc(state, &srq->srq_wqinfo, sleepflag);
254 	if (status != DDI_SUCCESS) {
255 		status = IBT_INSUFF_RESOURCE;
256 		goto srqalloc_fail4a;
257 	}
258 	buf = (uint32_t *)srq->srq_wqinfo.qa_buf_aligned;
259 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
260 
261 	/*
262 	 * Register the memory for the SRQ work queues.  The memory for the SRQ
263 	 * must be registered in the Hermon cMPT tables.  This gives us the LKey
264 	 * to specify in the SRQ context later.  Note: If the work queue is to
265 	 * be allocated from DDR memory, then only a "bypass" mapping is
266 	 * appropriate.  And if the SRQ memory is user-mappable, then we force
267 	 * DDI_DMA_CONSISTENT mapping.  Also, in order to meet the alignment
268 	 * restriction, we pass the "mro_bind_override_addr" flag in the call
269 	 * to hermon_mr_register().  This guarantees that the resulting IB vaddr
270 	 * will be zero-based (modulo the offset into the first page).  If we
271 	 * fail here, we still have the bunch of resource and reference count
272 	 * cleanup to do.
273 	 */
274 	flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP :
275 	    IBT_MR_NOSLEEP;
276 	mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
277 	mr_attr.mr_len   = srq->srq_wqinfo.qa_size;
278 	mr_attr.mr_as    = NULL;
279 	mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
280 	mr_op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
281 	mr_op.mro_bind_dmahdl = srq->srq_wqinfo.qa_dmahdl;
282 	mr_op.mro_bind_override_addr = 1;
283 	status = hermon_mr_register(state, pd, &mr_attr, &mr,
284 	    &mr_op, HERMON_SRQ_CMPT);
285 	if (status != DDI_SUCCESS) {
286 		status = IBT_INSUFF_RESOURCE;
287 		goto srqalloc_fail5;
288 	}
289 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
290 
291 	/*
292 	 * Calculate the offset between the kernel virtual address space
293 	 * and the IB virtual address space.  This will be used when
294 	 * posting work requests to properly initialize each WQE.
295 	 */
296 	srq_desc_off = (uint64_t)(uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
297 	    (uint64_t)mr->mr_bindinfo.bi_addr;
298 
299 	srq->srq_wq_wqhdr = hermon_wrid_wqhdr_create(1 << log_srq_size);
300 
301 	/*
302 	 * Fill in all the return arguments (if necessary).  This includes
303 	 * real queue size and real SGLs.
304 	 */
305 	if (real_sizes != NULL) {
306 		real_sizes->srq_wr_sz = (1 << log_srq_size) - 1;
307 		real_sizes->srq_sgl_sz = srq->srq_wq_sgl;
308 	}
309 
310 	/*
311 	 * Fill in the SRQC entry.  This is the final step before passing
312 	 * ownership of the SRQC entry to the Hermon hardware.  We use all of
313 	 * the information collected/calculated above to fill in the
314 	 * requisite portions of the SRQC.  Note: If this SRQ is going to be
315 	 * used for userland access, then we need to set the UAR page number
316 	 * appropriately (otherwise it's a "don't care")
317 	 */
318 	bzero(&srqc_entry, sizeof (hermon_hw_srqc_t));
319 	srqc_entry.state	   = HERMON_SRQ_STATE_HW_OWNER;
320 	srqc_entry.log_srq_size	   = log_srq_size;
321 	srqc_entry.srqn		   = srq->srq_srqnum;
322 	srqc_entry.log_rq_stride   = srq->srq_wq_log_wqesz - 4;
323 					/* 16-byte chunks */
324 
325 	srqc_entry.page_offs	   = srq->srq_wqinfo.qa_pgoffs >> 6;
326 	srqc_entry.log2_pgsz	   = mr->mr_log2_pgsz;
327 	srqc_entry.mtt_base_addrh  = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
328 	srqc_entry.mtt_base_addrl  = mr->mr_mttaddr >> 3;
329 	srqc_entry.pd		   = pd->pd_pdnum;
330 	srqc_entry.dbr_addrh = (uint32_t)((uint64_t)srq->srq_wq_pdbr >> 32);
331 	srqc_entry.dbr_addrl = (uint32_t)((uint64_t)srq->srq_wq_pdbr >> 2);
332 
333 	/*
334 	 * all others - specifically, xrcd, cqn_xrc, lwm, wqe_cnt, and wqe_cntr
335 	 * are zero thanks to the bzero of the structure
336 	 */
337 
338 	/*
339 	 * Write the SRQC entry to hardware.  Lastly, we pass ownership of
340 	 * the entry to the hardware (using the Hermon SW2HW_SRQ firmware
341 	 * command).  Note: In general, this operation shouldn't fail.  But
342 	 * if it does, we have to undo everything we've done above before
343 	 * returning error.
344 	 */
345 	status = hermon_cmn_ownership_cmd_post(state, SW2HW_SRQ, &srqc_entry,
346 	    sizeof (hermon_hw_srqc_t), srq->srq_srqnum,
347 	    sleepflag);
348 	if (status != HERMON_CMD_SUCCESS) {
349 		cmn_err(CE_CONT, "Hermon: SW2HW_SRQ command failed: %08x\n",
350 		    status);
351 		if (status == HERMON_CMD_INVALID_STATUS) {
352 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
353 		}
354 		status = ibc_get_ci_failure(0);
355 		goto srqalloc_fail8;
356 	}
357 
358 	/*
359 	 * Fill in the rest of the Hermon SRQ handle.  We can update
360 	 * the following fields for use in further operations on the SRQ.
361 	 */
362 	srq->srq_srqcrsrcp = srqc;
363 	srq->srq_rsrcp	   = rsrc;
364 	srq->srq_mrhdl	   = mr;
365 	srq->srq_refcnt	   = 0;
366 	srq->srq_is_umap   = srq_is_umap;
367 	srq->srq_uarpg	   = uarpg;
368 	srq->srq_umap_dhp  = (devmap_cookie_t)NULL;
369 	srq->srq_pdhdl	   = pd;
370 	srq->srq_wq_bufsz  = (1 << log_srq_size);
371 	srq->srq_wq_buf	   = buf;
372 	srq->srq_desc_off  = srq_desc_off;
373 	srq->srq_hdlrarg   = (void *)ibt_srqhdl;
374 	srq->srq_state	   = 0;
375 	srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
376 	srq->srq_real_sizes.srq_sgl_sz = srq->srq_wq_sgl;
377 
378 	/*
379 	 * Put SRQ handle in Hermon SRQNum-to-SRQhdl list.  Then fill in the
380 	 * "srqhdl" and return success
381 	 */
382 	hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, srq);
383 
384 	/*
385 	 * If this is a user-mappable SRQ, then we need to insert the
386 	 * previously allocated entry into the "userland resources database".
387 	 * This will allow for later lookup during devmap() (i.e. mmap())
388 	 * calls.
389 	 */
390 	if (srq->srq_is_umap) {
391 		hermon_umap_db_add(umapdb);
392 	} else {	/* initialize work queue for kernel SRQs */
393 		int i, len, last;
394 		uint16_t *desc;
395 
396 		desc = (uint16_t *)buf;
397 		len = wqesz / sizeof (*desc);
398 		last = srq->srq_wq_bufsz - 1;
399 		for (i = 0; i < last; i++) {
400 			desc[1] = htons(i + 1);
401 			desc += len;
402 		}
403 		srq->srq_wq_wqhdr->wq_tail = last;
404 		srq->srq_wq_wqhdr->wq_head = 0;
405 	}
406 
407 	*srqhdl = srq;
408 
409 	return (status);
410 
411 /*
412  * The following is cleanup for all possible failure cases in this routine
413  */
414 srqalloc_fail8:
415 	hermon_wrid_wqhdr_destroy(srq->srq_wq_wqhdr);
416 srqalloc_fail7:
417 	if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
418 	    HERMON_SLEEPFLAG_FOR_CONTEXT()) != DDI_SUCCESS) {
419 		HERMON_WARNING(state, "failed to deregister SRQ memory");
420 	}
421 srqalloc_fail5:
422 	hermon_queue_free(&srq->srq_wqinfo);
423 srqalloc_fail4a:
424 	hermon_dbr_free(state, uarpg, srq->srq_wq_vdbr);
425 srqalloc_fail4:
426 	if (srq_is_umap) {
427 		hermon_umap_db_free(umapdb);
428 	}
429 srqalloc_fail3:
430 	hermon_rsrc_free(state, &rsrc);
431 srqalloc_fail2:
432 	hermon_rsrc_free(state, &srqc);
433 srqalloc_fail1:
434 	hermon_pd_refcnt_dec(pd);
435 srqalloc_fail:
436 	return (status);
437 }
438 
439 
440 /*
441  * hermon_srq_free()
442  *    Context: Can be called only from user or kernel context.
443  */
444 /* ARGSUSED */
445 int
hermon_srq_free(hermon_state_t * state,hermon_srqhdl_t * srqhdl,uint_t sleepflag)446 hermon_srq_free(hermon_state_t *state, hermon_srqhdl_t *srqhdl,
447     uint_t sleepflag)
448 {
449 	hermon_rsrc_t		*srqc, *rsrc;
450 	hermon_umap_db_entry_t	*umapdb;
451 	uint64_t		value;
452 	hermon_srqhdl_t		srq;
453 	hermon_mrhdl_t		mr;
454 	hermon_pdhdl_t		pd;
455 	hermon_hw_srqc_t	srqc_entry;
456 	uint32_t		srqnum;
457 	uint_t			maxprot;
458 	int			status;
459 
460 	/*
461 	 * Pull all the necessary information from the Hermon Shared Receive
462 	 * Queue handle.  This is necessary here because the resource for the
463 	 * SRQ handle is going to be freed up as part of this operation.
464 	 */
465 	srq	= *srqhdl;
466 	mutex_enter(&srq->srq_lock);
467 	srqc	= srq->srq_srqcrsrcp;
468 	rsrc	= srq->srq_rsrcp;
469 	pd	= srq->srq_pdhdl;
470 	mr	= srq->srq_mrhdl;
471 	srqnum	= srq->srq_srqnum;
472 
473 	/*
474 	 * If there are work queues still associated with the SRQ, then return
475 	 * an error.  Otherwise, we will be holding the SRQ lock.
476 	 */
477 	if (srq->srq_refcnt != 0) {
478 		mutex_exit(&srq->srq_lock);
479 		return (IBT_SRQ_IN_USE);
480 	}
481 
482 	/*
483 	 * If this was a user-mappable SRQ, then we need to remove its entry
484 	 * from the "userland resources database".  If it is also currently
485 	 * mmap()'d out to a user process, then we need to call
486 	 * devmap_devmem_remap() to remap the SRQ memory to an invalid mapping.
487 	 * We also need to invalidate the SRQ tracking information for the
488 	 * user mapping.
489 	 */
490 	if (srq->srq_is_umap) {
491 		status = hermon_umap_db_find(state->hs_instance,
492 		    srq->srq_srqnum, MLNX_UMAP_SRQMEM_RSRC, &value,
493 		    HERMON_UMAP_DB_REMOVE, &umapdb);
494 		if (status != DDI_SUCCESS) {
495 			mutex_exit(&srq->srq_lock);
496 			HERMON_WARNING(state, "failed to find in database");
497 			return (ibc_get_ci_failure(0));
498 		}
499 		hermon_umap_db_free(umapdb);
500 		if (srq->srq_umap_dhp != NULL) {
501 			maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
502 			status = devmap_devmem_remap(srq->srq_umap_dhp,
503 			    state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size,
504 			    maxprot, DEVMAP_MAPPING_INVALID, NULL);
505 			if (status != DDI_SUCCESS) {
506 				mutex_exit(&srq->srq_lock);
507 				HERMON_WARNING(state, "failed in SRQ memory "
508 				    "devmap_devmem_remap()");
509 				return (ibc_get_ci_failure(0));
510 			}
511 			srq->srq_umap_dhp = (devmap_cookie_t)NULL;
512 		}
513 	}
514 
515 	/*
516 	 * Put NULL into the Hermon SRQNum-to-SRQHdl list.  This will allow any
517 	 * in-progress events to detect that the SRQ corresponding to this
518 	 * number has been freed.
519 	 */
520 	hermon_icm_set_num_to_hdl(state, HERMON_SRQC, srqc->hr_indx, NULL);
521 
522 	mutex_exit(&srq->srq_lock);
523 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq));
524 
525 	/*
526 	 * Reclaim SRQC entry from hardware (using the Hermon HW2SW_SRQ
527 	 * firmware command).  If the ownership transfer fails for any reason,
528 	 * then it is an indication that something (either in HW or SW) has
529 	 * gone seriously wrong.
530 	 */
531 	status = hermon_cmn_ownership_cmd_post(state, HW2SW_SRQ, &srqc_entry,
532 	    sizeof (hermon_hw_srqc_t), srqnum, sleepflag);
533 	if (status != HERMON_CMD_SUCCESS) {
534 		HERMON_WARNING(state, "failed to reclaim SRQC ownership");
535 		cmn_err(CE_CONT, "Hermon: HW2SW_SRQ command failed: %08x\n",
536 		    status);
537 		if (status == HERMON_CMD_INVALID_STATUS) {
538 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
539 		}
540 		return (ibc_get_ci_failure(0));
541 	}
542 
543 	/*
544 	 * Deregister the memory for the Shared Receive Queue.  If this fails
545 	 * for any reason, then it is an indication that something (either
546 	 * in HW or SW) has gone seriously wrong.  So we print a warning
547 	 * message and return.
548 	 */
549 	status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
550 	    sleepflag);
551 	if (status != DDI_SUCCESS) {
552 		HERMON_WARNING(state, "failed to deregister SRQ memory");
553 		return (IBT_FAILURE);
554 	}
555 
556 	hermon_wrid_wqhdr_destroy(srq->srq_wq_wqhdr);
557 
558 	/* Free the memory for the SRQ */
559 	hermon_queue_free(&srq->srq_wqinfo);
560 
561 	/* Free the dbr */
562 	hermon_dbr_free(state, srq->srq_uarpg, srq->srq_wq_vdbr);
563 
564 	/* Free the Hermon SRQ Handle */
565 	hermon_rsrc_free(state, &rsrc);
566 
567 	/* Free the SRQC entry resource */
568 	hermon_rsrc_free(state, &srqc);
569 
570 	/* Decrement the reference count on the protection domain (PD) */
571 	hermon_pd_refcnt_dec(pd);
572 
573 	/* Set the srqhdl pointer to NULL and return success */
574 	*srqhdl = NULL;
575 
576 	return (DDI_SUCCESS);
577 }
578 
579 
580 /*
581  * hermon_srq_modify()
582  *    Context: Can be called only from user or kernel context.
583  */
584 int
hermon_srq_modify(hermon_state_t * state,hermon_srqhdl_t srq,uint_t size,uint_t * real_size,uint_t sleepflag)585 hermon_srq_modify(hermon_state_t *state, hermon_srqhdl_t srq, uint_t size,
586     uint_t *real_size, uint_t sleepflag)
587 {
588 	hermon_qalloc_info_t	new_srqinfo, old_srqinfo;
589 	hermon_rsrc_t		*mtt, *old_mtt;
590 	hermon_bind_info_t	bind;
591 	hermon_bind_info_t	old_bind;
592 	hermon_mrhdl_t		mr;
593 	hermon_hw_srqc_t	srqc_entry;
594 	hermon_hw_dmpt_t	mpt_entry;
595 	uint64_t		*wre_new, *wre_old;
596 	uint64_t		mtt_addr;
597 	uint64_t		srq_pgoffs;
598 	uint64_t		srq_desc_off;
599 	uint32_t		*buf, srq_old_bufsz;
600 	uint32_t		wqesz;
601 	uint_t			max_srq_size;
602 	uint_t			mtt_pgsize_bits;
603 	uint_t			log_srq_size, maxprot;
604 	int			status;
605 
606 	if ((state->hs_devlim.mod_wr_srq == 0) ||
607 	    (state->hs_cfg_profile->cp_srq_resize_enabled == 0))
608 		return (IBT_NOT_SUPPORTED);
609 
610 	/*
611 	 * If size requested is larger than device capability, return
612 	 * Insufficient Resources
613 	 */
614 	max_srq_size = (1 << state->hs_cfg_profile->cp_log_max_srq_sz);
615 	if (size > max_srq_size) {
616 		return (IBT_HCA_WR_EXCEEDED);
617 	}
618 
619 	/*
620 	 * Calculate the appropriate size for the SRQ.
621 	 * Note:  All Hermon SRQs must be a power-of-2 in size.  Also
622 	 * they may not be any smaller than HERMON_SRQ_MIN_SIZE.  This step
623 	 * is to round the requested size up to the next highest power-of-2
624 	 */
625 	size = max(size, HERMON_SRQ_MIN_SIZE);
626 	log_srq_size = highbit(size);
627 	if (ISP2(size)) {
628 		log_srq_size = log_srq_size - 1;
629 	}
630 
631 	/*
632 	 * Next we verify that the rounded-up size is valid (i.e. consistent
633 	 * with the device limits and/or software-configured limits).
634 	 */
635 	if (log_srq_size > state->hs_cfg_profile->cp_log_max_srq_sz) {
636 		status = IBT_HCA_WR_EXCEEDED;
637 		goto srqmodify_fail;
638 	}
639 
640 	/*
641 	 * Allocate the memory for newly resized Shared Receive Queue.
642 	 *
643 	 * Note: If SRQ is not user-mappable, then it may come from either
644 	 * kernel system memory or from HCA-attached local DDR memory.
645 	 *
646 	 * Note2: We align this queue on a pagesize boundary.  This is required
647 	 * to make sure that all the resulting IB addresses will start at 0,
648 	 * for a zero-based queue.  By making sure we are aligned on at least a
649 	 * page, any offset we use into our queue will be the same as it was
650 	 * when we allocated it at hermon_srq_alloc() time.
651 	 */
652 	wqesz = (1 << srq->srq_wq_log_wqesz);
653 	new_srqinfo.qa_size = (1 << log_srq_size) * wqesz;
654 	new_srqinfo.qa_alloc_align = PAGESIZE;
655 	new_srqinfo.qa_bind_align  = PAGESIZE;
656 	if (srq->srq_is_umap) {
657 		new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
658 	} else {
659 		new_srqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
660 	}
661 	status = hermon_queue_alloc(state, &new_srqinfo, sleepflag);
662 	if (status != DDI_SUCCESS) {
663 		status = IBT_INSUFF_RESOURCE;
664 		goto srqmodify_fail;
665 	}
666 	buf = (uint32_t *)new_srqinfo.qa_buf_aligned;
667 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
668 
669 	/*
670 	 * Allocate the memory for the new WRE list.  This will be used later
671 	 * when we resize the wridlist based on the new SRQ size.
672 	 */
673 	wre_new = kmem_zalloc((1 << log_srq_size) * sizeof (uint64_t),
674 	    sleepflag);
675 	if (wre_new == NULL) {
676 		status = IBT_INSUFF_RESOURCE;
677 		goto srqmodify_fail;
678 	}
679 
680 	/*
681 	 * Fill in the "bind" struct.  This struct provides the majority
682 	 * of the information that will be used to distinguish between an
683 	 * "addr" binding (as is the case here) and a "buf" binding (see
684 	 * below).  The "bind" struct is later passed to hermon_mr_mem_bind()
685 	 * which does most of the "heavy lifting" for the Hermon memory
686 	 * registration routines.
687 	 */
688 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(bind))
689 	bzero(&bind, sizeof (hermon_bind_info_t));
690 	bind.bi_type  = HERMON_BINDHDL_VADDR;
691 	bind.bi_addr  = (uint64_t)(uintptr_t)buf;
692 	bind.bi_len   = new_srqinfo.qa_size;
693 	bind.bi_as    = NULL;
694 	bind.bi_flags = sleepflag == HERMON_SLEEP ? IBT_MR_SLEEP :
695 	    IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
696 	bind.bi_bypass = state->hs_cfg_profile->cp_iommu_bypass;
697 
698 	status = hermon_mr_mtt_bind(state, &bind, new_srqinfo.qa_dmahdl, &mtt,
699 	    &mtt_pgsize_bits, 0); /* no relaxed ordering */
700 	if (status != DDI_SUCCESS) {
701 		status = status;
702 		kmem_free(wre_new, (1 << log_srq_size) *
703 		    sizeof (uint64_t));
704 		hermon_queue_free(&new_srqinfo);
705 		goto srqmodify_fail;
706 	}
707 
708 	/*
709 	 * Calculate the offset between the kernel virtual address space
710 	 * and the IB virtual address space.  This will be used when
711 	 * posting work requests to properly initialize each WQE.
712 	 *
713 	 * Note: bind addr is zero-based (from alloc) so we calculate the
714 	 * correct new offset here.
715 	 */
716 	bind.bi_addr = bind.bi_addr & ((1 << mtt_pgsize_bits) - 1);
717 	srq_desc_off = (uint64_t)(uintptr_t)new_srqinfo.qa_buf_aligned -
718 	    (uint64_t)bind.bi_addr;
719 	srq_pgoffs   = (uint_t)
720 	    ((uintptr_t)new_srqinfo.qa_buf_aligned & HERMON_PAGEOFFSET);
721 
722 	/*
723 	 * Fill in the MPT entry.  This is the final step before passing
724 	 * ownership of the MPT entry to the Hermon hardware.  We use all of
725 	 * the information collected/calculated above to fill in the
726 	 * requisite portions of the MPT.
727 	 */
728 	bzero(&mpt_entry, sizeof (hermon_hw_dmpt_t));
729 	mpt_entry.reg_win_len	= bind.bi_len;
730 	mtt_addr = (mtt->hr_indx << HERMON_MTT_SIZE_SHIFT);
731 	mpt_entry.mtt_addr_h = mtt_addr >> 32;
732 	mpt_entry.mtt_addr_l = mtt_addr >> 3;
733 
734 	/*
735 	 * for hermon we build up a new srqc and pass that (partially filled
736 	 * to resize SRQ instead of modifying the (d)mpt directly
737 	 */
738 
739 
740 
741 	/*
742 	 * Now we grab the SRQ lock.  Since we will be updating the actual
743 	 * SRQ location and the producer/consumer indexes, we should hold
744 	 * the lock.
745 	 *
746 	 * We do a HERMON_NOSLEEP here (and below), though, because we are
747 	 * holding the "srq_lock" and if we got raised to interrupt level
748 	 * by priority inversion, we would not want to block in this routine
749 	 * waiting for success.
750 	 */
751 	mutex_enter(&srq->srq_lock);
752 
753 	/*
754 	 * Copy old entries to new buffer
755 	 */
756 	srq_old_bufsz = srq->srq_wq_bufsz;
757 	bcopy(srq->srq_wq_buf, buf, srq_old_bufsz * wqesz);
758 
759 	/*
760 	 * Setup MPT information for use in the MODIFY_MPT command
761 	 */
762 	mr = srq->srq_mrhdl;
763 	mutex_enter(&mr->mr_lock);
764 
765 	/*
766 	 * now, setup the srqc information needed for resize - limit the
767 	 * values, but use the same structure as the srqc
768 	 */
769 
770 	srqc_entry.log_srq_size	  = log_srq_size;
771 	srqc_entry.page_offs	  = srq_pgoffs >> 6;
772 	srqc_entry.log2_pgsz	  = mr->mr_log2_pgsz;
773 	srqc_entry.mtt_base_addrl = (uint64_t)mtt_addr >> 32;
774 	srqc_entry.mtt_base_addrh = mtt_addr >> 3;
775 
776 	/*
777 	 * RESIZE_SRQ
778 	 *
779 	 * If this fails for any reason, then it is an indication that
780 	 * something (either in HW or SW) has gone seriously wrong.  So we
781 	 * print a warning message and return.
782 	 */
783 	status = hermon_resize_srq_cmd_post(state, &srqc_entry,
784 	    srq->srq_srqnum, sleepflag);
785 	if (status != HERMON_CMD_SUCCESS) {
786 		cmn_err(CE_CONT, "Hermon: RESIZE_SRQ command failed: %08x\n",
787 		    status);
788 		if (status == HERMON_CMD_INVALID_STATUS) {
789 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
790 		}
791 		(void) hermon_mr_mtt_unbind(state, &bind, mtt);
792 		kmem_free(wre_new, (1 << log_srq_size) *
793 		    sizeof (uint64_t));
794 		hermon_queue_free(&new_srqinfo);
795 		mutex_exit(&mr->mr_lock);
796 		mutex_exit(&srq->srq_lock);
797 		return (ibc_get_ci_failure(0));
798 	}
799 	/*
800 	 * Update the Hermon Shared Receive Queue handle with all the new
801 	 * information.  At the same time, save away all the necessary
802 	 * information for freeing up the old resources
803 	 */
804 	old_srqinfo	   = srq->srq_wqinfo;
805 	old_mtt		   = srq->srq_mrhdl->mr_mttrsrcp;
806 	bcopy(&srq->srq_mrhdl->mr_bindinfo, &old_bind,
807 	    sizeof (hermon_bind_info_t));
808 
809 	/* Now set the new info */
810 	srq->srq_wqinfo	   = new_srqinfo;
811 	srq->srq_wq_buf	   = buf;
812 	srq->srq_wq_bufsz  = (1 << log_srq_size);
813 	bcopy(&bind, &srq->srq_mrhdl->mr_bindinfo, sizeof (hermon_bind_info_t));
814 	srq->srq_mrhdl->mr_mttrsrcp = mtt;
815 	srq->srq_desc_off  = srq_desc_off;
816 	srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
817 
818 	/* Update MR mtt pagesize */
819 	mr->mr_logmttpgsz = mtt_pgsize_bits;
820 	mutex_exit(&mr->mr_lock);
821 
822 	/*
823 	 * Initialize new wridlist, if needed.
824 	 *
825 	 * If a wridlist already is setup on an SRQ (the QP associated with an
826 	 * SRQ has moved "from_reset") then we must update this wridlist based
827 	 * on the new SRQ size.  We allocate the new size of Work Request ID
828 	 * Entries, copy over the old entries to the new list, and
829 	 * re-initialize the srq wridlist in non-umap case
830 	 */
831 	wre_old = srq->srq_wq_wqhdr->wq_wrid;
832 
833 	bcopy(wre_old, wre_new, srq_old_bufsz * sizeof (uint64_t));
834 
835 	/* Setup new sizes in wre */
836 	srq->srq_wq_wqhdr->wq_wrid = wre_new;
837 
838 	/*
839 	 * If "old" SRQ was a user-mappable SRQ that is currently mmap()'d out
840 	 * to a user process, then we need to call devmap_devmem_remap() to
841 	 * invalidate the mapping to the SRQ memory.  We also need to
842 	 * invalidate the SRQ tracking information for the user mapping.
843 	 *
844 	 * Note: On failure, the remap really shouldn't ever happen.  So, if it
845 	 * does, it is an indication that something has gone seriously wrong.
846 	 * So we print a warning message and return error (knowing, of course,
847 	 * that the "old" SRQ memory will be leaked)
848 	 */
849 	if ((srq->srq_is_umap) && (srq->srq_umap_dhp != NULL)) {
850 		maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
851 		status = devmap_devmem_remap(srq->srq_umap_dhp,
852 		    state->hs_dip, 0, 0, srq->srq_wqinfo.qa_size, maxprot,
853 		    DEVMAP_MAPPING_INVALID, NULL);
854 		if (status != DDI_SUCCESS) {
855 			mutex_exit(&srq->srq_lock);
856 			HERMON_WARNING(state, "failed in SRQ memory "
857 			    "devmap_devmem_remap()");
858 			/* We can, however, free the memory for old wre */
859 			kmem_free(wre_old, srq_old_bufsz * sizeof (uint64_t));
860 			return (ibc_get_ci_failure(0));
861 		}
862 		srq->srq_umap_dhp = (devmap_cookie_t)NULL;
863 	}
864 
865 	/*
866 	 * Drop the SRQ lock now.  The only thing left to do is to free up
867 	 * the old resources.
868 	 */
869 	mutex_exit(&srq->srq_lock);
870 
871 	/*
872 	 * Unbind the MTT entries.
873 	 */
874 	status = hermon_mr_mtt_unbind(state, &old_bind, old_mtt);
875 	if (status != DDI_SUCCESS) {
876 		HERMON_WARNING(state, "failed to unbind old SRQ memory");
877 		status = ibc_get_ci_failure(0);
878 		goto srqmodify_fail;
879 	}
880 
881 	/* Free the memory for old wre */
882 	kmem_free(wre_old, srq_old_bufsz * sizeof (uint64_t));
883 
884 	/* Free the memory for the old SRQ */
885 	hermon_queue_free(&old_srqinfo);
886 
887 	/*
888 	 * Fill in the return arguments (if necessary).  This includes the
889 	 * real new completion queue size.
890 	 */
891 	if (real_size != NULL) {
892 		*real_size = (1 << log_srq_size);
893 	}
894 
895 	return (DDI_SUCCESS);
896 
897 srqmodify_fail:
898 	return (status);
899 }
900 
901 
902 /*
903  * hermon_srq_refcnt_inc()
904  *    Context: Can be called from interrupt or base context.
905  */
906 void
hermon_srq_refcnt_inc(hermon_srqhdl_t srq)907 hermon_srq_refcnt_inc(hermon_srqhdl_t srq)
908 {
909 	mutex_enter(&srq->srq_lock);
910 	srq->srq_refcnt++;
911 	mutex_exit(&srq->srq_lock);
912 }
913 
914 
915 /*
916  * hermon_srq_refcnt_dec()
917  *    Context: Can be called from interrupt or base context.
918  */
919 void
hermon_srq_refcnt_dec(hermon_srqhdl_t srq)920 hermon_srq_refcnt_dec(hermon_srqhdl_t srq)
921 {
922 	mutex_enter(&srq->srq_lock);
923 	srq->srq_refcnt--;
924 	mutex_exit(&srq->srq_lock);
925 }
926 
927 
928 /*
929  * hermon_srqhdl_from_srqnum()
930  *    Context: Can be called from interrupt or base context.
931  *
932  *    This routine is important because changing the unconstrained
933  *    portion of the SRQ number is critical to the detection of a
934  *    potential race condition in the SRQ handler code (i.e. the case
935  *    where a SRQ is freed and alloc'd again before an event for the
936  *    "old" SRQ can be handled).
937  *
938  *    While this is not a perfect solution (not sure that one exists)
939  *    it does help to mitigate the chance that this race condition will
940  *    cause us to deliver a "stale" event to the new SRQ owner.  Note:
941  *    this solution does not scale well because the number of constrained
942  *    bits increases (and, hence, the number of unconstrained bits
943  *    decreases) as the number of supported SRQ grows.  For small and
944  *    intermediate values, it should hopefully provide sufficient
945  *    protection.
946  */
947 hermon_srqhdl_t
hermon_srqhdl_from_srqnum(hermon_state_t * state,uint_t srqnum)948 hermon_srqhdl_from_srqnum(hermon_state_t *state, uint_t srqnum)
949 {
950 	uint_t	srqindx, srqmask;
951 
952 	/* Calculate the SRQ table index from the srqnum */
953 	srqmask = (1 << state->hs_cfg_profile->cp_log_num_srq) - 1;
954 	srqindx = srqnum & srqmask;
955 	return (hermon_icm_num_to_hdl(state, HERMON_SRQC, srqindx));
956 }
957 
958 
959 /*
960  * hermon_srq_sgl_to_logwqesz()
961  *    Context: Can be called from interrupt or base context.
962  */
963 static void
hermon_srq_sgl_to_logwqesz(hermon_state_t * state,uint_t num_sgl,hermon_qp_wq_type_t wq_type,uint_t * logwqesz,uint_t * max_sgl)964 hermon_srq_sgl_to_logwqesz(hermon_state_t *state, uint_t num_sgl,
965     hermon_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl)
966 {
967 	uint_t	max_size, log2, actual_sgl;
968 
969 	switch (wq_type) {
970 	case HERMON_QP_WQ_TYPE_RECVQ:
971 		/*
972 		 * Use requested maximum SGL to calculate max descriptor size
973 		 * (while guaranteeing that the descriptor size is a
974 		 * power-of-2 cachelines).
975 		 */
976 		max_size = (HERMON_QP_WQE_MLX_SRQ_HDRS + (num_sgl << 4));
977 		log2 = highbit(max_size);
978 		if (ISP2(max_size)) {
979 			log2 = log2 - 1;
980 		}
981 
982 		/* Make sure descriptor is at least the minimum size */
983 		log2 = max(log2, HERMON_QP_WQE_LOG_MINIMUM);
984 
985 		/* Calculate actual number of SGL (given WQE size) */
986 		actual_sgl = ((1 << log2) - HERMON_QP_WQE_MLX_SRQ_HDRS) >> 4;
987 		break;
988 
989 	default:
990 		HERMON_WARNING(state, "unexpected work queue type");
991 		break;
992 	}
993 
994 	/* Fill in the return values */
995 	*logwqesz = log2;
996 	*max_sgl  = min(state->hs_cfg_profile->cp_srq_max_sgl, actual_sgl);
997 }
998