1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26
27 /*
28 * NAME: sol_uverbs_comp.c
29 *
30 * OFED User Verbs Kernel completion queue/processing implementation
31 *
32 */
33 #include <sys/file.h>
34 #include <sys/vfs.h>
35 #include <sys/vnode.h>
36 #include <sys/errno.h>
37 #include <sys/cred.h>
38 #include <sys/uio.h>
39 #include <sys/semaphore.h>
40 #include <sys/ddi.h>
41
42 #include <sys/ib/clients/of/ofa_solaris.h>
43 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
44 #include <sys/ib/ibtl/ibvti.h>
45 #include <sys/ib/clients/of/ofed_kernel.h>
46 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_comp.h>
48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>
49
50 extern char *sol_uverbs_dbg_str;
51
52 /*
53 * Function:
54 * uverbs_convert_wc
55 * Input:
56 * uctxt - Pointer to the callers user context.
57 * ibt_wc - Pointer to IBT work completion.
58 * Output:
59 * ofed_wc - Pointer to hold converted OFED work completion.
60 * Returns:
61 * None
62 * Description:
63 * Convert and IBT work completion to an OFED work completion.
64 */
65 /* ARGSUSED */
66 static void
uverbs_convert_wc(uverbs_uctxt_uobj_t * uctxt,ibt_wc_t * ibt_wc,struct ib_uverbs_wc * ofed_wc)67 uverbs_convert_wc(uverbs_uctxt_uobj_t *uctxt, ibt_wc_t *ibt_wc,
68 struct ib_uverbs_wc *ofed_wc)
69 {
70 ASSERT(uctxt != NULL);
71 ASSERT(ibt_wc != NULL);
72 ASSERT(ofed_wc != NULL);
73
74 ofed_wc->wr_id = ibt_wc->wc_id;
75
76 switch (ibt_wc->wc_status) {
77
78 case IBT_WC_SUCCESS:
79 ofed_wc->status = IB_WC_SUCCESS;
80 break;
81 case IBT_WC_LOCAL_LEN_ERR:
82 ofed_wc->status = IB_WC_LOC_LEN_ERR;
83 break;
84 case IBT_WC_LOCAL_CHAN_OP_ERR:
85 ofed_wc->status = IB_WC_LOC_QP_OP_ERR;
86 break;
87 case IBT_WC_LOCAL_PROTECT_ERR:
88 ofed_wc->status = IB_WC_LOC_PROT_ERR;
89 break;
90 case IBT_WC_WR_FLUSHED_ERR:
91 ofed_wc->status = IB_WC_WR_FLUSH_ERR;
92 break;
93 case IBT_WC_MEM_WIN_BIND_ERR:
94 ofed_wc->status = IB_WC_MW_BIND_ERR;
95 break;
96 case IBT_WC_BAD_RESPONSE_ERR:
97 ofed_wc->status = IB_WC_BAD_RESP_ERR;
98 break;
99 case IBT_WC_LOCAL_ACCESS_ERR:
100 ofed_wc->status = IB_WC_LOC_ACCESS_ERR;
101 break;
102 case IBT_WC_REMOTE_INVALID_REQ_ERR:
103 ofed_wc->status = IB_WC_REM_INV_REQ_ERR;
104 break;
105 case IBT_WC_REMOTE_ACCESS_ERR:
106 ofed_wc->status = IB_WC_REM_ACCESS_ERR;
107 break;
108 case IBT_WC_REMOTE_OP_ERR:
109 ofed_wc->status = IB_WC_REM_OP_ERR;
110 break;
111 case IBT_WC_TRANS_TIMEOUT_ERR:
112 case IBT_WC_RNR_NAK_TIMEOUT_ERR:
113 ofed_wc->status = IB_WC_RESP_TIMEOUT_ERR;
114 break;
115 default:
116 ofed_wc->status = IB_WC_FATAL_ERR;
117 break;
118 }
119
120 switch (ibt_wc->wc_type) {
121
122 case IBT_WRC_SEND:
123 ofed_wc->opcode = IB_WC_SEND;
124 break;
125 case IBT_WRC_RDMAR:
126 ofed_wc->opcode = IB_WC_RDMA_READ;
127 break;
128 case IBT_WRC_RDMAW:
129 ofed_wc->opcode = IB_WC_RDMA_WRITE;
130 break;
131 case IBT_WRC_CSWAP:
132 ofed_wc->opcode = IB_WC_COMP_SWAP;
133 break;
134 case IBT_WRC_FADD:
135 ofed_wc->opcode = IB_WC_FETCH_ADD;
136 break;
137 case IBT_WRC_BIND:
138 ofed_wc->opcode = IB_WC_BIND_MW;
139 break;
140 case IBT_WRC_RECV:
141 ofed_wc->opcode = IB_WC_RECV;
142 break;
143 case IBT_WRC_RECV_RDMAWI:
144 ofed_wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
145 break;
146
147 case IBT_WRC_FAST_REG_PMR:
148 case IBT_WRC_LOCAL_INVALIDATE:
149 default:
150 ofed_wc->opcode = -1; /* (?) What to do here */
151 break;
152 }
153
154 ofed_wc->vendor_err = 0;
155 ofed_wc->byte_len = ibt_wc->wc_bytes_xfer;
156 ofed_wc->imm_data = ibt_wc->wc_immed_data;
157 ofed_wc->qp_num = ibt_wc->wc_local_qpn;
158 ofed_wc->src_qp = ibt_wc->wc_qpn;
159 ofed_wc->wc_flags = 0;
160
161 if (ibt_wc->wc_flags & IBT_WC_GRH_PRESENT) {
162 ofed_wc->wc_flags |= IB_WC_GRH;
163 }
164
165 if (ibt_wc->wc_flags & IBT_WC_IMMED_DATA_PRESENT) {
166 ofed_wc->wc_flags |= IB_WC_WITH_IMM;
167 }
168
169 ofed_wc->pkey_index = ibt_wc->wc_pkey_ix;
170 ofed_wc->slid = ibt_wc->wc_slid;
171 ofed_wc->sl = ibt_wc->wc_sl;
172 ofed_wc->dlid_path_bits = ibt_wc->wc_path_bits;
173 ofed_wc->port_num = 0;
174 ofed_wc->reserved = 0;
175 }
176
177 /*
178 * Function:
179 * sol_uverbs_create_cq
180 * Input:
181 * uctxt - Pointer to the callers user context.
182 * buf - Pointer to kernel buffer containing create CQ command.
183 * in_len - Length in bytes of input command buffer.
184 * out_len - Length in bytes of output response buffer.
185 * Output:
186 * The command output buffer is updated with command results.
187 * Returns:
188 * DDI_SUCCESS on success, else error code.
189 * Description:
190 * User verbs entry point to create a device CQ.
191 */
192 /* ARGSUSED */
193 int
sol_uverbs_create_cq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)194 sol_uverbs_create_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
195 int in_len, int out_len)
196 {
197 struct ib_uverbs_create_cq cmd;
198 struct ib_uverbs_create_cq_resp resp;
199 uverbs_ucq_uobj_t *ucq;
200 ibt_cq_attr_t cq_attr;
201 uint_t real_size;
202 int rc;
203
204 (void) memcpy(&cmd, buf, sizeof (cmd));
205 (void) memset(&resp, 0, sizeof (resp));
206 (void) memset(&cq_attr, 0, sizeof (cq_attr));
207
208 cq_attr.cq_size = cmd.cqe;
209 cq_attr.cq_sched = 0;
210 cq_attr.cq_flags = IBT_CQ_USER_MAP;
211
212 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: "
213 "num_cqe=%d, sched=%x, flags=%x",
214 cq_attr.cq_size, cq_attr.cq_sched, cq_attr.cq_flags);
215
216 /*
217 * To be consistent with OFED semantics, we fail a CQ that is being
218 * created with 0 CQ entries.
219 */
220 if (!cmd.cqe) {
221 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "create_cq: 0 cqe");
222 rc = EINVAL;
223 goto err_out;
224 }
225
226 ucq = kmem_zalloc(sizeof (*ucq), KM_NOSLEEP);
227 if (!ucq) {
228 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
229 "create_cq: mem alloc failure");
230 rc = ENOMEM;
231 goto err_out;
232 }
233 sol_ofs_uobj_init(&ucq->uobj, cmd.user_handle,
234 SOL_UVERBS_UCQ_UOBJ_TYPE);
235 rw_enter(&ucq->uobj.uo_lock, RW_WRITER);
236 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
237 "create_cq: ucq %p, comp_chan %d", ucq, cmd.comp_channel);
238
239 /*
240 * If a event completion channel was specified look it up and
241 * assign the channel to the user CQ object. Note that this
242 * places a reference on the file itself.
243 */
244 if ((int)cmd.comp_channel > SOL_UVERBS_DRIVER_MAX_MINOR) {
245 uverbs_uctxt_uobj_t *compl_uctxt = NULL;
246 uverbs_ufile_uobj_t *ufile;
247
248 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: "
249 "cmd.comp_chan %d", cmd.comp_channel);
250 compl_uctxt = uverbs_uobj_get_uctxt_write(
251 cmd.comp_channel - SOL_UVERBS_DRIVER_MAX_MINOR);
252 if (!compl_uctxt) {
253 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
254 "create_cq: Invalid comp channel %d",
255 cmd.comp_channel);
256 rc = EINVAL;
257 goto chan_err;
258 }
259 if (compl_uctxt->uctxt_verbs_id != uctxt->uobj.uo_id +
260 SOL_UVERBS_DRIVER_MAX_MINOR) {
261 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
262 "create_cq: Invalid comp channel %d, "
263 "verbs id %d mismatch",
264 cmd.comp_channel,
265 compl_uctxt->uctxt_verbs_id);
266 rc = EINVAL;
267 sol_ofs_uobj_put(&compl_uctxt->uobj);
268 goto chan_err;
269 }
270 ufile = compl_uctxt->comp_evfile;
271 ucq->comp_chan = ufile;
272 rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
273 ufile->ufile_cq_cnt++;
274 rw_exit(&ufile->uobj.uo_lock);
275 sol_ofs_uobj_put(&compl_uctxt->uobj);
276
277 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: "
278 "ucq %p, comp_chan %p", ucq, ucq->comp_chan);
279 } else {
280 ucq->comp_chan = NULL;
281 }
282
283 llist_head_init(&ucq->async_list, NULL);
284 llist_head_init(&ucq->comp_list, NULL);
285
286 rc = ibt_alloc_cq(uctxt->hca->hdl, &cq_attr, &ucq->cq, &real_size);
287
288 if (rc != IBT_SUCCESS) {
289 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
290 "create_cq: ibt_alloc_cq() (rc=%d)", rc);
291 rc = sol_uverbs_ibt_to_kernel_status(rc);
292 ucq->uobj.uo_uobj_sz = sizeof (uverbs_ucq_uobj_t);
293 goto alloc_err;
294 }
295
296 ibt_set_cq_private(ucq->cq, ucq);
297
298 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
299 "create_cq: ib_alloc_cq() (rc=%d), real_size=%d",
300 rc, real_size);
301 /*
302 * Query underlying hardware for data used in mapping CQ back to user
303 * space, we will return this information in the user verbs command
304 * response.
305 */
306 rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_CQ,
307 (void *) ucq->cq, &resp.drv_out, sizeof (resp.drv_out));
308 if (rc != IBT_SUCCESS) {
309 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
310 "create_cq: ibt_ci_data_out() rc=%d", rc);
311 rc = EFAULT;
312 ucq->uobj.uo_uobj_sz = sizeof (uverbs_ucq_uobj_t);
313 goto err_cq_destroy;
314 }
315
316 if (sol_ofs_uobj_add(&uverbs_ucq_uo_tbl, &ucq->uobj) != 0) {
317 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
318 "create_cq: User object add failed");
319 rc = ENOMEM;
320 goto err_cq_destroy;
321 }
322
323 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
324 "create_cq: ibt_ci_data_out: 0x%16llx 0x%16llx "
325 "0x%16llx 0x%16llx", resp.drv_out[0], resp.drv_out[1],
326 resp.drv_out[2], resp.drv_out[3]);
327
328 resp.cqe = real_size;
329 resp.cq_handle = ucq->uobj.uo_id;
330
331 #ifdef _LP64
332 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
333 #else
334 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
335 #endif
336 if (rc != 0) {
337 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
338 "create_cq: copyout failed %x", rc);
339 rc = EFAULT;
340 goto err_uo_delete;
341 }
342
343 ucq->uctxt = uctxt;
344
345 mutex_enter(&uctxt->lock);
346 ucq->list_entry = add_genlist(&uctxt->cq_list, (uintptr_t)ucq, uctxt);
347 mutex_exit(&uctxt->lock);
348
349 if (!ucq->list_entry) {
350 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
351 "create_cq: Error adding ucq to cq_list");
352 rc = ENOMEM;
353 goto err_uo_delete;
354 }
355
356 if (ucq->comp_chan) {
357 ibt_set_cq_handler(ucq->cq, sol_uverbs_comp_event_handler, ucq);
358 }
359
360 ucq->uobj.uo_live = 1;
361 rw_exit(&ucq->uobj.uo_lock);
362
363 return (DDI_SUCCESS);
364
365 err_uo_delete:
366 /*
367 * Need to set uo_live, so sol_ofs_uobj_remove() will
368 * remove the object from the object table.
369 */
370 ucq->uobj.uo_live = 1;
371 (void) sol_ofs_uobj_remove(&uverbs_ucq_uo_tbl, &ucq->uobj);
372
373 err_cq_destroy:
374 (void) ibt_free_cq(ucq->cq);
375
376 alloc_err:
377 if (ucq->comp_chan) {
378 uverbs_release_ucq_channel(uctxt, ucq->comp_chan, ucq);
379 }
380
381 chan_err:
382 rw_exit(&ucq->uobj.uo_lock);
383 sol_ofs_uobj_deref(&ucq->uobj, sol_ofs_uobj_free);
384
385 err_out:
386 return (rc);
387 }
388
389 int
uverbs_ucq_free(uverbs_ucq_uobj_t * ucq,uverbs_uctxt_uobj_t * uctxt)390 uverbs_ucq_free(uverbs_ucq_uobj_t *ucq, uverbs_uctxt_uobj_t *uctxt)
391 {
392 int rc;
393
394 rc = ibt_free_cq(ucq->cq);
395 if (rc != IBT_SUCCESS) {
396 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
397 "destroy_id: ibt_free_cq() rc=%d",
398 rc);
399 rc = sol_uverbs_ibt_to_kernel_status(rc);
400 sol_ofs_uobj_put(&ucq->uobj);
401 return (rc);
402 }
403 (void) sol_ofs_uobj_remove(&uverbs_ucq_uo_tbl, &ucq->uobj);
404 sol_ofs_uobj_put(&ucq->uobj);
405
406 if (ucq->list_entry) {
407 mutex_enter(&uctxt->lock);
408 delete_genlist(&uctxt->cq_list, ucq->list_entry);
409 mutex_exit(&uctxt->lock);
410 }
411 sol_ofs_uobj_deref(&ucq->uobj, sol_ofs_uobj_free);
412 return (0);
413 }
414
415 /*
416 * Function:
417 * sol_uverbs_destroy_cq
418 * Input:
419 * uctxt - Pointer to the callers user context.
420 * buf - Pointer to kernel buffer containing a destroy CQ command.
421 * in_len - Length in bytes of input command buffer.
422 * out_len - Length in bytes of output response buffer.
423 * Output:
424 * The command output buffer is updated with command results.
425 * Returns:
426 * DDI_SUCCESS on success, else error code.
427 * Description:
428 * User verbs entry point to destroy a device CQ.
429 */
430 /* ARGSUSED */
431 int
sol_uverbs_destroy_cq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)432 sol_uverbs_destroy_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
433 int in_len, int out_len)
434 {
435 struct ib_uverbs_destroy_cq cmd;
436 struct ib_uverbs_destroy_cq_resp resp;
437 uverbs_ucq_uobj_t *ucq;
438 int rc;
439
440 (void) memcpy(&cmd, buf, sizeof (cmd));
441 (void) memset(&resp, 0, sizeof (resp));
442
443 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
444 "destroy_cq(cq_handle=%d)", cmd.cq_handle);
445
446 ucq = uverbs_uobj_get_ucq_write(cmd.cq_handle);
447 if (ucq == NULL) {
448 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
449 "destroy_cq: Invalid handle: %d",
450 cmd.cq_handle);
451 rc = EINVAL;
452 goto err_out;
453 }
454
455 uverbs_release_ucq_channel(uctxt, ucq->comp_chan, ucq);
456 cmd.cq_handle = 0;
457 resp.comp_events_reported = ucq->comp_events_reported;
458 resp.async_events_reported = ucq->async_events_reported;
459
460 if (ucq->active_qp_cnt) {
461 sol_ofs_uobj_put(&ucq->uobj);
462 return (EBUSY);
463 } else {
464 rc = uverbs_ucq_free(ucq, uctxt);
465 if (rc)
466 goto err_out;
467 }
468
469 #ifdef _LP64
470 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
471 #else
472 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
473 #endif
474 if (rc != 0) {
475 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
476 "destroy_cq: copuout failed %x", rc);
477 rc = EFAULT;
478 goto err_out;
479 }
480
481 return (DDI_SUCCESS);
482
483 err_out:
484 return (rc);
485 }
486
487 /*
488 * Function:
489 * sol_uverbs_resize_cq
490 * Input:
491 * uctxt - Pointer to the callers user context.
492 * buf - Pointer to kernel buffer containing a resize CQ command.
493 * in_len - Length in bytes of input command buffer.
494 * out_len - Length in bytes of output response buffer.
495 * Output:
496 * The command output buffer is updated with command results.
497 * Returns:
498 * DDI_SUCCESS on success, else error code.
499 * Description:
500 * User verbs entry point to resize a device CQ.
501 */
502 /* ARGSUSED */
503 int
sol_uverbs_resize_cq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)504 sol_uverbs_resize_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
505 int in_len, int out_len)
506 {
507 struct ib_uverbs_resize_cq cmd;
508 struct ib_uverbs_resize_cq_resp resp;
509 uverbs_ucq_uobj_t *ucq;
510 int rc;
511 int resize_status;
512
513 (void) memcpy(&cmd, buf, sizeof (cmd));
514 (void) memset(&resp, 0, sizeof (resp));
515
516 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
517 "resize_cq(cq_handle=%d)", cmd.cq_handle);
518
519 ucq = uverbs_uobj_get_ucq_write(cmd.cq_handle);
520 if (ucq == NULL) {
521 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
522 "resize_cq: Invalid handle: %d",
523 cmd.cq_handle);
524 rc = EINVAL;
525 goto err_out;
526 }
527
528 /*
529 * If CQ resize fails, note the error but keep going. In this case we
530 * expect the old CQ size to be returned, and when we extract the
531 * mapping data, we expect the offset and length are approrpriate for
532 * the old CQ.
533 */
534 resize_status = ibt_resize_cq(ucq->cq, cmd.cqe, &resp.cqe);
535 if (resize_status != IBT_SUCCESS) {
536 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
537 "resize_cq: ibt_resize_cq() (resize_status=%d), using "
538 "original CQ", resize_status);
539 rc = sol_uverbs_ibt_to_kernel_status(resize_status);
540 goto err_out;
541 }
542
543 sol_ofs_uobj_put(&ucq->uobj);
544
545 rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_CQ,
546 (void *) ucq->cq, &resp.drv_out, sizeof (resp.drv_out));
547 if (rc != IBT_SUCCESS) {
548 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
549 "resize_cq: Error in ibt_ci_data_out() "
550 "(rc=%d)", rc);
551 rc = EFAULT;
552 goto err_out;
553 }
554
555 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
556 "resize_cq: ibt_ci_data_out: 0x%16llx 0x%16llx "
557 "0x%16llx 0x%16llx", resp.drv_out[0], resp.drv_out[1],
558 resp.drv_out[2], resp.drv_out[3]);
559
560 #ifdef _LP64
561 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
562 #else
563 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
564 #endif
565 if (rc != 0) {
566 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
567 "resize_cq: copyout %d", rc);
568 rc = EFAULT;
569 goto err_out;
570 }
571
572 return (resize_status);
573
574 err_out:
575 return (rc);
576 }
577
578 /*
579 * Function:
580 * sol_uverbs_req_notify_cq
581 * Input:
582 * uctxt - Pointer to the callers user context.
583 * buf - Pointer to kernel buffer containing request notify
584 * command.
585 * in_len - Length in bytes of input command buffer.
586 * out_len - Length in bytes of output response buffer.
587 * Output:
588 * The command output buffer is updated with command results.
589 * Returns:
590 * DDI_SUCCESS on success, else error code.
591 * Description:
592 * User verbs entry point to request notification of CQ events.
593 */
594 /* ARGSUSED */
595 int
sol_uverbs_req_notify_cq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)596 sol_uverbs_req_notify_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
597 int in_len, int out_len)
598 {
599 struct ib_uverbs_req_notify_cq cmd;
600 ibt_cq_notify_flags_t flag;
601 uverbs_ucq_uobj_t *ucq;
602 int rc;
603
604 (void) memcpy(&cmd, buf, sizeof (cmd));
605
606 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
607 "req_notify_cq(cq_handle=%d)", cmd.cq_handle);
608
609 ucq = uverbs_uobj_get_ucq_read(cmd.cq_handle);
610 if (ucq == NULL) {
611 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
612 "req_notify_cq: List lookup failure");
613 rc = EINVAL;
614 goto err_out;
615 }
616
617 flag = IBT_NEXT_COMPLETION;
618
619 if (cmd.solicited_only != 0) {
620 flag = IBT_NEXT_SOLICITED;
621 }
622
623 rc = ibt_enable_cq_notify(ucq->cq, flag);
624 if (rc != IBT_SUCCESS) {
625 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
626 "req_notify_cq: ibt_enable_cq_notify() "
627 "(rc=%d)", rc);
628 rc = sol_uverbs_ibt_to_kernel_status(rc);
629 goto err_put;
630 }
631
632 sol_ofs_uobj_put(&ucq->uobj);
633 return (DDI_SUCCESS);
634
635 err_put:
636 sol_ofs_uobj_put(&ucq->uobj);
637
638 err_out:
639 return (rc);
640 }
641
642 /*
643 * Function:
644 * sol_uverbs_poll_cq
645 * Input:
646 * uctxt - Pointer to the callers user context.
647 * buf - Pointer to kernel buffer containing poll CQ command.
648 * in_len - Length in bytes of input command buffer.
649 * out_len - Length in bytes of output response buffer.
650 * Output:
651 * The command output buffer is updated with command results.
652 * Returns:
653 * DDI_SUCCESS on success, else error code.
654 * Description:
655 * User verbs entry point to poll a CQ for completion events. Note that
656 * this is a non OS Bypass version, the CQ normally would be polled
657 * directly from the user space driver.
658 */
659 /* ARGSUSED */
660 int
sol_uverbs_poll_cq(uverbs_uctxt_uobj_t * uctxt,char * buf,int in_len,int out_len)661 sol_uverbs_poll_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
662 int in_len, int out_len)
663 {
664 struct ib_uverbs_poll_cq cmd;
665 struct ib_uverbs_poll_cq_resp resp;
666 uverbs_ucq_uobj_t *ucq;
667 ibt_wc_t *completions;
668 struct ib_uverbs_wc ofed_wc;
669 int rc;
670 int i;
671
672 (void) memcpy(&cmd, buf, sizeof (cmd));
673 (void) memset(&resp, 0, sizeof (resp));
674
675 #ifdef DEBUG
676 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
677 "poll_cq(cq_handle=%d)", cmd.cq_handle);
678 #endif
679
680 ucq = uverbs_uobj_get_ucq_read(cmd.cq_handle);
681 if (ucq == NULL) {
682 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
683 "poll_cq: List lookup failure");
684 rc = EINVAL;
685 goto err_find;
686 }
687
688 completions = (ibt_wc_t *)kmem_zalloc(sizeof (ibt_wc_t) * cmd.ne,
689 KM_NOSLEEP);
690 if (completions == NULL) {
691 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
692 "poll_cq: Allocation Error");
693 rc = ENOMEM;
694 goto err_alloc;
695 }
696
697 rc = ibt_poll_cq(ucq->cq, completions, cmd.ne, &resp.count);
698 if (rc != IBT_SUCCESS) {
699 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
700 "poll_cq: ibt_poll_cq() (rc=%d)", rc);
701 rc = sol_uverbs_ibt_to_kernel_status(rc);
702 goto err_poll;
703 }
704
705 #ifdef _LP64
706 rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
707 #else
708 rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
709 #endif
710 if (rc != 0) {
711 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
712 "poll_cq: copyout %x", rc);
713 rc = EFAULT;
714 goto err_poll;
715 }
716
717 for (i = 0; i < resp.count; i++) {
718 (void) memset(&ofed_wc, 0, sizeof (ofed_wc));
719 uverbs_convert_wc(uctxt, &completions[i], &ofed_wc);
720
721 #ifdef _LP64
722 rc = copyout((void*)&ofed_wc,
723 (void *)cmd.response.r_laddr, sizeof (ofed_wc));
724 #else
725 rc = copyout((void*)&ofed_wc,
726 (void *)cmd.response.r_addr, sizeof (ofed_wc));
727 #endif
728 if (rc != 0) {
729 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
730 "poll_cq: copyout wc data %x", rc);
731 rc = EFAULT;
732 goto err_poll;
733 }
734 }
735
736 kmem_free((void*)completions, sizeof (ibt_wc_t) * cmd.ne);
737 sol_ofs_uobj_put(&ucq->uobj);
738
739 return (DDI_SUCCESS);
740
741 err_poll:
742 kmem_free((void *)completions, sizeof (ibt_wc_t) * cmd.ne);
743
744 err_alloc:
745 sol_ofs_uobj_put(&ucq->uobj);
746
747 err_find:
748 return (rc);
749 }
750
751 /*
752 * Function:
753 * sol_uverbs_comp_event_handler
754 * Input:
755 * ibt_cq - Handle to the IBT CQ.
756 * arg - Address of the associated Solaris User Verbs CQ
757 * object.
758 * Output:
759 * None
760 * Returns:
761 * None
762 * Description:
763 * Solaris User Verbs completion channel IBT CQ callback. Queue
764 * the completion event and wakeup/notify consumers that may be
765 * blocked waiting for the completion.
766 */
767 /* ARGSUSED */
768 void
sol_uverbs_comp_event_handler(ibt_cq_hdl_t ibt_cq,void * arg)769 sol_uverbs_comp_event_handler(ibt_cq_hdl_t ibt_cq, void *arg)
770 {
771 uverbs_ucq_uobj_t *ucq = arg;
772 uverbs_ufile_uobj_t *ufile;
773 uverbs_event_t *entry;
774
775 if (!ucq || !ucq->comp_chan) {
776 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr "
777 "ucq %p ucq->comp_chan %p", ucq, (ucq) ? ucq->comp_chan :
778 NULL);
779 return;
780 }
781
782 #ifdef DEBUG
783 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "comp_evt_hdlr(%p, %p) - ",
784 "ucq = %p, ucq->cq = %p, ucq->uctxt = %p, ucq->comp_chan =%p",
785 ibt_cq, arg, ucq, ucq->cq, ucq->uctxt, ucq->comp_chan);
786 #endif
787
788 ufile = ucq->comp_chan;
789
790 mutex_enter(&ufile->lock);
791 if (!ufile->uctxt) {
792 mutex_exit(&ufile->lock);
793 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr "
794 "ufile->uctxt %p", ufile->uctxt);
795 return;
796 }
797
798 entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
799 if (!entry) {
800 mutex_exit(&ufile->lock);
801 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr() "
802 "memory allocation error");
803 return;
804 }
805
806 entry->ev_desc.comp.cq_handle = ucq->uobj.uo_user_handle;
807 entry->ev_counter = &ucq->comp_events_reported;
808
809 /*
810 * Add to list of entries associated with completion channel
811 * and the list associated with the specific CQ.
812 */
813 llist_head_init(&entry->ev_list, entry);
814 llist_head_init(&entry->ev_obj_list, entry);
815
816 #ifdef DEBUG
817 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "comp_evt_hdlr() "
818 "Add COMP entry->ev_list=%p, &entry->ev_obj_list, entry=%p",
819 &entry->ev_list, &entry->ev_obj_list, entry);
820 #endif
821
822 llist_add_tail(&entry->ev_list, &ufile->event_list);
823 llist_add_tail(&entry->ev_obj_list, &ucq->comp_list);
824
825 /* Do not notify users if sol_ucma has disabled CQ notify */
826 if (ufile->ufile_notify_enabled ==
827 SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE) {
828 mutex_exit(&ufile->lock);
829 return;
830 }
831
832 cv_signal(&ufile->poll_wait);
833 mutex_exit(&ufile->lock);
834 pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM);
835 }
836