1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_qpmod.c
29 * Tavor Queue Pair Modify Routines
30 *
31 * This contains all the routines necessary to implement the Tavor
32 * ModifyQP() verb. This includes all the code for legal transitions to
33 * and from Reset, Init, RTR, RTS, SQD, SQErr, and Error.
34 */
35
36 #include <sys/sysmacros.h>
37 #include <sys/types.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/bitmap.h>
43
44 #include <sys/ib/adapters/tavor/tavor.h>
45 #include <sys/ib/ib_pkt_hdrs.h>
46
47 static int tavor_qp_reset2init(tavor_state_t *state, tavor_qphdl_t qp,
48 ibt_qp_info_t *info_p);
49 static int tavor_qp_init2init(tavor_state_t *state, tavor_qphdl_t qp,
50 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
51 static int tavor_qp_init2rtr(tavor_state_t *state, tavor_qphdl_t qp,
52 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
53 static int tavor_qp_rtr2rts(tavor_state_t *state, tavor_qphdl_t qp,
54 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
55 static int tavor_qp_rts2rts(tavor_state_t *state, tavor_qphdl_t qp,
56 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
57 static int tavor_qp_rts2sqd(tavor_state_t *state, tavor_qphdl_t qp,
58 ibt_cep_modify_flags_t flags);
59 static int tavor_qp_sqd2rts(tavor_state_t *state, tavor_qphdl_t qp,
60 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
61 static int tavor_qp_sqd2sqd(tavor_state_t *state, tavor_qphdl_t qp,
62 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
63 static int tavor_qp_sqerr2rts(tavor_state_t *state, tavor_qphdl_t qp,
64 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
65 static int tavor_qp_to_error(tavor_state_t *state, tavor_qphdl_t qp);
66 static int tavor_qp_reset2err(tavor_state_t *state, tavor_qphdl_t qp);
67
68 static uint_t tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,
69 ibt_qp_info_t *info_p, tavor_hw_qpc_t *qpc);
70 static int tavor_qp_validate_resp_rsrc(tavor_state_t *state,
71 ibt_qp_rc_attr_t *rc, uint_t *rra_max);
72 static int tavor_qp_validate_init_depth(tavor_state_t *state,
73 ibt_qp_rc_attr_t *rc, uint_t *sra_max);
74 static int tavor_qp_validate_mtu(tavor_state_t *state, uint_t mtu);
75
76 /*
77 * tavor_qp_modify()
78 * Context: Can be called from interrupt or base context.
79 */
80 /* ARGSUSED */
81 int
tavor_qp_modify(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p,ibt_queue_sizes_t * actual_sz)82 tavor_qp_modify(tavor_state_t *state, tavor_qphdl_t qp,
83 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p,
84 ibt_queue_sizes_t *actual_sz)
85 {
86 ibt_cep_state_t cur_state, mod_state;
87 ibt_cep_modify_flags_t okflags;
88 int status;
89
90 /*
91 * Lock the QP so that we can modify it atomically. After grabbing
92 * the lock, get the current QP state. We will use this current QP
93 * state to determine the legal transitions (and the checks that need
94 * to be performed.)
95 * Below you will find a case for every possible QP state. In each
96 * case we check that no flags are set which are not valid for the
97 * possible transitions from that state. If these tests pass (and if
98 * the state transition we are attempting is legal), then we call
99 * one of the helper functions. Each of these functions does some
100 * additional setup before posting a Tavor firmware command for the
101 * appropriate state transition.
102 */
103 mutex_enter(&qp->qp_lock);
104
105 /*
106 * Verify that the transport type matches between the serv_type and the
107 * qp_trans. A caller to IBT must specify the qp_trans field as
108 * IBT_UD_SRV, IBT_RC_SRV, or IBT_UC_SRV, depending on the QP. We
109 * check here that the correct value was specified, based on our
110 * understanding of the QP serv type.
111 *
112 * Because callers specify part of a 'union' based on what QP type they
113 * think they're working with, this ensures that we do not pickup bogus
114 * data if the caller thought they were working with a different QP
115 * type.
116 */
117 if (!(TAVOR_QP_TYPE_VALID(info_p->qp_trans, qp->qp_serv_type))) {
118 mutex_exit(&qp->qp_lock);
119 return (IBT_QP_SRV_TYPE_INVALID);
120 }
121
122 /*
123 * If this is a transition to RTS (which is valid from RTR, RTS,
124 * SQError, and SQ Drain) then we should honor the "current QP state"
125 * specified by the consumer. This means converting the IBTF QP state
126 * in "info_p->qp_current_state" to a Tavor QP state. Otherwise, we
127 * assume that we already know the current state (i.e. whatever it was
128 * last modified to or queried as - in "qp->qp_state").
129 */
130 mod_state = info_p->qp_state;
131
132 if (flags & IBT_CEP_SET_RTR_RTS) {
133 cur_state = TAVOR_QP_RTR; /* Ready to Receive */
134
135 } else if ((flags & IBT_CEP_SET_STATE) &&
136 (mod_state == IBT_STATE_RTS)) {
137
138 /* Convert the current IBTF QP state to a Tavor QP state */
139 switch (info_p->qp_current_state) {
140 case IBT_STATE_RTR:
141 cur_state = TAVOR_QP_RTR; /* Ready to Receive */
142 break;
143 case IBT_STATE_RTS:
144 cur_state = TAVOR_QP_RTS; /* Ready to Send */
145 break;
146 case IBT_STATE_SQE:
147 cur_state = TAVOR_QP_SQERR; /* Send Queue Error */
148 break;
149 case IBT_STATE_SQD:
150 cur_state = TAVOR_QP_SQD; /* SQ Drained */
151 break;
152 default:
153 mutex_exit(&qp->qp_lock);
154 return (IBT_QP_STATE_INVALID);
155 }
156 } else {
157 cur_state = qp->qp_state;
158 }
159
160 switch (cur_state) {
161 case TAVOR_QP_RESET:
162 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RESET_INIT |
163 IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
164 IBT_CEP_SET_ATOMIC | IBT_CEP_SET_PKEY_IX |
165 IBT_CEP_SET_PORT | IBT_CEP_SET_QKEY);
166
167 /*
168 * Check for attempts to modify invalid attributes from the
169 * "Reset" state
170 */
171 if (flags & ~okflags) {
172 mutex_exit(&qp->qp_lock);
173 goto qpmod_fail;
174 }
175
176 /*
177 * Verify state transition is to either "Init", back to
178 * "Reset", or to "Error".
179 */
180 if ((flags & IBT_CEP_SET_RESET_INIT) &&
181 (flags & IBT_CEP_SET_STATE) &&
182 (mod_state != IBT_STATE_INIT)) {
183 /* Invalid transition - ambiguous flags */
184 mutex_exit(&qp->qp_lock);
185 goto qpmod_fail;
186
187 } else if ((flags & IBT_CEP_SET_RESET_INIT) ||
188 ((flags & IBT_CEP_SET_STATE) &&
189 (mod_state == IBT_STATE_INIT))) {
190 /*
191 * Attempt to transition from "Reset" to "Init"
192 */
193 status = tavor_qp_reset2init(state, qp, info_p);
194 if (status != DDI_SUCCESS) {
195 mutex_exit(&qp->qp_lock);
196 goto qpmod_fail;
197 }
198 qp->qp_state = TAVOR_QP_INIT;
199
200 } else if ((flags & IBT_CEP_SET_STATE) &&
201 (mod_state == IBT_STATE_RESET)) {
202 /*
203 * Attempt to transition from "Reset" back to "Reset"
204 * Nothing to do here really... just drop the lock
205 * and return success. The qp->qp_state should
206 * already be set to TAVOR_QP_RESET.
207 *
208 * Note: We return here because we do not want to fall
209 * through to the tavor_wrid_from_reset_handling()
210 * routine below (since we are not really moving
211 * _out_ of the "Reset" state.
212 */
213 mutex_exit(&qp->qp_lock);
214 return (DDI_SUCCESS);
215
216 } else if ((flags & IBT_CEP_SET_STATE) &&
217 (mod_state == IBT_STATE_ERROR)) {
218 /*
219 * Attempt to transition from "Reset" to "Error"
220 */
221 status = tavor_qp_reset2err(state, qp);
222 if (status != DDI_SUCCESS) {
223 mutex_exit(&qp->qp_lock);
224 goto qpmod_fail;
225 }
226 qp->qp_state = TAVOR_QP_ERR;
227
228 } else {
229 /* Invalid transition - return error */
230 mutex_exit(&qp->qp_lock);
231 goto qpmod_fail;
232 }
233
234 /*
235 * Do any additional handling necessary here for the transition
236 * from the "Reset" state (e.g. re-initialize the workQ WRID
237 * lists). Note: If tavor_wrid_from_reset_handling() fails,
238 * then we attempt to transition the QP back to the "Reset"
239 * state. If that fails, then it is an indication of a serious
240 * problem (either HW or SW). So we print out a warning
241 * message and return failure.
242 */
243 status = tavor_wrid_from_reset_handling(state, qp);
244 if (status != DDI_SUCCESS) {
245 if (tavor_qp_to_reset(state, qp) != DDI_SUCCESS) {
246 TAVOR_WARNING(state, "failed to reset QP");
247 }
248 qp->qp_state = TAVOR_QP_RESET;
249
250 mutex_exit(&qp->qp_lock);
251 goto qpmod_fail;
252 }
253 break;
254
255 case TAVOR_QP_INIT:
256 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_INIT_RTR |
257 IBT_CEP_SET_ADDS_VECT | IBT_CEP_SET_RDMARA_IN |
258 IBT_CEP_SET_MIN_RNR_NAK | IBT_CEP_SET_ALT_PATH |
259 IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
260 IBT_CEP_SET_ATOMIC | IBT_CEP_SET_PKEY_IX |
261 IBT_CEP_SET_QKEY | IBT_CEP_SET_PORT);
262
263 /*
264 * Check for attempts to modify invalid attributes from the
265 * "Init" state
266 */
267 if (flags & ~okflags) {
268 mutex_exit(&qp->qp_lock);
269 goto qpmod_fail;
270 }
271
272 /*
273 * Verify state transition is to either "RTR", back to "Init",
274 * to "Reset", or to "Error"
275 */
276 if ((flags & IBT_CEP_SET_INIT_RTR) &&
277 (flags & IBT_CEP_SET_STATE) &&
278 (mod_state != IBT_STATE_RTR)) {
279 /* Invalid transition - ambiguous flags */
280 mutex_exit(&qp->qp_lock);
281 goto qpmod_fail;
282
283 } else if ((flags & IBT_CEP_SET_INIT_RTR) ||
284 ((flags & IBT_CEP_SET_STATE) &&
285 (mod_state == IBT_STATE_RTR))) {
286 /*
287 * Attempt to transition from "Init" to "RTR"
288 */
289 status = tavor_qp_init2rtr(state, qp, flags, info_p);
290 if (status != DDI_SUCCESS) {
291 mutex_exit(&qp->qp_lock);
292 goto qpmod_fail;
293 }
294 qp->qp_state = TAVOR_QP_RTR;
295
296 } else if ((flags & IBT_CEP_SET_STATE) &&
297 (mod_state == IBT_STATE_INIT)) {
298 /*
299 * Attempt to transition from "Init" to "Init"
300 */
301 status = tavor_qp_init2init(state, qp, flags, info_p);
302 if (status != DDI_SUCCESS) {
303 mutex_exit(&qp->qp_lock);
304 goto qpmod_fail;
305 }
306 qp->qp_state = TAVOR_QP_INIT;
307
308 } else if ((flags & IBT_CEP_SET_STATE) &&
309 (mod_state == IBT_STATE_RESET)) {
310 /*
311 * Attempt to transition from "Init" to "Reset"
312 */
313 status = tavor_qp_to_reset(state, qp);
314 if (status != DDI_SUCCESS) {
315 mutex_exit(&qp->qp_lock);
316 goto qpmod_fail;
317 }
318 qp->qp_state = TAVOR_QP_RESET;
319
320 /*
321 * Do any additional handling necessary for the
322 * transition _to_ the "Reset" state (e.g. update the
323 * workQ WRID lists)
324 */
325 tavor_wrid_to_reset_handling(state, qp);
326
327 } else if ((flags & IBT_CEP_SET_STATE) &&
328 (mod_state == IBT_STATE_ERROR)) {
329 /*
330 * Attempt to transition from "Init" to "Error"
331 */
332 status = tavor_qp_to_error(state, qp);
333 if (status != DDI_SUCCESS) {
334 mutex_exit(&qp->qp_lock);
335 goto qpmod_fail;
336 }
337 qp->qp_state = TAVOR_QP_ERR;
338
339 } else {
340 /* Invalid transition - return error */
341 mutex_exit(&qp->qp_lock);
342 goto qpmod_fail;
343 }
344 break;
345
346 case TAVOR_QP_RTR:
347 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RTR_RTS |
348 IBT_CEP_SET_TIMEOUT | IBT_CEP_SET_RETRY |
349 IBT_CEP_SET_RNR_NAK_RETRY | IBT_CEP_SET_RDMARA_OUT |
350 IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
351 IBT_CEP_SET_ATOMIC | IBT_CEP_SET_QKEY |
352 IBT_CEP_SET_ALT_PATH | IBT_CEP_SET_MIG |
353 IBT_CEP_SET_MIN_RNR_NAK);
354
355 /*
356 * Check for attempts to modify invalid attributes from the
357 * "RTR" state
358 */
359 if (flags & ~okflags) {
360 mutex_exit(&qp->qp_lock);
361 goto qpmod_fail;
362 }
363
364 /*
365 * Verify state transition is to either "RTS", "Reset",
366 * or "Error"
367 */
368 if ((flags & IBT_CEP_SET_RTR_RTS) &&
369 (flags & IBT_CEP_SET_STATE) &&
370 (mod_state != IBT_STATE_RTS)) {
371 /* Invalid transition - ambiguous flags */
372 mutex_exit(&qp->qp_lock);
373 goto qpmod_fail;
374
375 } else if ((flags & IBT_CEP_SET_RTR_RTS) ||
376 ((flags & IBT_CEP_SET_STATE) &&
377 (mod_state == IBT_STATE_RTS))) {
378 /*
379 * Attempt to transition from "RTR" to "RTS"
380 */
381 status = tavor_qp_rtr2rts(state, qp, flags, info_p);
382 if (status != DDI_SUCCESS) {
383 mutex_exit(&qp->qp_lock);
384 goto qpmod_fail;
385 }
386 qp->qp_state = TAVOR_QP_RTS;
387
388 } else if ((flags & IBT_CEP_SET_STATE) &&
389 (mod_state == IBT_STATE_RESET)) {
390 /*
391 * Attempt to transition from "RTR" to "Reset"
392 */
393 status = tavor_qp_to_reset(state, qp);
394 if (status != DDI_SUCCESS) {
395 mutex_exit(&qp->qp_lock);
396 goto qpmod_fail;
397 }
398 qp->qp_state = TAVOR_QP_RESET;
399
400 /*
401 * Do any additional handling necessary for the
402 * transition _to_ the "Reset" state (e.g. update the
403 * workQ WRID lists)
404 */
405 tavor_wrid_to_reset_handling(state, qp);
406
407 } else if ((flags & IBT_CEP_SET_STATE) &&
408 (mod_state == IBT_STATE_ERROR)) {
409 /*
410 * Attempt to transition from "RTR" to "Error"
411 */
412 status = tavor_qp_to_error(state, qp);
413 if (status != DDI_SUCCESS) {
414 mutex_exit(&qp->qp_lock);
415 goto qpmod_fail;
416 }
417 qp->qp_state = TAVOR_QP_ERR;
418
419 } else {
420 /* Invalid transition - return error */
421 mutex_exit(&qp->qp_lock);
422 goto qpmod_fail;
423 }
424 break;
425
426 case TAVOR_QP_RTS:
427 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RDMA_R |
428 IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC |
429 IBT_CEP_SET_QKEY | IBT_CEP_SET_ALT_PATH |
430 IBT_CEP_SET_MIG | IBT_CEP_SET_MIN_RNR_NAK |
431 IBT_CEP_SET_SQD_EVENT);
432
433 /*
434 * Check for attempts to modify invalid attributes from the
435 * "RTS" state
436 */
437 if (flags & ~okflags) {
438 mutex_exit(&qp->qp_lock);
439 goto qpmod_fail;
440 }
441
442 /*
443 * Verify state transition is to either "RTS", "SQD", "Reset",
444 * or "Error"
445 */
446 if ((flags & IBT_CEP_SET_STATE) &&
447 (mod_state == IBT_STATE_RTS)) {
448 /*
449 * Attempt to transition from "RTS" to "RTS"
450 */
451 status = tavor_qp_rts2rts(state, qp, flags, info_p);
452 if (status != DDI_SUCCESS) {
453 mutex_exit(&qp->qp_lock);
454 goto qpmod_fail;
455 }
456 /* qp->qp_state = TAVOR_QP_RTS; */
457
458 } else if ((flags & IBT_CEP_SET_STATE) &&
459 (mod_state == IBT_STATE_SQD)) {
460 /*
461 * Attempt to transition from "RTS" to "SQD"
462 */
463 status = tavor_qp_rts2sqd(state, qp, flags);
464 if (status != DDI_SUCCESS) {
465 mutex_exit(&qp->qp_lock);
466 goto qpmod_fail;
467 }
468 qp->qp_state = TAVOR_QP_SQD;
469
470 } else if ((flags & IBT_CEP_SET_STATE) &&
471 (mod_state == IBT_STATE_RESET)) {
472 /*
473 * Attempt to transition from "RTS" to "Reset"
474 */
475 status = tavor_qp_to_reset(state, qp);
476 if (status != DDI_SUCCESS) {
477 mutex_exit(&qp->qp_lock);
478 goto qpmod_fail;
479 }
480 qp->qp_state = TAVOR_QP_RESET;
481
482 /*
483 * Do any additional handling necessary for the
484 * transition _to_ the "Reset" state (e.g. update the
485 * workQ WRID lists)
486 */
487 tavor_wrid_to_reset_handling(state, qp);
488
489 } else if ((flags & IBT_CEP_SET_STATE) &&
490 (mod_state == IBT_STATE_ERROR)) {
491 /*
492 * Attempt to transition from "RTS" to "Error"
493 */
494 status = tavor_qp_to_error(state, qp);
495 if (status != DDI_SUCCESS) {
496 mutex_exit(&qp->qp_lock);
497 goto qpmod_fail;
498 }
499 qp->qp_state = TAVOR_QP_ERR;
500
501 } else {
502 /* Invalid transition - return error */
503 mutex_exit(&qp->qp_lock);
504 goto qpmod_fail;
505 }
506 break;
507
508 case TAVOR_QP_SQERR:
509 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RDMA_R |
510 IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC |
511 IBT_CEP_SET_QKEY | IBT_CEP_SET_MIN_RNR_NAK);
512
513 /*
514 * Check for attempts to modify invalid attributes from the
515 * "SQErr" state
516 */
517 if (flags & ~okflags) {
518 mutex_exit(&qp->qp_lock);
519 goto qpmod_fail;
520 }
521
522 /*
523 * Verify state transition is to either "RTS", "Reset", or
524 * "Error"
525 */
526 if ((flags & IBT_CEP_SET_STATE) &&
527 (mod_state == IBT_STATE_RTS)) {
528 /*
529 * Attempt to transition from "SQErr" to "RTS"
530 */
531 status = tavor_qp_sqerr2rts(state, qp, flags, info_p);
532 if (status != DDI_SUCCESS) {
533 mutex_exit(&qp->qp_lock);
534 goto qpmod_fail;
535 }
536 qp->qp_state = TAVOR_QP_RTS;
537
538 } else if ((flags & IBT_CEP_SET_STATE) &&
539 (mod_state == IBT_STATE_RESET)) {
540 /*
541 * Attempt to transition from "SQErr" to "Reset"
542 */
543 status = tavor_qp_to_reset(state, qp);
544 if (status != DDI_SUCCESS) {
545 mutex_exit(&qp->qp_lock);
546 goto qpmod_fail;
547 }
548 qp->qp_state = TAVOR_QP_RESET;
549
550 /*
551 * Do any additional handling necessary for the
552 * transition _to_ the "Reset" state (e.g. update the
553 * workQ WRID lists)
554 */
555 tavor_wrid_to_reset_handling(state, qp);
556
557 } else if ((flags & IBT_CEP_SET_STATE) &&
558 (mod_state == IBT_STATE_ERROR)) {
559 /*
560 * Attempt to transition from "SQErr" to "Error"
561 */
562 status = tavor_qp_to_error(state, qp);
563 if (status != DDI_SUCCESS) {
564 mutex_exit(&qp->qp_lock);
565 goto qpmod_fail;
566 }
567 qp->qp_state = TAVOR_QP_ERR;
568
569 } else {
570 /* Invalid transition - return error */
571 mutex_exit(&qp->qp_lock);
572 goto qpmod_fail;
573 }
574 break;
575
576 case TAVOR_QP_SQD:
577 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_ADDS_VECT |
578 IBT_CEP_SET_ALT_PATH | IBT_CEP_SET_MIG |
579 IBT_CEP_SET_RDMARA_OUT | IBT_CEP_SET_RDMARA_IN |
580 IBT_CEP_SET_QKEY | IBT_CEP_SET_PKEY_IX |
581 IBT_CEP_SET_TIMEOUT | IBT_CEP_SET_RETRY |
582 IBT_CEP_SET_RNR_NAK_RETRY | IBT_CEP_SET_PORT |
583 IBT_CEP_SET_MIN_RNR_NAK | IBT_CEP_SET_RDMA_R |
584 IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC);
585
586 /*
587 * Check for attempts to modify invalid attributes from the
588 * "SQD" state
589 */
590 if (flags & ~okflags) {
591 mutex_exit(&qp->qp_lock);
592 goto qpmod_fail;
593 }
594
595 /*
596 * Verify state transition is to either "SQD", "RTS", "Reset",
597 * or "Error"
598 */
599
600 if ((flags & IBT_CEP_SET_STATE) &&
601 (mod_state == IBT_STATE_SQD)) {
602 /*
603 * Attempt to transition from "SQD" to "SQD"
604 */
605 status = tavor_qp_sqd2sqd(state, qp, flags, info_p);
606 if (status != DDI_SUCCESS) {
607 mutex_exit(&qp->qp_lock);
608 goto qpmod_fail;
609 }
610 qp->qp_state = TAVOR_QP_SQD;
611
612 } else if ((flags & IBT_CEP_SET_STATE) &&
613 (mod_state == IBT_STATE_RTS)) {
614 /*
615 * If still draining SQ, then fail transition attempt
616 * to RTS.
617 */
618 if (qp->qp_sqd_still_draining) {
619 mutex_exit(&qp->qp_lock);
620 status = IBT_QP_STATE_INVALID;
621 goto qpmod_fail;
622 }
623
624 /*
625 * Attempt to transition from "SQD" to "RTS"
626 */
627 status = tavor_qp_sqd2rts(state, qp, flags, info_p);
628 if (status != DDI_SUCCESS) {
629 mutex_exit(&qp->qp_lock);
630 goto qpmod_fail;
631 }
632 qp->qp_state = TAVOR_QP_RTS;
633
634 } else if ((flags & IBT_CEP_SET_STATE) &&
635 (mod_state == IBT_STATE_RESET)) {
636 /*
637 * Attempt to transition from "SQD" to "Reset"
638 */
639 status = tavor_qp_to_reset(state, qp);
640 if (status != DDI_SUCCESS) {
641 mutex_exit(&qp->qp_lock);
642 goto qpmod_fail;
643 }
644 qp->qp_state = TAVOR_QP_RESET;
645
646 /*
647 * Do any additional handling necessary for the
648 * transition _to_ the "Reset" state (e.g. update the
649 * workQ WRID lists)
650 */
651 tavor_wrid_to_reset_handling(state, qp);
652
653 } else if ((flags & IBT_CEP_SET_STATE) &&
654 (mod_state == IBT_STATE_ERROR)) {
655 /*
656 * Attempt to transition from "SQD" to "Error"
657 */
658 status = tavor_qp_to_error(state, qp);
659 if (status != DDI_SUCCESS) {
660 mutex_exit(&qp->qp_lock);
661 goto qpmod_fail;
662 }
663 qp->qp_state = TAVOR_QP_ERR;
664
665 } else {
666 /* Invalid transition - return error */
667 mutex_exit(&qp->qp_lock);
668 goto qpmod_fail;
669 }
670 break;
671
672 case TAVOR_QP_ERR:
673 /*
674 * Verify state transition is to either "Reset" or back to
675 * "Error"
676 */
677 if ((flags & IBT_CEP_SET_STATE) &&
678 (mod_state == IBT_STATE_RESET)) {
679 /*
680 * Attempt to transition from "Error" to "Reset"
681 */
682 status = tavor_qp_to_reset(state, qp);
683 if (status != DDI_SUCCESS) {
684 mutex_exit(&qp->qp_lock);
685 goto qpmod_fail;
686 }
687 qp->qp_state = TAVOR_QP_RESET;
688
689 /*
690 * Do any additional handling necessary for the
691 * transition _to_ the "Reset" state (e.g. update the
692 * workQ WRID lists)
693 */
694 tavor_wrid_to_reset_handling(state, qp);
695
696 } else if ((flags & IBT_CEP_SET_STATE) &&
697 (mod_state == IBT_STATE_ERROR)) {
698 /*
699 * Attempt to transition from "Error" back to "Error"
700 * Nothing to do here really... just drop the lock
701 * and return success. The qp->qp_state should
702 * already be set to TAVOR_QP_ERR.
703 *
704 */
705 mutex_exit(&qp->qp_lock);
706 return (DDI_SUCCESS);
707
708 } else {
709 /* Invalid transition - return error */
710 mutex_exit(&qp->qp_lock);
711 goto qpmod_fail;
712 }
713 break;
714
715 default:
716 /*
717 * Invalid QP state. If we got here then it's a warning of
718 * a probably serious problem. So print a message and return
719 * failure
720 */
721 mutex_exit(&qp->qp_lock);
722 TAVOR_WARNING(state, "unknown QP state in modify");
723 goto qpmod_fail;
724 }
725
726 mutex_exit(&qp->qp_lock);
727 return (DDI_SUCCESS);
728
729 qpmod_fail:
730 return (status);
731 }
732
733
734 /*
735 * tavor_qp_reset2init()
736 * Context: Can be called from interrupt or base context.
737 */
738 static int
tavor_qp_reset2init(tavor_state_t * state,tavor_qphdl_t qp,ibt_qp_info_t * info_p)739 tavor_qp_reset2init(tavor_state_t *state, tavor_qphdl_t qp,
740 ibt_qp_info_t *info_p)
741 {
742 tavor_hw_qpc_t *qpc;
743 ibt_qp_rc_attr_t *rc;
744 ibt_qp_ud_attr_t *ud;
745 ibt_qp_uc_attr_t *uc;
746 uint_t portnum, pkeyindx;
747 int status;
748
749 ASSERT(MUTEX_HELD(&qp->qp_lock));
750
751 /*
752 * Grab the temporary QPC entry from QP software state
753 */
754 qpc = &qp->qpc;
755
756 /*
757 * Fill in the common and/or Tavor-specific fields in the QPC
758 */
759 if (qp->qp_is_special) {
760 qpc->serv_type = TAVOR_QP_MLX;
761 } else {
762 qpc->serv_type = qp->qp_serv_type;
763 }
764 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
765 qpc->de = TAVOR_QP_DESC_EVT_ENABLED;
766 qpc->sched_q = TAVOR_QP_SCHEDQ_GET(qp->qp_qpnum);
767 if (qp->qp_is_umap) {
768 qpc->usr_page = qp->qp_uarpg;
769 } else {
770 qpc->usr_page = 0;
771 }
772 qpc->pd = qp->qp_pdhdl->pd_pdnum;
773 qpc->wqe_baseaddr = 0;
774 qpc->wqe_lkey = qp->qp_mrhdl->mr_lkey;
775 qpc->ssc = qp->qp_sq_sigtype;
776 qpc->cqn_snd = qp->qp_sq_cqhdl->cq_cqnum;
777 qpc->rsc = TAVOR_QP_RQ_ALL_SIGNALED;
778 qpc->cqn_rcv = qp->qp_rq_cqhdl->cq_cqnum;
779 qpc->srq_en = qp->qp_srq_en;
780
781 if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
782 qpc->srq_number = qp->qp_srqhdl->srq_srqnum;
783 } else {
784 qpc->srq_number = 0;
785 }
786
787 /*
788 * Now fill in the QPC fields which are specific to transport type
789 */
790 if (qp->qp_serv_type == TAVOR_QP_UD) {
791 ud = &info_p->qp_transport.ud;
792
793 /* Set the QKey */
794 qpc->qkey = ud->ud_qkey;
795
796 /* Check for valid port number and fill it in */
797 portnum = ud->ud_port;
798 if (tavor_portnum_is_valid(state, portnum)) {
799 qpc->pri_addr_path.portnum = portnum;
800 } else {
801 return (IBT_HCA_PORT_INVALID);
802 }
803
804 /* Check for valid PKey index and fill it in */
805 pkeyindx = ud->ud_pkey_ix;
806 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
807 qpc->pri_addr_path.pkey_indx = pkeyindx;
808 qp->qp_pkeyindx = pkeyindx;
809 } else {
810 return (IBT_PKEY_IX_ILLEGAL);
811 }
812
813 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
814 rc = &info_p->qp_transport.rc;
815
816 /* Set the RDMA (recv) enable/disable flags */
817 qpc->rre = (info_p->qp_flags & IBT_CEP_RDMA_RD) ? 1 : 0;
818 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
819 qpc->rae = (info_p->qp_flags & IBT_CEP_ATOMIC) ? 1 : 0;
820
821 /* Check for valid port number and fill it in */
822 portnum = rc->rc_path.cep_hca_port_num;
823 if (tavor_portnum_is_valid(state, portnum)) {
824 qpc->pri_addr_path.portnum = portnum;
825 } else {
826 return (IBT_HCA_PORT_INVALID);
827 }
828
829 /* Check for valid PKey index and fill it in */
830 pkeyindx = rc->rc_path.cep_pkey_ix;
831 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
832 qpc->pri_addr_path.pkey_indx = pkeyindx;
833 } else {
834 return (IBT_PKEY_IX_ILLEGAL);
835 }
836
837 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
838 uc = &info_p->qp_transport.uc;
839
840 /*
841 * Set the RDMA (recv) enable/disable flags. Note: RDMA Read
842 * and Atomic are ignored by default.
843 */
844 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
845
846 /* Check for valid port number and fill it in */
847 portnum = uc->uc_path.cep_hca_port_num;
848 if (tavor_portnum_is_valid(state, portnum)) {
849 qpc->pri_addr_path.portnum = portnum;
850 } else {
851 return (IBT_HCA_PORT_INVALID);
852 }
853
854 /* Check for valid PKey index and fill it in */
855 pkeyindx = uc->uc_path.cep_pkey_ix;
856 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
857 qpc->pri_addr_path.pkey_indx = pkeyindx;
858 } else {
859 return (IBT_PKEY_IX_ILLEGAL);
860 }
861 } else {
862 /*
863 * Invalid QP transport type. If we got here then it's a
864 * warning of a probably serious problem. So print a message
865 * and return failure
866 */
867 TAVOR_WARNING(state, "unknown QP transport type in rst2init");
868 return (ibc_get_ci_failure(0));
869 }
870
871 /*
872 * Post the RST2INIT_QP command to the Tavor firmware
873 *
874 * We do a TAVOR_NOSLEEP here because we are still holding the
875 * "qp_lock". If we got raised to interrupt level by priority
876 * inversion, we do not want to block in this routine waiting for
877 * success.
878 */
879 status = tavor_cmn_qp_cmd_post(state, RST2INIT_QP, qpc, qp->qp_qpnum,
880 0, TAVOR_CMD_NOSLEEP_SPIN);
881 if (status != TAVOR_CMD_SUCCESS) {
882 cmn_err(CE_CONT, "Tavor: RST2INIT_QP command failed: %08x\n",
883 status);
884 return (ibc_get_ci_failure(0));
885 }
886
887 return (DDI_SUCCESS);
888 }
889
890
891 /*
892 * tavor_qp_init2init()
893 * Context: Can be called from interrupt or base context.
894 */
895 static int
tavor_qp_init2init(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)896 tavor_qp_init2init(tavor_state_t *state, tavor_qphdl_t qp,
897 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
898 {
899 tavor_hw_qpc_t *qpc;
900 ibt_qp_rc_attr_t *rc;
901 ibt_qp_ud_attr_t *ud;
902 ibt_qp_uc_attr_t *uc;
903 uint_t portnum, pkeyindx;
904 uint32_t opmask = 0;
905 int status;
906
907 ASSERT(MUTEX_HELD(&qp->qp_lock));
908
909 /*
910 * Grab the temporary QPC entry from QP software state
911 */
912 qpc = &qp->qpc;
913
914 /*
915 * Since there are no common and/or Tavor-specific fields to be filled
916 * in for this command, we begin with the QPC fields which are
917 * specific to transport type.
918 */
919 if (qp->qp_serv_type == TAVOR_QP_UD) {
920 ud = &info_p->qp_transport.ud;
921
922 /*
923 * If we are attempting to modify the port for this QP, then
924 * check for valid port number and fill it in. Also set the
925 * appropriate flag in the "opmask" parameter.
926 */
927 if (flags & IBT_CEP_SET_PORT) {
928 portnum = ud->ud_port;
929 if (tavor_portnum_is_valid(state, portnum)) {
930 qpc->pri_addr_path.portnum = portnum;
931 } else {
932 return (IBT_HCA_PORT_INVALID);
933 }
934 opmask |= TAVOR_CMD_OP_PRIM_PORT;
935 }
936
937 /*
938 * If we are attempting to modify the PKey index for this QP,
939 * then check for valid PKey index and fill it in. Also set
940 * the appropriate flag in the "opmask" parameter.
941 */
942 if (flags & IBT_CEP_SET_PKEY_IX) {
943 pkeyindx = ud->ud_pkey_ix;
944 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
945 qpc->pri_addr_path.pkey_indx = pkeyindx;
946 opmask |= TAVOR_CMD_OP_PKEYINDX;
947 qp->qp_pkeyindx = pkeyindx;
948 } else {
949 return (IBT_PKEY_IX_ILLEGAL);
950 }
951 }
952
953 /*
954 * If we are attempting to modify the QKey for this QP, then
955 * fill it in and set the appropriate flag in the "opmask"
956 * parameter.
957 */
958 if (flags & IBT_CEP_SET_QKEY) {
959 qpc->qkey = ud->ud_qkey;
960 opmask |= TAVOR_CMD_OP_QKEY;
961 }
962
963 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
964 rc = &info_p->qp_transport.rc;
965
966 /*
967 * If we are attempting to modify the port for this QP, then
968 * check for valid port number and fill it in. Also set the
969 * appropriate flag in the "opmask" parameter.
970 */
971 if (flags & IBT_CEP_SET_PORT) {
972 portnum = rc->rc_path.cep_hca_port_num;
973 if (tavor_portnum_is_valid(state, portnum)) {
974 qpc->pri_addr_path.portnum = portnum;
975 } else {
976 return (IBT_HCA_PORT_INVALID);
977 }
978 opmask |= TAVOR_CMD_OP_PRIM_PORT;
979 }
980
981 /*
982 * If we are attempting to modify the PKey index for this QP,
983 * then check for valid PKey index and fill it in. Also set
984 * the appropriate flag in the "opmask" parameter.
985 */
986 if (flags & IBT_CEP_SET_PKEY_IX) {
987 pkeyindx = rc->rc_path.cep_pkey_ix;
988 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
989 qpc->pri_addr_path.pkey_indx = pkeyindx;
990 opmask |= TAVOR_CMD_OP_PKEYINDX;
991 } else {
992 return (IBT_PKEY_IX_ILLEGAL);
993 }
994 }
995
996 /*
997 * Check if any of the flags indicate a change in the RDMA
998 * (recv) enable/disable flags and set the appropriate flag in
999 * the "opmask" parameter
1000 */
1001 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1002
1003 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
1004 uc = &info_p->qp_transport.uc;
1005
1006 /*
1007 * If we are attempting to modify the port for this QP, then
1008 * check for valid port number and fill it in. Also set the
1009 * appropriate flag in the "opmask" parameter.
1010 */
1011 if (flags & IBT_CEP_SET_PORT) {
1012 portnum = uc->uc_path.cep_hca_port_num;
1013 if (tavor_portnum_is_valid(state, portnum)) {
1014 qpc->pri_addr_path.portnum = portnum;
1015 } else {
1016 return (IBT_HCA_PORT_INVALID);
1017 }
1018 opmask |= TAVOR_CMD_OP_PRIM_PORT;
1019 }
1020
1021 /*
1022 * If we are attempting to modify the PKey index for this QP,
1023 * then check for valid PKey index and fill it in. Also set
1024 * the appropriate flag in the "opmask" parameter.
1025 */
1026 if (flags & IBT_CEP_SET_PKEY_IX) {
1027 pkeyindx = uc->uc_path.cep_pkey_ix;
1028 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1029 qpc->pri_addr_path.pkey_indx = pkeyindx;
1030 opmask |= TAVOR_CMD_OP_PKEYINDX;
1031 } else {
1032 return (IBT_PKEY_IX_ILLEGAL);
1033 }
1034 }
1035
1036 /*
1037 * Check if any of the flags indicate a change in the RDMA
1038 * Write (recv) enable/disable and set the appropriate flag
1039 * in the "opmask" parameter. Note: RDMA Read and Atomic are
1040 * not valid for UC transport.
1041 */
1042 if (flags & IBT_CEP_SET_RDMA_W) {
1043 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1044 opmask |= TAVOR_CMD_OP_RWE;
1045 }
1046 } else {
1047 /*
1048 * Invalid QP transport type. If we got here then it's a
1049 * warning of a probably serious problem. So print a message
1050 * and return failure
1051 */
1052 TAVOR_WARNING(state, "unknown QP transport type in init2init");
1053 return (ibc_get_ci_failure(0));
1054 }
1055
1056 /*
1057 * Post the INIT2INIT_QP command to the Tavor firmware
1058 *
1059 * We do a TAVOR_NOSLEEP here because we are still holding the
1060 * "qp_lock". If we got raised to interrupt level by priority
1061 * inversion, we do not want to block in this routine waiting for
1062 * success.
1063 */
1064 status = tavor_cmn_qp_cmd_post(state, INIT2INIT_QP, qpc, qp->qp_qpnum,
1065 opmask, TAVOR_CMD_NOSLEEP_SPIN);
1066 if (status != TAVOR_CMD_SUCCESS) {
1067 if (status != TAVOR_CMD_BAD_QP_STATE) {
1068 cmn_err(CE_CONT, "Tavor: INIT2INIT_QP command failed: "
1069 "%08x\n", status);
1070 return (ibc_get_ci_failure(0));
1071 } else {
1072 return (IBT_QP_STATE_INVALID);
1073 }
1074 }
1075
1076 return (DDI_SUCCESS);
1077 }
1078
1079
1080 /*
1081 * tavor_qp_init2rtr()
1082 * Context: Can be called from interrupt or base context.
1083 */
1084 static int
tavor_qp_init2rtr(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)1085 tavor_qp_init2rtr(tavor_state_t *state, tavor_qphdl_t qp,
1086 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1087 {
1088 tavor_hw_qpc_t *qpc;
1089 ibt_qp_rc_attr_t *rc;
1090 ibt_qp_ud_attr_t *ud;
1091 ibt_qp_uc_attr_t *uc;
1092 tavor_hw_addr_path_t *qpc_path;
1093 ibt_adds_vect_t *adds_vect;
1094 uint_t portnum, pkeyindx, rra_max;
1095 uint_t mtu;
1096 uint32_t opmask = 0;
1097 int status;
1098
1099 ASSERT(MUTEX_HELD(&qp->qp_lock));
1100
1101 /*
1102 * Grab the temporary QPC entry from QP software state
1103 */
1104 qpc = &qp->qpc;
1105
1106 /*
1107 * Since there are no common and/or Tavor-specific fields to be filled
1108 * in for this command, we begin with the QPC fields which are
1109 * specific to transport type.
1110 */
1111 if (qp->qp_serv_type == TAVOR_QP_UD) {
1112 ud = &info_p->qp_transport.ud;
1113
1114 /*
1115 * If this UD QP is also a "special QP" (QP0 or QP1), then
1116 * the MTU is 256 bytes. However, Tavor HW requires us to
1117 * set the MTU to 4 (which is the IB code for a 2K MTU).
1118 * If this is not a special QP, then we set the MTU to the
1119 * configured maximum (which defaults to 2K). Note: the
1120 * QPC "msg_max" must also be set so as to correspond with
1121 * the specified MTU value.
1122 */
1123 if (qp->qp_is_special) {
1124 qpc->mtu = 4;
1125 } else {
1126 qpc->mtu = state->ts_cfg_profile->cp_max_mtu;
1127 }
1128 qpc->msg_max = qpc->mtu + 7; /* must equal MTU plus seven */
1129
1130 /*
1131 * Save away the MTU value. This is used in future sqd2sqd
1132 * transitions, as the MTU must remain the same in future
1133 * changes.
1134 */
1135 qp->qp_save_mtu = qpc->mtu;
1136
1137 /*
1138 * If we are attempting to modify the PKey index for this QP,
1139 * then check for valid PKey index and fill it in. Also set
1140 * the appropriate flag in the "opmask" parameter.
1141 */
1142 if (flags & IBT_CEP_SET_PKEY_IX) {
1143 pkeyindx = ud->ud_pkey_ix;
1144 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1145 qpc->pri_addr_path.pkey_indx = pkeyindx;
1146 opmask |= TAVOR_CMD_OP_PKEYINDX;
1147 qp->qp_pkeyindx = pkeyindx;
1148 } else {
1149 return (IBT_PKEY_IX_ILLEGAL);
1150 }
1151 }
1152
1153 /*
1154 * If we are attempting to modify the QKey for this QP, then
1155 * fill it in and set the appropriate flag in the "opmask"
1156 * parameter.
1157 */
1158 if (flags & IBT_CEP_SET_QKEY) {
1159 qpc->qkey = ud->ud_qkey;
1160 opmask |= TAVOR_CMD_OP_QKEY;
1161 }
1162
1163 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
1164 rc = &info_p->qp_transport.rc;
1165 qpc_path = &qpc->pri_addr_path;
1166 adds_vect = &rc->rc_path.cep_adds_vect;
1167
1168 /*
1169 * Set the common primary address path fields
1170 */
1171 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1172 TAVOR_ADDRPATH_QP, qp);
1173 if (status != DDI_SUCCESS) {
1174 return (status);
1175 }
1176
1177 /*
1178 * The following values are apparently "required" here (as
1179 * they are part of the IBA-defined "Remote Node Address
1180 * Vector"). However, they are also going to be "required"
1181 * later - at RTR2RTS_QP time. Not sure why. But we set
1182 * them here anyway.
1183 */
1184 qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1185 qpc->retry_cnt = rc->rc_retry_cnt;
1186 qpc_path->ack_timeout = rc->rc_path.cep_timeout;
1187
1188 /*
1189 * Setup the destination QP, recv PSN, MTU, max msg size,etc.
1190 * Note max message size is defined to be the maximum IB
1191 * allowed message size (which is 2^31 bytes). Also max
1192 * MTU is defined by HCA port properties.
1193 */
1194 qpc->rem_qpn = rc->rc_dst_qpn;
1195 qpc->next_rcv_psn = rc->rc_rq_psn;
1196 qpc->msg_max = TAVOR_QP_LOG_MAX_MSGSZ;
1197
1198 /*
1199 * If this QP is using an SRQ, 'ric' must be set to 1.
1200 */
1201 qpc->ric = (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) ? 1 : 0;
1202 mtu = rc->rc_path_mtu;
1203 if (tavor_qp_validate_mtu(state, mtu) != DDI_SUCCESS) {
1204 return (IBT_HCA_PORT_MTU_EXCEEDED);
1205 }
1206 qpc->mtu = mtu;
1207
1208 /*
1209 * Save away the MTU value. This is used in future sqd2sqd
1210 * transitions, as the MTU must remain the same in future
1211 * changes.
1212 */
1213 qp->qp_save_mtu = qpc->mtu;
1214
1215 /*
1216 * Though it is a "required" parameter, "min_rnr_nak" is
1217 * optionally specifiable in Tavor. So we hardcode the
1218 * optional flag here.
1219 */
1220 qpc->min_rnr_nak = rc->rc_min_rnr_nak;
1221 opmask |= TAVOR_CMD_OP_MINRNRNAK;
1222
1223 /*
1224 * Check that the number of specified "incoming RDMA resources"
1225 * is valid. And if it is, then setup the "rra_max" and
1226 * "ra_buf_index" fields in the QPC to point to the
1227 * pre-allocated RDB resources (in DDR)
1228 */
1229 if (tavor_qp_validate_resp_rsrc(state, rc, &rra_max) !=
1230 DDI_SUCCESS) {
1231 return (IBT_INVALID_PARAM);
1232 }
1233 qpc->rra_max = rra_max;
1234 qpc->ra_buff_indx = qp->qp_rdb_ddraddr >> TAVOR_RDB_SIZE_SHIFT;
1235
1236 /*
1237 * If we are attempting to modify the PKey index for this QP,
1238 * then check for valid PKey index and fill it in. Also set
1239 * the appropriate flag in the "opmask" parameter.
1240 */
1241 if (flags & IBT_CEP_SET_PKEY_IX) {
1242 pkeyindx = rc->rc_path.cep_pkey_ix;
1243 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1244 qpc->pri_addr_path.pkey_indx = pkeyindx;
1245 opmask |= TAVOR_CMD_OP_PKEYINDX;
1246 } else {
1247 return (IBT_PKEY_IX_ILLEGAL);
1248 }
1249 }
1250
1251 /*
1252 * Check if any of the flags indicate a change in the RDMA
1253 * (recv) enable/disable flags and set the appropriate flag in
1254 * the "opmask" parameter
1255 */
1256 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1257
1258 /*
1259 * Check for optional alternate path and fill in the
1260 * appropriate QPC fields if one is specified
1261 */
1262 if (flags & IBT_CEP_SET_ALT_PATH) {
1263 qpc_path = &qpc->alt_addr_path;
1264 adds_vect = &rc->rc_alt_path.cep_adds_vect;
1265
1266 /* Set the common alternate address path fields */
1267 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1268 TAVOR_ADDRPATH_QP, qp);
1269 if (status != DDI_SUCCESS) {
1270 return (status);
1271 }
1272 qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
1273
1274 /*
1275 * Copy the "RNR Retry count" value from the primary
1276 * path. Just as we did above, we need to hardcode
1277 * the optional flag here (see below).
1278 */
1279 qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1280
1281 /*
1282 * Check for valid alternate path port number and fill
1283 * it in
1284 */
1285 portnum = rc->rc_alt_path.cep_hca_port_num;
1286 if (tavor_portnum_is_valid(state, portnum)) {
1287 qpc->alt_addr_path.portnum = portnum;
1288 } else {
1289 return (IBT_HCA_PORT_INVALID);
1290 }
1291
1292 /*
1293 * Check for valid alternate path PKey index and fill
1294 * it in
1295 */
1296 pkeyindx = rc->rc_alt_path.cep_pkey_ix;
1297 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1298 qpc->alt_addr_path.pkey_indx = pkeyindx;
1299 } else {
1300 return (IBT_PKEY_IX_ILLEGAL);
1301 }
1302 opmask |= (TAVOR_CMD_OP_ALT_PATH |
1303 TAVOR_CMD_OP_ALT_RNRRETRY);
1304 }
1305
1306 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
1307 uc = &info_p->qp_transport.uc;
1308 qpc_path = &qpc->pri_addr_path;
1309 adds_vect = &uc->uc_path.cep_adds_vect;
1310
1311 /*
1312 * Set the common primary address path fields
1313 */
1314 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1315 TAVOR_ADDRPATH_QP, qp);
1316 if (status != DDI_SUCCESS) {
1317 return (status);
1318 }
1319
1320 /*
1321 * Setup the destination QP, recv PSN, MTU, max msg size,etc.
1322 * Note max message size is defined to be the maximum IB
1323 * allowed message size (which is 2^31 bytes). Also max
1324 * MTU is defined by HCA port properties.
1325 */
1326 qpc->rem_qpn = uc->uc_dst_qpn;
1327 qpc->next_rcv_psn = uc->uc_rq_psn;
1328 qpc->msg_max = TAVOR_QP_LOG_MAX_MSGSZ;
1329 mtu = uc->uc_path_mtu;
1330 if (tavor_qp_validate_mtu(state, mtu) != DDI_SUCCESS) {
1331 return (IBT_HCA_PORT_MTU_EXCEEDED);
1332 }
1333 qpc->mtu = mtu;
1334
1335 /*
1336 * Save away the MTU value. This is used in future sqd2sqd
1337 * transitions, as the MTU must remain the same in future
1338 * changes.
1339 */
1340 qp->qp_save_mtu = qpc->mtu;
1341
1342 /*
1343 * If we are attempting to modify the PKey index for this QP,
1344 * then check for valid PKey index and fill it in. Also set
1345 * the appropriate flag in the "opmask" parameter.
1346 */
1347 if (flags & IBT_CEP_SET_PKEY_IX) {
1348 pkeyindx = uc->uc_path.cep_pkey_ix;
1349 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1350 qpc->pri_addr_path.pkey_indx = pkeyindx;
1351 opmask |= TAVOR_CMD_OP_PKEYINDX;
1352 } else {
1353 return (IBT_PKEY_IX_ILLEGAL);
1354 }
1355 }
1356
1357 /*
1358 * Check if any of the flags indicate a change in the RDMA
1359 * Write (recv) enable/disable and set the appropriate flag
1360 * in the "opmask" parameter. Note: RDMA Read and Atomic are
1361 * not valid for UC transport.
1362 */
1363 if (flags & IBT_CEP_SET_RDMA_W) {
1364 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1365 opmask |= TAVOR_CMD_OP_RWE;
1366 }
1367
1368 /*
1369 * Check for optional alternate path and fill in the
1370 * appropriate QPC fields if one is specified
1371 */
1372 if (flags & IBT_CEP_SET_ALT_PATH) {
1373 qpc_path = &qpc->alt_addr_path;
1374 adds_vect = &uc->uc_alt_path.cep_adds_vect;
1375
1376 /* Set the common alternate address path fields */
1377 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1378 TAVOR_ADDRPATH_QP, qp);
1379 if (status != DDI_SUCCESS) {
1380 return (status);
1381 }
1382
1383 /*
1384 * Check for valid alternate path port number and fill
1385 * it in
1386 */
1387 portnum = uc->uc_alt_path.cep_hca_port_num;
1388 if (tavor_portnum_is_valid(state, portnum)) {
1389 qpc->alt_addr_path.portnum = portnum;
1390 } else {
1391 return (IBT_HCA_PORT_INVALID);
1392 }
1393
1394 /*
1395 * Check for valid alternate path PKey index and fill
1396 * it in
1397 */
1398 pkeyindx = uc->uc_alt_path.cep_pkey_ix;
1399 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1400 qpc->alt_addr_path.pkey_indx = pkeyindx;
1401 } else {
1402 return (IBT_PKEY_IX_ILLEGAL);
1403 }
1404 opmask |= TAVOR_CMD_OP_ALT_PATH;
1405 }
1406 } else {
1407 /*
1408 * Invalid QP transport type. If we got here then it's a
1409 * warning of a probably serious problem. So print a message
1410 * and return failure
1411 */
1412 TAVOR_WARNING(state, "unknown QP transport type in init2rtr");
1413 return (ibc_get_ci_failure(0));
1414 }
1415
1416 /*
1417 * Post the INIT2RTR_QP command to the Tavor firmware
1418 *
1419 * We do a TAVOR_NOSLEEP here because we are still holding the
1420 * "qp_lock". If we got raised to interrupt level by priority
1421 * inversion, we do not want to block in this routine waiting for
1422 * success.
1423 */
1424 status = tavor_cmn_qp_cmd_post(state, INIT2RTR_QP, qpc, qp->qp_qpnum,
1425 opmask, TAVOR_CMD_NOSLEEP_SPIN);
1426 if (status != TAVOR_CMD_SUCCESS) {
1427 if (status != TAVOR_CMD_BAD_QP_STATE) {
1428 cmn_err(CE_CONT, "Tavor: INIT2RTR_QP command failed: "
1429 "%08x\n", status);
1430 return (ibc_get_ci_failure(0));
1431 } else {
1432 return (IBT_QP_STATE_INVALID);
1433 }
1434 }
1435
1436 return (DDI_SUCCESS);
1437 }
1438
1439
1440 /*
1441 * tavor_qp_rtr2rts()
1442 * Context: Can be called from interrupt or base context.
1443 */
1444 static int
tavor_qp_rtr2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)1445 tavor_qp_rtr2rts(tavor_state_t *state, tavor_qphdl_t qp,
1446 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1447 {
1448 tavor_hw_qpc_t *qpc;
1449 ibt_qp_rc_attr_t *rc;
1450 ibt_qp_ud_attr_t *ud;
1451 ibt_qp_uc_attr_t *uc;
1452 tavor_hw_addr_path_t *qpc_path;
1453 ibt_adds_vect_t *adds_vect;
1454 uint_t portnum, pkeyindx, sra_max;
1455 uint32_t opmask = 0;
1456 int status;
1457
1458 ASSERT(MUTEX_HELD(&qp->qp_lock));
1459
1460 /*
1461 * Grab the temporary QPC entry from QP software state
1462 */
1463 qpc = &qp->qpc;
1464
1465 /*
1466 * Fill in the common and/or Tavor-specific fields in the QPC
1467 */
1468 qpc->flight_lim = TAVOR_QP_FLIGHT_LIM_UNLIMITED;
1469
1470 /*
1471 * Now fill in the QPC fields which are specific to transport type
1472 */
1473 if (qp->qp_serv_type == TAVOR_QP_UD) {
1474 ud = &info_p->qp_transport.ud;
1475
1476 /* Set the send PSN */
1477 qpc->next_snd_psn = ud->ud_sq_psn;
1478
1479 /*
1480 * If we are attempting to modify the QKey for this QP, then
1481 * fill it in and set the appropriate flag in the "opmask"
1482 * parameter.
1483 */
1484 if (flags & IBT_CEP_SET_QKEY) {
1485 qpc->qkey = ud->ud_qkey;
1486 opmask |= TAVOR_CMD_OP_QKEY;
1487 }
1488
1489 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
1490 rc = &info_p->qp_transport.rc;
1491 qpc_path = &qpc->pri_addr_path;
1492
1493 /*
1494 * Setup the send PSN, ACK timeout, and retry counts
1495 */
1496 qpc->next_snd_psn = rc->rc_sq_psn;
1497 qpc_path->ack_timeout = rc->rc_path.cep_timeout;
1498 qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1499 qpc->retry_cnt = rc->rc_retry_cnt;
1500
1501 /*
1502 * Set "ack_req_freq" based on the configuration variable
1503 */
1504 qpc->ack_req_freq = state->ts_cfg_profile->cp_ackreq_freq;
1505
1506 /*
1507 * Check that the number of specified "outgoing RDMA resources"
1508 * is valid. And if it is, then setup the "sra_max"
1509 * appropriately
1510 */
1511 if (tavor_qp_validate_init_depth(state, rc, &sra_max) !=
1512 DDI_SUCCESS) {
1513 return (IBT_INVALID_PARAM);
1514 }
1515 qpc->sra_max = sra_max;
1516
1517 /*
1518 * Configure the QP to allow (sending of) all types of RC
1519 * traffic. Tavor hardware allows these bits to be set to
1520 * zero (thereby disabling certain outgoing RDMA types), but
1521 * we do not desire to do this.
1522 */
1523 qpc->sre = qpc->swe = qpc->sae = 1;
1524 qpc->sic = 0;
1525
1526 /*
1527 * Check if any of the flags indicate a change in the RDMA
1528 * (recv) enable/disable flags and set the appropriate flag in
1529 * the "opmask" parameter
1530 */
1531 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1532
1533 /*
1534 * If we are attempting to modify the path migration state for
1535 * this QP, then check for valid state and fill it in. Also
1536 * set the appropriate flag in the "opmask" parameter.
1537 */
1538 if (flags & IBT_CEP_SET_MIG) {
1539 if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
1540 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1541 } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
1542 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1543 } else {
1544 return (IBT_QP_APM_STATE_INVALID);
1545 }
1546 opmask |= TAVOR_CMD_OP_PM_STATE;
1547 }
1548
1549 /*
1550 * If we are attempting to modify the "Minimum RNR NAK" value
1551 * for this QP, then fill it in and set the appropriate flag
1552 * in the "opmask" parameter.
1553 */
1554 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
1555 qpc->min_rnr_nak = rc->rc_min_rnr_nak;
1556 opmask |= TAVOR_CMD_OP_MINRNRNAK;
1557 }
1558
1559 /*
1560 * Check for optional alternate path and fill in the
1561 * appropriate QPC fields if one is specified
1562 */
1563 if (flags & IBT_CEP_SET_ALT_PATH) {
1564 qpc_path = &qpc->alt_addr_path;
1565 adds_vect = &rc->rc_alt_path.cep_adds_vect;
1566
1567 /* Set the common alternate address path fields */
1568 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1569 TAVOR_ADDRPATH_QP, qp);
1570 if (status != DDI_SUCCESS) {
1571 return (status);
1572 }
1573
1574 qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
1575
1576 /*
1577 * Copy the "RNR Retry count" value from the primary
1578 * path. Just as we did above, we need to hardcode
1579 * the optional flag here (see below).
1580 */
1581 qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1582
1583 /*
1584 * Check for valid alternate path port number and fill
1585 * it in
1586 */
1587 portnum = rc->rc_alt_path.cep_hca_port_num;
1588 if (tavor_portnum_is_valid(state, portnum)) {
1589 qpc->alt_addr_path.portnum = portnum;
1590 } else {
1591 return (IBT_HCA_PORT_INVALID);
1592 }
1593
1594 /*
1595 * Check for valid alternate path PKey index and fill
1596 * it in
1597 */
1598 pkeyindx = rc->rc_alt_path.cep_pkey_ix;
1599 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1600 qpc->alt_addr_path.pkey_indx = pkeyindx;
1601 } else {
1602 return (IBT_PKEY_IX_ILLEGAL);
1603 }
1604 opmask |= (TAVOR_CMD_OP_ALT_PATH |
1605 TAVOR_CMD_OP_ALT_RNRRETRY);
1606 }
1607
1608 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
1609 uc = &info_p->qp_transport.uc;
1610
1611 /* Set the send PSN */
1612 qpc->next_snd_psn = uc->uc_sq_psn;
1613
1614 /*
1615 * Configure the QP to allow (sending of) all types of allowable
1616 * UC traffic (i.e. RDMA Write).
1617 */
1618 qpc->swe = 1;
1619
1620 /*
1621 * Check if any of the flags indicate a change in the RDMA
1622 * Write (recv) enable/disable and set the appropriate flag
1623 * in the "opmask" parameter. Note: RDMA Read and Atomic are
1624 * not valid for UC transport.
1625 */
1626 if (flags & IBT_CEP_SET_RDMA_W) {
1627 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1628 opmask |= TAVOR_CMD_OP_RWE;
1629 }
1630
1631 /*
1632 * If we are attempting to modify the path migration state for
1633 * this QP, then check for valid state and fill it in. Also
1634 * set the appropriate flag in the "opmask" parameter.
1635 */
1636 if (flags & IBT_CEP_SET_MIG) {
1637 if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
1638 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1639 } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
1640 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1641 } else {
1642 return (IBT_QP_APM_STATE_INVALID);
1643 }
1644 opmask |= TAVOR_CMD_OP_PM_STATE;
1645 }
1646
1647 /*
1648 * Check for optional alternate path and fill in the
1649 * appropriate QPC fields if one is specified
1650 */
1651 if (flags & IBT_CEP_SET_ALT_PATH) {
1652 qpc_path = &qpc->alt_addr_path;
1653 adds_vect = &uc->uc_alt_path.cep_adds_vect;
1654
1655 /* Set the common alternate address path fields */
1656 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1657 TAVOR_ADDRPATH_QP, qp);
1658 if (status != DDI_SUCCESS) {
1659 return (status);
1660 }
1661
1662 /*
1663 * Check for valid alternate path port number and fill
1664 * it in
1665 */
1666 portnum = uc->uc_alt_path.cep_hca_port_num;
1667 if (tavor_portnum_is_valid(state, portnum)) {
1668 qpc->alt_addr_path.portnum = portnum;
1669 } else {
1670 return (IBT_HCA_PORT_INVALID);
1671 }
1672
1673 /*
1674 * Check for valid alternate path PKey index and fill
1675 * it in
1676 */
1677 pkeyindx = uc->uc_alt_path.cep_pkey_ix;
1678 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1679 qpc->alt_addr_path.pkey_indx = pkeyindx;
1680 } else {
1681 return (IBT_PKEY_IX_ILLEGAL);
1682 }
1683 opmask |= TAVOR_CMD_OP_ALT_PATH;
1684 }
1685 } else {
1686 /*
1687 * Invalid QP transport type. If we got here then it's a
1688 * warning of a probably serious problem. So print a message
1689 * and return failure
1690 */
1691 TAVOR_WARNING(state, "unknown QP transport type in rtr2rts");
1692 return (ibc_get_ci_failure(0));
1693 }
1694
1695 /*
1696 * Post the RTR2RTS_QP command to the Tavor firmware
1697 *
1698 * We do a TAVOR_NOSLEEP here because we are still holding the
1699 * "qp_lock". If we got raised to interrupt level by priority
1700 * inversion, we do not want to block in this routine waiting for
1701 * success.
1702 */
1703 status = tavor_cmn_qp_cmd_post(state, RTR2RTS_QP, qpc, qp->qp_qpnum,
1704 opmask, TAVOR_CMD_NOSLEEP_SPIN);
1705 if (status != TAVOR_CMD_SUCCESS) {
1706 if (status != TAVOR_CMD_BAD_QP_STATE) {
1707 cmn_err(CE_CONT, "Tavor: RTR2RTS_QP command failed: "
1708 "%08x\n", status);
1709 return (ibc_get_ci_failure(0));
1710 } else {
1711 return (IBT_QP_STATE_INVALID);
1712 }
1713 }
1714
1715 return (DDI_SUCCESS);
1716 }
1717
1718
1719 /*
1720 * tavor_qp_rts2rts()
1721 * Context: Can be called from interrupt or base context.
1722 */
1723 static int
tavor_qp_rts2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)1724 tavor_qp_rts2rts(tavor_state_t *state, tavor_qphdl_t qp,
1725 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1726 {
1727 tavor_hw_qpc_t *qpc;
1728 ibt_qp_rc_attr_t *rc;
1729 ibt_qp_ud_attr_t *ud;
1730 ibt_qp_uc_attr_t *uc;
1731 tavor_hw_addr_path_t *qpc_path;
1732 ibt_adds_vect_t *adds_vect;
1733 uint_t portnum, pkeyindx;
1734 uint32_t opmask = 0;
1735 int status;
1736
1737 ASSERT(MUTEX_HELD(&qp->qp_lock));
1738
1739 /*
1740 * Grab the temporary QPC entry from QP software state
1741 */
1742 qpc = &qp->qpc;
1743
1744 /*
1745 * Since there are no common and/or Tavor-specific fields to be filled
1746 * in for this command, we begin with the QPC fields which are
1747 * specific to transport type.
1748 */
1749 if (qp->qp_serv_type == TAVOR_QP_UD) {
1750 ud = &info_p->qp_transport.ud;
1751
1752 /*
1753 * If we are attempting to modify the QKey for this QP, then
1754 * fill it in and set the appropriate flag in the "opmask"
1755 * parameter.
1756 */
1757 if (flags & IBT_CEP_SET_QKEY) {
1758 qpc->qkey = ud->ud_qkey;
1759 opmask |= TAVOR_CMD_OP_QKEY;
1760 }
1761
1762 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
1763 rc = &info_p->qp_transport.rc;
1764
1765 /*
1766 * Check if any of the flags indicate a change in the RDMA
1767 * (recv) enable/disable flags and set the appropriate flag in
1768 * the "opmask" parameter
1769 */
1770 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1771
1772 /*
1773 * If we are attempting to modify the path migration state for
1774 * this QP, then check for valid state and fill it in. Also
1775 * set the appropriate flag in the "opmask" parameter.
1776 */
1777 if (flags & IBT_CEP_SET_MIG) {
1778 if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
1779 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1780 } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
1781 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1782 } else {
1783 return (IBT_QP_APM_STATE_INVALID);
1784 }
1785 opmask |= TAVOR_CMD_OP_PM_STATE;
1786 }
1787
1788 /*
1789 * If we are attempting to modify the "Minimum RNR NAK" value
1790 * for this QP, then fill it in and set the appropriate flag
1791 * in the "opmask" parameter.
1792 */
1793 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
1794 qpc->min_rnr_nak = rc->rc_min_rnr_nak;
1795 opmask |= TAVOR_CMD_OP_MINRNRNAK;
1796 }
1797
1798 /*
1799 * Check for optional alternate path and fill in the
1800 * appropriate QPC fields if one is specified
1801 */
1802 if (flags & IBT_CEP_SET_ALT_PATH) {
1803 qpc_path = &qpc->alt_addr_path;
1804 adds_vect = &rc->rc_alt_path.cep_adds_vect;
1805
1806 /* Set the common alternate address path fields */
1807 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1808 TAVOR_ADDRPATH_QP, qp);
1809 if (status != DDI_SUCCESS) {
1810 return (status);
1811 }
1812 qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
1813
1814 /*
1815 * Check for valid alternate path port number and fill
1816 * it in
1817 */
1818 portnum = rc->rc_alt_path.cep_hca_port_num;
1819 if (tavor_portnum_is_valid(state, portnum)) {
1820 qpc->alt_addr_path.portnum = portnum;
1821 } else {
1822 return (IBT_HCA_PORT_INVALID);
1823 }
1824
1825 /*
1826 * Check for valid alternate path PKey index and fill
1827 * it in
1828 */
1829 pkeyindx = rc->rc_alt_path.cep_pkey_ix;
1830 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1831 qpc->alt_addr_path.pkey_indx = pkeyindx;
1832 } else {
1833 return (IBT_PKEY_IX_ILLEGAL);
1834 }
1835 opmask |= TAVOR_CMD_OP_ALT_PATH;
1836 }
1837
1838 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
1839 uc = &info_p->qp_transport.uc;
1840
1841 /*
1842 * Check if any of the flags indicate a change in the RDMA
1843 * Write (recv) enable/disable and set the appropriate flag
1844 * in the "opmask" parameter. Note: RDMA Read and Atomic are
1845 * not valid for UC transport.
1846 */
1847 if (flags & IBT_CEP_SET_RDMA_W) {
1848 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1849 opmask |= TAVOR_CMD_OP_RWE;
1850 }
1851
1852 /*
1853 * If we are attempting to modify the path migration state for
1854 * this QP, then check for valid state and fill it in. Also
1855 * set the appropriate flag in the "opmask" parameter.
1856 */
1857 if (flags & IBT_CEP_SET_MIG) {
1858 if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
1859 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1860 } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
1861 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1862 } else {
1863 return (IBT_QP_APM_STATE_INVALID);
1864 }
1865 opmask |= TAVOR_CMD_OP_PM_STATE;
1866 }
1867
1868 /*
1869 * Check for optional alternate path and fill in the
1870 * appropriate QPC fields if one is specified
1871 */
1872 if (flags & IBT_CEP_SET_ALT_PATH) {
1873 qpc_path = &qpc->alt_addr_path;
1874 adds_vect = &uc->uc_alt_path.cep_adds_vect;
1875
1876 /* Set the common alternate address path fields */
1877 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1878 TAVOR_ADDRPATH_QP, qp);
1879 if (status != DDI_SUCCESS) {
1880 return (status);
1881 }
1882
1883 /*
1884 * Check for valid alternate path port number and fill
1885 * it in
1886 */
1887 portnum = uc->uc_alt_path.cep_hca_port_num;
1888 if (tavor_portnum_is_valid(state, portnum)) {
1889 qpc->alt_addr_path.portnum = portnum;
1890 } else {
1891 return (IBT_HCA_PORT_INVALID);
1892 }
1893
1894 /*
1895 * Check for valid alternate path PKey index and fill
1896 * it in
1897 */
1898 pkeyindx = uc->uc_alt_path.cep_pkey_ix;
1899 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1900 qpc->alt_addr_path.pkey_indx = pkeyindx;
1901 } else {
1902 return (IBT_PKEY_IX_ILLEGAL);
1903 }
1904 opmask |= TAVOR_CMD_OP_ALT_PATH;
1905 }
1906 } else {
1907 /*
1908 * Invalid QP transport type. If we got here then it's a
1909 * warning of a probably serious problem. So print a message
1910 * and return failure
1911 */
1912 TAVOR_WARNING(state, "unknown QP transport type in rts2rts");
1913 return (ibc_get_ci_failure(0));
1914 }
1915
1916 /*
1917 * Post the RTS2RTS_QP command to the Tavor firmware
1918 *
1919 * We do a TAVOR_NOSLEEP here because we are still holding the
1920 * "qp_lock". If we got raised to interrupt level by priority
1921 * inversion, we do not want to block in this routine waiting for
1922 * success.
1923 */
1924 status = tavor_cmn_qp_cmd_post(state, RTS2RTS_QP, qpc, qp->qp_qpnum,
1925 opmask, TAVOR_CMD_NOSLEEP_SPIN);
1926 if (status != TAVOR_CMD_SUCCESS) {
1927 if (status != TAVOR_CMD_BAD_QP_STATE) {
1928 cmn_err(CE_CONT, "Tavor: RTS2RTS_QP command failed: "
1929 "%08x\n", status);
1930 return (ibc_get_ci_failure(0));
1931 } else {
1932 return (IBT_QP_STATE_INVALID);
1933 }
1934 }
1935
1936 return (DDI_SUCCESS);
1937 }
1938
1939
1940 /*
1941 * tavor_qp_rts2sqd()
1942 * Context: Can be called from interrupt or base context.
1943 */
1944 static int
tavor_qp_rts2sqd(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags)1945 tavor_qp_rts2sqd(tavor_state_t *state, tavor_qphdl_t qp,
1946 ibt_cep_modify_flags_t flags)
1947 {
1948 int status;
1949
1950 ASSERT(MUTEX_HELD(&qp->qp_lock));
1951
1952 /*
1953 * Set a flag to indicate whether or not the consumer is interested
1954 * in receiving the SQ drained event. Since we are going to always
1955 * request hardware generation of the SQD event, we use the value in
1956 * "qp_forward_sqd_event" to determine whether or not to pass the event
1957 * to the IBTF or to silently consume it.
1958 */
1959 qp->qp_forward_sqd_event = (flags & IBT_CEP_SET_SQD_EVENT) ? 1 : 0;
1960
1961 /*
1962 * Post the RTS2SQD_QP command to the Tavor firmware
1963 *
1964 * We do a TAVOR_NOSLEEP here because we are still holding the
1965 * "qp_lock". If we got raised to interrupt level by priority
1966 * inversion, we do not want to block in this routine waiting for
1967 * success.
1968 */
1969 status = tavor_cmn_qp_cmd_post(state, RTS2SQD_QP, NULL, qp->qp_qpnum,
1970 0, TAVOR_CMD_NOSLEEP_SPIN);
1971 if (status != TAVOR_CMD_SUCCESS) {
1972 if (status != TAVOR_CMD_BAD_QP_STATE) {
1973 cmn_err(CE_CONT, "Tavor: RTS2SQD_QP command failed: "
1974 "%08x\n", status);
1975 return (ibc_get_ci_failure(0));
1976 } else {
1977 return (IBT_QP_STATE_INVALID);
1978 }
1979 }
1980
1981 /*
1982 * Mark the current QP state as "SQ Draining". This allows us to
1983 * distinguish between the two underlying states in SQD. (see QueryQP()
1984 * code in tavor_qp.c)
1985 */
1986 qp->qp_sqd_still_draining = 1;
1987
1988 return (DDI_SUCCESS);
1989 }
1990
1991
1992 /*
1993 * tavor_qp_sqd2rts()
1994 * Context: Can be called from interrupt or base context.
1995 */
1996 static int
tavor_qp_sqd2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)1997 tavor_qp_sqd2rts(tavor_state_t *state, tavor_qphdl_t qp,
1998 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1999 {
2000 tavor_hw_qpc_t *qpc;
2001 ibt_qp_rc_attr_t *rc;
2002 ibt_qp_ud_attr_t *ud;
2003 ibt_qp_uc_attr_t *uc;
2004 tavor_hw_addr_path_t *qpc_path;
2005 ibt_adds_vect_t *adds_vect;
2006 uint_t portnum, pkeyindx;
2007 uint32_t opmask = 0;
2008 int status;
2009
2010 ASSERT(MUTEX_HELD(&qp->qp_lock));
2011
2012 /*
2013 * Grab the temporary QPC entry from QP software state
2014 */
2015 qpc = &qp->qpc;
2016
2017 /*
2018 * Since there are no common and/or Tavor-specific fields to be filled
2019 * in for this command, we begin with the QPC fields which are
2020 * specific to transport type.
2021 */
2022 if (qp->qp_serv_type == TAVOR_QP_UD) {
2023 ud = &info_p->qp_transport.ud;
2024
2025 /*
2026 * If we are attempting to modify the QKey for this QP, then
2027 * fill it in and set the appropriate flag in the "opmask"
2028 * parameter.
2029 */
2030 if (flags & IBT_CEP_SET_QKEY) {
2031 qpc->qkey = ud->ud_qkey;
2032 opmask |= TAVOR_CMD_OP_QKEY;
2033 }
2034
2035 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
2036 rc = &info_p->qp_transport.rc;
2037
2038 /*
2039 * Check if any of the flags indicate a change in the RDMA
2040 * (recv) enable/disable flags and set the appropriate flag in
2041 * the "opmask" parameter
2042 */
2043 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2044
2045 /*
2046 * If we are attempting to modify the path migration state for
2047 * this QP, then check for valid state and fill it in. Also
2048 * set the appropriate flag in the "opmask" parameter.
2049 */
2050 if (flags & IBT_CEP_SET_MIG) {
2051 if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2052 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2053 } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2054 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2055 } else {
2056 return (IBT_QP_APM_STATE_INVALID);
2057 }
2058 opmask |= TAVOR_CMD_OP_PM_STATE;
2059 }
2060
2061 /*
2062 * Check for optional alternate path and fill in the
2063 * appropriate QPC fields if one is specified
2064 */
2065 if (flags & IBT_CEP_SET_ALT_PATH) {
2066 qpc_path = &qpc->alt_addr_path;
2067 adds_vect = &rc->rc_alt_path.cep_adds_vect;
2068
2069 /* Set the common alternate address path fields */
2070 status = tavor_set_addr_path(state, adds_vect, qpc_path,
2071 TAVOR_ADDRPATH_QP, qp);
2072 if (status != DDI_SUCCESS) {
2073 return (status);
2074 }
2075 qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2076
2077 /*
2078 * Check for valid alternate path port number and fill
2079 * it in
2080 */
2081 portnum = rc->rc_alt_path.cep_hca_port_num;
2082 if (tavor_portnum_is_valid(state, portnum)) {
2083 qpc->alt_addr_path.portnum = portnum;
2084 } else {
2085 return (IBT_HCA_PORT_INVALID);
2086 }
2087
2088 /*
2089 * Check for valid alternate path PKey index and fill
2090 * it in
2091 */
2092 pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2093 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2094 qpc->alt_addr_path.pkey_indx = pkeyindx;
2095 } else {
2096 return (IBT_PKEY_IX_ILLEGAL);
2097 }
2098 opmask |= TAVOR_CMD_OP_ALT_PATH;
2099 }
2100
2101 /*
2102 * If we are attempting to modify the "Minimum RNR NAK" value
2103 * for this QP, then fill it in and set the appropriate flag
2104 * in the "opmask" parameter.
2105 */
2106 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2107 qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2108 opmask |= TAVOR_CMD_OP_MINRNRNAK;
2109 }
2110
2111 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
2112 uc = &info_p->qp_transport.uc;
2113
2114 /*
2115 * Check if any of the flags indicate a change in the RDMA
2116 * Write (recv) enable/disable and set the appropriate flag
2117 * in the "opmask" parameter. Note: RDMA Read and Atomic are
2118 * not valid for UC transport.
2119 */
2120 if (flags & IBT_CEP_SET_RDMA_W) {
2121 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2122 opmask |= TAVOR_CMD_OP_RWE;
2123 }
2124
2125 /*
2126 * If we are attempting to modify the path migration state for
2127 * this QP, then check for valid state and fill it in. Also
2128 * set the appropriate flag in the "opmask" parameter.
2129 */
2130 if (flags & IBT_CEP_SET_MIG) {
2131 if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2132 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2133 } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2134 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2135 } else {
2136 return (IBT_QP_APM_STATE_INVALID);
2137 }
2138 opmask |= TAVOR_CMD_OP_PM_STATE;
2139 }
2140
2141 /*
2142 * Check for optional alternate path and fill in the
2143 * appropriate QPC fields if one is specified
2144 */
2145 if (flags & IBT_CEP_SET_ALT_PATH) {
2146 qpc_path = &qpc->alt_addr_path;
2147 adds_vect = &uc->uc_alt_path.cep_adds_vect;
2148
2149 /* Set the common alternate address path fields */
2150 status = tavor_set_addr_path(state, adds_vect, qpc_path,
2151 TAVOR_ADDRPATH_QP, qp);
2152 if (status != DDI_SUCCESS) {
2153 return (status);
2154 }
2155
2156 /*
2157 * Check for valid alternate path port number and fill
2158 * it in
2159 */
2160 portnum = uc->uc_alt_path.cep_hca_port_num;
2161 if (tavor_portnum_is_valid(state, portnum)) {
2162 qpc->alt_addr_path.portnum = portnum;
2163 } else {
2164 return (IBT_HCA_PORT_INVALID);
2165 }
2166
2167 /*
2168 * Check for valid alternate path PKey index and fill
2169 * it in
2170 */
2171 pkeyindx = uc->uc_alt_path.cep_pkey_ix;
2172 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2173 qpc->alt_addr_path.pkey_indx = pkeyindx;
2174 } else {
2175 return (IBT_PKEY_IX_ILLEGAL);
2176 }
2177 opmask |= TAVOR_CMD_OP_ALT_PATH;
2178 }
2179 } else {
2180 /*
2181 * Invalid QP transport type. If we got here then it's a
2182 * warning of a probably serious problem. So print a message
2183 * and return failure
2184 */
2185 TAVOR_WARNING(state, "unknown QP transport type in sqd2rts");
2186 return (ibc_get_ci_failure(0));
2187 }
2188
2189 /*
2190 * Post the SQD2RTS_QP command to the Tavor firmware
2191 *
2192 * We do a TAVOR_NOSLEEP here because we are still holding the
2193 * "qp_lock". If we got raised to interrupt level by priority
2194 * inversion, we do not want to block in this routine waiting for
2195 * success.
2196 */
2197 status = tavor_cmn_qp_cmd_post(state, SQD2RTS_QP, qpc, qp->qp_qpnum,
2198 opmask, TAVOR_CMD_NOSLEEP_SPIN);
2199 if (status != TAVOR_CMD_SUCCESS) {
2200 if (status != TAVOR_CMD_BAD_QP_STATE) {
2201 cmn_err(CE_CONT, "Tavor: SQD2RTS_QP command failed: "
2202 "%08x\n", status);
2203 return (ibc_get_ci_failure(0));
2204 } else {
2205 return (IBT_QP_STATE_INVALID);
2206 }
2207 }
2208
2209 return (DDI_SUCCESS);
2210 }
2211
2212
2213 /*
2214 * tavor_qp_sqd2sqd()
2215 * Context: Can be called from interrupt or base context.
2216 */
2217 static int
tavor_qp_sqd2sqd(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)2218 tavor_qp_sqd2sqd(tavor_state_t *state, tavor_qphdl_t qp,
2219 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2220 {
2221 tavor_hw_qpc_t *qpc;
2222 ibt_qp_rc_attr_t *rc;
2223 ibt_qp_ud_attr_t *ud;
2224 ibt_qp_uc_attr_t *uc;
2225 tavor_hw_addr_path_t *qpc_path;
2226 ibt_adds_vect_t *adds_vect;
2227 uint_t portnum, pkeyindx;
2228 uint_t rra_max, sra_max;
2229 uint32_t opmask = 0;
2230 int status;
2231
2232 ASSERT(MUTEX_HELD(&qp->qp_lock));
2233
2234 /*
2235 * Grab the temporary QPC entry from QP software state
2236 */
2237 qpc = &qp->qpc;
2238
2239 /*
2240 * Since there are no common and/or Tavor-specific fields to be filled
2241 * in for this command, we begin with the QPC fields which are
2242 * specific to transport type.
2243 */
2244 if (qp->qp_serv_type == TAVOR_QP_UD) {
2245 ud = &info_p->qp_transport.ud;
2246
2247 /*
2248 * If we are attempting to modify the PKey index for this QP,
2249 * then check for valid PKey index and fill it in. Also set
2250 * the appropriate flag in the "opmask" parameter.
2251 */
2252 if (flags & IBT_CEP_SET_PKEY_IX) {
2253 pkeyindx = ud->ud_pkey_ix;
2254 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2255 qpc->pri_addr_path.pkey_indx = pkeyindx;
2256 opmask |= TAVOR_CMD_OP_PKEYINDX;
2257 qp->qp_pkeyindx = pkeyindx;
2258 } else {
2259 return (IBT_PKEY_IX_ILLEGAL);
2260 }
2261 }
2262
2263 /*
2264 * If we are attempting to modify the QKey for this QP, then
2265 * fill it in and set the appropriate flag in the "opmask"
2266 * parameter.
2267 */
2268 if (flags & IBT_CEP_SET_QKEY) {
2269 qpc->qkey = ud->ud_qkey;
2270 opmask |= TAVOR_CMD_OP_QKEY;
2271 }
2272
2273 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
2274 rc = &info_p->qp_transport.rc;
2275
2276 /*
2277 * Check if any of the flags indicate a change in the RDMA
2278 * (recv) enable/disable flags and set the appropriate flag in
2279 * the "opmask" parameter
2280 */
2281 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2282
2283 /*
2284 * Check for optional primary path and fill in the
2285 * appropriate QPC fields if one is specified
2286 */
2287 if (flags & IBT_CEP_SET_ADDS_VECT) {
2288 qpc_path = &qpc->pri_addr_path;
2289 adds_vect = &rc->rc_path.cep_adds_vect;
2290
2291 /* Set the common primary address path fields */
2292 status = tavor_set_addr_path(state, adds_vect, qpc_path,
2293 TAVOR_ADDRPATH_QP, qp);
2294 if (status != DDI_SUCCESS) {
2295 return (status);
2296 }
2297 qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
2298 qpc_path->ack_timeout = rc->rc_path.cep_timeout;
2299 qpc->retry_cnt = rc->rc_retry_cnt;
2300
2301 /*
2302 * MTU changes as part of sqd2sqd are not allowed.
2303 * Simply keep the same MTU value here, stored in the
2304 * qphdl from init2rtr time.
2305 */
2306 qpc->mtu = qp->qp_save_mtu;
2307
2308 opmask |= (TAVOR_CMD_OP_PRIM_PATH |
2309 TAVOR_CMD_OP_RETRYCNT | TAVOR_CMD_OP_ACKTIMEOUT |
2310 TAVOR_CMD_OP_PRIM_RNRRETRY);
2311 }
2312
2313 /*
2314 * If we are attempting to modify the path migration state for
2315 * this QP, then check for valid state and fill it in. Also
2316 * set the appropriate flag in the "opmask" parameter.
2317 */
2318 if (flags & IBT_CEP_SET_MIG) {
2319 if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2320 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2321 } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2322 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2323 } else {
2324 return (IBT_QP_APM_STATE_INVALID);
2325 }
2326 opmask |= TAVOR_CMD_OP_PM_STATE;
2327 }
2328
2329 /*
2330 * If we are attempting to modify the PKey index for this QP,
2331 * then check for valid PKey index and fill it in. Also set
2332 * the appropriate flag in the "opmask" parameter.
2333 */
2334 if (flags & IBT_CEP_SET_PKEY_IX) {
2335 pkeyindx = rc->rc_path.cep_pkey_ix;
2336 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2337 qpc->pri_addr_path.pkey_indx = pkeyindx;
2338 opmask |= TAVOR_CMD_OP_PKEYINDX;
2339 } else {
2340 return (IBT_PKEY_IX_ILLEGAL);
2341 }
2342 }
2343
2344 /*
2345 * If we are attempting to modify the port for this QP, then
2346 * check for valid port number and fill it in. Also set the
2347 * appropriate flag in the "opmask" parameter.
2348 */
2349 if (flags & IBT_CEP_SET_PORT) {
2350 portnum = rc->rc_path.cep_hca_port_num;
2351 if (tavor_portnum_is_valid(state, portnum)) {
2352 qpc->pri_addr_path.portnum = portnum;
2353 } else {
2354 return (IBT_HCA_PORT_INVALID);
2355 }
2356 opmask |= TAVOR_CMD_OP_PRIM_PORT;
2357 }
2358
2359 /*
2360 * Check for optional alternate path and fill in the
2361 * appropriate QPC fields if one is specified
2362 */
2363 if (flags & IBT_CEP_SET_ALT_PATH) {
2364 qpc_path = &qpc->alt_addr_path;
2365 adds_vect = &rc->rc_alt_path.cep_adds_vect;
2366
2367 /* Set the common alternate address path fields */
2368 status = tavor_set_addr_path(state, adds_vect, qpc_path,
2369 TAVOR_ADDRPATH_QP, qp);
2370 if (status != DDI_SUCCESS) {
2371 return (status);
2372 }
2373 qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2374
2375 /*
2376 * Check for valid alternate path port number and fill
2377 * it in
2378 */
2379 portnum = rc->rc_alt_path.cep_hca_port_num;
2380 if (tavor_portnum_is_valid(state, portnum)) {
2381 qpc->alt_addr_path.portnum = portnum;
2382 } else {
2383 return (IBT_HCA_PORT_INVALID);
2384 }
2385
2386 /*
2387 * Check for valid alternate path PKey index and fill
2388 * it in
2389 */
2390 pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2391 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2392 qpc->alt_addr_path.pkey_indx = pkeyindx;
2393 } else {
2394 return (IBT_PKEY_IX_ILLEGAL);
2395 }
2396 opmask |= TAVOR_CMD_OP_ALT_PATH;
2397 }
2398
2399 /*
2400 * If we are attempting to modify the number of "outgoing
2401 * RDMA resources" for this QP, then check for valid value and
2402 * fill it in. Also set the appropriate flag in the "opmask"
2403 * parameter.
2404 */
2405 if (flags & IBT_CEP_SET_RDMARA_OUT) {
2406 if (tavor_qp_validate_init_depth(state, rc,
2407 &sra_max) != DDI_SUCCESS) {
2408 return (IBT_INVALID_PARAM);
2409 }
2410 qpc->sra_max = sra_max;
2411 opmask |= TAVOR_CMD_OP_SRA_SET;
2412 }
2413
2414 /*
2415 * If we are attempting to modify the number of "incoming
2416 * RDMA resources" for this QP, then check for valid value and
2417 * update the "rra_max" and "ra_buf_index" fields in the QPC to
2418 * point to the pre-allocated RDB resources (in DDR). Also set
2419 * the appropriate flag in the "opmask" parameter.
2420 */
2421 if (flags & IBT_CEP_SET_RDMARA_IN) {
2422 if (tavor_qp_validate_resp_rsrc(state, rc,
2423 &rra_max) != DDI_SUCCESS) {
2424 return (IBT_INVALID_PARAM);
2425 }
2426 qpc->rra_max = rra_max;
2427 qpc->ra_buff_indx = qp->qp_rdb_ddraddr >>
2428 TAVOR_RDB_SIZE_SHIFT;
2429 opmask |= TAVOR_CMD_OP_RRA_SET;
2430 }
2431
2432 /*
2433 * If we are attempting to modify the "Local Ack Timeout" value
2434 * for this QP, then fill it in and set the appropriate flag in
2435 * the "opmask" parameter.
2436 */
2437 if (flags & IBT_CEP_SET_TIMEOUT) {
2438 qpc_path = &qpc->pri_addr_path;
2439 qpc_path->ack_timeout = rc->rc_path.cep_timeout;
2440 opmask |= TAVOR_CMD_OP_ACKTIMEOUT;
2441 }
2442
2443 /*
2444 * If we are attempting to modify the "Retry Count" for this QP,
2445 * then fill it in and set the appropriate flag in the "opmask"
2446 * parameter.
2447 */
2448 if (flags & IBT_CEP_SET_RETRY) {
2449 qpc->retry_cnt = rc->rc_retry_cnt;
2450 opmask |= TAVOR_CMD_OP_PRIM_RNRRETRY;
2451 }
2452
2453 /*
2454 * If we are attempting to modify the "RNR Retry Count" for this
2455 * QP, then fill it in and set the appropriate flag in the
2456 * "opmask" parameter.
2457 */
2458 if (flags & IBT_CEP_SET_RNR_NAK_RETRY) {
2459 qpc_path = &qpc->pri_addr_path;
2460 qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
2461 opmask |= TAVOR_CMD_OP_RETRYCNT;
2462 }
2463
2464 /*
2465 * If we are attempting to modify the "Minimum RNR NAK" value
2466 * for this QP, then fill it in and set the appropriate flag
2467 * in the "opmask" parameter.
2468 */
2469 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2470 qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2471 opmask |= TAVOR_CMD_OP_MINRNRNAK;
2472 }
2473
2474 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
2475 uc = &info_p->qp_transport.uc;
2476
2477 /*
2478 * Check if any of the flags indicate a change in the RDMA
2479 * Write (recv) enable/disable and set the appropriate flag
2480 * in the "opmask" parameter. Note: RDMA Read and Atomic are
2481 * not valid for UC transport.
2482 */
2483 if (flags & IBT_CEP_SET_RDMA_W) {
2484 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2485 opmask |= TAVOR_CMD_OP_RWE;
2486 }
2487
2488 /*
2489 * Check for optional primary path and fill in the
2490 * appropriate QPC fields if one is specified
2491 */
2492 if (flags & IBT_CEP_SET_ADDS_VECT) {
2493 qpc_path = &qpc->pri_addr_path;
2494 adds_vect = &uc->uc_path.cep_adds_vect;
2495
2496 /* Set the common primary address path fields */
2497 status = tavor_set_addr_path(state, adds_vect, qpc_path,
2498 TAVOR_ADDRPATH_QP, qp);
2499 if (status != DDI_SUCCESS) {
2500 return (status);
2501 }
2502
2503 /*
2504 * MTU changes as part of sqd2sqd are not allowed.
2505 * Simply keep the same MTU value here, stored in the
2506 * qphdl from init2rtr time.
2507 */
2508 qpc->mtu = qp->qp_save_mtu;
2509
2510 opmask |= TAVOR_CMD_OP_PRIM_PATH;
2511 }
2512
2513 /*
2514 * If we are attempting to modify the path migration state for
2515 * this QP, then check for valid state and fill it in. Also
2516 * set the appropriate flag in the "opmask" parameter.
2517 */
2518 if (flags & IBT_CEP_SET_MIG) {
2519 if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2520 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2521 } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2522 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2523 } else {
2524 return (IBT_QP_APM_STATE_INVALID);
2525 }
2526 opmask |= TAVOR_CMD_OP_PM_STATE;
2527 }
2528
2529 /*
2530 * If we are attempting to modify the PKey index for this QP,
2531 * then check for valid PKey index and fill it in. Also set
2532 * the appropriate flag in the "opmask" parameter.
2533 */
2534 if (flags & IBT_CEP_SET_PKEY_IX) {
2535 pkeyindx = uc->uc_path.cep_pkey_ix;
2536 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2537 qpc->pri_addr_path.pkey_indx = pkeyindx;
2538 opmask |= TAVOR_CMD_OP_PKEYINDX;
2539 } else {
2540 return (IBT_PKEY_IX_ILLEGAL);
2541 }
2542 }
2543
2544 /*
2545 * Check for optional alternate path and fill in the
2546 * appropriate QPC fields if one is specified
2547 */
2548 if (flags & IBT_CEP_SET_ALT_PATH) {
2549 qpc_path = &qpc->alt_addr_path;
2550 adds_vect = &uc->uc_alt_path.cep_adds_vect;
2551
2552 /* Set the common alternate address path fields */
2553 status = tavor_set_addr_path(state, adds_vect, qpc_path,
2554 TAVOR_ADDRPATH_QP, qp);
2555 if (status != DDI_SUCCESS) {
2556 return (status);
2557 }
2558
2559 /*
2560 * Check for valid alternate path port number and fill
2561 * it in
2562 */
2563 portnum = uc->uc_alt_path.cep_hca_port_num;
2564 if (tavor_portnum_is_valid(state, portnum)) {
2565 qpc->alt_addr_path.portnum = portnum;
2566 } else {
2567 return (IBT_HCA_PORT_INVALID);
2568 }
2569
2570 /*
2571 * Check for valid alternate path PKey index and fill
2572 * it in
2573 */
2574 pkeyindx = uc->uc_alt_path.cep_pkey_ix;
2575 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2576 qpc->alt_addr_path.pkey_indx = pkeyindx;
2577 } else {
2578 return (IBT_PKEY_IX_ILLEGAL);
2579 }
2580 opmask |= TAVOR_CMD_OP_ALT_PATH;
2581 }
2582 } else {
2583 /*
2584 * Invalid QP transport type. If we got here then it's a
2585 * warning of a probably serious problem. So print a message
2586 * and return failure
2587 */
2588 TAVOR_WARNING(state, "unknown QP transport type in sqd2sqd");
2589 return (ibc_get_ci_failure(0));
2590 }
2591
2592 /*
2593 * Post the SQD2SQD_QP command to the Tavor firmware
2594 *
2595 * We do a TAVOR_NOSLEEP here because we are still holding the
2596 * "qp_lock". If we got raised to interrupt level by priority
2597 * inversion, we do not want to block in this routine waiting for
2598 * success.
2599 */
2600 status = tavor_cmn_qp_cmd_post(state, SQD2SQD_QP, qpc, qp->qp_qpnum,
2601 opmask, TAVOR_CMD_NOSLEEP_SPIN);
2602 if (status != TAVOR_CMD_SUCCESS) {
2603 if (status != TAVOR_CMD_BAD_QP_STATE) {
2604 cmn_err(CE_CONT, "Tavor: SQD2SQD_QP command failed: "
2605 "%08x\n", status);
2606 return (ibc_get_ci_failure(0));
2607 } else {
2608 return (IBT_QP_STATE_INVALID);
2609 }
2610 }
2611
2612 return (DDI_SUCCESS);
2613 }
2614
2615
2616 /*
2617 * tavor_qp_sqerr2rts()
2618 * Context: Can be called from interrupt or base context.
2619 */
2620 static int
tavor_qp_sqerr2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)2621 tavor_qp_sqerr2rts(tavor_state_t *state, tavor_qphdl_t qp,
2622 ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2623 {
2624 tavor_hw_qpc_t *qpc;
2625 ibt_qp_ud_attr_t *ud;
2626 uint32_t opmask = 0;
2627 int status;
2628
2629 ASSERT(MUTEX_HELD(&qp->qp_lock));
2630
2631 /*
2632 * Grab the temporary QPC entry from QP software state
2633 */
2634 qpc = &qp->qpc;
2635
2636 /*
2637 * Since there are no common and/or Tavor-specific fields to be filled
2638 * in for this command, we begin with the QPC fields which are
2639 * specific to transport type.
2640 */
2641 if (qp->qp_serv_type == TAVOR_QP_UD) {
2642 ud = &info_p->qp_transport.ud;
2643
2644 /*
2645 * If we are attempting to modify the QKey for this QP, then
2646 * fill it in and set the appropriate flag in the "opmask"
2647 * parameter.
2648 */
2649 if (flags & IBT_CEP_SET_QKEY) {
2650 qpc->qkey = ud->ud_qkey;
2651 opmask |= TAVOR_CMD_OP_QKEY;
2652 }
2653
2654 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
2655
2656 /*
2657 * Check if any of the flags indicate a change in the RDMA
2658 * Write (recv) enable/disable and set the appropriate flag
2659 * in the "opmask" parameter. Note: RDMA Read and Atomic are
2660 * not valid for UC transport.
2661 */
2662 if (flags & IBT_CEP_SET_RDMA_W) {
2663 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2664 opmask |= TAVOR_CMD_OP_RWE;
2665 }
2666 } else {
2667 /*
2668 * Invalid QP transport type. If we got here then it's a
2669 * warning of a probably serious problem. So print a message
2670 * and return failure
2671 */
2672 TAVOR_WARNING(state, "unknown QP transport type in sqerr2rts");
2673 return (ibc_get_ci_failure(0));
2674 }
2675
2676 /*
2677 * Post the SQERR2RTS_QP command to the Tavor firmware
2678 *
2679 * We do a TAVOR_NOSLEEP here because we are still holding the
2680 * "qp_lock". If we got raised to interrupt level by priority
2681 * inversion, we do not want to block in this routine waiting for
2682 * success.
2683 */
2684 status = tavor_cmn_qp_cmd_post(state, SQERR2RTS_QP, qpc, qp->qp_qpnum,
2685 opmask, TAVOR_CMD_NOSLEEP_SPIN);
2686 if (status != TAVOR_CMD_SUCCESS) {
2687 if (status != TAVOR_CMD_BAD_QP_STATE) {
2688 cmn_err(CE_CONT, "Tavor: SQERR2RTS_QP command failed: "
2689 "%08x\n", status);
2690 return (ibc_get_ci_failure(0));
2691 } else {
2692 return (IBT_QP_STATE_INVALID);
2693 }
2694 }
2695
2696 return (DDI_SUCCESS);
2697 }
2698
2699
2700 /*
2701 * tavor_qp_to_error()
2702 * Context: Can be called from interrupt or base context.
2703 */
2704 static int
tavor_qp_to_error(tavor_state_t * state,tavor_qphdl_t qp)2705 tavor_qp_to_error(tavor_state_t *state, tavor_qphdl_t qp)
2706 {
2707 int status;
2708
2709 ASSERT(MUTEX_HELD(&qp->qp_lock));
2710
2711 /*
2712 * Post the TOERR_QP command to the Tavor firmware
2713 *
2714 * We do a TAVOR_NOSLEEP here because we are still holding the
2715 * "qp_lock". If we got raised to interrupt level by priority
2716 * inversion, we do not want to block in this routine waiting for
2717 * success.
2718 */
2719 status = tavor_cmn_qp_cmd_post(state, TOERR_QP, NULL, qp->qp_qpnum,
2720 0, TAVOR_CMD_NOSLEEP_SPIN);
2721 if (status != TAVOR_CMD_SUCCESS) {
2722 cmn_err(CE_CONT, "Tavor: TOERR_QP command failed: %08x\n",
2723 status);
2724 return (ibc_get_ci_failure(0));
2725 }
2726
2727 return (DDI_SUCCESS);
2728 }
2729
2730
2731 /*
2732 * tavor_qp_to_reset()
2733 * Context: Can be called from interrupt or base context.
2734 */
2735 int
tavor_qp_to_reset(tavor_state_t * state,tavor_qphdl_t qp)2736 tavor_qp_to_reset(tavor_state_t *state, tavor_qphdl_t qp)
2737 {
2738 tavor_hw_qpc_t *qpc;
2739 int status;
2740
2741 ASSERT(MUTEX_HELD(&qp->qp_lock));
2742
2743 /*
2744 * Grab the temporary QPC entry from QP software state
2745 */
2746 qpc = &qp->qpc;
2747
2748 /*
2749 * Post the TORST_QP command to the Tavor firmware
2750 *
2751 * We do a TAVOR_NOSLEEP here because we are still holding the
2752 * "qp_lock". If we got raised to interrupt level by priority
2753 * inversion, we do not want to block in this routine waiting for
2754 * success.
2755 */
2756 status = tavor_cmn_qp_cmd_post(state, TORST_QP, qpc, qp->qp_qpnum,
2757 0, TAVOR_CMD_NOSLEEP_SPIN);
2758 if (status != TAVOR_CMD_SUCCESS) {
2759 cmn_err(CE_CONT, "Tavor: TORST_QP command failed: %08x\n",
2760 status);
2761 return (ibc_get_ci_failure(0));
2762 }
2763
2764 return (DDI_SUCCESS);
2765 }
2766
2767
2768 /*
2769 * tavor_qp_reset2err()
2770 * Context: Can be called from interrupt or base context.
2771 */
2772 static int
tavor_qp_reset2err(tavor_state_t * state,tavor_qphdl_t qp)2773 tavor_qp_reset2err(tavor_state_t *state, tavor_qphdl_t qp)
2774 {
2775 tavor_hw_qpc_t *qpc;
2776 int status;
2777
2778 ASSERT(MUTEX_HELD(&qp->qp_lock));
2779
2780 /*
2781 * In order to implement the transition from "Reset" directly to the
2782 * "Error" state, it is necessary to first give ownership of the QP
2783 * context to the Tavor hardware. This is accomplished by transitioning
2784 * the QP to "Init" as an intermediate step and then, immediately
2785 * transitioning to "Error".
2786 *
2787 * When this function returns success, the QP context will be owned by
2788 * the Tavor hardware and will be in the "Error" state.
2789 */
2790
2791 /*
2792 * Grab the temporary QPC entry from QP software state
2793 */
2794 qpc = &qp->qpc;
2795
2796 /*
2797 * Fill in the common and/or Tavor-specific fields in the QPC
2798 */
2799 if (qp->qp_is_special) {
2800 qpc->serv_type = TAVOR_QP_MLX;
2801 } else {
2802 qpc->serv_type = qp->qp_serv_type;
2803 }
2804 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2805 qpc->de = TAVOR_QP_DESC_EVT_ENABLED;
2806 qpc->sched_q = TAVOR_QP_SCHEDQ_GET(qp->qp_qpnum);
2807 if (qp->qp_is_umap) {
2808 qpc->usr_page = qp->qp_uarpg;
2809 } else {
2810 qpc->usr_page = 0;
2811 }
2812 qpc->pd = qp->qp_pdhdl->pd_pdnum;
2813 qpc->wqe_baseaddr = 0;
2814 qpc->wqe_lkey = qp->qp_mrhdl->mr_lkey;
2815 qpc->ssc = qp->qp_sq_sigtype;
2816 qpc->cqn_snd = qp->qp_sq_cqhdl->cq_cqnum;
2817 qpc->rsc = TAVOR_QP_RQ_ALL_SIGNALED;
2818 qpc->cqn_rcv = qp->qp_rq_cqhdl->cq_cqnum;
2819 qpc->srq_en = qp->qp_srq_en;
2820
2821 if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
2822 qpc->srq_number = qp->qp_srqhdl->srq_srqnum;
2823 } else {
2824 qpc->srq_number = 0;
2825 }
2826
2827 /*
2828 * Now fill in the QPC fields which are specific to transport type
2829 */
2830 if (qp->qp_serv_type == TAVOR_QP_UD) {
2831 /* Set the UD parameters to an invalid default */
2832 qpc->qkey = 0;
2833 qpc->pri_addr_path.portnum = 1;
2834 qpc->pri_addr_path.pkey_indx = 0;
2835
2836 } else if (qp->qp_serv_type == TAVOR_QP_RC) {
2837 /* Set the RC parameters to invalid default */
2838 qpc->rre = 0;
2839 qpc->rwe = 0;
2840 qpc->rae = 0;
2841 qpc->pri_addr_path.portnum = 1;
2842 qpc->pri_addr_path.pkey_indx = 0;
2843
2844 } else if (qp->qp_serv_type == TAVOR_QP_UC) {
2845 /* Set the UC parameters to invalid default */
2846 qpc->rwe = 0;
2847 qpc->pri_addr_path.portnum = 1;
2848 qpc->pri_addr_path.pkey_indx = 0;
2849
2850 } else {
2851 /*
2852 * Invalid QP transport type. If we got here then it's a
2853 * warning of a probably serious problem. So print a message
2854 * and return failure
2855 */
2856 TAVOR_WARNING(state, "unknown QP transport type in rst2err");
2857 return (ibc_get_ci_failure(0));
2858 }
2859
2860 /*
2861 * Post the RST2INIT_QP command to the Tavor firmware
2862 *
2863 * We do a TAVOR_NOSLEEP here because we are still holding the
2864 * "qp_lock". If we got raised to interrupt level by priority
2865 * inversion, we do not want to block in this routine waiting for
2866 * success.
2867 */
2868 status = tavor_cmn_qp_cmd_post(state, RST2INIT_QP, qpc, qp->qp_qpnum,
2869 0, TAVOR_CMD_NOSLEEP_SPIN);
2870 if (status != TAVOR_CMD_SUCCESS) {
2871 cmn_err(CE_CONT, "Tavor: RST2INIT_QP command failed: %08x\n",
2872 status);
2873 return (ibc_get_ci_failure(0));
2874 }
2875
2876 /*
2877 * Now post the TOERR_QP command to the Tavor firmware
2878 *
2879 * We still do a TAVOR_NOSLEEP here because we are still holding the
2880 * "qp_lock". Note: If this fails (which it really never should),
2881 * it indicates a serious problem in the HW or SW. We try to move
2882 * the QP back to the "Reset" state if possible and print a warning
2883 * message if not. In any case, we return an error here.
2884 */
2885 status = tavor_cmn_qp_cmd_post(state, TOERR_QP, NULL, qp->qp_qpnum,
2886 0, TAVOR_CMD_NOSLEEP_SPIN);
2887 if (status != TAVOR_CMD_SUCCESS) {
2888 cmn_err(CE_CONT, "Tavor: TOERR_QP command failed: %08x\n",
2889 status);
2890 if (tavor_qp_to_reset(state, qp) != DDI_SUCCESS) {
2891 TAVOR_WARNING(state, "failed to reset QP context");
2892 }
2893 return (ibc_get_ci_failure(0));
2894 }
2895
2896 return (DDI_SUCCESS);
2897 }
2898
2899
2900 /*
2901 * tavor_check_rdma_enable_flags()
2902 * Context: Can be called from interrupt or base context.
2903 */
2904 static uint_t
tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p,tavor_hw_qpc_t * qpc)2905 tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,
2906 ibt_qp_info_t *info_p, tavor_hw_qpc_t *qpc)
2907 {
2908 uint_t opmask = 0;
2909
2910 if (flags & IBT_CEP_SET_RDMA_R) {
2911 qpc->rre = (info_p->qp_flags & IBT_CEP_RDMA_RD) ? 1 : 0;
2912 opmask |= TAVOR_CMD_OP_RRE;
2913 }
2914
2915 if (flags & IBT_CEP_SET_RDMA_W) {
2916 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2917 opmask |= TAVOR_CMD_OP_RWE;
2918 }
2919
2920 if (flags & IBT_CEP_SET_ATOMIC) {
2921 qpc->rae = (info_p->qp_flags & IBT_CEP_ATOMIC) ? 1 : 0;
2922 opmask |= TAVOR_CMD_OP_RAE;
2923 }
2924
2925 return (opmask);
2926 }
2927
2928 /*
2929 * tavor_qp_validate_resp_rsrc()
2930 * Context: Can be called from interrupt or base context.
2931 */
2932 static int
tavor_qp_validate_resp_rsrc(tavor_state_t * state,ibt_qp_rc_attr_t * rc,uint_t * rra_max)2933 tavor_qp_validate_resp_rsrc(tavor_state_t *state, ibt_qp_rc_attr_t *rc,
2934 uint_t *rra_max)
2935 {
2936 uint_t rdma_ra_in;
2937
2938 rdma_ra_in = rc->rc_rdma_ra_in;
2939
2940 /*
2941 * Check if number of responder resources is too large. Return an
2942 * error if it is
2943 */
2944 if (rdma_ra_in > state->ts_cfg_profile->cp_hca_max_rdma_in_qp) {
2945 return (IBT_INVALID_PARAM);
2946 }
2947
2948 /*
2949 * If the number of responder resources is too small, round it up.
2950 * Then find the next highest power-of-2
2951 */
2952 if (rdma_ra_in == 0) {
2953 rdma_ra_in = 1;
2954 }
2955 if (ISP2(rdma_ra_in)) {
2956 *rra_max = highbit(rdma_ra_in) - 1;
2957 } else {
2958 *rra_max = highbit(rdma_ra_in);
2959 }
2960 return (DDI_SUCCESS);
2961 }
2962
2963
2964 /*
2965 * tavor_qp_validate_init_depth()
2966 * Context: Can be called from interrupt or base context.
2967 */
2968 static int
tavor_qp_validate_init_depth(tavor_state_t * state,ibt_qp_rc_attr_t * rc,uint_t * sra_max)2969 tavor_qp_validate_init_depth(tavor_state_t *state, ibt_qp_rc_attr_t *rc,
2970 uint_t *sra_max)
2971 {
2972 uint_t rdma_ra_out;
2973
2974 rdma_ra_out = rc->rc_rdma_ra_out;
2975
2976 /*
2977 * Check if requested initiator depth is too large. Return an error
2978 * if it is
2979 */
2980 if (rdma_ra_out > state->ts_cfg_profile->cp_hca_max_rdma_out_qp) {
2981 return (IBT_INVALID_PARAM);
2982 }
2983
2984 /*
2985 * If the requested initiator depth is too small, round it up.
2986 * Then find the next highest power-of-2
2987 */
2988 if (rdma_ra_out == 0) {
2989 rdma_ra_out = 1;
2990 }
2991 if (ISP2(rdma_ra_out)) {
2992 *sra_max = highbit(rdma_ra_out) - 1;
2993 } else {
2994 *sra_max = highbit(rdma_ra_out);
2995 }
2996 return (DDI_SUCCESS);
2997 }
2998
2999
3000 /*
3001 * tavor_qp_validate_mtu()
3002 * Context: Can be called from interrupt or base context.
3003 */
3004 static int
tavor_qp_validate_mtu(tavor_state_t * state,uint_t mtu)3005 tavor_qp_validate_mtu(tavor_state_t *state, uint_t mtu)
3006 {
3007 /*
3008 * Check for invalid MTU values (i.e. zero or any value larger than
3009 * the HCA's port maximum).
3010 */
3011 if ((mtu == 0) || (mtu > state->ts_cfg_profile->cp_max_mtu)) {
3012 return (IBT_HCA_PORT_MTU_EXCEEDED);
3013 }
3014 return (DDI_SUCCESS);
3015 }
3016