xref: /illumos-gate/usr/src/uts/common/io/ib/adapters/tavor/tavor_cmd.c (revision 2bc987325e3ded1865bff043128661815c4690b9)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * tavor_cmd.c
29  *    Tavor Firmware Command Routines
30  *
31  *    Implements all the routines necessary for allocating, posting, and
32  *    freeing commands for the Tavor firmware.  These routines manage a
33  *    preallocated list of command mailboxes and provide interfaces to post
34  *    each of the several dozen commands to the Tavor firmware.
35  */
36 
37 #include <sys/types.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 
43 #include <sys/ib/adapters/tavor/tavor.h>
44 
45 static int tavor_impl_mbox_alloc(tavor_state_t *state, tavor_mboxlist_t *mblist,
46     tavor_mbox_t **mb, uint_t mbox_wait);
47 static void tavor_impl_mbox_free(tavor_mboxlist_t *mblist, tavor_mbox_t **mb);
48 static int tavor_impl_mboxlist_init(tavor_state_t *state,
49     tavor_mboxlist_t *mblist, uint_t num_mbox, tavor_rsrc_type_t type);
50 static void tavor_impl_mboxlist_fini(tavor_state_t *state,
51     tavor_mboxlist_t *mblist);
52 static int tavor_outstanding_cmd_alloc(tavor_state_t *state,
53     tavor_cmd_t **cmd_ptr, uint_t cmd_wait);
54 static void tavor_outstanding_cmd_free(tavor_state_t *state,
55     tavor_cmd_t **cmd_ptr);
56 static int tavor_write_hcr(tavor_state_t *state, tavor_cmd_post_t *cmdpost,
57     uint16_t token);
58 static void tavor_mbox_sync(tavor_mbox_t *mbox, uint_t offset,
59     uint_t length, uint_t flag);
60 
61 /*
62  * tavor_cmd_post()
63  *    Context: Can be called from interrupt or base context.
64  *
65  *    The "cp_flags" field in cmdpost
66  *    is used to determine whether to wait for an available
67  *    outstanding command (if necessary) or to return error.
68  */
69 int
70 tavor_cmd_post(tavor_state_t *state, tavor_cmd_post_t *cmdpost)
71 {
72 	tavor_cmd_t	*cmdptr;
73 	int		status;
74 	uint16_t	token;
75 
76 	TAVOR_TNF_ENTER(tavor_cmd_post);
77 
78 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdpost))
79 
80 	/* Determine if we are going to spin until completion */
81 	if (cmdpost->cp_flags == TAVOR_CMD_NOSLEEP_SPIN) {
82 
83 		TNF_PROBE_0_DEBUG(tavor_cmd_post_spin, TAVOR_TNF_TRACE, "");
84 
85 		/* Write the command to the HCR */
86 		status = tavor_write_hcr(state, cmdpost, 0);
87 		if (status != TAVOR_CMD_SUCCESS) {
88 			TNF_PROBE_0(tavor_cmd_post_fail,
89 			    TAVOR_TNF_ERROR, "");
90 			TAVOR_TNF_EXIT(tavor_cmd_post);
91 			return (status);
92 		}
93 
94 		TAVOR_TNF_EXIT(tavor_cmd_post);
95 		return (TAVOR_CMD_SUCCESS);
96 
97 	} else {  /* "TAVOR_CMD_SLEEP_NOSPIN" */
98 
99 		TNF_PROBE_0_DEBUG(tavor_cmd_post_nospin, TAVOR_TNF_TRACE, "");
100 
101 		ASSERT(TAVOR_SLEEPFLAG_FOR_CONTEXT() != TAVOR_NOSLEEP);
102 
103 		/* NOTE: Expect threads to be waiting in here */
104 		status = tavor_outstanding_cmd_alloc(state, &cmdptr,
105 		    cmdpost->cp_flags);
106 		if (status != TAVOR_CMD_SUCCESS) {
107 			TNF_PROBE_0(tavor_cmd_alloc_fail, TAVOR_TNF_ERROR, "");
108 			TAVOR_TNF_EXIT(tavor_cmd_post);
109 			return (status);
110 		}
111 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdptr))
112 
113 		/*
114 		 * Set status to "TAVOR_CMD_INVALID_STATUS".  It is
115 		 * appropriate to do this here without the "cmd_comp_lock"
116 		 * because this register is overloaded.  Later it will be
117 		 * used to indicate - through a change from this invalid
118 		 * value to some other value - that the condition variable
119 		 * has been signaled.  Once it has, status will then contain
120 		 * the _real_ completion status
121 		 */
122 		cmdptr->cmd_status = TAVOR_CMD_INVALID_STATUS;
123 
124 		/* Write the command to the HCR */
125 		token = (uint16_t)cmdptr->cmd_indx;
126 		status = tavor_write_hcr(state, cmdpost, token);
127 		if (status != TAVOR_CMD_SUCCESS) {
128 			tavor_outstanding_cmd_free(state, &cmdptr);
129 			TNF_PROBE_0(tavor_cmd_post_fail, TAVOR_TNF_ERROR, "");
130 			TAVOR_TNF_EXIT(tavor_cmd_post);
131 			return (status);
132 		}
133 
134 		/*
135 		 * cv_wait() on the "command_complete" condition variable.
136 		 * Note: We have the "__lock_lint" here to workaround warlock.
137 		 * Since warlock doesn't know that other parts of the Tavor
138 		 * may occasionally call this routine while holding their own
139 		 * locks, it complains about this cv_wait.  In reality,
140 		 * however, the rest of the driver never calls this routine
141 		 * with a lock held unless they pass TAVOR_CMD_NOSLEEP.
142 		 */
143 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*cmdptr))
144 		mutex_enter(&cmdptr->cmd_comp_lock);
145 		while (cmdptr->cmd_status == TAVOR_CMD_INVALID_STATUS) {
146 #ifndef	__lock_lint
147 			cv_wait(&cmdptr->cmd_comp_cv, &cmdptr->cmd_comp_lock);
148 			/* NOTE: EXPECT SEVERAL THREADS TO BE WAITING HERE */
149 #endif
150 		}
151 		mutex_exit(&cmdptr->cmd_comp_lock);
152 		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cmdptr))
153 
154 		/*
155 		 * Wake up after command completes (cv_signal).  Read status
156 		 * from the command (success, fail, etc.).  It is appropriate
157 		 * here (as above) to read the status field without the
158 		 * "cmd_comp_lock" because it is no longer being used to
159 		 * indicate whether the condition variable has been signaled
160 		 * (i.e. at this point we are certain that it already has).
161 		 */
162 		status = cmdptr->cmd_status;
163 
164 		/* Save the "outparam" values into the cmdpost struct */
165 		cmdpost->cp_outparm = cmdptr->cmd_outparm;
166 
167 		/*
168 		 * Add the command back to the "outstanding commands list".
169 		 * Signal the "cmd_list" condition variable, if necessary.
170 		 */
171 		tavor_outstanding_cmd_free(state, &cmdptr);
172 
173 		if (status != TAVOR_CMD_SUCCESS) {
174 			TNF_PROBE_0(tavor_cmd_post_fail, TAVOR_TNF_ERROR, "");
175 			TAVOR_TNF_EXIT(tavor_cmd_post);
176 			return (status);
177 		}
178 
179 		TAVOR_TNF_EXIT(tavor_cmd_post);
180 		return (TAVOR_CMD_SUCCESS);
181 	}
182 }
183 
184 
185 /*
186  * tavor_mbox_alloc()
187  *    Context: Can be called from interrupt or base context.
188  *
189  *    The "mbox_wait" parameter is used to determine whether to
190  *    wait for a mailbox to become available or not.
191  */
192 int
193 tavor_mbox_alloc(tavor_state_t *state, tavor_mbox_info_t *mbox_info,
194     uint_t mbox_wait)
195 {
196 	int			status;
197 	uint_t			sleep_context;
198 
199 	TAVOR_TNF_ENTER(tavor_mbox_alloc);
200 
201 	sleep_context = TAVOR_SLEEPFLAG_FOR_CONTEXT();
202 
203 	/* Allocate an "In" mailbox */
204 	if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
205 		/* Determine correct mboxlist based on calling context */
206 		if (sleep_context == TAVOR_NOSLEEP) {
207 			status = tavor_impl_mbox_alloc(state,
208 			    &state->ts_in_intr_mblist,
209 			    &mbox_info->mbi_in, mbox_wait);
210 
211 			ASSERT(status == TAVOR_CMD_SUCCESS);
212 		} else {
213 			/* NOTE: Expect threads to be waiting in here */
214 			status = tavor_impl_mbox_alloc(state,
215 			    &state->ts_in_mblist, &mbox_info->mbi_in,
216 			    mbox_wait);
217 			if (status != TAVOR_CMD_SUCCESS) {
218 				TAVOR_TNF_EXIT(tavor_mbox_alloc);
219 				return (status);
220 			}
221 		}
222 
223 	}
224 
225 	/* Allocate an "Out" mailbox */
226 	if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
227 		/* Determine correct mboxlist based on calling context */
228 		if (sleep_context == TAVOR_NOSLEEP) {
229 			status = tavor_impl_mbox_alloc(state,
230 			    &state->ts_out_intr_mblist,
231 			    &mbox_info->mbi_out, mbox_wait);
232 
233 			ASSERT(status == TAVOR_CMD_SUCCESS);
234 		} else {
235 			/* NOTE: Expect threads to be waiting in here */
236 			status = tavor_impl_mbox_alloc(state,
237 			    &state->ts_out_mblist, &mbox_info->mbi_out,
238 			    mbox_wait);
239 			if (status != TAVOR_CMD_SUCCESS) {
240 				/* If we allocated an "In" mailbox, free it */
241 				if (mbox_info->mbi_alloc_flags &
242 				    TAVOR_ALLOC_INMBOX) {
243 					tavor_impl_mbox_free(
244 					    &state->ts_in_mblist,
245 					    &mbox_info->mbi_in);
246 				}
247 				TAVOR_TNF_EXIT(tavor_mbox_alloc);
248 				return (status);
249 			}
250 		}
251 	}
252 
253 	/* Store appropriate context in mbox_info */
254 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mbox_info->mbi_sleep_context))
255 	mbox_info->mbi_sleep_context = sleep_context;
256 
257 	TAVOR_TNF_EXIT(tavor_mbox_alloc);
258 	return (TAVOR_CMD_SUCCESS);
259 }
260 
261 
262 /*
263  * tavor_mbox_free()
264  *    Context: Can be called from interrupt or base context.
265  */
266 void
267 tavor_mbox_free(tavor_state_t *state, tavor_mbox_info_t *mbox_info)
268 {
269 	TAVOR_TNF_ENTER(tavor_mbox_free);
270 
271 	/*
272 	 * The mailbox has to be freed in the same context from which it was
273 	 * allocated.  The context is stored in the mbox_info at
274 	 * tavor_mbox_alloc() time.  We check the stored context against the
275 	 * current context here.
276 	 */
277 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mbox_info->mbi_sleep_context))
278 	ASSERT(mbox_info->mbi_sleep_context == TAVOR_SLEEPFLAG_FOR_CONTEXT());
279 
280 	/* Determine correct mboxlist based on calling context */
281 	if (mbox_info->mbi_sleep_context == TAVOR_NOSLEEP) {
282 		/* Free the intr "In" mailbox */
283 		if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
284 			tavor_impl_mbox_free(&state->ts_in_intr_mblist,
285 			    &mbox_info->mbi_in);
286 		}
287 
288 		/* Free the intr "Out" mailbox */
289 		if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
290 			tavor_impl_mbox_free(&state->ts_out_intr_mblist,
291 			    &mbox_info->mbi_out);
292 		}
293 	} else {
294 		/* Free the "In" mailbox */
295 		if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_INMBOX) {
296 			tavor_impl_mbox_free(&state->ts_in_mblist,
297 			    &mbox_info->mbi_in);
298 		}
299 
300 		/* Free the "Out" mailbox */
301 		if (mbox_info->mbi_alloc_flags & TAVOR_ALLOC_OUTMBOX) {
302 			tavor_impl_mbox_free(&state->ts_out_mblist,
303 			    &mbox_info->mbi_out);
304 		}
305 	}
306 
307 	TAVOR_TNF_EXIT(tavor_mbox_free);
308 }
309 
310 
311 /*
312  * tavor_cmd_complete_handler()
313  *    Context: Called only from interrupt context.
314  */
315 int
316 tavor_cmd_complete_handler(tavor_state_t *state, tavor_eqhdl_t eq,
317     tavor_hw_eqe_t *eqe)
318 {
319 	tavor_cmd_t		*cmdp;
320 	uint_t			eqe_evttype;
321 
322 	TAVOR_TNF_ENTER(tavor_cmd_complete_handler);
323 
324 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
325 
326 	ASSERT(eqe_evttype == TAVOR_EVT_COMMAND_INTF_COMP ||
327 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
328 
329 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
330 		TNF_PROBE_0(tavor_cmd_complete_overflow_condition,
331 		    TAVOR_TNF_ERROR, "");
332 		tavor_eq_overflow_handler(state, eq, eqe);
333 
334 		TAVOR_TNF_EXIT(tavor_cmd_complete_handler);
335 		return (DDI_FAILURE);
336 	}
337 
338 	/*
339 	 * Find the outstanding command pointer based on value returned
340 	 * in "token"
341 	 */
342 	cmdp = &state->ts_cmd_list.cml_cmd[TAVOR_EQE_CMDTOKEN_GET(eq, eqe)];
343 
344 	/* Signal the waiting thread */
345 	mutex_enter(&cmdp->cmd_comp_lock);
346 	cmdp->cmd_outparm = ((uint64_t)TAVOR_EQE_CMDOUTP0_GET(eq, eqe) << 32) |
347 	    TAVOR_EQE_CMDOUTP1_GET(eq, eqe);
348 	cmdp->cmd_status = TAVOR_EQE_CMDSTATUS_GET(eq, eqe);
349 
350 	cv_signal(&cmdp->cmd_comp_cv);
351 	mutex_exit(&cmdp->cmd_comp_lock);
352 
353 	TAVOR_TNF_EXIT(tavor_cmd_complete_handler);
354 	return (DDI_SUCCESS);
355 }
356 
357 
358 /*
359  * tavor_inmbox_list_init()
360  *    Context: Only called from attach() path context
361  */
362 int
363 tavor_inmbox_list_init(tavor_state_t *state)
364 {
365 	int		status;
366 	uint_t		num_inmbox;
367 
368 	TAVOR_TNF_ENTER(tavor_inmbox_list_init);
369 
370 	/* Initialize the "In" mailbox list */
371 	num_inmbox  =  (1 << state->ts_cfg_profile->cp_log_num_inmbox);
372 	status = tavor_impl_mboxlist_init(state, &state->ts_in_mblist,
373 	    num_inmbox, TAVOR_IN_MBOX);
374 	if (status != DDI_SUCCESS) {
375 		TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
376 		    TAVOR_TNF_ERROR, "");
377 		TAVOR_TNF_EXIT(tavor_inmbox_list_init);
378 		return (DDI_FAILURE);
379 	}
380 
381 	TAVOR_TNF_EXIT(tavor_inmbox_list_init);
382 	return (DDI_SUCCESS);
383 }
384 
385 
386 /*
387  * tavor_intr_inmbox_list_init()
388  *    Context: Only called from attach() path context
389  */
390 int
391 tavor_intr_inmbox_list_init(tavor_state_t *state)
392 {
393 	int		status;
394 	uint_t		num_inmbox;
395 
396 	TAVOR_TNF_ENTER(tavor_intr_inmbox_list_init);
397 
398 	/* Initialize the interrupt "In" mailbox list */
399 	num_inmbox  =  (1 << state->ts_cfg_profile->cp_log_num_intr_inmbox);
400 	status = tavor_impl_mboxlist_init(state, &state->ts_in_intr_mblist,
401 	    num_inmbox, TAVOR_INTR_IN_MBOX);
402 	if (status != DDI_SUCCESS) {
403 		TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
404 		    TAVOR_TNF_ERROR, "");
405 		TAVOR_TNF_EXIT(tavor_intr_inmbox_list_init);
406 		return (DDI_FAILURE);
407 	}
408 
409 	TAVOR_TNF_EXIT(tavor_intr_inmbox_list_init);
410 	return (DDI_SUCCESS);
411 }
412 
413 
414 /*
415  * tavor_outmbox_list_init()
416  *    Context: Only called from attach() path context
417  */
418 int
419 tavor_outmbox_list_init(tavor_state_t *state)
420 {
421 	int		status;
422 	uint_t		num_outmbox;
423 
424 	TAVOR_TNF_ENTER(tavor_outmbox_list_init);
425 
426 	/* Initialize the "Out" mailbox list */
427 	num_outmbox  =  (1 << state->ts_cfg_profile->cp_log_num_outmbox);
428 	status = tavor_impl_mboxlist_init(state, &state->ts_out_mblist,
429 	    num_outmbox, TAVOR_OUT_MBOX);
430 	if (status != DDI_SUCCESS) {
431 		TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
432 		    TAVOR_TNF_ERROR, "");
433 		TAVOR_TNF_EXIT(tavor_outmbox_list_init);
434 		return (DDI_FAILURE);
435 	}
436 
437 	TAVOR_TNF_EXIT(tavor_outmbox_list_init);
438 	return (DDI_SUCCESS);
439 }
440 
441 
442 /*
443  * tavor_intr_outmbox_list_init()
444  *    Context: Only called from attach() path context
445  */
446 int
447 tavor_intr_outmbox_list_init(tavor_state_t *state)
448 {
449 	int		status;
450 	uint_t		num_outmbox;
451 
452 	TAVOR_TNF_ENTER(tavor_intr_outmbox_list_init);
453 
454 	/* Initialize the interrupts "Out" mailbox list */
455 	num_outmbox  =  (1 << state->ts_cfg_profile->cp_log_num_intr_outmbox);
456 	status = tavor_impl_mboxlist_init(state, &state->ts_out_intr_mblist,
457 	    num_outmbox, TAVOR_INTR_OUT_MBOX);
458 	if (status != DDI_SUCCESS) {
459 		TNF_PROBE_0(tavor_impl_mboxlist_init_fail,
460 		    TAVOR_TNF_ERROR, "");
461 		TAVOR_TNF_EXIT(tavor_intr_outmbox_list_init);
462 		return (DDI_FAILURE);
463 	}
464 
465 	TAVOR_TNF_EXIT(tavor_intr_outmbox_list_init);
466 	return (DDI_SUCCESS);
467 }
468 
469 
470 /*
471  * tavor_inmbox_list_fini()
472  *    Context: Only called from attach() and/or detach() path contexts
473  */
474 void
475 tavor_inmbox_list_fini(tavor_state_t *state)
476 {
477 	TAVOR_TNF_ENTER(tavor_inmbox_list_fini);
478 
479 	/* Free up the "In" mailbox list */
480 	tavor_impl_mboxlist_fini(state, &state->ts_in_mblist);
481 
482 	TAVOR_TNF_EXIT(tavor_inmbox_list_fini);
483 }
484 
485 
486 /*
487  * tavor_intr_inmbox_list_fini()
488  *    Context: Only called from attach() and/or detach() path contexts
489  */
490 void
491 tavor_intr_inmbox_list_fini(tavor_state_t *state)
492 {
493 	TAVOR_TNF_ENTER(tavor_intr_inmbox_list_fini);
494 
495 	/* Free up the interupts "In" mailbox list */
496 	tavor_impl_mboxlist_fini(state, &state->ts_in_intr_mblist);
497 
498 	TAVOR_TNF_EXIT(tavor_intr_inmbox_list_fini);
499 }
500 
501 
502 /*
503  * tavor_outmbox_list_fini()
504  *    Context: Only called from attach() and/or detach() path contexts
505  */
506 void
507 tavor_outmbox_list_fini(tavor_state_t *state)
508 {
509 	TAVOR_TNF_ENTER(tavor_outmbox_list_fini);
510 
511 	/* Free up the "Out" mailbox list */
512 	tavor_impl_mboxlist_fini(state, &state->ts_out_mblist);
513 
514 	TAVOR_TNF_EXIT(tavor_outmbox_list_fini);
515 }
516 
517 
518 /*
519  * tavor_intr_outmbox_list_fini()
520  *    Context: Only called from attach() and/or detach() path contexts
521  */
522 void
523 tavor_intr_outmbox_list_fini(tavor_state_t *state)
524 {
525 	TAVOR_TNF_ENTER(tavor_intr_outmbox_list_fini);
526 
527 	/* Free up the interrupt "Out" mailbox list */
528 	tavor_impl_mboxlist_fini(state, &state->ts_out_intr_mblist);
529 
530 	TAVOR_TNF_EXIT(tavor_intr_outmbox_list_fini);
531 }
532 
533 
534 /*
535  * tavor_impl_mbox_alloc()
536  *    Context: Can be called from interrupt or base context.
537  */
538 static int
539 tavor_impl_mbox_alloc(tavor_state_t *state, tavor_mboxlist_t *mblist,
540     tavor_mbox_t **mb, uint_t mbox_wait)
541 {
542 	tavor_mbox_t	*mbox_ptr;
543 	uint_t		index, next, prev;
544 	uint_t		count, countmax;
545 
546 	TAVOR_TNF_ENTER(tavor_impl_mbox_alloc);
547 
548 	/*
549 	 * If the mailbox list is empty, then wait (if appropriate in the
550 	 * current context).  Otherwise, grab the next available mailbox.
551 	 */
552 	if (mbox_wait == TAVOR_NOSLEEP) {
553 		count	 = 0;
554 		countmax = state->ts_cfg_profile->cp_cmd_poll_max;
555 
556 		mutex_enter(&mblist->mbl_lock);
557 		mblist->mbl_pollers++;
558 		while (mblist->mbl_entries_free == 0) {
559 			mutex_exit(&mblist->mbl_lock);
560 			/* Delay loop polling for an available mbox */
561 			if (++count > countmax) {
562 				TNF_PROBE_0(tavor_impl_mbox_alloc_fail,
563 				    TAVOR_TNF_ERROR, "");
564 				TAVOR_TNF_EXIT(tavor_impl_mbox_alloc);
565 				return (TAVOR_CMD_INSUFF_RSRC);
566 			}
567 
568 			/* Delay before polling for mailbox again */
569 			drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
570 			mutex_enter(&mblist->mbl_lock);
571 		}
572 		mblist->mbl_pollers--;
573 
574 	/* TAVOR_SLEEP */
575 	} else {
576 		/*
577 		 * Grab lock here as we prepare to cv_wait if needed.
578 		 */
579 		mutex_enter(&mblist->mbl_lock);
580 		while (mblist->mbl_entries_free == 0) {
581 			/*
582 			 * Wait (on cv) for a mailbox to become free.  Note:
583 			 * Just as we do above in tavor_cmd_post(), we also
584 			 * have the "__lock_lint" here to workaround warlock.
585 			 * Warlock doesn't know that other parts of the Tavor
586 			 * may occasionally call this routine while holding
587 			 * their own locks, so it complains about this cv_wait.
588 			 * In reality, however, the rest of the driver never
589 			 * calls this routine with a lock held unless they pass
590 			 * TAVOR_CMD_NOSLEEP.
591 			 */
592 			mblist->mbl_waiters++;
593 #ifndef	__lock_lint
594 			cv_wait(&mblist->mbl_cv, &mblist->mbl_lock);
595 #endif
596 		}
597 	}
598 
599 	/* Grab the next available mailbox from list */
600 	mbox_ptr = mblist->mbl_mbox;
601 	index	 = mblist->mbl_head_indx;
602 	next	 = mbox_ptr[index].mb_next;
603 	prev	 = mbox_ptr[index].mb_prev;
604 
605 	/* Remove it from the mailbox list */
606 	mblist->mbl_mbox[next].mb_prev	= prev;
607 	mblist->mbl_mbox[prev].mb_next	= next;
608 	mblist->mbl_head_indx		= next;
609 
610 	/* Update the "free" count and return the mailbox pointer */
611 	mblist->mbl_entries_free--;
612 	*mb = &mbox_ptr[index];
613 
614 	mutex_exit(&mblist->mbl_lock);
615 
616 	TAVOR_TNF_EXIT(tavor_impl_mbox_alloc);
617 	return (TAVOR_CMD_SUCCESS);
618 }
619 
620 
621 /*
622  * tavor_impl_mbox_free()
623  *    Context: Can be called from interrupt or base context.
624  */
625 static void
626 tavor_impl_mbox_free(tavor_mboxlist_t *mblist, tavor_mbox_t **mb)
627 {
628 	uint_t		mbox_indx;
629 
630 	TAVOR_TNF_ENTER(tavor_impl_mbox_free);
631 
632 	mutex_enter(&mblist->mbl_lock);
633 
634 	/* Pull the "index" from mailbox entry */
635 	mbox_indx = (*mb)->mb_indx;
636 
637 	/*
638 	 * If mailbox list is not empty, then insert the entry.  Otherwise,
639 	 * this is the only entry.  So update the pointers appropriately.
640 	 */
641 	if (mblist->mbl_entries_free++ != 0) {
642 		/* Update the current mailbox */
643 		(*mb)->mb_next = mblist->mbl_head_indx;
644 		(*mb)->mb_prev = mblist->mbl_tail_indx;
645 
646 		/* Update head and tail mailboxes */
647 		mblist->mbl_mbox[mblist->mbl_head_indx].mb_prev = mbox_indx;
648 		mblist->mbl_mbox[mblist->mbl_tail_indx].mb_next = mbox_indx;
649 
650 		/* Update tail index */
651 		mblist->mbl_tail_indx = mbox_indx;
652 
653 	} else {
654 		/* Update the current mailbox */
655 		(*mb)->mb_next = mbox_indx;
656 		(*mb)->mb_prev = mbox_indx;
657 
658 		/* Update head and tail indexes */
659 		mblist->mbl_tail_indx = mbox_indx;
660 		mblist->mbl_head_indx = mbox_indx;
661 	}
662 
663 	/*
664 	 * Because we can have both waiters (SLEEP treads waiting for a
665 	 * cv_signal to continue processing) and pollers (NOSLEEP treads
666 	 * polling for a mailbox to become available), we try to share CPU time
667 	 * between them.  We do this by signalling the waiters only every other
668 	 * call to mbox_free.  This gives the pollers a chance to get some CPU
669 	 * time to do their command.  If we signalled every time, the pollers
670 	 * would have a much harder time getting CPU time.
671 	 *
672 	 * If there are waiters and no pollers, then we signal always.
673 	 *
674 	 * Otherwise, if there are either no waiters, there may in fact be
675 	 * pollers, so we do not signal in that case.
676 	 */
677 	if (mblist->mbl_pollers > 0 && mblist->mbl_waiters > 0) {
678 		/* flip the signal value */
679 		mblist->mbl_signal = (mblist->mbl_signal + 1) % 2;
680 	} else if (mblist->mbl_waiters > 0) {
681 		mblist->mbl_signal = 1;
682 	} else {
683 		mblist->mbl_signal = 0;
684 	}
685 
686 	/*
687 	 * Depending on the conditions in the previous check, we signal only if
688 	 * we are supposed to.
689 	 */
690 	if (mblist->mbl_signal) {
691 		mblist->mbl_waiters--;
692 		cv_signal(&mblist->mbl_cv);
693 	}
694 
695 	/* Clear out the mailbox entry pointer */
696 	*mb = NULL;
697 
698 	mutex_exit(&mblist->mbl_lock);
699 
700 	TAVOR_TNF_EXIT(tavor_impl_mbox_free);
701 }
702 
703 
704 /*
705  * tavor_impl_mboxlist_init()
706  *    Context: Only called from attach() path context
707  */
708 static int
709 tavor_impl_mboxlist_init(tavor_state_t *state, tavor_mboxlist_t *mblist,
710     uint_t num_mbox, tavor_rsrc_type_t type)
711 {
712 	tavor_rsrc_t		*rsrc;
713 	ddi_dma_cookie_t	dma_cookie;
714 	uint_t			dma_cookiecnt, flag, sync;
715 	int			status, i;
716 
717 	TAVOR_TNF_ENTER(tavor_impl_mboxlist_init);
718 
719 	/* Allocate the memory for the mailbox entries list */
720 	mblist->mbl_list_sz = num_mbox;
721 	mblist->mbl_mbox = kmem_zalloc(mblist->mbl_list_sz *
722 	    sizeof (tavor_mbox_t), KM_SLEEP);
723 
724 	/* Initialize the mailbox entries list */
725 	mblist->mbl_head_indx	 = 0;
726 	mblist->mbl_tail_indx	 = mblist->mbl_list_sz - 1;
727 	mblist->mbl_entries_free = mblist->mbl_list_sz;
728 	mblist->mbl_waiters	 = 0;
729 	mblist->mbl_num_alloc	 = 0;
730 
731 	/* Set up the mailbox list's cv and mutex */
732 	mutex_init(&mblist->mbl_lock, NULL, MUTEX_DRIVER,
733 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
734 	cv_init(&mblist->mbl_cv, NULL, CV_DRIVER, NULL);
735 
736 	/* Determine if syncs will be necessary */
737 	sync = TAVOR_MBOX_IS_SYNC_REQ(state, type);
738 
739 	/* Determine whether to map DDI_DMA_STREAMING or DDI_DMA_CONSISTENT */
740 	flag = state->ts_cfg_profile->cp_streaming_consistent;
741 
742 	/* Initialize the mailbox list entries */
743 	for (i = 0; i < mblist->mbl_list_sz; i++) {
744 		/* Allocate resources for the mailbox */
745 		status = tavor_rsrc_alloc(state, type, 1, TAVOR_SLEEP,
746 		    &rsrc);
747 		if (status != DDI_SUCCESS) {
748 			/* Jump to cleanup and return error */
749 			TNF_PROBE_0(tavor_impl_mbox_init_rsrcalloc_fail,
750 			    TAVOR_TNF_ERROR, "");
751 			goto mboxlist_init_fail;
752 		}
753 
754 		/* Save away the mailbox resource info */
755 		mblist->mbl_mbox[i].mb_rsrcptr	= rsrc;
756 		mblist->mbl_mbox[i].mb_addr	= rsrc->tr_addr;
757 		mblist->mbl_mbox[i].mb_acchdl	= rsrc->tr_acchdl;
758 
759 		/*
760 		 * Get a PCI mapped address for each mailbox.  Note: this
761 		 * uses the ddi_dma_handle return from the resource
762 		 * allocation routine
763 		 */
764 		status = ddi_dma_addr_bind_handle(rsrc->tr_dmahdl, NULL,
765 		    rsrc->tr_addr, rsrc->tr_len, (DDI_DMA_RDWR | flag),
766 		    DDI_DMA_SLEEP, NULL, &dma_cookie, &dma_cookiecnt);
767 		if (status != DDI_SUCCESS) {
768 			/* Jump to cleanup and return error */
769 			tavor_rsrc_free(state, &rsrc);
770 			TNF_PROBE_0(tavor_impl_mbox_init_dmabind_fail,
771 			    TAVOR_TNF_ERROR, "");
772 			goto mboxlist_init_fail;
773 		}
774 
775 		/* Save away the mapped address for the mailbox */
776 		mblist->mbl_mbox[i].mb_mapaddr	= dma_cookie.dmac_laddress;
777 
778 		/* Set sync flag appropriately */
779 		mblist->mbl_mbox[i].mb_sync	= sync;
780 
781 		/* Make each entry point to the "next" and "prev" entries */
782 		mblist->mbl_mbox[i].mb_next	= i+1;
783 		mblist->mbl_mbox[i].mb_prev	= i-1;
784 		mblist->mbl_mbox[i].mb_indx	= i;
785 		mblist->mbl_num_alloc		= i + 1;
786 	}
787 
788 	/* Make the "head" and "tail" entries point to each other */
789 	mblist->mbl_mbox[mblist->mbl_head_indx].mb_prev =
790 	    mblist->mbl_tail_indx;
791 	mblist->mbl_mbox[mblist->mbl_tail_indx].mb_next =
792 	    mblist->mbl_head_indx;
793 
794 	TAVOR_TNF_EXIT(tavor_impl_mboxlist_init);
795 	return (DDI_SUCCESS);
796 
797 mboxlist_init_fail:
798 	tavor_impl_mboxlist_fini(state, mblist);
799 
800 	TAVOR_TNF_EXIT(tavor_impl_mboxlist_init);
801 	return (DDI_FAILURE);
802 }
803 
804 
805 /*
806  * tavor_impl_mboxlist_fini()
807  *    Context: Only called from attach() and/or detach() path contexts
808  */
809 static void
810 tavor_impl_mboxlist_fini(tavor_state_t *state, tavor_mboxlist_t *mblist)
811 {
812 	tavor_rsrc_t	*rsrc;
813 	int		i, status;
814 
815 	TAVOR_TNF_ENTER(tavor_impl_mboxlist_fini);
816 
817 	/* Release the resources for each of the mailbox list entries */
818 	for (i = 0; i < mblist->mbl_num_alloc; i++) {
819 		rsrc = mblist->mbl_mbox[i].mb_rsrcptr;
820 
821 		/*
822 		 * First, unbind the DMA memory for the mailbox
823 		 *
824 		 * Note: The only way ddi_dma_unbind_handle() currently
825 		 * can return an error is if the handle passed in is invalid.
826 		 * Since this should never happen, we choose to return void
827 		 * from this function!  If this does return an error,
828 		 * however, then we print a warning message to the console.
829 		 */
830 		status = ddi_dma_unbind_handle(rsrc->tr_dmahdl);
831 		if (status != DDI_SUCCESS) {
832 			TAVOR_WARNING(state, "failed to unbind DMA mapping");
833 			TNF_PROBE_0(tavor_impl_mboxlist_fini_dmaunbind_fail,
834 			    TAVOR_TNF_ERROR, "");
835 			TAVOR_TNF_EXIT(tavor_impl_mboxlist_fini);
836 			return;
837 		}
838 
839 		/* Next, free the mailbox resource */
840 		tavor_rsrc_free(state, &rsrc);
841 	}
842 
843 	/* Destroy the mailbox list mutex and cv */
844 	mutex_destroy(&mblist->mbl_lock);
845 	cv_destroy(&mblist->mbl_cv);
846 
847 	/* Free up the memory for tracking the mailbox list */
848 	kmem_free(mblist->mbl_mbox, mblist->mbl_list_sz *
849 	    sizeof (tavor_mbox_t));
850 
851 	TAVOR_TNF_EXIT(tavor_impl_mboxlist_fini);
852 }
853 
854 
855 /*
856  * tavor_outstanding_cmd_alloc()
857  *    Context: Can be called only from base context.
858  */
859 static int
860 tavor_outstanding_cmd_alloc(tavor_state_t *state, tavor_cmd_t **cmd_ptr,
861     uint_t cmd_wait)
862 {
863 	tavor_cmdlist_t	*cmd_list;
864 	uint_t		next, prev, head;
865 
866 	TAVOR_TNF_ENTER(tavor_outstanding_cmd_alloc);
867 
868 	cmd_list = &state->ts_cmd_list;
869 	mutex_enter(&cmd_list->cml_lock);
870 
871 	/* Ensure that outstanding commands are supported */
872 	ASSERT(cmd_list->cml_num_alloc != 0);
873 
874 	/*
875 	 * If the outstanding command list is empty, then wait (if
876 	 * appropriate in the current context).  Otherwise, grab the
877 	 * next available command.
878 	 */
879 	while (cmd_list->cml_entries_free == 0) {
880 		/* No free commands */
881 		if (cmd_wait == TAVOR_NOSLEEP) {
882 			mutex_exit(&cmd_list->cml_lock);
883 			TNF_PROBE_0(tavor_outstanding_cmd_alloc_fail,
884 			    TAVOR_TNF_ERROR, "");
885 			TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
886 			return (TAVOR_CMD_INSUFF_RSRC);
887 		}
888 
889 		/*
890 		 * Wait (on cv) for a command to become free.  Note: Just
891 		 * as we do above in tavor_cmd_post(), we also have the
892 		 * "__lock_lint" here to workaround warlock.  Warlock doesn't
893 		 * know that other parts of the Tavor may occasionally call
894 		 * this routine while holding their own locks, so it complains
895 		 * about this cv_wait.  In reality, however, the rest of the
896 		 * driver never calls this routine with a lock held unless
897 		 * they pass TAVOR_CMD_NOSLEEP.
898 		 */
899 		cmd_list->cml_waiters++;
900 #ifndef	__lock_lint
901 		cv_wait(&cmd_list->cml_cv, &cmd_list->cml_lock);
902 #endif
903 	}
904 
905 	/* Grab the next available command from the list */
906 	head = cmd_list->cml_head_indx;
907 	*cmd_ptr = &cmd_list->cml_cmd[head];
908 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(**cmd_ptr))
909 	next = (*cmd_ptr)->cmd_next;
910 	prev = (*cmd_ptr)->cmd_prev;
911 	(*cmd_ptr)->cmd_status = TAVOR_CMD_INVALID_STATUS;
912 
913 	/* Remove it from the command list */
914 	cmd_list->cml_cmd[next].cmd_prev = prev;
915 	cmd_list->cml_cmd[prev].cmd_next = next;
916 	cmd_list->cml_head_indx		 = next;
917 
918 	/* Update the "free" count and return */
919 	cmd_list->cml_entries_free--;
920 
921 	mutex_exit(&cmd_list->cml_lock);
922 
923 	TAVOR_TNF_EXIT(tavor_outstanding_cmd_alloc);
924 	return (TAVOR_CMD_SUCCESS);
925 }
926 
927 
928 /*
929  * tavor_outstanding_cmd_free()
930  *    Context: Can be called only from base context.
931  */
932 static void
933 tavor_outstanding_cmd_free(tavor_state_t *state, tavor_cmd_t **cmd_ptr)
934 {
935 	tavor_cmdlist_t	*cmd_list;
936 	uint_t		cmd_indx;
937 
938 	TAVOR_TNF_ENTER(tavor_outstanding_cmd_free);
939 
940 	cmd_list = &state->ts_cmd_list;
941 	mutex_enter(&cmd_list->cml_lock);
942 
943 	/* Pull the "index" from command entry */
944 	cmd_indx = (*cmd_ptr)->cmd_indx;
945 
946 	/*
947 	 * If outstanding command list is not empty, then insert the entry.
948 	 * Otherwise, this is the only entry.  So update the pointers
949 	 * appropriately.
950 	 */
951 	if (cmd_list->cml_entries_free++ != 0) {
952 		/* Update the current command */
953 		(*cmd_ptr)->cmd_next = cmd_list->cml_head_indx;
954 		(*cmd_ptr)->cmd_prev = cmd_list->cml_tail_indx;
955 
956 		/* Update head and tail commands */
957 		cmd_list->cml_cmd[cmd_list->cml_head_indx].cmd_prev = cmd_indx;
958 		cmd_list->cml_cmd[cmd_list->cml_tail_indx].cmd_next = cmd_indx;
959 
960 		/* Update tail index */
961 		cmd_list->cml_tail_indx = cmd_indx;
962 
963 	} else {
964 		/* Update the current command */
965 		(*cmd_ptr)->cmd_next = cmd_indx;
966 		(*cmd_ptr)->cmd_prev = cmd_indx;
967 
968 		/* Update head and tail indexes */
969 		cmd_list->cml_head_indx = cmd_indx;
970 		cmd_list->cml_tail_indx = cmd_indx;
971 	}
972 
973 	/* If there are threads waiting, signal one of them */
974 	if (cmd_list->cml_waiters > 0) {
975 		cmd_list->cml_waiters--;
976 		cv_signal(&cmd_list->cml_cv);
977 	}
978 
979 	/* Clear out the command entry pointer */
980 	*cmd_ptr = NULL;
981 
982 	mutex_exit(&cmd_list->cml_lock);
983 
984 	TAVOR_TNF_EXIT(tavor_outstanding_cmd_free);
985 }
986 
987 
988 /*
989  * tavor_write_hcr()
990  *    Context: Can be called from interrupt or base context.
991  */
992 static int
993 tavor_write_hcr(tavor_state_t *state, tavor_cmd_post_t *cmdpost,
994     uint16_t token)
995 {
996 	tavor_hw_hcr_t	*hcr;
997 	uint_t		status, count, countmax;
998 	uint64_t	hcrreg;
999 
1000 	TAVOR_TNF_ENTER(tavor_write_hcr);
1001 
1002 	/*
1003 	 * Grab the "HCR access" lock if the driver is not in
1004 	 * fastreboot. In fastreboot, this function is called
1005 	 * with the single thread but in high interrupt context
1006 	 * (so that this mutex lock cannot be used).
1007 	 */
1008 #ifdef __lock_lint
1009 	mutex_enter(&state->ts_cmd_regs.hcr_lock);
1010 #else
1011 	if (!TAVOR_IN_FASTREBOOT(state)) {
1012 		mutex_enter(&state->ts_cmd_regs.hcr_lock);
1013 	}
1014 #endif
1015 
1016 	hcr = state->ts_cmd_regs.hcr;
1017 
1018 	/*
1019 	 * First, check the "go" bit to see if the previous hcr usage is
1020 	 * complete.  As long as it is set then we must continue to poll.
1021 	 */
1022 	count	 = 0;
1023 	countmax = state->ts_cfg_profile->cp_cmd_poll_max;
1024 	for (;;) {
1025 		hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
1026 
1027 		/* If "go" bit is clear, then done */
1028 		if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
1029 			TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
1030 			    TAVOR_TNF_ERROR, "", tnf_uint, nospinloopcount,
1031 			    count);
1032 			break;
1033 		}
1034 		/* Delay before polling the "go" bit again */
1035 		drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
1036 
1037 		/*
1038 		 * If we poll more than the maximum number of times, then
1039 		 * return a "timeout" error.
1040 		 */
1041 		if (++count > countmax) {
1042 #ifdef __lock_lint
1043 			mutex_exit(&state->ts_cmd_regs.hcr_lock);
1044 #else
1045 			if (!TAVOR_IN_FASTREBOOT(state)) {
1046 				mutex_exit(&state->ts_cmd_regs.hcr_lock);
1047 			}
1048 #endif
1049 			TNF_PROBE_0(tavor_write_hcr_timeout1, TAVOR_TNF_ERROR,
1050 			    "");
1051 			TAVOR_TNF_EXIT(tavor_write_hcr);
1052 			return (TAVOR_CMD_TIMEOUT);
1053 		}
1054 	}
1055 
1056 	/* Write "inparam" as a 64-bit quantity */
1057 	ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->in_param0,
1058 	    cmdpost->cp_inparm);
1059 
1060 	/* Write "inmod" and 32-bits of "outparam" as 64-bit */
1061 	hcrreg = ((uint64_t)cmdpost->cp_inmod << 32);
1062 	hcrreg = hcrreg | (cmdpost->cp_outparm >> 32);
1063 	ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->input_modifier,
1064 	    hcrreg);
1065 
1066 	/* Write the other 32-bits of "outparam" and "token" as 64-bit */
1067 	hcrreg = (cmdpost->cp_outparm << 32);
1068 	hcrreg = hcrreg | ((uint32_t)token << TAVOR_HCR_TOKEN_SHIFT);
1069 	ddi_put64(state->ts_reg_cmdhdl, (uint64_t *)&hcr->out_param1,
1070 	    hcrreg);
1071 
1072 	/* Then setup the final hcrreg to hit doorbell (i.e. "go" bit) */
1073 	hcrreg = TAVOR_HCR_CMD_GO_MASK;
1074 	if (cmdpost->cp_flags == TAVOR_CMD_SLEEP_NOSPIN)
1075 		hcrreg = hcrreg | TAVOR_HCR_CMD_E_MASK;
1076 	hcrreg = hcrreg | (cmdpost->cp_opmod << TAVOR_HCR_CMD_OPMOD_SHFT);
1077 	hcrreg = hcrreg | (cmdpost->cp_opcode);
1078 
1079 	/* Write the doorbell to the HCR */
1080 	ddi_put32(state->ts_reg_cmdhdl, &hcr->cmd, hcrreg);
1081 
1082 	/*
1083 	 * In the SPIN case we read the HCR and check the "go" bit.  For the
1084 	 * NOSPIN case we do not have to poll, we simply release the HCR lock
1085 	 * and return.
1086 	 */
1087 	if (cmdpost->cp_flags == TAVOR_CMD_NOSLEEP_SPIN) {
1088 		count	 = 0;
1089 		countmax = state->ts_cfg_profile->cp_cmd_poll_max;
1090 
1091 		for (;;) {
1092 			hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->cmd);
1093 
1094 			/* If "go" bit is clear, then done */
1095 			if ((hcrreg & TAVOR_HCR_CMD_GO_MASK) == 0) {
1096 				TNF_PROBE_1_DEBUG(tavor_write_hcr_loop_count,
1097 				    TAVOR_TNF_ERROR, "", tnf_uint,
1098 				    spinloopcount, count);
1099 				break;
1100 			}
1101 			/* Delay before polling the "go" bit again */
1102 			drv_usecwait(state->ts_cfg_profile->cp_cmd_poll_delay);
1103 
1104 			/*
1105 			 * If we poll more than the maximum number of times,
1106 			 * then return a "timeout" error.
1107 			 */
1108 			if (++count > countmax) {
1109 #ifdef __lock_lint
1110 				mutex_exit(&state-> ts_cmd_regs.hcr_lock);
1111 #else
1112 				if (!TAVOR_IN_FASTREBOOT(state)) {
1113 					mutex_exit(&state->
1114 					    ts_cmd_regs.hcr_lock);
1115 				}
1116 #endif
1117 				TNF_PROBE_0(tavor_write_hcr_timeout2,
1118 				    TAVOR_TNF_ERROR, "");
1119 				TAVOR_TNF_EXIT(tavor_write_hcr);
1120 				return (TAVOR_CMD_TIMEOUT);
1121 			}
1122 		}
1123 
1124 		/* Pull out the "status" bits from the HCR */
1125 		status = (hcrreg >> TAVOR_HCR_CMD_STATUS_SHFT);
1126 
1127 		/*
1128 		 * Read the "outparam" value.  Note: we have to read "outparam"
1129 		 * as two separate 32-bit reads because the field in the HCR is
1130 		 * not 64-bit aligned.
1131 		 */
1132 		hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param0);
1133 		cmdpost->cp_outparm = hcrreg << 32;
1134 		hcrreg = ddi_get32(state->ts_reg_cmdhdl, &hcr->out_param1);
1135 		cmdpost->cp_outparm |= hcrreg;
1136 
1137 	/* NOSPIN */
1138 	} else {
1139 		status = TAVOR_CMD_SUCCESS;
1140 	}
1141 
1142 	/* Drop the "HCR access" lock */
1143 #ifdef __lock_lint
1144 	mutex_exit(&state->ts_cmd_regs.hcr_lock);
1145 #else
1146 	if (!TAVOR_IN_FASTREBOOT(state)) {
1147 		mutex_exit(&state->ts_cmd_regs.hcr_lock);
1148 	}
1149 #endif
1150 
1151 	TAVOR_TNF_EXIT(tavor_write_hcr);
1152 	return (status);
1153 }
1154 
1155 
1156 /*
1157  * tavor_outstanding_cmdlist_init()
1158  *    Context: Only called from attach() path context
1159  */
1160 int
1161 tavor_outstanding_cmdlist_init(tavor_state_t *state)
1162 {
1163 	uint_t		num_outstanding_cmds, head, tail;
1164 	int		i;
1165 
1166 	TAVOR_TNF_ENTER(tavor_outstanding_cmdlist_init);
1167 
1168 	/*
1169 	 * Determine the number of the outstanding commands supported
1170 	 * by the Tavor device (obtained from the QUERY_FW command).  Note:
1171 	 * Because we handle both SLEEP and NOSLEEP cases around the tavor HCR,
1172 	 * we know that when an interrupt comes in it will be next on the
1173 	 * command register, and will at most have to wait one commands time.
1174 	 * We do not have to reserve an outstanding command here for
1175 	 * interrupts.
1176 	 */
1177 	num_outstanding_cmds = (1 << state->ts_fw.log_max_cmd);
1178 
1179 	/* Initialize the outstanding command list */
1180 	state->ts_cmd_list.cml_list_sz	 = num_outstanding_cmds;
1181 	state->ts_cmd_list.cml_head_indx = 0;
1182 	state->ts_cmd_list.cml_tail_indx = state->ts_cmd_list.cml_list_sz - 1;
1183 	state->ts_cmd_list.cml_entries_free = state->ts_cmd_list.cml_list_sz;
1184 	state->ts_cmd_list.cml_waiters	 = 0;
1185 	state->ts_cmd_list.cml_num_alloc = 0;
1186 
1187 	/* Allocate the memory for the outstanding command list */
1188 	if (num_outstanding_cmds) {
1189 		state->ts_cmd_list.cml_cmd =
1190 		    kmem_zalloc(state->ts_cmd_list.cml_list_sz *
1191 		    sizeof (tavor_cmd_t), KM_SLEEP);
1192 	}
1193 	mutex_init(&state->ts_cmd_list.cml_lock, NULL, MUTEX_DRIVER,
1194 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
1195 	cv_init(&state->ts_cmd_list.cml_cv, NULL, CV_DRIVER, NULL);
1196 
1197 	/* Initialize the individual outstanding command list entries */
1198 	for (i = 0; i < state->ts_cmd_list.cml_list_sz; i++) {
1199 		mutex_init(&state->ts_cmd_list.cml_cmd[i].cmd_comp_lock,
1200 		    NULL, MUTEX_DRIVER, DDI_INTR_PRI(state->ts_intrmsi_pri));
1201 		cv_init(&state->ts_cmd_list.cml_cmd[i].cmd_comp_cv, NULL,
1202 		    CV_DRIVER, NULL);
1203 
1204 		state->ts_cmd_list.cml_cmd[i].cmd_next	= i+1;
1205 		state->ts_cmd_list.cml_cmd[i].cmd_prev	= i-1;
1206 		state->ts_cmd_list.cml_cmd[i].cmd_indx	= i;
1207 		state->ts_cmd_list.cml_num_alloc	= i + 1;
1208 	}
1209 	if (num_outstanding_cmds) {
1210 		head = state->ts_cmd_list.cml_head_indx;
1211 		tail = state->ts_cmd_list.cml_tail_indx;
1212 		state->ts_cmd_list.cml_cmd[head].cmd_prev =
1213 		    state->ts_cmd_list.cml_tail_indx;
1214 		state->ts_cmd_list.cml_cmd[tail].cmd_next =
1215 		    state->ts_cmd_list.cml_head_indx;
1216 	}
1217 
1218 	TAVOR_TNF_EXIT(tavor_outstanding_cmdlist_init);
1219 	return (DDI_SUCCESS);
1220 }
1221 
1222 
1223 /*
1224  * tavor_outstanding_cmdlist_fini()
1225  *    Context: Only called from attach() and/or detach() path contexts
1226  */
1227 void
1228 tavor_outstanding_cmdlist_fini(tavor_state_t *state)
1229 {
1230 	int		i;
1231 
1232 	TAVOR_TNF_ENTER(tavor_outstanding_cmdlist_fini);
1233 
1234 	/* Destroy the outstanding command list entries */
1235 	for (i = 0; i < state->ts_cmd_list.cml_num_alloc; i++) {
1236 		mutex_destroy(&state->ts_cmd_list.cml_cmd[i].cmd_comp_lock);
1237 		cv_destroy(&state->ts_cmd_list.cml_cmd[i].cmd_comp_cv);
1238 	}
1239 
1240 	/* Destroy the lock (and cv) and free up memory for list */
1241 	mutex_destroy(&state->ts_cmd_list.cml_lock);
1242 	cv_destroy(&state->ts_cmd_list.cml_cv);
1243 	if (state->ts_cmd_list.cml_num_alloc) {
1244 		kmem_free(state->ts_cmd_list.cml_cmd,
1245 		    state->ts_cmd_list.cml_list_sz * sizeof (tavor_cmd_t));
1246 	}
1247 
1248 	TAVOR_TNF_EXIT(tavor_outstanding_cmdlist_fini);
1249 }
1250 
1251 
1252 /*
1253  * tavor_mbox_sync()
1254  */
1255 static void
1256 tavor_mbox_sync(tavor_mbox_t *mbox, uint_t offset, uint_t length,
1257     uint_t flag)
1258 {
1259 	ddi_dma_handle_t	dmahdl;
1260 	int			status;
1261 
1262 	TAVOR_TNF_ENTER(tavor_mbox_sync);
1263 
1264 	/* Determine if mailbox needs to be synced or not */
1265 	if (mbox->mb_sync == 0) {
1266 		TAVOR_TNF_EXIT(tavor_mbox_sync);
1267 		return;
1268 	}
1269 
1270 	/* Get the DMA handle from mailbox */
1271 	dmahdl = mbox->mb_rsrcptr->tr_dmahdl;
1272 
1273 	/* Calculate offset into mailbox */
1274 	status = ddi_dma_sync(dmahdl, (off_t)offset, (size_t)length, flag);
1275 	if (status != DDI_SUCCESS) {
1276 		TNF_PROBE_0(tavor_mbox_sync_fail, TAVOR_TNF_ERROR, "");
1277 		TAVOR_TNF_EXIT(tavor_mbox_sync);
1278 		return;
1279 	}
1280 
1281 	TAVOR_TNF_EXIT(tavor_mbox_sync);
1282 }
1283 
1284 
1285 /*
1286  * tavor_sys_en_cmd_post()
1287  *    Context: Can be called from interrupt or base context.
1288  *    (Currently called only from attach() path context)
1289  */
1290 int
1291 tavor_sys_en_cmd_post(tavor_state_t *state, uint_t flags,
1292     uint64_t *errorcode, uint_t sleepflag)
1293 {
1294 	tavor_cmd_post_t	cmd;
1295 	int			status;
1296 
1297 	TAVOR_TNF_ENTER(tavor_sys_en_cmd_post);
1298 
1299 	/* Make sure we are called with the correct flag */
1300 	ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1301 
1302 	/* Setup and post the Tavor "SYS_EN" command */
1303 	cmd.cp_inparm	= 0;
1304 	cmd.cp_outparm	= 0;
1305 	cmd.cp_inmod	= 0;
1306 	cmd.cp_opcode	= SYS_EN;
1307 	cmd.cp_opmod	= flags;
1308 	cmd.cp_flags	= sleepflag;
1309 	status = tavor_cmd_post(state, &cmd);
1310 	if (status != TAVOR_CMD_SUCCESS) {
1311 		TNF_PROBE_0(tavor_sys_en_cmd_post_fail, TAVOR_TNF_ERROR, "");
1312 		/*
1313 		 * When the SYS_EN command fails, the "outparam" field may
1314 		 * contain more detailed information about what caused the
1315 		 * failure.
1316 		 */
1317 		*errorcode = cmd.cp_outparm;
1318 	}
1319 
1320 	TAVOR_TNF_EXIT(tavor_sys_en_cmd_post);
1321 	return (status);
1322 }
1323 
1324 
1325 /*
1326  * tavor_sys_dis_cmd_post()
1327  *    Context: Can be called from interrupt or base context.
1328  *    (Currently called only from attach() and/or detach() path contexts)
1329  */
1330 int
1331 tavor_sys_dis_cmd_post(tavor_state_t *state, uint_t sleepflag)
1332 {
1333 	tavor_cmd_post_t	cmd;
1334 	int			status;
1335 
1336 	TAVOR_TNF_ENTER(tavor_sys_dis_cmd_post);
1337 
1338 	/* Make sure we are called with the correct flag */
1339 	ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1340 
1341 	/* Setup and post the Tavor "SYS_DIS" command */
1342 	cmd.cp_inparm	= 0;
1343 	cmd.cp_outparm	= 0;
1344 	cmd.cp_inmod	= 0;
1345 	cmd.cp_opcode	= SYS_DIS;
1346 	cmd.cp_opmod	= 0;
1347 	cmd.cp_flags	= sleepflag;
1348 	status = tavor_cmd_post(state, &cmd);
1349 	if (status != TAVOR_CMD_SUCCESS) {
1350 		TNF_PROBE_0(tavor_sys_dis_cmd_post_fail,
1351 		    TAVOR_TNF_ERROR, "");
1352 	}
1353 
1354 	TAVOR_TNF_EXIT(tavor_sys_dis_cmd_post);
1355 	return (status);
1356 }
1357 
1358 
1359 /*
1360  * tavor_init_hca_cmd_post()
1361  *    Context: Can be called from interrupt or base context.
1362  *    (Currently called only from attach() path context)
1363  */
1364 int
1365 tavor_init_hca_cmd_post(tavor_state_t *state,
1366     tavor_hw_initqueryhca_t *inithca, uint_t sleepflag)
1367 {
1368 	tavor_mbox_info_t	mbox_info;
1369 	tavor_cmd_post_t	cmd;
1370 	uint64_t		data;
1371 	uint_t			size;
1372 	int			status, i;
1373 
1374 	TAVOR_TNF_ENTER(tavor_init_hca_cmd_post);
1375 
1376 	/* Make sure we are called with the correct flag */
1377 	ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1378 
1379 	/* Get an "In" mailbox for the command */
1380 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
1381 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1382 	if (status != TAVOR_CMD_SUCCESS) {
1383 		TNF_PROBE_0(tavor_init_hca_mbox_fail, TAVOR_TNF_ERROR, "");
1384 		TAVOR_TNF_EXIT(tavor_init_hca_cmd_post);
1385 		return (status);
1386 	}
1387 
1388 	/* Copy the Tavor "INIT_HCA" command into the mailbox */
1389 	size = sizeof (tavor_hw_initqueryhca_t);
1390 	for (i = 0; i < (size >> 3); i++) {
1391 		data = ((uint64_t *)inithca)[i];
1392 		ddi_put64(mbox_info.mbi_in->mb_acchdl,
1393 		    ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
1394 	}
1395 
1396 	/* Sync the mailbox for the device to read */
1397 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1398 
1399 	/* Setup and post the Tavor "INIT_HCA" command */
1400 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
1401 	cmd.cp_outparm	= 0;
1402 	cmd.cp_inmod	= 0;
1403 	cmd.cp_opcode	= INIT_HCA;
1404 	cmd.cp_opmod	= 0;
1405 	cmd.cp_flags	= sleepflag;
1406 	status = tavor_cmd_post(state, &cmd);
1407 	if (status != TAVOR_CMD_SUCCESS) {
1408 		TNF_PROBE_0(tavor_init_hca_cmd_post_fail,
1409 		    TAVOR_TNF_ERROR, "");
1410 	}
1411 
1412 	/* Free the mailbox */
1413 	tavor_mbox_free(state, &mbox_info);
1414 
1415 	TAVOR_TNF_EXIT(tavor_init_hca_cmd_post);
1416 	return (status);
1417 }
1418 
1419 
1420 /*
1421  * tavor_close_hca_cmd_post()
1422  *    Context: Can be called from interrupt or base context.
1423  *    (Currently called only from attach() and/or detach() path contexts)
1424  */
1425 int
1426 tavor_close_hca_cmd_post(tavor_state_t *state, uint_t sleepflag)
1427 {
1428 	tavor_cmd_post_t	cmd;
1429 	int			status;
1430 
1431 	TAVOR_TNF_ENTER(tavor_close_hca_cmd_post);
1432 
1433 	/* Make sure we are called with the correct flag */
1434 	ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1435 
1436 	/* Setup and post the Tavor "CLOSE_HCA" command */
1437 	cmd.cp_inparm	= 0;
1438 	cmd.cp_outparm	= 0;
1439 	cmd.cp_inmod	= 0;
1440 	cmd.cp_opcode	= CLOSE_HCA;
1441 	cmd.cp_opmod	= 0;
1442 	cmd.cp_flags	= sleepflag;
1443 	status = tavor_cmd_post(state, &cmd);
1444 	if (status != TAVOR_CMD_SUCCESS) {
1445 		TNF_PROBE_0(tavor_close_hca_cmd_post_fail,
1446 		    TAVOR_TNF_ERROR, "");
1447 	}
1448 
1449 	TAVOR_TNF_EXIT(tavor_close_hca_cmd_post);
1450 	return (status);
1451 }
1452 
1453 
1454 /*
1455  * tavor_init_ib_cmd_post()
1456  *    Context: Can be called from interrupt or base context.
1457  *    (Currently called only from attach() path context)
1458  */
1459 int
1460 tavor_init_ib_cmd_post(tavor_state_t *state, tavor_hw_initib_t *initib,
1461     uint_t port, uint_t sleepflag)
1462 {
1463 	tavor_mbox_info_t	mbox_info;
1464 	tavor_cmd_post_t	cmd;
1465 	uint64_t		data;
1466 	uint_t			size;
1467 	int			status, i;
1468 
1469 	TAVOR_TNF_ENTER(tavor_init_ib_cmd_post);
1470 
1471 	/* Make sure we are called with the correct flag */
1472 	ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1473 
1474 	/* Get an "In" mailbox for the command */
1475 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
1476 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1477 	if (status != TAVOR_CMD_SUCCESS) {
1478 		TNF_PROBE_0(tavor_init_ib_mbox_fail, TAVOR_TNF_ERROR, "");
1479 		TAVOR_TNF_EXIT(tavor_init_ib_cmd_post);
1480 		return (status);
1481 	}
1482 
1483 	/* Copy the Tavor "INIT_IB" command into the mailbox */
1484 	size = sizeof (tavor_hw_initib_t);
1485 	for (i = 0; i < (size >> 3); i++) {
1486 		data = ((uint64_t *)initib)[i];
1487 		ddi_put64(mbox_info.mbi_in->mb_acchdl,
1488 		    ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
1489 	}
1490 
1491 	/* Sync the mailbox for the device to read */
1492 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1493 
1494 	/* Setup and post the Tavor "INIT_IB" command */
1495 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
1496 	cmd.cp_outparm	= 0;
1497 	cmd.cp_inmod	= port;
1498 	cmd.cp_opcode	= INIT_IB;
1499 	cmd.cp_opmod	= 0;
1500 	cmd.cp_flags	= sleepflag;
1501 	status = tavor_cmd_post(state, &cmd);
1502 	if (status != TAVOR_CMD_SUCCESS) {
1503 		TNF_PROBE_0(tavor_init_ib_cmd_post_fail,
1504 		    TAVOR_TNF_ERROR, "");
1505 	}
1506 
1507 	/* Free the mailbox */
1508 	tavor_mbox_free(state, &mbox_info);
1509 
1510 	TAVOR_TNF_EXIT(tavor_init_ib_cmd_post);
1511 	return (status);
1512 }
1513 
1514 
1515 /*
1516  * tavor_close_ib_cmd_post()
1517  *    Context: Can be called from interrupt or base context.
1518  *    (Currently called only from attach() and/or detach() path contexts)
1519  */
1520 int
1521 tavor_close_ib_cmd_post(tavor_state_t *state, uint_t port, uint_t sleepflag)
1522 {
1523 	tavor_cmd_post_t	cmd;
1524 	int			status;
1525 
1526 	TAVOR_TNF_ENTER(tavor_close_ib_cmd_post);
1527 
1528 	/* Setup and post the Tavor "CLOSE_IB" command */
1529 	cmd.cp_inparm	= 0;
1530 	cmd.cp_outparm	= 0;
1531 	cmd.cp_inmod	= port;
1532 	cmd.cp_opcode	= CLOSE_IB;
1533 	cmd.cp_opmod	= 0;
1534 	cmd.cp_flags	= sleepflag;
1535 	status = tavor_cmd_post(state, &cmd);
1536 	if (status != TAVOR_CMD_SUCCESS) {
1537 		TNF_PROBE_0(tavor_close_ib_cmd_post_fail, TAVOR_TNF_ERROR, "");
1538 	}
1539 
1540 	TAVOR_TNF_EXIT(tavor_close_ib_cmd_post);
1541 	return (status);
1542 }
1543 
1544 
1545 /*
1546  * tavor_set_ib_cmd_post()
1547  *    Context: Can be called from interrupt or base context.
1548  */
1549 int
1550 tavor_set_ib_cmd_post(tavor_state_t *state, uint32_t capmask, uint_t port,
1551     uint_t reset_qkey, uint_t sleepflag)
1552 {
1553 	tavor_mbox_info_t	mbox_info;
1554 	tavor_cmd_post_t	cmd;
1555 	int			status;
1556 
1557 	TAVOR_TNF_ENTER(tavor_set_ib_cmd_post);
1558 
1559 	/* Get an "In" mailbox for the command */
1560 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
1561 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1562 	if (status != TAVOR_CMD_SUCCESS) {
1563 		TNF_PROBE_0(tavor_set_ib_mbox_fail, TAVOR_TNF_ERROR, "");
1564 		TAVOR_TNF_EXIT(tavor_set_ib_cmd_post);
1565 		return (status);
1566 	}
1567 
1568 	/* Copy the Tavor "SET_IB" command into mailbox */
1569 	ddi_put32(mbox_info.mbi_in->mb_acchdl,
1570 	    ((uint32_t *)mbox_info.mbi_in->mb_addr + 0), reset_qkey);
1571 	ddi_put32(mbox_info.mbi_in->mb_acchdl,
1572 	    ((uint32_t *)mbox_info.mbi_in->mb_addr + 1), capmask);
1573 
1574 	/* Sync the mailbox for the device to read */
1575 	tavor_mbox_sync(mbox_info.mbi_in, 0, TAVOR_CMD_SETIB_SZ,
1576 	    DDI_DMA_SYNC_FORDEV);
1577 
1578 	/* Setup and post the Tavor "SET_IB" command */
1579 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
1580 	cmd.cp_outparm	= 0;
1581 	cmd.cp_inmod	= port;
1582 	cmd.cp_opcode	= SET_IB;
1583 	cmd.cp_opmod	= 0;
1584 	cmd.cp_flags	= sleepflag;
1585 	status = tavor_cmd_post(state, &cmd);
1586 	if (status != TAVOR_CMD_SUCCESS) {
1587 		TNF_PROBE_0(tavor_set_ib_cmd_post_fail, TAVOR_TNF_ERROR, "");
1588 	}
1589 
1590 	/* Free the mailbox */
1591 	tavor_mbox_free(state, &mbox_info);
1592 
1593 	TAVOR_TNF_EXIT(tavor_set_ib_cmd_post);
1594 	return (status);
1595 }
1596 
1597 
1598 /*
1599  * tavor_mod_stat_cfg_cmd_post()
1600  *    Context: Can be called only from attach() path
1601  */
1602 int
1603 tavor_mod_stat_cfg_cmd_post(tavor_state_t *state)
1604 {
1605 	tavor_mbox_info_t	mbox_info;
1606 	tavor_cmd_post_t	cmd;
1607 	tavor_hw_mod_stat_cfg_t	*mod;
1608 	uint64_t		data;
1609 	uint_t			size;
1610 	int			status, i;
1611 
1612 	TAVOR_TNF_ENTER(tavor_mod_stat_cfg_cmd_post);
1613 
1614 	/*
1615 	 * "MOD_STAT_CFG" needs an INMBOX parameter, to specify what operations
1616 	 * to do.  However, at the point in time that we call this command, the
1617 	 * DDR has not yet been initialized, and all INMBOX'es are located in
1618 	 * DDR.  Because we want to call MOD_STAT_CFG before QUERY_DEVLIM is
1619 	 * called, and thus call it before DDR is setup, we simply use an
1620 	 * OUTMBOX memory location here as our INMBOX parameter.
1621 	 */
1622 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
1623 	status = tavor_mbox_alloc(state, &mbox_info, TAVOR_NOSLEEP);
1624 	if (status != TAVOR_CMD_SUCCESS) {
1625 		TNF_PROBE_0(tavor_mod_stat_cfg_mbox_fail, TAVOR_TNF_ERROR, "");
1626 		TAVOR_TNF_EXIT(tavor_mod_stat_cfg_cmd_post);
1627 		return (status);
1628 	}
1629 
1630 	/*
1631 	 * Allocate on the heap our 'mod_stat_cfg' structure.  We want to
1632 	 * ideally move all of this on to the stack in the future, but this
1633 	 * works well for now.
1634 	 */
1635 	mod = (tavor_hw_mod_stat_cfg_t *)kmem_zalloc(
1636 	    sizeof (tavor_hw_mod_stat_cfg_t), KM_SLEEP);
1637 
1638 	/* Setup "MOD_STAT_CFG" settings */
1639 	mod->srq_m	= 1;
1640 	mod->srq	= state->ts_cfg_profile->cp_srq_enable;
1641 
1642 	if (mod->srq) {
1643 		mod->log_max_srq = state->ts_cfg_profile->cp_log_num_srq;
1644 	} else {
1645 		mod->log_max_srq = 0;
1646 	}
1647 
1648 	/* Copy the "MOD_STAT_CFG" command into the "In" mailbox */
1649 	size = sizeof (tavor_hw_mod_stat_cfg_t);
1650 	for (i = 0; i < (size >> 3); i++) {
1651 		data = ((uint64_t *)mod)[i];
1652 		ddi_put64(mbox_info.mbi_out->mb_acchdl,
1653 		    ((uint64_t *)mbox_info.mbi_out->mb_addr + i), data);
1654 	}
1655 
1656 	/* Sync the mailbox for the device to read */
1657 	tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORDEV);
1658 
1659 	/* Setup and post the Tavor "MOD_STAT_CFG" command */
1660 	cmd.cp_inparm	= mbox_info.mbi_out->mb_mapaddr;
1661 	cmd.cp_outparm	= 0;
1662 	cmd.cp_inmod	= 0;
1663 	cmd.cp_opcode	= MOD_STAT_CFG;
1664 	cmd.cp_opmod	= 0;
1665 	cmd.cp_flags	= TAVOR_CMD_NOSLEEP_SPIN;
1666 	status = tavor_cmd_post(state, &cmd);
1667 	if (status != TAVOR_CMD_SUCCESS) {
1668 		TNF_PROBE_0(tavor_mod_stat_cfg_cmd_post_fail, TAVOR_TNF_ERROR,
1669 		    "");
1670 	}
1671 
1672 	/* Free "MOD_STAT_CFG" struct */
1673 	kmem_free(mod, sizeof (tavor_hw_mod_stat_cfg_t));
1674 
1675 	/* Free the mailbox */
1676 	tavor_mbox_free(state, &mbox_info);
1677 
1678 	TAVOR_TNF_EXIT(tavor_mod_stat_cfg_cmd_post);
1679 	return (status);
1680 }
1681 
1682 
1683 /*
1684  * tavor_mad_ifc_cmd_post()
1685  *    Context: Can be called from interrupt or base context.
1686  */
1687 int
1688 tavor_mad_ifc_cmd_post(tavor_state_t *state, uint_t port,
1689     uint_t sleepflag, uint32_t *mad, uint32_t *resp)
1690 {
1691 	tavor_mbox_info_t	mbox_info;
1692 	tavor_cmd_post_t	cmd;
1693 	uint_t			size;
1694 	int			status;
1695 
1696 	TAVOR_TNF_ENTER(tavor_mad_ifc_cmd_post);
1697 
1698 	/* Get "In" and "Out" mailboxes for the command */
1699 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1700 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1701 	if (status != TAVOR_CMD_SUCCESS) {
1702 		TNF_PROBE_0(tavor_mad_ifc_mbox_fail, TAVOR_TNF_ERROR, "");
1703 		TAVOR_TNF_EXIT(tavor_mad_ifc_cmd_post);
1704 		return (status);
1705 	}
1706 
1707 	/* Copy the request MAD into the "In" mailbox */
1708 	size = TAVOR_CMD_MAD_IFC_SIZE;
1709 	bcopy(mad, mbox_info.mbi_in->mb_addr, size);
1710 
1711 	/* Sync the mailbox for the device to read */
1712 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1713 
1714 	/* Setup the Tavor "MAD_IFC" command */
1715 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
1716 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
1717 	cmd.cp_inmod	= port;
1718 	cmd.cp_opcode	= MAD_IFC;
1719 	cmd.cp_opmod	= TAVOR_CMD_MKEY_CHECK;  /* Enable MKey checking */
1720 	cmd.cp_flags	= sleepflag;
1721 	status = tavor_cmd_post(state, &cmd);
1722 	if (status != TAVOR_CMD_SUCCESS) {
1723 		TNF_PROBE_0(tavor_mad_ifc_cmd_post_fail,
1724 		    TAVOR_TNF_ERROR, "");
1725 		goto mad_ifc_fail;
1726 	}
1727 
1728 	/* Sync the mailbox to read the results */
1729 	tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
1730 
1731 	/* Copy the response MAD into "resp" */
1732 	bcopy(mbox_info.mbi_out->mb_addr, resp, size);
1733 
1734 mad_ifc_fail:
1735 	/* Free the mailbox */
1736 	tavor_mbox_free(state, &mbox_info);
1737 
1738 	TAVOR_TNF_EXIT(tavor_mad_ifc_cmd_post);
1739 	return (status);
1740 }
1741 
1742 
1743 /*
1744  * tavor_getportinfo_cmd_post()
1745  *    Context: Can be called from interrupt or base context.
1746  */
1747 int
1748 tavor_getportinfo_cmd_post(tavor_state_t *state, uint_t port,
1749     uint_t sleepflag, sm_portinfo_t *portinfo)
1750 {
1751 	tavor_mbox_info_t	mbox_info;
1752 	tavor_cmd_post_t	cmd;
1753 	uint32_t		*mbox;
1754 	uint_t			size;
1755 	int			status, i;
1756 
1757 	TAVOR_TNF_ENTER(tavor_getportinfo_cmd_post);
1758 
1759 	/* Get "In" and "Out" mailboxes for the command */
1760 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1761 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1762 	if (status != TAVOR_CMD_SUCCESS) {
1763 		TNF_PROBE_0(tavor_getportinfo_mbox_fail,
1764 		    TAVOR_TNF_ERROR, "");
1765 		TAVOR_TNF_EXIT(tavor_getportinfo_cmd_post);
1766 		return (status);
1767 	}
1768 
1769 	/* Build the GetPortInfo request MAD in the "In" mailbox */
1770 	size = TAVOR_CMD_MAD_IFC_SIZE;
1771 	mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
1772 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
1773 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
1774 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
1775 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
1776 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_PORTINFO);
1777 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], port);
1778 	for (i = 6; i < (size >> 2); i++) {
1779 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
1780 	}
1781 
1782 	/* Sync the mailbox for the device to read */
1783 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1784 
1785 	/* Setup the Tavor "MAD_IFC" command */
1786 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
1787 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
1788 	cmd.cp_inmod	= port;
1789 	cmd.cp_opcode	= MAD_IFC;
1790 	cmd.cp_opmod	= TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
1791 	cmd.cp_flags	= sleepflag;
1792 	status = tavor_cmd_post(state, &cmd);
1793 	if (status != TAVOR_CMD_SUCCESS) {
1794 		TNF_PROBE_0(tavor_getportinfo_cmd_post_fail,
1795 		    TAVOR_TNF_ERROR, "");
1796 		goto getportinfo_fail;
1797 	}
1798 
1799 	/* Sync the mailbox to read the results */
1800 	size = sizeof (sm_portinfo_t);
1801 	tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1802 	    size, DDI_DMA_SYNC_FORCPU);
1803 
1804 	/*
1805 	 * Copy GetPortInfo response MAD into "portinfo".  Do any endian
1806 	 * swapping that may be necessary to flip any of the "portinfo"
1807 	 * fields
1808 	 */
1809 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*portinfo))
1810 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1811 	    TAVOR_CMD_MADDATA_OFFSET), portinfo, size);
1812 	TAVOR_GETPORTINFO_SWAP(portinfo);
1813 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*portinfo))
1814 
1815 getportinfo_fail:
1816 	/* Free the mailbox */
1817 	tavor_mbox_free(state, &mbox_info);
1818 
1819 	TAVOR_TNF_EXIT(tavor_getportinfo_cmd_post);
1820 	return (status);
1821 }
1822 
1823 
1824 /*
1825  * tavor_getnodeinfo_cmd_post()
1826  *    Context: Can be called from interrupt or base context.
1827  *    (Currently called only from attach() and detach() path contexts)
1828  */
1829 int
1830 tavor_getnodeinfo_cmd_post(tavor_state_t *state, uint_t sleepflag,
1831     sm_nodeinfo_t *nodeinfo)
1832 {
1833 	tavor_mbox_info_t	mbox_info;
1834 	tavor_cmd_post_t	cmd;
1835 	uint32_t		*mbox;
1836 	uint_t			size;
1837 	int			status, i;
1838 
1839 	TAVOR_TNF_ENTER(tavor_getnodeinfo_cmd_post);
1840 
1841 	/* Make sure we are called with the correct flag */
1842 	ASSERT(sleepflag == TAVOR_CMD_NOSLEEP_SPIN);
1843 
1844 	/* Get "In" and "Out" mailboxes for the command */
1845 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1846 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1847 	if (status != TAVOR_CMD_SUCCESS) {
1848 		TNF_PROBE_0(tavor_getnodeinfo_mbox_fail,
1849 		    TAVOR_TNF_ERROR, "");
1850 		TAVOR_TNF_EXIT(tavor_getnodeinfo_cmd_post);
1851 		return (status);
1852 	}
1853 
1854 	/* Build the GetNodeInfo request MAD into the "In" mailbox */
1855 	size = TAVOR_CMD_MAD_IFC_SIZE;
1856 	mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
1857 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
1858 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
1859 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
1860 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
1861 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_NODEINFO);
1862 	for (i = 5; i < (size >> 2); i++) {
1863 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
1864 	}
1865 
1866 	/* Sync the mailbox for the device to read */
1867 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1868 
1869 	/* Setup the Tavor "MAD_IFC" command */
1870 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
1871 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
1872 	cmd.cp_inmod	= 1;  /* Get NodeInfo from port #1 */
1873 	cmd.cp_opcode	= MAD_IFC;
1874 	cmd.cp_opmod	= TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
1875 	cmd.cp_flags	= sleepflag;
1876 	status = tavor_cmd_post(state, &cmd);
1877 	if (status != TAVOR_CMD_SUCCESS) {
1878 		TNF_PROBE_0(tavor_getnodeinfo_cmd_post_fail,
1879 		    TAVOR_TNF_ERROR, "");
1880 		goto getnodeinfo_fail;
1881 	}
1882 
1883 	/* Sync the mailbox to read the results */
1884 	size = sizeof (sm_nodeinfo_t);
1885 	tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1886 	    size, DDI_DMA_SYNC_FORCPU);
1887 
1888 	/*
1889 	 * Copy GetNodeInfo response MAD into "nodeinfo".  Do any endian
1890 	 * swapping that may be necessary to flip any of the "nodeinfo"
1891 	 * fields
1892 	 */
1893 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1894 	    TAVOR_CMD_MADDATA_OFFSET), nodeinfo, size);
1895 	TAVOR_GETNODEINFO_SWAP(nodeinfo);
1896 
1897 getnodeinfo_fail:
1898 	/* Free the mailbox */
1899 	tavor_mbox_free(state, &mbox_info);
1900 
1901 	TAVOR_TNF_EXIT(tavor_getnodeinfo_cmd_post);
1902 	return (status);
1903 }
1904 
1905 
1906 /*
1907  * tavor_getnodedesc_cmd_post()
1908  *    Context: Can be called from interrupt or base context.
1909  *    (Currently called only from attach() and detach() path contexts)
1910  */
1911 int
1912 tavor_getnodedesc_cmd_post(tavor_state_t *state, uint_t sleepflag,
1913     sm_nodedesc_t *nodedesc)
1914 {
1915 	tavor_mbox_info_t	mbox_info;
1916 	tavor_cmd_post_t	cmd;
1917 	uint32_t		*mbox;
1918 	uint_t			size;
1919 	int			status, i;
1920 
1921 	TAVOR_TNF_ENTER(tavor_getnodedesc_cmd_post);
1922 
1923 	/* Get "In" and "Out" mailboxes for the command */
1924 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1925 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1926 	if (status != TAVOR_CMD_SUCCESS) {
1927 		TNF_PROBE_0(tavor_getnodedesc_mbox_fail, TAVOR_TNF_ERROR, "");
1928 		TAVOR_TNF_EXIT(tavor_getnodedesc_cmd_post);
1929 		return (status);
1930 	}
1931 
1932 	/* Build the GetNodeDesc request MAD into the "In" mailbox */
1933 	size = TAVOR_CMD_MAD_IFC_SIZE;
1934 	mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
1935 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
1936 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
1937 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
1938 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
1939 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_NODEDESC);
1940 	for (i = 5; i < (size >> 2); i++) {
1941 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
1942 	}
1943 
1944 	/* Sync the mailbox for the device to read */
1945 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
1946 
1947 	/* Setup the Tavor "MAD_IFC" command */
1948 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
1949 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
1950 	cmd.cp_inmod	= 1;  /* Get NodeDesc from port #1 */
1951 	cmd.cp_opcode	= MAD_IFC;
1952 	cmd.cp_opmod	= TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
1953 	cmd.cp_flags	= sleepflag;
1954 	status = tavor_cmd_post(state, &cmd);
1955 	if (status != TAVOR_CMD_SUCCESS) {
1956 		TNF_PROBE_0(tavor_getnodedesc_cmd_post_fail,
1957 		    TAVOR_TNF_ERROR, "");
1958 		goto getnodedesc_fail;
1959 	}
1960 
1961 	/* Sync the mailbox to read the results */
1962 	size = sizeof (sm_nodedesc_t);
1963 	tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
1964 	    size, DDI_DMA_SYNC_FORCPU);
1965 
1966 	/* Copy GetNodeDesc response MAD into "nodedesc" */
1967 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
1968 	    TAVOR_CMD_MADDATA_OFFSET), nodedesc, size);
1969 
1970 getnodedesc_fail:
1971 	/* Free the mailbox */
1972 	tavor_mbox_free(state, &mbox_info);
1973 
1974 	TAVOR_TNF_EXIT(tavor_getnodedesc_cmd_post);
1975 	return (status);
1976 }
1977 
1978 
1979 /*
1980  * tavor_getguidinfo_cmd_post()
1981  *    Context: Can be called from interrupt or base context.
1982  */
1983 int
1984 tavor_getguidinfo_cmd_post(tavor_state_t *state, uint_t port,
1985     uint_t guidblock, uint_t sleepflag, sm_guidinfo_t *guidinfo)
1986 {
1987 	tavor_mbox_info_t	mbox_info;
1988 	tavor_cmd_post_t	cmd;
1989 	uint32_t		*mbox;
1990 	uint_t			size;
1991 	int			status, i;
1992 
1993 	TAVOR_TNF_ENTER(tavor_getguidinfo_cmd_post);
1994 
1995 	/* Get "In" and "Out" mailboxes for the command */
1996 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
1997 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
1998 	if (status != TAVOR_CMD_SUCCESS) {
1999 		TNF_PROBE_0(tavor_getguidinfo_mbox_fail, TAVOR_TNF_ERROR, "");
2000 		TAVOR_TNF_EXIT(tavor_getguidinfo_cmd_post);
2001 		return (status);
2002 	}
2003 
2004 	/* Build the GetGUIDInfo request MAD into the "In" mailbox */
2005 	size = TAVOR_CMD_MAD_IFC_SIZE;
2006 	mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
2007 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
2008 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
2009 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
2010 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
2011 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_GUIDINFO);
2012 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], guidblock);
2013 	for (i = 6; i < (size >> 2); i++) {
2014 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
2015 	}
2016 
2017 	/* Sync the mailbox for the device to read */
2018 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2019 
2020 	/* Setup the Tavor "MAD_IFC" command */
2021 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
2022 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
2023 	cmd.cp_inmod	= port;
2024 	cmd.cp_opcode	= MAD_IFC;
2025 	cmd.cp_opmod	= TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
2026 	cmd.cp_flags	= sleepflag;
2027 	status = tavor_cmd_post(state, &cmd);
2028 	if (status != TAVOR_CMD_SUCCESS) {
2029 		TNF_PROBE_0(tavor_getguidinfo_cmd_post_fail,
2030 		    TAVOR_TNF_ERROR, "");
2031 		goto getguidinfo_fail;
2032 	}
2033 
2034 	/* Sync the mailbox to read the results */
2035 	size = sizeof (sm_guidinfo_t);
2036 	tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
2037 	    size, DDI_DMA_SYNC_FORCPU);
2038 
2039 	/*
2040 	 * Copy GetGUIDInfo response MAD into "guidinfo".  Do any endian
2041 	 * swapping that may be necessary to flip the "guidinfo" fields
2042 	 */
2043 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*guidinfo))
2044 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
2045 	    TAVOR_CMD_MADDATA_OFFSET), guidinfo, size);
2046 	TAVOR_GETGUIDINFO_SWAP(guidinfo);
2047 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*guidinfo))
2048 
2049 getguidinfo_fail:
2050 	/* Free the mailbox */
2051 	tavor_mbox_free(state, &mbox_info);
2052 
2053 	TAVOR_TNF_EXIT(tavor_getguidinfo_cmd_post);
2054 	return (status);
2055 }
2056 
2057 
2058 /*
2059  * tavor_getpkeytable_cmd_post()
2060  *    Context: Can be called from interrupt or base context.
2061  */
2062 int
2063 tavor_getpkeytable_cmd_post(tavor_state_t *state, uint_t port,
2064     uint_t pkeyblock, uint_t sleepflag, sm_pkey_table_t *pkeytable)
2065 {
2066 	tavor_mbox_info_t	mbox_info;
2067 	tavor_cmd_post_t	cmd;
2068 	uint32_t		*mbox;
2069 	uint_t			size;
2070 	int			status, i;
2071 
2072 	TAVOR_TNF_ENTER(tavor_getpkeytable_cmd_post);
2073 
2074 	/* Get "In" and "Out" mailboxes for the command */
2075 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
2076 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2077 	if (status != TAVOR_CMD_SUCCESS) {
2078 		TNF_PROBE_0(tavor_getpkeytable_mbox_fail, TAVOR_TNF_ERROR, "");
2079 		TAVOR_TNF_EXIT(tavor_getpkeytable_cmd_post);
2080 		return (status);
2081 	}
2082 
2083 	/* Build the GetPkeyTable request MAD into the "In" mailbox */
2084 	size = TAVOR_CMD_MAD_IFC_SIZE;
2085 	mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
2086 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0], TAVOR_CMD_MADHDR0);
2087 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
2088 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
2089 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
2090 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_PKEYTBLE);
2091 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], pkeyblock);
2092 	for (i = 6; i < (size >> 2); i++) {
2093 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[i], 0);
2094 	}
2095 
2096 	/* Sync the mailbox for the device to read */
2097 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2098 
2099 	/* Setup the Tavor "MAD_IFC" command */
2100 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
2101 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
2102 	cmd.cp_inmod	= port;
2103 	cmd.cp_opcode	= MAD_IFC;
2104 	cmd.cp_opmod	= TAVOR_CMD_MKEY_DONTCHECK;  /* No MKey checking */
2105 	cmd.cp_flags	= sleepflag;
2106 	status = tavor_cmd_post(state, &cmd);
2107 	if (status != TAVOR_CMD_SUCCESS) {
2108 		TNF_PROBE_0(tavor_getpkeytable_cmd_post_fail,
2109 		    TAVOR_TNF_ERROR, "");
2110 		goto getpkeytable_fail;
2111 	}
2112 
2113 	/* Sync the mailbox to read the results */
2114 	size = sizeof (sm_pkey_table_t);
2115 	tavor_mbox_sync(mbox_info.mbi_out, TAVOR_CMD_MADDATA_OFFSET,
2116 	    size, DDI_DMA_SYNC_FORCPU);
2117 
2118 	/*
2119 	 * Copy GetPKeyTable response MAD into "pkeytable".  Do any endian
2120 	 * swapping that may be necessary to flip the "pkeytable" fields
2121 	 */
2122 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pkeytable))
2123 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
2124 	    TAVOR_CMD_MADDATA_OFFSET), pkeytable, size);
2125 	TAVOR_GETPKEYTABLE_SWAP(pkeytable);
2126 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*pkeytable))
2127 
2128 getpkeytable_fail:
2129 	/* Free the mailbox */
2130 	tavor_mbox_free(state, &mbox_info);
2131 
2132 	TAVOR_TNF_EXIT(tavor_getpkeytable_cmd_post);
2133 	return (status);
2134 }
2135 
2136 
2137 /*
2138  * tavor_write_mtt_cmd_post()
2139  *    Context: Can be called from interrupt or base context.
2140  */
2141 int
2142 tavor_write_mtt_cmd_post(tavor_state_t *state, tavor_mbox_info_t *mbox_info,
2143     uint_t num_mtt, uint_t sleepflag)
2144 {
2145 	tavor_cmd_post_t	cmd;
2146 	uint_t			size;
2147 	int			status;
2148 
2149 	TAVOR_TNF_ENTER(tavor_write_mtt_cmd_post);
2150 
2151 	/*
2152 	 * The WRITE_MTT command is unlike the other commands we use, in that
2153 	 * we have intentionally separated the mailbox allocation step from
2154 	 * the rest of the command posting steps.  At this point (when this
2155 	 * function is called) the "In" mailbox already contains all the MTT
2156 	 * entries to be copied into the Tavor tables (starting at offset
2157 	 * 0x10) _and_ the 64-bit address of the destination for the first
2158 	 * MTT entry in the MTT table.
2159 	 */
2160 
2161 	/* Sync the mailbox for the device to read */
2162 	size = (num_mtt << TAVOR_MTT_SIZE_SHIFT) + TAVOR_CMD_WRITEMTT_RSVD_SZ;
2163 	tavor_mbox_sync(mbox_info->mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2164 
2165 	/* Setup and post Tavor "WRITE_MTT" command */
2166 	cmd.cp_inparm	= mbox_info->mbi_in->mb_mapaddr;
2167 	cmd.cp_outparm	= 0;
2168 	cmd.cp_inmod	= num_mtt;
2169 	cmd.cp_opcode	= WRITE_MTT;
2170 	cmd.cp_opmod	= 0;
2171 	cmd.cp_flags	= sleepflag;
2172 	status = tavor_cmd_post(state, &cmd);
2173 	if (status != TAVOR_CMD_SUCCESS) {
2174 		TNF_PROBE_0(tavor_write_mtt_cmd_fail, TAVOR_TNF_ERROR, "");
2175 	}
2176 
2177 	TAVOR_TNF_EXIT(tavor_write_mtt_cmd_post);
2178 	return (status);
2179 }
2180 
2181 
2182 /*
2183  * tavor_sync_tpt_cmd_post()
2184  *    Context: Can be called from interrupt or base context.
2185  */
2186 int
2187 tavor_sync_tpt_cmd_post(tavor_state_t *state, uint_t sleepflag)
2188 {
2189 	tavor_cmd_post_t	cmd;
2190 	int			status;
2191 
2192 	TAVOR_TNF_ENTER(tavor_sync_tpt_cmd_post);
2193 
2194 	/* Setup and post the Tavor "SYNC_TPT" command */
2195 	cmd.cp_inparm	= 0;
2196 	cmd.cp_outparm	= 0;
2197 	cmd.cp_inmod	= 0;
2198 	cmd.cp_opcode	= SYNC_TPT;
2199 	cmd.cp_opmod	= 0;
2200 	cmd.cp_flags	= sleepflag;
2201 	status = tavor_cmd_post(state, &cmd);
2202 	if (status != TAVOR_CMD_SUCCESS) {
2203 		TNF_PROBE_0(tavor_sync_tpt_cmd_post_fail, TAVOR_TNF_ERROR, "");
2204 	}
2205 
2206 	TAVOR_TNF_EXIT(tavor_sync_tpt_cmd_post);
2207 	return (status);
2208 }
2209 
2210 /*
2211  * tavor_map_eq_cmd_post()
2212  *    Context: Can be called from interrupt or base context.
2213  *    (Currently called only from attach() and/or detach() path contexts)
2214  */
2215 int
2216 tavor_map_eq_cmd_post(tavor_state_t *state, uint_t map, uint_t eqcindx,
2217     uint64_t eqmapmask, uint_t sleepflag)
2218 {
2219 	tavor_cmd_post_t	cmd;
2220 	int			status;
2221 
2222 	TAVOR_TNF_ENTER(tavor_map_eq_cmd_post);
2223 
2224 	/* Setup and post Tavor "MAP_EQ" command */
2225 	cmd.cp_inparm	= eqmapmask;
2226 	cmd.cp_outparm	= 0;
2227 	cmd.cp_inmod	= eqcindx;
2228 	if (map != TAVOR_CMD_MAP_EQ_EVT_MAP) {
2229 		cmd.cp_inmod |= TAVOR_CMD_UNMAP_EQ_MASK;
2230 	}
2231 	cmd.cp_opcode	= MAP_EQ;
2232 	cmd.cp_opmod	= 0;
2233 	cmd.cp_flags	= sleepflag;
2234 	status = tavor_cmd_post(state, &cmd);
2235 	if (status != TAVOR_CMD_SUCCESS) {
2236 		TNF_PROBE_0(tavor_map_eq_cmd_post_fail, TAVOR_TNF_ERROR, "");
2237 	}
2238 
2239 	TAVOR_TNF_EXIT(tavor_map_eq_cmd_post);
2240 	return (status);
2241 }
2242 
2243 
2244 /*
2245  * tavor_resize_cq_cmd_post()
2246  *    Context: Can be called from interrupt or base context.
2247  */
2248 int
2249 tavor_resize_cq_cmd_post(tavor_state_t *state, tavor_hw_cqc_t *cqc,
2250     uint_t cqcindx, uint32_t *prod_indx, uint_t sleepflag)
2251 {
2252 	tavor_mbox_info_t	mbox_info;
2253 	tavor_cmd_post_t	cmd;
2254 	uint64_t		data;
2255 	uint_t			size;
2256 	int			status, i;
2257 
2258 	TAVOR_TNF_ENTER(tavor_resize_cq_cmd_post);
2259 
2260 	/* Get an "In" mailbox for the command */
2261 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2262 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2263 	if (status != TAVOR_CMD_SUCCESS) {
2264 		TNF_PROBE_0(tavor_resize_cq_mbox_fail, TAVOR_TNF_ERROR, "");
2265 		TAVOR_TNF_EXIT(tavor_resize_cq_cmd_post);
2266 		return (status);
2267 	}
2268 
2269 	/* Copy the Tavor "RESIZE_CQ" command into mailbox */
2270 	size = sizeof (tavor_hw_cqc_t);
2271 	for (i = 0; i < (size >> 3); i++) {
2272 		data = ((uint64_t *)cqc)[i];
2273 		ddi_put64(mbox_info.mbi_in->mb_acchdl,
2274 		    ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
2275 	}
2276 
2277 	/* Sync the mailbox for the device to read */
2278 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2279 
2280 	/* Setup and post Tavor "RESIZE_CQ" command */
2281 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
2282 	cmd.cp_outparm	= 0;
2283 	cmd.cp_inmod	= cqcindx;
2284 	cmd.cp_opcode	= RESIZE_CQ;
2285 	cmd.cp_opmod	= 0;
2286 	cmd.cp_flags	= sleepflag;
2287 	status = tavor_cmd_post(state, &cmd);
2288 	if (status != TAVOR_CMD_SUCCESS) {
2289 		TNF_PROBE_0(tavor_resize_cq_cmd_post_fail, TAVOR_TNF_ERROR, "");
2290 	}
2291 
2292 	/*
2293 	 * New "producer index" is returned in the upper 32 bits of
2294 	 * command "outparam"
2295 	 */
2296 	*prod_indx = (cmd.cp_outparm >> 32);
2297 
2298 	/* Free the mailbox */
2299 	tavor_mbox_free(state, &mbox_info);
2300 
2301 	TAVOR_TNF_EXIT(tavor_resize_cq_cmd_post);
2302 	return (status);
2303 }
2304 
2305 
2306 /*
2307  * tavor_cmn_qp_cmd_post()
2308  *    Context: Can be called from interrupt or base context.
2309  *
2310  *    This is the common function for posting all the various types of
2311  *    QP state transition related Tavor commands.  Since some of the
2312  *    commands differ from the others in the number (and type) of arguments
2313  *    that each require, this routine does checks based on opcode type
2314  *    (explained in more detail below).
2315  *
2316  * Note: This common function should be used only with the following
2317  *    opcodes: RTS2SQD_QP, TOERR_QP, TORST_QP, RST2INIT_QP, INIT2INIT_QP,
2318  *    INIT2RTR_QP, RTR2RTS_QP, RTS2RTS_QP, SQD2RTS_QP, and SQERR2RTS_QP.
2319  */
2320 int
2321 tavor_cmn_qp_cmd_post(tavor_state_t *state, uint_t opcode,
2322     tavor_hw_qpc_t *qp, uint_t qpindx, uint32_t opmask,
2323     uint_t sleepflag)
2324 {
2325 	tavor_mbox_info_t	mbox_info;
2326 	tavor_cmd_post_t	cmd;
2327 	uint64_t		data, in_mapaddr, out_mapaddr;
2328 	uint_t			size, flags, opmod;
2329 	int			status, i;
2330 
2331 	TAVOR_TNF_ENTER(tavor_cmn_qp_cmd_post);
2332 
2333 	/*
2334 	 * Use the specified opcode type to set the appropriate parameters.
2335 	 * Specifically, we need to set in_mapaddr, out_mapaddr, flags, and
2336 	 * opmod (as necessary).  Setting these parameters may also require
2337 	 * us to allocate an "In" or "Out" mailbox depending on the command
2338 	 * type.
2339 	 */
2340 	if (opcode == RTS2SQD_QP) {
2341 		/*
2342 		 * Note: For RTS-to-SendQueueDrain state transitions we
2343 		 * always want to request the event generation from the
2344 		 * hardware.  Though we may not notify the consumer of the
2345 		 * drained event, the decision to forward (or not) is made
2346 		 * later in the SQD event handler.
2347 		 */
2348 		flags = TAVOR_CMD_REQ_SQD_EVENT;
2349 
2350 		/*
2351 		 * The RTS2SQD_QP command uses no "In" or "Out" mailboxes (and
2352 		 * has no special opcode modifiers).
2353 		 */
2354 		in_mapaddr  = 0;
2355 		out_mapaddr = 0;
2356 		opmod = 0;
2357 
2358 	} else if (opcode == TOERR_QP) {
2359 		/*
2360 		 * The TOERR_QP command uses no "In" or "Out" mailboxes, has no
2361 		 * special opcode modifiers, and takes no special flags.
2362 		 */
2363 		in_mapaddr  = 0;
2364 		out_mapaddr = 0;
2365 		opmod = 0;
2366 		flags = 0;
2367 
2368 	} else if (opcode == TORST_QP) {
2369 		/*
2370 		 * The TORST_QP command could take an "Out" mailbox, but we do
2371 		 * not require it here.  It also does not takes any special
2372 		 * flags.  It does however, take a TAVOR_CMD_DIRECT_TO_RESET
2373 		 * opcode modifier, which indicates that the transition to
2374 		 * reset should happen without first moving the QP through the
2375 		 * Error state (and, hence, without generating any unnecessary
2376 		 * "flushed-in-error" completions).
2377 		 */
2378 		in_mapaddr  = 0;
2379 		out_mapaddr = 0;
2380 		opmod = TAVOR_CMD_DIRECT_TO_RESET | TAVOR_CMD_NO_OUTMBOX;
2381 		flags = 0;
2382 
2383 	} else {
2384 		/*
2385 		 * All the other QP state transition commands (RST2INIT_QP,
2386 		 * INIT2INIT_QP, INIT2RTR_QP, RTR2RTS_QP, RTS2RTS_QP,
2387 		 * SQD2RTS_QP, and SQERR2RTS_QP) require an "In" mailbox.
2388 		 * None of these require any special flags or opcode modifiers.
2389 		 */
2390 		mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2391 		status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2392 		if (status != TAVOR_CMD_SUCCESS) {
2393 			TNF_PROBE_0(tavor_cmn_qp_mbox_fail,
2394 			    TAVOR_TNF_ERROR, "");
2395 			TAVOR_TNF_EXIT(tavor_cmn_qp_cmd_post);
2396 			return (status);
2397 		}
2398 		in_mapaddr  = mbox_info.mbi_in->mb_mapaddr;
2399 		out_mapaddr = 0;
2400 		flags = 0;
2401 		opmod = 0;
2402 
2403 		/* Copy the Tavor command into the "In" mailbox */
2404 		size = sizeof (tavor_hw_qpc_t);
2405 		for (i = 0; i < (size >> 3); i++) {
2406 			data = ((uint64_t *)qp)[i];
2407 			ddi_put64(mbox_info.mbi_in->mb_acchdl,
2408 			    ((uint64_t *)mbox_info.mbi_in->mb_addr + i + 1),
2409 			    data);
2410 		}
2411 		ddi_put32(mbox_info.mbi_in->mb_acchdl,
2412 		    ((uint32_t *)mbox_info.mbi_in->mb_addr), opmask);
2413 
2414 		/*
2415 		 * Sync the mailbox for the device to read.  We have to add
2416 		 * eight bytes here to account for "opt_param_mask" and
2417 		 * proper alignment.
2418 		 */
2419 		tavor_mbox_sync(mbox_info.mbi_in, 0, size + 8,
2420 		    DDI_DMA_SYNC_FORDEV);
2421 	}
2422 
2423 	/* Setup and post Tavor QP state transition command */
2424 	cmd.cp_inparm	= in_mapaddr;
2425 	cmd.cp_outparm	= out_mapaddr;
2426 	cmd.cp_inmod	= qpindx | flags;
2427 	cmd.cp_opcode	= opcode;
2428 	cmd.cp_opmod	= opmod;
2429 	cmd.cp_flags	= sleepflag;
2430 	status = tavor_cmd_post(state, &cmd);
2431 	if (status != TAVOR_CMD_SUCCESS) {
2432 		TNF_PROBE_0(tavor_cmn_qp_cmd_post_fail, TAVOR_TNF_ERROR, "");
2433 	}
2434 
2435 	/*
2436 	 * If we allocated a mailbox (either an "In" or an "Out") above,
2437 	 * then free it now before returning.
2438 	 */
2439 	if ((opcode != RTS2SQD_QP) && (opcode != TOERR_QP) &&
2440 	    (opcode != TORST_QP)) {
2441 		/* Free the mailbox */
2442 		tavor_mbox_free(state, &mbox_info);
2443 	}
2444 
2445 	TAVOR_TNF_EXIT(tavor_cmn_qp_cmd_post);
2446 	return (status);
2447 }
2448 
2449 
2450 /*
2451  * tavor_cmn_query_cmd_post()
2452  *    Context: Can be called from interrupt or base context.
2453  *
2454  *    This is the common function for posting all the various types of
2455  *    Tavor query commands.  All Tavor query commands require an "Out"
2456  *    mailbox to be allocated for the resulting queried data.
2457  *
2458  * Note: This common function should be used only with the following
2459  *    opcodes: QUERY_DEV_LIM, QUERY_FW, QUERY_DDR, QUERY_ADAPTER,
2460  *     QUERY_HCA, QUERY_MPT, QUERY_EQ, QUERY_CQ, and QUERY_QP.
2461  */
2462 int
2463 tavor_cmn_query_cmd_post(tavor_state_t *state, uint_t opcode,
2464     uint_t queryindx, void *query, uint_t size, uint_t sleepflag)
2465 {
2466 	tavor_mbox_info_t	mbox_info;
2467 	tavor_cmd_post_t	cmd;
2468 	uint64_t		data;
2469 	uint_t			offset;
2470 	int			status, i;
2471 
2472 	TAVOR_TNF_ENTER(tavor_cmn_query_cmd_post);
2473 
2474 	/* Get an "Out" mailbox for the command */
2475 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
2476 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2477 	if (status != TAVOR_CMD_SUCCESS) {
2478 		TNF_PROBE_0(tavor_cmn_query_mbox_fail, TAVOR_TNF_ERROR, "");
2479 		TAVOR_TNF_EXIT(tavor_cmn_query_cmd_post);
2480 		return (status);
2481 	}
2482 
2483 	/* Setup and post the Tavor query command */
2484 	cmd.cp_inparm	= 0;
2485 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
2486 	cmd.cp_inmod	= queryindx;
2487 	cmd.cp_opcode	= opcode;
2488 	cmd.cp_opmod	= 0;
2489 	cmd.cp_flags	= sleepflag;
2490 	status = tavor_cmd_post(state, &cmd);
2491 	if (status != TAVOR_CMD_SUCCESS) {
2492 		TNF_PROBE_0(tavor_cmn_query_cmd_post_fail, TAVOR_TNF_ERROR, "");
2493 		goto cmn_query_fail;
2494 	}
2495 
2496 	/* Sync the mailbox to read the results */
2497 	tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
2498 
2499 	/*
2500 	 * QUERY_QP is handled somewhat differently than the other query
2501 	 * commands.  For QUERY_QP, the actual queried data is offset into
2502 	 * the mailbox (by one 64-bit word).
2503 	 */
2504 	offset = (opcode == QUERY_QP) ? 1 : 0;
2505 
2506 	/* Copy query command results into "query" */
2507 	for (i = 0; i < (size >> 3); i++) {
2508 		data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
2509 		    ((uint64_t *)mbox_info.mbi_out->mb_addr + i + offset));
2510 		((uint64_t *)query)[i] = data;
2511 	}
2512 
2513 cmn_query_fail:
2514 	/* Free the mailbox */
2515 	tavor_mbox_free(state, &mbox_info);
2516 
2517 	TAVOR_TNF_EXIT(tavor_cmn_query_cmd_post);
2518 	return (status);
2519 }
2520 
2521 
2522 /*
2523  * tavor_cmn_ownership_cmd_post()
2524  *    Context: Can be called from interrupt or base context.
2525  *
2526  *    This is the common function for posting all the various types of
2527  *    Tavor HW/SW resource ownership commands.  Since some of the commands
2528  *    differ from the others in the direction of ownership change (i.e.
2529  *    from HW ownership to SW, or vice versa), they differ in the type of
2530  *    mailbox and specific handling that each requires.  This routine does
2531  *    certain checks based on opcode type to determine the direction of
2532  *    the transition and to correctly handle the request.
2533  *
2534  * Note: This common function should be used only with the following
2535  *    opcodes: HW2SW_MPT, HW2SW_EQ, HW2SW_CQ, SW2HW_MPT, SW2HW_EQ, and
2536  *    SW2HW_CQ
2537  */
2538 int
2539 tavor_cmn_ownership_cmd_post(tavor_state_t *state, uint_t opcode,
2540     void *hwrsrc, uint_t size, uint_t hwrsrcindx, uint_t sleepflag)
2541 {
2542 	tavor_mbox_info_t	mbox_info;
2543 	tavor_cmd_post_t	cmd;
2544 	uint64_t		data, in_mapaddr, out_mapaddr;
2545 	uint_t			direction, opmod;
2546 	int			status, i;
2547 
2548 	TAVOR_TNF_ENTER(tavor_cmn_ownership_cmd_post);
2549 
2550 	/*
2551 	 * Determine the direction of the ownership transfer based on the
2552 	 * provided opcode
2553 	 */
2554 	if ((opcode == HW2SW_MPT) || (opcode == HW2SW_EQ) ||
2555 	    (opcode == HW2SW_CQ) || (opcode == HW2SW_SRQ)) {
2556 		direction = TAVOR_CMD_RSRC_HW2SW;
2557 
2558 	} else if ((opcode == SW2HW_MPT) || (opcode == SW2HW_EQ) ||
2559 	    (opcode == SW2HW_CQ) || (opcode == SW2HW_SRQ)) {
2560 		direction = TAVOR_CMD_RSRC_SW2HW;
2561 
2562 	} else {
2563 		TNF_PROBE_0(tavor_cmn_ownership_dir_fail,
2564 		    TAVOR_TNF_ERROR, "");
2565 		TAVOR_TNF_EXIT(tavor_cmn_ownership_cmd_post);
2566 		return (TAVOR_CMD_INVALID_STATUS);
2567 	}
2568 
2569 	/*
2570 	 * If hwrsrc is NULL then we do not allocate a mailbox.  This is used
2571 	 * in the case of memory deregister where the out mailbox is not
2572 	 * needed.  In the case of re-register, we do use the hwrsrc.
2573 	 *
2574 	 * Otherwise, If ownership transfer is going from hardware to software,
2575 	 * then allocate an "Out" mailbox.  This will be filled in later as a
2576 	 * result of the Tavor command.
2577 	 *
2578 	 * And if the ownership transfer is going from software to hardware,
2579 	 * then we need an "In" mailbox, and we need to fill it in and sync it
2580 	 * (if necessary).  Then the mailbox can be passed to the Tavor
2581 	 * firmware.
2582 	 *
2583 	 * For the HW2SW (dereg) case, we only use an out mbox if hwrsrc is !=
2584 	 * NULL.  This implies a re-reg, and the out mbox must be used.  If
2585 	 * hwrsrc is == NULL, then we can save some time and resources by not
2586 	 * using an out mbox at all.  We must set opmod to TAVOR_CMD_DO_OUTMBOX
2587 	 * and TAVOR_CMD_NO_OUTMBOX appropriately in this case.
2588 	 *
2589 	 * For the SW2HW (reg) case, no out mbox is possible.  We set opmod to
2590 	 * 0 anyway, but this field is not used in this case.
2591 	 */
2592 	if (direction == TAVOR_CMD_RSRC_HW2SW) {
2593 		if (hwrsrc != NULL) {
2594 			mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
2595 			status = tavor_mbox_alloc(state, &mbox_info,
2596 			    sleepflag);
2597 			if (status != TAVOR_CMD_SUCCESS) {
2598 				TNF_PROBE_0(tavor_cmn_ownership_mbox_fail,
2599 				    TAVOR_TNF_ERROR, "");
2600 				TAVOR_TNF_EXIT(tavor_cmn_ownership_cmd_post);
2601 				return (status);
2602 			}
2603 			in_mapaddr  = 0;
2604 			out_mapaddr = mbox_info.mbi_out->mb_mapaddr;
2605 			opmod = TAVOR_CMD_DO_OUTMBOX;
2606 		} else {
2607 			in_mapaddr = 0;
2608 			out_mapaddr = 0;
2609 			opmod = TAVOR_CMD_NO_OUTMBOX;
2610 		}
2611 	} else {  /* TAVOR_CMD_RSRC_SW2HW */
2612 		mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2613 		status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2614 		if (status != TAVOR_CMD_SUCCESS) {
2615 			TNF_PROBE_0(tavor_cmn_ownership_mbox_fail,
2616 			    TAVOR_TNF_ERROR, "");
2617 			TAVOR_TNF_EXIT(tavor_sw2hw_mpt_cmd_post);
2618 			return (status);
2619 		}
2620 
2621 		/* Copy the SW2HW ownership command into mailbox */
2622 		for (i = 0; i < (size >> 3); i++) {
2623 			data = ((uint64_t *)hwrsrc)[i];
2624 			ddi_put64(mbox_info.mbi_in->mb_acchdl,
2625 			    ((uint64_t *)mbox_info.mbi_in->mb_addr + i),
2626 			    data);
2627 		}
2628 
2629 		/* Sync the mailbox for the device to read */
2630 		tavor_mbox_sync(mbox_info.mbi_in, 0, size,
2631 		    DDI_DMA_SYNC_FORDEV);
2632 
2633 		in_mapaddr  = mbox_info.mbi_in->mb_mapaddr;
2634 		out_mapaddr = 0;
2635 		opmod = 0;
2636 	}
2637 
2638 
2639 	/* Setup and post the Tavor ownership command */
2640 	cmd.cp_inparm	= in_mapaddr;
2641 	cmd.cp_outparm	= out_mapaddr;
2642 	cmd.cp_inmod	= hwrsrcindx;
2643 	cmd.cp_opcode	= opcode;
2644 	cmd.cp_opmod	= opmod;
2645 	cmd.cp_flags	= sleepflag;
2646 	status = tavor_cmd_post(state, &cmd);
2647 	if (status != TAVOR_CMD_SUCCESS) {
2648 		TNF_PROBE_0(tavor_cmn_ownership_cmd_post_fail,
2649 		    TAVOR_TNF_ERROR, "");
2650 		goto cmn_ownership_fail;
2651 	}
2652 
2653 	/*
2654 	 * As mentioned above, for HW2SW ownership transfers we need to
2655 	 * sync (if necessary) and copy out the resulting data from the
2656 	 * "Out" mailbox" (assuming the above command was successful).
2657 	 */
2658 	if (direction == TAVOR_CMD_RSRC_HW2SW && hwrsrc != NULL) {
2659 		/* Sync the mailbox to read the results */
2660 		tavor_mbox_sync(mbox_info.mbi_out, 0, size,
2661 		    DDI_DMA_SYNC_FORCPU);
2662 
2663 		/* Copy HW2SW ownership command results into "hwrsrc" */
2664 		for (i = 0; i < (size >> 3); i++) {
2665 			data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
2666 			    ((uint64_t *)mbox_info.mbi_out->mb_addr + i));
2667 			((uint64_t *)hwrsrc)[i] = data;
2668 		}
2669 	}
2670 
2671 cmn_ownership_fail:
2672 	if (hwrsrc != NULL) {
2673 		/* Free the mailbox */
2674 		tavor_mbox_free(state, &mbox_info);
2675 	}
2676 
2677 	TAVOR_TNF_EXIT(tavor_cmn_ownership_cmd_post);
2678 	return (status);
2679 }
2680 
2681 
2682 /*
2683  * tavor_conf_special_qp_cmd_post()
2684  *    Context: Can be called from interrupt or base context.
2685  */
2686 int
2687 tavor_conf_special_qp_cmd_post(tavor_state_t *state, uint_t qpindx,
2688     uint_t qptype, uint_t sleepflag)
2689 {
2690 	tavor_cmd_post_t	cmd;
2691 	int			status;
2692 
2693 	TAVOR_TNF_ENTER(tavor_conf_special_qp_cmd_post);
2694 
2695 	/* Setup and post Tavor "CONF_SPECIAL_QP" command */
2696 	cmd.cp_inparm	= 0;
2697 	cmd.cp_outparm	= 0;
2698 	cmd.cp_inmod	= qpindx;
2699 	cmd.cp_opcode	= CONF_SPECIAL_QP;
2700 	cmd.cp_opmod	= qptype;
2701 	cmd.cp_flags	= sleepflag;
2702 	status = tavor_cmd_post(state, &cmd);
2703 	if (status != TAVOR_CMD_SUCCESS) {
2704 		TNF_PROBE_0(tavor_conf_special_qp_cmd_post_fail,
2705 		    TAVOR_TNF_ERROR, "");
2706 	}
2707 
2708 	TAVOR_TNF_EXIT(tavor_conf_special_qp_cmd_post);
2709 	return (status);
2710 }
2711 
2712 
2713 /*
2714  * tavor_mgid_hash_cmd_post()
2715  *    Context: Can be called from interrupt or base context.
2716  */
2717 int
2718 tavor_mgid_hash_cmd_post(tavor_state_t *state, uint64_t mgid_h,
2719     uint64_t mgid_l, uint64_t *mgid_hash, uint_t sleepflag)
2720 {
2721 	tavor_mbox_info_t	mbox_info;
2722 	tavor_cmd_post_t	cmd;
2723 	int			status;
2724 
2725 	TAVOR_TNF_ENTER(tavor_mgid_hash_cmd_post);
2726 
2727 	/* Get an "In" mailbox for the command */
2728 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2729 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2730 	if (status != TAVOR_CMD_SUCCESS) {
2731 		TNF_PROBE_0(tavor_mgid_hash_mbox_fail, TAVOR_TNF_ERROR, "");
2732 		TAVOR_TNF_EXIT(tavor_mgid_hash_cmd_post);
2733 		return (status);
2734 	}
2735 
2736 	/* Copy the Tavor "MGID_HASH" command into mailbox */
2737 	ddi_put64(mbox_info.mbi_in->mb_acchdl,
2738 	    ((uint64_t *)mbox_info.mbi_in->mb_addr + 0), mgid_h);
2739 	ddi_put64(mbox_info.mbi_in->mb_acchdl,
2740 	    ((uint64_t *)mbox_info.mbi_in->mb_addr + 1), mgid_l);
2741 
2742 	/* Sync the mailbox for the device to read */
2743 	tavor_mbox_sync(mbox_info.mbi_in, 0, TAVOR_CMD_MGIDHASH_SZ,
2744 	    DDI_DMA_SYNC_FORDEV);
2745 
2746 	/* Setup and post the Tavor "MGID_HASH" command */
2747 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
2748 	cmd.cp_outparm	= 0;
2749 	cmd.cp_inmod	= 0;
2750 	cmd.cp_opcode	= MGID_HASH;
2751 	cmd.cp_opmod	= 0;
2752 	cmd.cp_flags	= sleepflag;
2753 	status = tavor_cmd_post(state, &cmd);
2754 	if (status != TAVOR_CMD_SUCCESS) {
2755 		TNF_PROBE_0(tavor_mgid_hash_cmd_post_fail, TAVOR_TNF_ERROR, "");
2756 	}
2757 
2758 	/* MGID hash value is returned in command "outparam" */
2759 	*mgid_hash = cmd.cp_outparm;
2760 
2761 	/* Free the mailbox */
2762 	tavor_mbox_free(state, &mbox_info);
2763 
2764 	TAVOR_TNF_EXIT(tavor_mgid_hash_cmd_post);
2765 	return (status);
2766 }
2767 
2768 
2769 /*
2770  * tavor_read_mgm_cmd_post()
2771  *    Context: Can be called from interrupt or base context.
2772  *
2773  * Note: It is assumed that the "mcg" parameter is actually a pointer to a
2774  *    "tavor_hw_mcg_t" struct and some number of "tavor_hw_mcg_qp_list_t"
2775  *    structs.  Combined size should be equal to result of TAVOR_MCGMEM_SZ()
2776  *    macro.
2777  */
2778 int
2779 tavor_read_mgm_cmd_post(tavor_state_t *state, tavor_hw_mcg_t *mcg,
2780     uint_t mcgindx, uint_t sleepflag)
2781 {
2782 	tavor_mbox_info_t	mbox_info;
2783 	tavor_cmd_post_t	cmd;
2784 	uint64_t		data;
2785 	uint_t			size, hdrsz, qplistsz;
2786 	int			status, i;
2787 
2788 	TAVOR_TNF_ENTER(tavor_read_mgm_cmd_post);
2789 
2790 	/* Get an "Out" mailbox for the results */
2791 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_OUTMBOX;
2792 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2793 	if (status != TAVOR_CMD_SUCCESS) {
2794 		TNF_PROBE_0(tavor_read_mgm_mbox_fail, TAVOR_TNF_ERROR, "");
2795 		TAVOR_TNF_EXIT(tavor_read_mgm_cmd_post);
2796 		return (status);
2797 	}
2798 
2799 	/* Setup and post Tavor "READ_MGM" command */
2800 	cmd.cp_inparm	= 0;
2801 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
2802 	cmd.cp_inmod	= mcgindx;
2803 	cmd.cp_opcode	= READ_MGM;
2804 	cmd.cp_opmod	= 0;
2805 	cmd.cp_flags	= sleepflag;
2806 	status = tavor_cmd_post(state, &cmd);
2807 	if (status != TAVOR_CMD_SUCCESS) {
2808 		TNF_PROBE_0(tavor_read_mgm_cmd_post_fail, TAVOR_TNF_ERROR, "");
2809 		goto read_mgm_fail;
2810 	}
2811 
2812 	/* Sync the mailbox to read the results */
2813 	size = TAVOR_MCGMEM_SZ(state);
2814 	tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
2815 
2816 	/* Copy the READ_MGM command results into "mcg" */
2817 	hdrsz = sizeof (tavor_hw_mcg_t);
2818 	for (i = 0; i < (hdrsz >> 3); i++) {
2819 		data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
2820 		    ((uint64_t *)mbox_info.mbi_out->mb_addr + i));
2821 		((uint64_t *)mcg)[i] = data;
2822 	}
2823 	qplistsz = size - hdrsz;
2824 	for (i = 0; i < (qplistsz >> 2); i++) {
2825 		data = ddi_get32(mbox_info.mbi_out->mb_acchdl,
2826 		    ((uint32_t *)mbox_info.mbi_out->mb_addr + i + 8));
2827 		((uint32_t *)mcg)[i + 8] = data;
2828 	}
2829 
2830 read_mgm_fail:
2831 	/* Free the mailbox */
2832 	tavor_mbox_free(state, &mbox_info);
2833 
2834 	TAVOR_TNF_EXIT(tavor_read_mgm_cmd_post);
2835 	return (status);
2836 }
2837 
2838 
2839 /*
2840  * tavor_write_mgm_cmd_post()
2841  *    Context: Can be called from interrupt or base context.
2842  *
2843  * Note: It is assumed that the "mcg" parameter is actually a pointer to a
2844  *    "tavor_hw_mcg_t" struct and some number of "tavor_hw_mcg_qp_list_t"
2845  *    structs.  Combined size should be equal to result of TAVOR_MCGMEM_SZ()
2846  *    macro.
2847  */
2848 int
2849 tavor_write_mgm_cmd_post(tavor_state_t *state, tavor_hw_mcg_t *mcg,
2850     uint_t mcgindx, uint_t sleepflag)
2851 {
2852 	tavor_mbox_info_t	mbox_info;
2853 	tavor_cmd_post_t	cmd;
2854 	uint64_t		data;
2855 	uint_t			size, hdrsz, qplistsz;
2856 	int			status, i;
2857 
2858 	TAVOR_TNF_ENTER(tavor_write_mgm_cmd_post);
2859 
2860 	/* Get an "In" mailbox for the command */
2861 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2862 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2863 	if (status != TAVOR_CMD_SUCCESS) {
2864 		TNF_PROBE_0(tavor_write_mcg_mbox_fail, TAVOR_TNF_ERROR, "");
2865 		TAVOR_TNF_EXIT(tavor_write_mgm_cmd_post);
2866 		return (status);
2867 	}
2868 
2869 	/* Copy the Tavor "WRITE_MGM" command into mailbox */
2870 	size  = TAVOR_MCGMEM_SZ(state);
2871 	hdrsz = sizeof (tavor_hw_mcg_t);
2872 	for (i = 0; i < (hdrsz >> 3); i++) {
2873 		data = ((uint64_t *)mcg)[i];
2874 		ddi_put64(mbox_info.mbi_in->mb_acchdl,
2875 		    ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
2876 	}
2877 	qplistsz = size - hdrsz;
2878 	for (i = 0; i < (qplistsz >> 2); i++) {
2879 		data = ((uint32_t *)mcg)[i + 8];
2880 		ddi_put32(mbox_info.mbi_in->mb_acchdl,
2881 		    ((uint32_t *)mbox_info.mbi_in->mb_addr + i + 8), data);
2882 	}
2883 
2884 	/* Sync the mailbox for the device to read */
2885 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2886 
2887 	/* Setup and post Tavor "WRITE_MGM" command */
2888 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
2889 	cmd.cp_outparm	= 0;
2890 	cmd.cp_inmod	= mcgindx;
2891 	cmd.cp_opcode	= WRITE_MGM;
2892 	cmd.cp_opmod	= 0;
2893 	cmd.cp_flags	= sleepflag;
2894 	status = tavor_cmd_post(state, &cmd);
2895 	if (status != TAVOR_CMD_SUCCESS) {
2896 		TNF_PROBE_0(tavor_write_mgm_cmd_post_fail, TAVOR_TNF_ERROR, "");
2897 	}
2898 
2899 	/* Free the mailbox */
2900 	tavor_mbox_free(state, &mbox_info);
2901 
2902 	TAVOR_TNF_EXIT(tavor_write_mgm_cmd_post);
2903 	return (status);
2904 
2905 }
2906 
2907 
2908 /*
2909  * tavor_modify_mpt_cmd_post()
2910  *    Context: Can be called from interrupt or base context.
2911  */
2912 int
2913 tavor_modify_mpt_cmd_post(tavor_state_t *state, tavor_hw_mpt_t *mpt,
2914     uint_t mptindx, uint_t flags, uint_t sleepflag)
2915 {
2916 	tavor_mbox_info_t	mbox_info;
2917 	tavor_cmd_post_t	cmd;
2918 	uint64_t		data;
2919 	uint_t			size;
2920 	int			status, i;
2921 
2922 	TAVOR_TNF_ENTER(tavor_modify_mpt_cmd_post);
2923 
2924 	/* Get an "In" mailbox for the command */
2925 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX;
2926 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2927 	if (status != TAVOR_CMD_SUCCESS) {
2928 		TNF_PROBE_0(tavor_modify_mpt_mbox_fail, TAVOR_TNF_ERROR, "");
2929 		TAVOR_TNF_EXIT(tavor_modify_mpt_cmd_post);
2930 		return (status);
2931 	}
2932 
2933 	/* Copy the Tavor "MODIFY_MPT" command into mailbox */
2934 	size = sizeof (tavor_hw_mpt_t);
2935 	for (i = 0; i < (size >> 3); i++) {
2936 		data = ((uint64_t *)mpt)[i];
2937 		ddi_put64(mbox_info.mbi_in->mb_acchdl,
2938 		    ((uint64_t *)mbox_info.mbi_in->mb_addr + i), data);
2939 	}
2940 
2941 	/* Sync the mailbox for the device to read */
2942 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
2943 
2944 	/* Setup and post Tavor "MODIFY_MPT" command */
2945 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
2946 	cmd.cp_outparm	= 0;
2947 	cmd.cp_inmod	= mptindx;
2948 	cmd.cp_opcode	= MODIFY_MPT;
2949 	cmd.cp_opmod	= flags;
2950 	cmd.cp_flags	= sleepflag;
2951 	status = tavor_cmd_post(state, &cmd);
2952 	if (status != TAVOR_CMD_SUCCESS) {
2953 		TNF_PROBE_0(tavor_modify_mpt_cmd_post_fail,
2954 		    TAVOR_TNF_ERROR, "");
2955 	}
2956 
2957 	/* Free the mailbox */
2958 	tavor_mbox_free(state, &mbox_info);
2959 
2960 	TAVOR_TNF_EXIT(tavor_modify_mpt_cmd_post);
2961 	return (status);
2962 }
2963 
2964 /*
2965  * tavor_getpefcntr_cmd_post()
2966  *    Context: Can be called from interrupt or base context.
2967  *
2968  * If reset is zero, read the performance counters of the specified port and
2969  * copy them into perfinfo.
2970  * If reset is non-zero reset the performance counters of the specified port.
2971  */
2972 int
2973 tavor_getperfcntr_cmd_post(tavor_state_t *state, uint_t port,
2974     uint_t sleepflag, tavor_hw_sm_perfcntr_t *perfinfo, int reset)
2975 {
2976 	tavor_mbox_info_t	mbox_info;
2977 	tavor_cmd_post_t	cmd;
2978 	uint64_t		data;
2979 	uint32_t		*mbox;
2980 	uint_t			size;
2981 	int			status, i;
2982 
2983 	bzero((void *)&cmd, sizeof (tavor_cmd_post_t));
2984 
2985 	/* Get "In" and "Out" mailboxes for the command */
2986 	mbox_info.mbi_alloc_flags = TAVOR_ALLOC_INMBOX | TAVOR_ALLOC_OUTMBOX;
2987 	status = tavor_mbox_alloc(state, &mbox_info, sleepflag);
2988 	if (status != TAVOR_CMD_SUCCESS) {
2989 		return (status);
2990 	}
2991 
2992 	/* Build request MAD in the "In" mailbox */
2993 	size = TAVOR_CMD_MAD_IFC_SIZE;
2994 	mbox = (uint32_t *)mbox_info.mbi_in->mb_addr;
2995 
2996 	if (reset) {
2997 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0],
2998 		    TAVOR_CMD_PERF_SET);
2999 	} else {
3000 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[0],
3001 		    TAVOR_CMD_PERF_GET);
3002 	}
3003 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[1], TAVOR_CMD_MADHDR1);
3004 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[2], TAVOR_CMD_MADHDR2);
3005 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[3], TAVOR_CMD_MADHDR3);
3006 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[4], TAVOR_CMD_PERFCNTRS);
3007 	ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[5], TAVOR_CMD_PERFATTR);
3008 
3009 	if (reset) {
3010 		/* reset counters for XmitData, RcvData, XmitPkts, RcvPkts */
3011 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[16],
3012 		    ((port << 16) | 0xf000));
3013 
3014 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[22], 0);
3015 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[23], 0);
3016 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[24], 0);
3017 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[25], 0);
3018 	} else
3019 		ddi_put32(mbox_info.mbi_in->mb_acchdl, &mbox[16], (port << 16));
3020 
3021 	/* Sync the mailbox for the device to read */
3022 	tavor_mbox_sync(mbox_info.mbi_in, 0, size, DDI_DMA_SYNC_FORDEV);
3023 
3024 	/* Setup the Hermon "MAD_IFC" command */
3025 	cmd.cp_inparm	= mbox_info.mbi_in->mb_mapaddr;
3026 	cmd.cp_outparm	= mbox_info.mbi_out->mb_mapaddr;
3027 	cmd.cp_inmod	= port;
3028 	cmd.cp_opcode	= MAD_IFC;
3029 	/* No MKey and BKey checking */
3030 	cmd.cp_opmod	= TAVOR_CMD_MKEY_DONTCHECK | TAVOR_CMD_BKEY_DONTCHECK;
3031 	cmd.cp_flags	= TAVOR_CMD_NOSLEEP_SPIN; /* NO SLEEP */
3032 	status = tavor_cmd_post(state, &cmd);
3033 	if (status != TAVOR_CMD_SUCCESS) {
3034 		goto getperfinfo_fail;
3035 	}
3036 
3037 	/* Sync the mailbox to read the results */
3038 	size = TAVOR_CMD_MAD_IFC_SIZE;
3039 	tavor_mbox_sync(mbox_info.mbi_out, 0, size, DDI_DMA_SYNC_FORCPU);
3040 
3041 	if (reset == 0) {
3042 		size = sizeof (tavor_hw_sm_perfcntr_t); /* for the copy */
3043 		/*
3044 		 * Copy Perfcounters into "perfinfo".  We can discard the MAD
3045 		 * header and the 8 Quadword reserved area of the PERM mgmt
3046 		 * class MAD
3047 		 */
3048 
3049 		for (i = 0; i < size >> 3; i++) {
3050 			data = ddi_get64(mbox_info.mbi_out->mb_acchdl,
3051 			    ((uint64_t *)mbox_info.mbi_out->mb_addr + i + 8));
3052 			((uint64_t *)(void *)perfinfo)[i] = data;
3053 		}
3054 	}
3055 
3056 getperfinfo_fail:
3057 	/* Free the mailbox */
3058 	tavor_mbox_free(state, &mbox_info);
3059 	return (status);
3060 }
3061