19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
239e39c5baSBill Taylor  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
249e39c5baSBill Taylor  * Use is subject to license terms.
259e39c5baSBill Taylor  */
269e39c5baSBill Taylor 
279e39c5baSBill Taylor /*
289e39c5baSBill Taylor  * tavor_event.c
299e39c5baSBill Taylor  *    Tavor Interrupt and Event Processing Routines
309e39c5baSBill Taylor  *
319e39c5baSBill Taylor  *    Implements all the routines necessary for allocating, freeing, and
329e39c5baSBill Taylor  *    handling all of the various event types that the Tavor hardware can
339e39c5baSBill Taylor  *    generate.
349e39c5baSBill Taylor  *    These routines include the main Tavor interrupt service routine
359e39c5baSBill Taylor  *    (tavor_isr()) as well as all the code necessary to setup and handle
369e39c5baSBill Taylor  *    events from each of the many event queues used by the Tavor device.
379e39c5baSBill Taylor  */
389e39c5baSBill Taylor 
399e39c5baSBill Taylor #include <sys/types.h>
409e39c5baSBill Taylor #include <sys/conf.h>
419e39c5baSBill Taylor #include <sys/ddi.h>
429e39c5baSBill Taylor #include <sys/sunddi.h>
439e39c5baSBill Taylor #include <sys/modctl.h>
449e39c5baSBill Taylor 
459e39c5baSBill Taylor #include <sys/ib/adapters/tavor/tavor.h>
469e39c5baSBill Taylor 
479e39c5baSBill Taylor static void tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq);
489e39c5baSBill Taylor static void tavor_eq_catastrophic(tavor_state_t *state);
499e39c5baSBill Taylor static int tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size,
509e39c5baSBill Taylor     uint_t intr, tavor_eqhdl_t *eqhdl);
519e39c5baSBill Taylor static int tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl);
529e39c5baSBill Taylor static int tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
539e39c5baSBill Taylor     uint_t evt_type_mask, int (*eqfunc)(tavor_state_t *state,
549e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe));
559e39c5baSBill Taylor static int tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq);
569e39c5baSBill Taylor static void tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
579e39c5baSBill Taylor     uint_t force_sync);
589e39c5baSBill Taylor static int tavor_port_state_change_handler(tavor_state_t *state,
599e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
609e39c5baSBill Taylor static int tavor_comm_estbl_handler(tavor_state_t *state,
619e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
629e39c5baSBill Taylor static int tavor_local_wq_cat_err_handler(tavor_state_t *state,
639e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
649e39c5baSBill Taylor static int tavor_invreq_local_wq_err_handler(tavor_state_t *state,
659e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
669e39c5baSBill Taylor static int tavor_local_acc_vio_wq_err_handler(tavor_state_t *state,
679e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
689e39c5baSBill Taylor static int tavor_sendq_drained_handler(tavor_state_t *state,
699e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
709e39c5baSBill Taylor static int tavor_path_mig_handler(tavor_state_t *state,
719e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
729e39c5baSBill Taylor static int tavor_path_mig_err_handler(tavor_state_t *state,
739e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
749e39c5baSBill Taylor static int tavor_srq_catastrophic_handler(tavor_state_t *state,
759e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
769e39c5baSBill Taylor static int tavor_srq_last_wqe_reached_handler(tavor_state_t *state,
779e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
789e39c5baSBill Taylor static int tavor_ecc_detection_handler(tavor_state_t *state,
799e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
809e39c5baSBill Taylor static int tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
819e39c5baSBill Taylor     tavor_hw_eqe_t *eqe);
829e39c5baSBill Taylor 
839e39c5baSBill Taylor 
849e39c5baSBill Taylor /*
859e39c5baSBill Taylor  * tavor_eq_init_all
869e39c5baSBill Taylor  *    Context: Only called from attach() path context
879e39c5baSBill Taylor  */
889e39c5baSBill Taylor int
tavor_eq_init_all(tavor_state_t * state)899e39c5baSBill Taylor tavor_eq_init_all(tavor_state_t *state)
909e39c5baSBill Taylor {
919e39c5baSBill Taylor 	uint_t		log_eq_size, intr_num;
929e39c5baSBill Taylor 	uint_t		num_eq, num_eq_init, num_eq_unmap;
939e39c5baSBill Taylor 	int		status, i;
949e39c5baSBill Taylor 
959e39c5baSBill Taylor 	/*
969e39c5baSBill Taylor 	 * For now, all Event Queues default to the same size (pulled from
979e39c5baSBill Taylor 	 * the current configuration profile) and are all assigned to the
989e39c5baSBill Taylor 	 * same interrupt or MSI.  In the future we may support assigning
999e39c5baSBill Taylor 	 * EQs to specific interrupts or MSIs XXX
1009e39c5baSBill Taylor 	 */
1019e39c5baSBill Taylor 	log_eq_size = state->ts_cfg_profile->cp_log_default_eq_sz;
1029e39c5baSBill Taylor 
1039e39c5baSBill Taylor 	/*
1049e39c5baSBill Taylor 	 * If MSI is to be used, then set intr_num to the MSI number
1059e39c5baSBill Taylor 	 * (currently zero because we're using only one) or'd with the
1069e39c5baSBill Taylor 	 * MSI enable flag.  Otherwise, for regular (i.e. 'legacy') interrupt,
1079e39c5baSBill Taylor 	 * use the 'inta_pin' value returned by QUERY_ADAPTER.
1089e39c5baSBill Taylor 	 */
1099e39c5baSBill Taylor 	if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
1109e39c5baSBill Taylor 		intr_num = TAVOR_EQ_MSI_ENABLE_FLAG | 0;
1119e39c5baSBill Taylor 	} else {
1129e39c5baSBill Taylor 		intr_num = state->ts_adapter.inta_pin;
1139e39c5baSBill Taylor 	}
1149e39c5baSBill Taylor 
1159e39c5baSBill Taylor 	/*
1169e39c5baSBill Taylor 	 * Total number of supported EQs is hardcoded.  Tavor hardware
1179e39c5baSBill Taylor 	 * supports up to 64 EQs.  We are currently using only 45 of them
1189e39c5baSBill Taylor 	 * We will set aside the first 32 for use with Completion Queues (CQ)
1199e39c5baSBill Taylor 	 * and reserve a few of the other 32 for each specific class of event
1209e39c5baSBill Taylor 	 * (see below for more details).
1219e39c5baSBill Taylor 	 */
1229e39c5baSBill Taylor 	num_eq = TAVOR_NUM_EQ_USED;
1239e39c5baSBill Taylor 
1249e39c5baSBill Taylor 	/*
1259e39c5baSBill Taylor 	 * The "num_eq_unmap" variable is used in any possible failure
1269e39c5baSBill Taylor 	 * cleanup (below) to indicate which events queues might require
1279e39c5baSBill Taylor 	 * possible event class unmapping.
1289e39c5baSBill Taylor 	 */
1299e39c5baSBill Taylor 	num_eq_unmap = 0;
1309e39c5baSBill Taylor 
1319e39c5baSBill Taylor 	/*
1329e39c5baSBill Taylor 	 * Allocate and initialize all the Event Queues.  If any of these
1339e39c5baSBill Taylor 	 * EQ allocations fail then jump to the end, cleanup what had been
1349e39c5baSBill Taylor 	 * successfully initialized, and return an error.
1359e39c5baSBill Taylor 	 */
1369e39c5baSBill Taylor 	for (i = 0; i < num_eq; i++) {
1379e39c5baSBill Taylor 		status = tavor_eq_alloc(state, log_eq_size, intr_num,
1389e39c5baSBill Taylor 		    &state->ts_eqhdl[i]);
1399e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
1409e39c5baSBill Taylor 			num_eq_init = i;
1419e39c5baSBill Taylor 			goto all_eq_init_fail;
1429e39c5baSBill Taylor 		}
1439e39c5baSBill Taylor 	}
1449e39c5baSBill Taylor 	num_eq_init = num_eq;
1459e39c5baSBill Taylor 
1469e39c5baSBill Taylor 	/*
1479e39c5baSBill Taylor 	 * Setup EQ0-EQ31 for use with Completion Queues.  Note: We can
1489e39c5baSBill Taylor 	 * cast the return value to void here because, when we use the
1499e39c5baSBill Taylor 	 * TAVOR_EVT_NO_MASK flag, it is not possible for
1509e39c5baSBill Taylor 	 * tavor_eq_handler_init() to return an error.
1519e39c5baSBill Taylor 	 */
1529e39c5baSBill Taylor 	for (i = 0; i < 32; i++) {
1539e39c5baSBill Taylor 		(void) tavor_eq_handler_init(state, state->ts_eqhdl[i],
1549e39c5baSBill Taylor 		    TAVOR_EVT_NO_MASK, tavor_cq_handler);
1559e39c5baSBill Taylor 	}
1569e39c5baSBill Taylor 	num_eq_unmap = 32;
1579e39c5baSBill Taylor 
1589e39c5baSBill Taylor 	/*
1599e39c5baSBill Taylor 	 * Setup EQ32 for handling Completion Queue Error Events.
1609e39c5baSBill Taylor 	 *
1619e39c5baSBill Taylor 	 * These events include things like CQ overflow or CQ access
1629e39c5baSBill Taylor 	 * violation errors.  If this setup fails for any reason (which, in
1639e39c5baSBill Taylor 	 * general, it really never should), then jump to the end, cleanup
1649e39c5baSBill Taylor 	 * everything that has been successfully initialized, and return an
1659e39c5baSBill Taylor 	 * error.
1669e39c5baSBill Taylor 	 */
1679e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[32],
1689e39c5baSBill Taylor 	    TAVOR_EVT_MSK_CQ_ERRORS, tavor_cq_err_handler);
1699e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
1709e39c5baSBill Taylor 		goto all_eq_init_fail;
1719e39c5baSBill Taylor 	}
1729e39c5baSBill Taylor 	num_eq_unmap = 33;
1739e39c5baSBill Taylor 
1749e39c5baSBill Taylor 	/*
1759e39c5baSBill Taylor 	 * Setup EQ33 for handling Port State Change Events
1769e39c5baSBill Taylor 	 *
1779e39c5baSBill Taylor 	 * These events include things like Port Up and Port Down events.
1789e39c5baSBill Taylor 	 * If this setup fails for any reason (which, in general, it really
1799e39c5baSBill Taylor 	 * never should), then undo all previous EQ mapping, jump to the end,
1809e39c5baSBill Taylor 	 * cleanup everything that has been successfully initialized, and
1819e39c5baSBill Taylor 	 * return an error.
1829e39c5baSBill Taylor 	 */
1839e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[33],
1849e39c5baSBill Taylor 	    TAVOR_EVT_MSK_PORT_STATE_CHANGE, tavor_port_state_change_handler);
1859e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
1869e39c5baSBill Taylor 		goto all_eq_init_fail;
1879e39c5baSBill Taylor 	}
1889e39c5baSBill Taylor 	num_eq_unmap = 34;
1899e39c5baSBill Taylor 
1909e39c5baSBill Taylor 	/*
1919e39c5baSBill Taylor 	 * Setup EQ34 for handling Communication Established Events
1929e39c5baSBill Taylor 	 *
1939e39c5baSBill Taylor 	 * These events correspond to the IB affiliated asynchronous events
1949e39c5baSBill Taylor 	 * that are used for connection management.  If this setup fails for
1959e39c5baSBill Taylor 	 * any reason (which, in general, it really never should), then undo
1969e39c5baSBill Taylor 	 * all previous EQ mapping, jump to the end, cleanup everything that
1979e39c5baSBill Taylor 	 * has been successfully initialized, and return an error.
1989e39c5baSBill Taylor 	 */
1999e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[34],
2009e39c5baSBill Taylor 	    TAVOR_EVT_MSK_COMM_ESTABLISHED, tavor_comm_estbl_handler);
2019e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2029e39c5baSBill Taylor 		goto all_eq_init_fail;
2039e39c5baSBill Taylor 	}
2049e39c5baSBill Taylor 	num_eq_unmap = 35;
2059e39c5baSBill Taylor 
2069e39c5baSBill Taylor 	/*
2079e39c5baSBill Taylor 	 * Setup EQ35 for handling Command Completion Events
2089e39c5baSBill Taylor 	 *
2099e39c5baSBill Taylor 	 * These events correspond to the Tavor generated events that are used
2109e39c5baSBill Taylor 	 * to indicate Tavor firmware command completion.  These events are
2119e39c5baSBill Taylor 	 * only generated when Tavor firmware commands are posted using the
2129e39c5baSBill Taylor 	 * asynchronous completion mechanism.  If this setup fails for any
2139e39c5baSBill Taylor 	 * reason (which, in general, it really never should), then undo all
2149e39c5baSBill Taylor 	 * previous EQ mapping, jump to the end, cleanup everything that has
2159e39c5baSBill Taylor 	 * been successfully initialized, and return an error.
2169e39c5baSBill Taylor 	 */
2179e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[35],
2189e39c5baSBill Taylor 	    TAVOR_EVT_MSK_COMMAND_INTF_COMP, tavor_cmd_complete_handler);
2199e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2209e39c5baSBill Taylor 		goto all_eq_init_fail;
2219e39c5baSBill Taylor 	}
2229e39c5baSBill Taylor 	num_eq_unmap = 36;
2239e39c5baSBill Taylor 
2249e39c5baSBill Taylor 	/*
2259e39c5baSBill Taylor 	 * Setup EQ36 for handling Local WQ Catastrophic Error Events
2269e39c5baSBill Taylor 	 *
2279e39c5baSBill Taylor 	 * These events correspond to the similarly-named IB affiliated
2289e39c5baSBill Taylor 	 * asynchronous error type.  If this setup fails for any reason
2299e39c5baSBill Taylor 	 * (which, in general, it really never should), then undo all previous
2309e39c5baSBill Taylor 	 * EQ mapping, jump to the end, cleanup everything that has been
2319e39c5baSBill Taylor 	 * successfully initialized, and return an error.
2329e39c5baSBill Taylor 	 */
2339e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[36],
2349e39c5baSBill Taylor 	    TAVOR_EVT_MSK_LOCAL_WQ_CAT_ERROR, tavor_local_wq_cat_err_handler);
2359e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2369e39c5baSBill Taylor 		goto all_eq_init_fail;
2379e39c5baSBill Taylor 	}
2389e39c5baSBill Taylor 	num_eq_unmap = 37;
2399e39c5baSBill Taylor 
2409e39c5baSBill Taylor 	/*
2419e39c5baSBill Taylor 	 * Setup EQ37 for handling Invalid Req Local WQ Error Events
2429e39c5baSBill Taylor 	 *
2439e39c5baSBill Taylor 	 * These events also correspond to the similarly-named IB affiliated
2449e39c5baSBill Taylor 	 * asynchronous error type.  If this setup fails for any reason
2459e39c5baSBill Taylor 	 * (which, in general, it really never should), then undo all previous
2469e39c5baSBill Taylor 	 * EQ mapping, jump to the end, cleanup everything that has been
2479e39c5baSBill Taylor 	 * successfully initialized, and return an error.
2489e39c5baSBill Taylor 	 */
2499e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[37],
2509e39c5baSBill Taylor 	    TAVOR_EVT_MSK_INV_REQ_LOCAL_WQ_ERROR,
2519e39c5baSBill Taylor 	    tavor_invreq_local_wq_err_handler);
2529e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2539e39c5baSBill Taylor 		goto all_eq_init_fail;
2549e39c5baSBill Taylor 	}
2559e39c5baSBill Taylor 	num_eq_unmap = 38;
2569e39c5baSBill Taylor 
2579e39c5baSBill Taylor 	/*
2589e39c5baSBill Taylor 	 * Setup EQ38 for handling Local Access Violation WQ Error Events
2599e39c5baSBill Taylor 	 *
2609e39c5baSBill Taylor 	 * These events also correspond to the similarly-named IB affiliated
2619e39c5baSBill Taylor 	 * asynchronous error type.  If this setup fails for any reason
2629e39c5baSBill Taylor 	 * (which, in general, it really never should), then undo all previous
2639e39c5baSBill Taylor 	 * EQ mapping, jump to the end, cleanup everything that has been
2649e39c5baSBill Taylor 	 * successfully initialized, and return an error.
2659e39c5baSBill Taylor 	 */
2669e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[38],
2679e39c5baSBill Taylor 	    TAVOR_EVT_MSK_LOCAL_ACC_VIO_WQ_ERROR,
2689e39c5baSBill Taylor 	    tavor_local_acc_vio_wq_err_handler);
2699e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2709e39c5baSBill Taylor 		goto all_eq_init_fail;
2719e39c5baSBill Taylor 	}
2729e39c5baSBill Taylor 	num_eq_unmap = 39;
2739e39c5baSBill Taylor 
2749e39c5baSBill Taylor 	/*
2759e39c5baSBill Taylor 	 * Setup EQ39 for handling Send Queue Drained Events
2769e39c5baSBill Taylor 	 *
2779e39c5baSBill Taylor 	 * These events correspond to the IB affiliated asynchronous events
2789e39c5baSBill Taylor 	 * that are used to indicate completion of a Send Queue Drained QP
2799e39c5baSBill Taylor 	 * state transition.  If this setup fails for any reason (which, in
2809e39c5baSBill Taylor 	 * general, it really never should), then undo all previous EQ
2819e39c5baSBill Taylor 	 * mapping, jump to the end, cleanup everything that has been
2829e39c5baSBill Taylor 	 * successfully initialized, and return an error.
2839e39c5baSBill Taylor 	 */
2849e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[39],
2859e39c5baSBill Taylor 	    TAVOR_EVT_MSK_SEND_QUEUE_DRAINED, tavor_sendq_drained_handler);
2869e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
2879e39c5baSBill Taylor 		goto all_eq_init_fail;
2889e39c5baSBill Taylor 	}
2899e39c5baSBill Taylor 	num_eq_unmap = 40;
2909e39c5baSBill Taylor 
2919e39c5baSBill Taylor 	/*
2929e39c5baSBill Taylor 	 * Setup EQ40 for handling Path Migration Succeeded Events
2939e39c5baSBill Taylor 	 *
2949e39c5baSBill Taylor 	 * These events correspond to the IB affiliated asynchronous events
2959e39c5baSBill Taylor 	 * that are used to indicate successful completion of a path
2969e39c5baSBill Taylor 	 * migration.  If this setup fails for any reason (which, in general,
2979e39c5baSBill Taylor 	 * it really never should), then undo all previous EQ mapping, jump
2989e39c5baSBill Taylor 	 * to the end, cleanup everything that has been successfully
2999e39c5baSBill Taylor 	 * initialized, and return an error.
3009e39c5baSBill Taylor 	 */
3019e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[40],
3029e39c5baSBill Taylor 	    TAVOR_EVT_MSK_PATH_MIGRATED, tavor_path_mig_handler);
3039e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
3049e39c5baSBill Taylor 		goto all_eq_init_fail;
3059e39c5baSBill Taylor 	}
3069e39c5baSBill Taylor 	num_eq_unmap = 41;
3079e39c5baSBill Taylor 
3089e39c5baSBill Taylor 	/*
3099e39c5baSBill Taylor 	 * Setup EQ41 for handling Path Migration Failed Events
3109e39c5baSBill Taylor 	 *
3119e39c5baSBill Taylor 	 * These events correspond to the IB affiliated asynchronous events
3129e39c5baSBill Taylor 	 * that are used to indicate that path migration was not successful.
3139e39c5baSBill Taylor 	 * If this setup fails for any reason (which, in general, it really
3149e39c5baSBill Taylor 	 * never should), then undo all previous EQ mapping, jump to the end,
3159e39c5baSBill Taylor 	 * cleanup everything that has been successfully initialized, and
3169e39c5baSBill Taylor 	 * return an error.
3179e39c5baSBill Taylor 	 */
3189e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[41],
3199e39c5baSBill Taylor 	    TAVOR_EVT_MSK_PATH_MIGRATE_FAILED, tavor_path_mig_err_handler);
3209e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
3219e39c5baSBill Taylor 		goto all_eq_init_fail;
3229e39c5baSBill Taylor 	}
3239e39c5baSBill Taylor 	num_eq_unmap = 42;
3249e39c5baSBill Taylor 
3259e39c5baSBill Taylor 	/*
3269e39c5baSBill Taylor 	 * Setup EQ42 for handling Local Catastrophic Error Events
3279e39c5baSBill Taylor 	 *
3289e39c5baSBill Taylor 	 * These events correspond to the similarly-named IB unaffiliated
3299e39c5baSBill Taylor 	 * asynchronous error type.  If this setup fails for any reason
3309e39c5baSBill Taylor 	 * (which, in general, it really never should), then undo all previous
3319e39c5baSBill Taylor 	 * EQ mapping, jump to the end, cleanup everything that has been
3329e39c5baSBill Taylor 	 * successfully initialized, and return an error.
3339e39c5baSBill Taylor 	 *
3349e39c5baSBill Taylor 	 * This error is unique, in that an EQE is not generated if this event
3359e39c5baSBill Taylor 	 * occurs.  Instead, an interrupt is called and we must poll the
3369e39c5baSBill Taylor 	 * Catastrophic Error buffer in CR-Space.  This mapping is setup simply
3379e39c5baSBill Taylor 	 * to enable this error reporting.  We pass in a NULL handler since it
3389e39c5baSBill Taylor 	 * will never be called.
3399e39c5baSBill Taylor 	 */
3409e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[42],
3419e39c5baSBill Taylor 	    TAVOR_EVT_MSK_LOCAL_CAT_ERROR, NULL);
3429e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
3439e39c5baSBill Taylor 		goto all_eq_init_fail;
3449e39c5baSBill Taylor 	}
3459e39c5baSBill Taylor 	num_eq_unmap = 43;
3469e39c5baSBill Taylor 
3479e39c5baSBill Taylor 	/*
3489e39c5baSBill Taylor 	 * Setup EQ43 for handling SRQ Catastrophic Error Events
3499e39c5baSBill Taylor 	 *
3509e39c5baSBill Taylor 	 * These events correspond to the similarly-named IB affiliated
3519e39c5baSBill Taylor 	 * asynchronous error type.  If this setup fails for any reason
3529e39c5baSBill Taylor 	 * (which, in general, it really never should), then undo all previous
3539e39c5baSBill Taylor 	 * EQ mapping, jump to the end, cleanup everything that has been
3549e39c5baSBill Taylor 	 * successfully initialized, and return an error.
3559e39c5baSBill Taylor 	 */
3569e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[43],
3579e39c5baSBill Taylor 	    TAVOR_EVT_MSK_SRQ_CATASTROPHIC_ERROR,
3589e39c5baSBill Taylor 	    tavor_srq_catastrophic_handler);
3599e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
3609e39c5baSBill Taylor 		goto all_eq_init_fail;
3619e39c5baSBill Taylor 	}
3629e39c5baSBill Taylor 	num_eq_unmap = 44;
3639e39c5baSBill Taylor 
3649e39c5baSBill Taylor 	/*
3659e39c5baSBill Taylor 	 * Setup EQ44 for handling SRQ Last WQE Reached Events
3669e39c5baSBill Taylor 	 *
3679e39c5baSBill Taylor 	 * These events correspond to the similarly-named IB affiliated
3689e39c5baSBill Taylor 	 * asynchronous event type.  If this setup fails for any reason
3699e39c5baSBill Taylor 	 * (which, in general, it really never should), then undo all previous
3709e39c5baSBill Taylor 	 * EQ mapping, jump to the end, cleanup everything that has been
3719e39c5baSBill Taylor 	 * successfully initialized, and return an error.
3729e39c5baSBill Taylor 	 */
3739e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[44],
3749e39c5baSBill Taylor 	    TAVOR_EVT_MSK_SRQ_LAST_WQE_REACHED,
3759e39c5baSBill Taylor 	    tavor_srq_last_wqe_reached_handler);
3769e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
3779e39c5baSBill Taylor 		goto all_eq_init_fail;
3789e39c5baSBill Taylor 	}
3799e39c5baSBill Taylor 	num_eq_unmap = 45;
3809e39c5baSBill Taylor 
3819e39c5baSBill Taylor 	/*
3829e39c5baSBill Taylor 	 * Setup EQ45 for handling ECC error detection events
3839e39c5baSBill Taylor 	 *
3849e39c5baSBill Taylor 	 * These events correspond to the similarly-named IB affiliated
3859e39c5baSBill Taylor 	 * asynchronous event type.  If this setup fails for any reason
3869e39c5baSBill Taylor 	 * (which, in general, it really never should), then undo all previous
3879e39c5baSBill Taylor 	 * EQ mapping, jump to the end, cleanup everything that has been
3889e39c5baSBill Taylor 	 * successfully initialized, and return an error.
3899e39c5baSBill Taylor 	 */
3909e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[45],
3919e39c5baSBill Taylor 	    TAVOR_EVT_MSK_ECC_DETECTION,
3929e39c5baSBill Taylor 	    tavor_ecc_detection_handler);
3939e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
3949e39c5baSBill Taylor 		goto all_eq_init_fail;
3959e39c5baSBill Taylor 	}
3969e39c5baSBill Taylor 	num_eq_unmap = 46;
3979e39c5baSBill Taylor 
3989e39c5baSBill Taylor 	/*
3999e39c5baSBill Taylor 	 * Setup EQ46 to catch all other types of events.  Specifically, we
4009e39c5baSBill Taylor 	 * do not catch the "Local EEC Catastrophic Error Event" because we
4019e39c5baSBill Taylor 	 * should have no EEC (the Tavor driver does not support RD).  We also
4029e39c5baSBill Taylor 	 * choose not to handle any of the address translation page fault
4039e39c5baSBill Taylor 	 * event types.  Since we are not doing any page fault handling (and
4049e39c5baSBill Taylor 	 * since the Tavor firmware does not currently support any such
4059e39c5baSBill Taylor 	 * handling), we allow these events to go to the catch-all handler.
4069e39c5baSBill Taylor 	 */
4079e39c5baSBill Taylor 	status = tavor_eq_handler_init(state, state->ts_eqhdl[46],
4089e39c5baSBill Taylor 	    TAVOR_EVT_CATCHALL_MASK, tavor_no_eqhandler);
4099e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
4109e39c5baSBill Taylor 		goto all_eq_init_fail;
4119e39c5baSBill Taylor 	}
4129e39c5baSBill Taylor 
4139e39c5baSBill Taylor 	return (DDI_SUCCESS);
4149e39c5baSBill Taylor 
4159e39c5baSBill Taylor all_eq_init_fail:
4169e39c5baSBill Taylor 	/* Unmap any of the partially mapped EQs from above */
4179e39c5baSBill Taylor 	for (i = 0; i < num_eq_unmap; i++) {
4189e39c5baSBill Taylor 		(void) tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
4199e39c5baSBill Taylor 	}
4209e39c5baSBill Taylor 
4219e39c5baSBill Taylor 	/* Free up any of the partially allocated EQs from above */
4229e39c5baSBill Taylor 	for (i = 0; i < num_eq_init; i++) {
4239e39c5baSBill Taylor 		(void) tavor_eq_free(state, &state->ts_eqhdl[i]);
4249e39c5baSBill Taylor 	}
4259e39c5baSBill Taylor 	return (status);
4269e39c5baSBill Taylor }
4279e39c5baSBill Taylor 
4289e39c5baSBill Taylor 
4299e39c5baSBill Taylor /*
4309e39c5baSBill Taylor  * tavor_eq_fini_all
4319e39c5baSBill Taylor  *    Context: Only called from attach() and/or detach() path contexts
4329e39c5baSBill Taylor  */
4339e39c5baSBill Taylor int
tavor_eq_fini_all(tavor_state_t * state)4349e39c5baSBill Taylor tavor_eq_fini_all(tavor_state_t *state)
4359e39c5baSBill Taylor {
4369e39c5baSBill Taylor 	uint_t		num_eq;
4379e39c5baSBill Taylor 	int		status, i;
4389e39c5baSBill Taylor 
4399e39c5baSBill Taylor 	/*
4409e39c5baSBill Taylor 	 * Grab the total number of supported EQs again.  This is the same
4419e39c5baSBill Taylor 	 * hardcoded value that was used above (during the event queue
4429e39c5baSBill Taylor 	 * initialization.)
4439e39c5baSBill Taylor 	 */
4449e39c5baSBill Taylor 	num_eq = TAVOR_NUM_EQ_USED;
4459e39c5baSBill Taylor 
4469e39c5baSBill Taylor 	/*
4479e39c5baSBill Taylor 	 * For each of the event queues that we initialized and mapped
4489e39c5baSBill Taylor 	 * earlier, attempt to unmap the events from the EQ.
4499e39c5baSBill Taylor 	 */
4509e39c5baSBill Taylor 	for (i = 0; i < num_eq; i++) {
4519e39c5baSBill Taylor 		status = tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
4529e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
4539e39c5baSBill Taylor 			return (DDI_FAILURE);
4549e39c5baSBill Taylor 		}
4559e39c5baSBill Taylor 	}
4569e39c5baSBill Taylor 
4579e39c5baSBill Taylor 	/*
4589e39c5baSBill Taylor 	 * Teardown and free up all the Event Queues that were allocated
4599e39c5baSBill Taylor 	 * earlier.
4609e39c5baSBill Taylor 	 */
4619e39c5baSBill Taylor 	for (i = 0; i < num_eq; i++) {
4629e39c5baSBill Taylor 		status = tavor_eq_free(state, &state->ts_eqhdl[i]);
4639e39c5baSBill Taylor 		if (status != DDI_SUCCESS) {
4649e39c5baSBill Taylor 			return (DDI_FAILURE);
4659e39c5baSBill Taylor 		}
4669e39c5baSBill Taylor 	}
4679e39c5baSBill Taylor 
4689e39c5baSBill Taylor 	return (DDI_SUCCESS);
4699e39c5baSBill Taylor }
4709e39c5baSBill Taylor 
4719e39c5baSBill Taylor 
4729e39c5baSBill Taylor /*
4739e39c5baSBill Taylor  * tavor_eq_arm_all
4749e39c5baSBill Taylor  *    Context: Only called from attach() and/or detach() path contexts
4759e39c5baSBill Taylor  */
4769e39c5baSBill Taylor void
tavor_eq_arm_all(tavor_state_t * state)4779e39c5baSBill Taylor tavor_eq_arm_all(tavor_state_t *state)
4789e39c5baSBill Taylor {
4799e39c5baSBill Taylor 	uint_t		num_eq;
4809e39c5baSBill Taylor 	int		i;
4819e39c5baSBill Taylor 
4829e39c5baSBill Taylor 	/*
4839e39c5baSBill Taylor 	 * Grab the total number of supported EQs again.  This is the same
4849e39c5baSBill Taylor 	 * hardcoded value that was used above (during the event queue
4859e39c5baSBill Taylor 	 * initialization.)
4869e39c5baSBill Taylor 	 */
4879e39c5baSBill Taylor 	num_eq = TAVOR_NUM_EQ_USED;
4889e39c5baSBill Taylor 
4899e39c5baSBill Taylor 	/*
4909e39c5baSBill Taylor 	 * For each of the event queues that we initialized and mapped
4919e39c5baSBill Taylor 	 * earlier, attempt to arm it for event generation.
4929e39c5baSBill Taylor 	 */
4939e39c5baSBill Taylor 	for (i = 0; i < num_eq; i++) {
4949e39c5baSBill Taylor 		tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, i, 0);
4959e39c5baSBill Taylor 	}
4969e39c5baSBill Taylor }
4979e39c5baSBill Taylor 
4989e39c5baSBill Taylor 
4999e39c5baSBill Taylor /*
5009e39c5baSBill Taylor  * tavor_isr()
5019e39c5baSBill Taylor  *    Context: Only called from interrupt context (and during panic)
5029e39c5baSBill Taylor  */
5039e39c5baSBill Taylor /* ARGSUSED */
5049e39c5baSBill Taylor uint_t
tavor_isr(caddr_t arg1,caddr_t arg2)5059e39c5baSBill Taylor tavor_isr(caddr_t arg1, caddr_t arg2)
5069e39c5baSBill Taylor {
5079e39c5baSBill Taylor 	tavor_state_t	*state;
5089e39c5baSBill Taylor 	uint64_t	*ecr, *clr_int;
5099e39c5baSBill Taylor 	uint64_t	ecrreg, int_mask;
5109e39c5baSBill Taylor 	uint_t		status;
5119e39c5baSBill Taylor 	int		i;
5129e39c5baSBill Taylor 
5139e39c5baSBill Taylor 	/*
5149e39c5baSBill Taylor 	 * Grab the Tavor softstate pointer from the input parameter
5159e39c5baSBill Taylor 	 */
5169e39c5baSBill Taylor 	state	= (tavor_state_t *)arg1;
5179e39c5baSBill Taylor 
5189e39c5baSBill Taylor 	/*
5199e39c5baSBill Taylor 	 * Find the pointers to the ECR and clr_INT registers
5209e39c5baSBill Taylor 	 */
5219e39c5baSBill Taylor 	ecr	= state->ts_cmd_regs.ecr;
5229e39c5baSBill Taylor 	clr_int = state->ts_cmd_regs.clr_int;
5239e39c5baSBill Taylor 
5249e39c5baSBill Taylor 	/*
5259e39c5baSBill Taylor 	 * Read the ECR register.  Each of the 64 bits in the ECR register
5269e39c5baSBill Taylor 	 * corresponds to an event queue.  If a bit is set, then the
5279e39c5baSBill Taylor 	 * corresponding event queue has fired.
5289e39c5baSBill Taylor 	 */
5299e39c5baSBill Taylor 	ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
5309e39c5baSBill Taylor 
5319e39c5baSBill Taylor 	/*
5329e39c5baSBill Taylor 	 * As long as there are bits set (i.e. as long as there are still
5339e39c5baSBill Taylor 	 * EQs in the "fired" state), call tavor_eq_poll() to process each
5349e39c5baSBill Taylor 	 * fired EQ.  If no ECR bits are set, do not claim the interrupt.
5359e39c5baSBill Taylor 	 */
5369e39c5baSBill Taylor 	status = DDI_INTR_UNCLAIMED;
5379e39c5baSBill Taylor 	do {
5389e39c5baSBill Taylor 		i = 0;
5399e39c5baSBill Taylor 		while (ecrreg != 0x0) {
5409e39c5baSBill Taylor 			if (ecrreg & 0x1) {
5419e39c5baSBill Taylor 				tavor_eq_poll(state, state->ts_eqhdl[i]);
5429e39c5baSBill Taylor 				status = DDI_INTR_CLAIMED;
5439e39c5baSBill Taylor 			}
5449e39c5baSBill Taylor 			ecrreg = ecrreg >> 1;
5459e39c5baSBill Taylor 			i++;
5469e39c5baSBill Taylor 		}
5479e39c5baSBill Taylor 
5489e39c5baSBill Taylor 		/*
5499e39c5baSBill Taylor 		 * Clear the interrupt.  Note: Depending on the type of
5509e39c5baSBill Taylor 		 * event (interrupt or MSI), we need to use a different
5519e39c5baSBill Taylor 		 * mask to clear the event.  In the case of MSI, the bit
5529e39c5baSBill Taylor 		 * to clear corresponds to the MSI number, and for legacy
5539e39c5baSBill Taylor 		 * interrupts the bit corresponds to the value in 'inta_pin'.
5549e39c5baSBill Taylor 		 */
5559e39c5baSBill Taylor 		if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
5569e39c5baSBill Taylor 			int_mask = ((uint64_t)1 << 0);
5579e39c5baSBill Taylor 		} else {
5589e39c5baSBill Taylor 			int_mask = ((uint64_t)1 << state->ts_adapter.inta_pin);
5599e39c5baSBill Taylor 		}
5609e39c5baSBill Taylor 		ddi_put64(state->ts_reg_cmdhdl, clr_int, int_mask);
5619e39c5baSBill Taylor 
5629e39c5baSBill Taylor 		/* Reread the ECR register */
5639e39c5baSBill Taylor 		ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
5649e39c5baSBill Taylor 
5659e39c5baSBill Taylor 	} while (ecrreg != 0x0);
5669e39c5baSBill Taylor 
5679e39c5baSBill Taylor 	return (status);
5689e39c5baSBill Taylor }
5699e39c5baSBill Taylor 
5709e39c5baSBill Taylor 
5719e39c5baSBill Taylor /*
5729e39c5baSBill Taylor  * tavor_eq_doorbell
5739e39c5baSBill Taylor  *    Context: Only called from interrupt context
5749e39c5baSBill Taylor  */
5759e39c5baSBill Taylor void
tavor_eq_doorbell(tavor_state_t * state,uint32_t eq_cmd,uint32_t eqn,uint32_t eq_param)5769e39c5baSBill Taylor tavor_eq_doorbell(tavor_state_t *state, uint32_t eq_cmd, uint32_t eqn,
5779e39c5baSBill Taylor     uint32_t eq_param)
5789e39c5baSBill Taylor {
5799e39c5baSBill Taylor 	uint64_t	doorbell = 0;
5809e39c5baSBill Taylor 
5819e39c5baSBill Taylor 	/* Build the doorbell from the parameters */
5829e39c5baSBill Taylor 	doorbell = ((uint64_t)eq_cmd << TAVOR_EQDB_CMD_SHIFT) |
5839e39c5baSBill Taylor 	    ((uint64_t)eqn << TAVOR_EQDB_EQN_SHIFT) | eq_param;
5849e39c5baSBill Taylor 
5859e39c5baSBill Taylor 	/* Write the doorbell to UAR */
5869e39c5baSBill Taylor 	TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->eq,
5879e39c5baSBill Taylor 	    doorbell);
5889e39c5baSBill Taylor }
5899e39c5baSBill Taylor 
5909e39c5baSBill Taylor /*
5919e39c5baSBill Taylor  * tavor_eq_poll
5929e39c5baSBill Taylor  *    Context: Only called from interrupt context (and during panic)
5939e39c5baSBill Taylor  */
5949e39c5baSBill Taylor static void
tavor_eq_poll(tavor_state_t * state,tavor_eqhdl_t eq)5959e39c5baSBill Taylor tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq)
5969e39c5baSBill Taylor {
5979e39c5baSBill Taylor 	uint64_t	*clr_ecr;
5989e39c5baSBill Taylor 	tavor_hw_eqe_t	*eqe;
5999e39c5baSBill Taylor 	uint64_t	ecr_mask;
6009e39c5baSBill Taylor 	uint32_t	cons_indx, wrap_around_mask;
6019e39c5baSBill Taylor 	int (*eqfunction)(tavor_state_t *state, tavor_eqhdl_t eq,
6029e39c5baSBill Taylor 	    tavor_hw_eqe_t *eqe);
6039e39c5baSBill Taylor 
6049e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eq))
6059e39c5baSBill Taylor 
6069e39c5baSBill Taylor 	/* Find the pointer to the clr_ECR register */
6079e39c5baSBill Taylor 	clr_ecr = state->ts_cmd_regs.clr_ecr;
6089e39c5baSBill Taylor 
6099e39c5baSBill Taylor 	/*
6109e39c5baSBill Taylor 	 * Check for Local Catastrophic Error If we have this kind of error,
6119e39c5baSBill Taylor 	 * then we don't need to do anything else here, as this kind of
6129e39c5baSBill Taylor 	 * catastrophic error is handled separately.  So we call the
6139e39c5baSBill Taylor 	 * catastrophic handler, clear the ECR and then return.
6149e39c5baSBill Taylor 	 */
6159e39c5baSBill Taylor 	if (eq->eq_evttypemask == TAVOR_EVT_MSK_LOCAL_CAT_ERROR) {
6169e39c5baSBill Taylor 		/*
6179e39c5baSBill Taylor 		 * Call Catastrophic Error handler
6189e39c5baSBill Taylor 		 */
6199e39c5baSBill Taylor 		tavor_eq_catastrophic(state);
6209e39c5baSBill Taylor 
6219e39c5baSBill Taylor 		/*
6229e39c5baSBill Taylor 		 * Clear the ECR.  Specifically, clear the bit corresponding
6239e39c5baSBill Taylor 		 * to the event queue just processed.
6249e39c5baSBill Taylor 		 */
6259e39c5baSBill Taylor 		ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
6269e39c5baSBill Taylor 		ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
6279e39c5baSBill Taylor 
6289e39c5baSBill Taylor 		return;
6299e39c5baSBill Taylor 	}
6309e39c5baSBill Taylor 
6319e39c5baSBill Taylor 	/* Get the consumer pointer index */
6329e39c5baSBill Taylor 	cons_indx = eq->eq_consindx;
6339e39c5baSBill Taylor 
6349e39c5baSBill Taylor 	/*
6359e39c5baSBill Taylor 	 * Calculate the wrap around mask.  Note: This operation only works
6369e39c5baSBill Taylor 	 * because all Tavor event queues have power-of-2 sizes
6379e39c5baSBill Taylor 	 */
6389e39c5baSBill Taylor 	wrap_around_mask = (eq->eq_bufsz - 1);
6399e39c5baSBill Taylor 
6409e39c5baSBill Taylor 	/* Calculate the pointer to the first EQ entry */
6419e39c5baSBill Taylor 	eqe = &eq->eq_buf[cons_indx];
6429e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eqe))
6439e39c5baSBill Taylor 
6449e39c5baSBill Taylor 	/*
6459e39c5baSBill Taylor 	 * Sync the current EQE to read
6469e39c5baSBill Taylor 	 *    We need to force a ddi_dma_sync() here (independent of how the
6479e39c5baSBill Taylor 	 *    EQ was mapped) because it is possible for us to receive the
6489e39c5baSBill Taylor 	 *    interrupt, do a read of the ECR, and have each of these
6499e39c5baSBill Taylor 	 *    operations complete successfully even though the hardware's DMA
6509e39c5baSBill Taylor 	 *    to the EQ has not yet completed.
6519e39c5baSBill Taylor 	 */
6529e39c5baSBill Taylor 	tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU, TAVOR_EQ_SYNC_FORCE);
6539e39c5baSBill Taylor 
6549e39c5baSBill Taylor 	/*
6559e39c5baSBill Taylor 	 * Pull the handler function for this EQ from the Tavor Event Queue
6569e39c5baSBill Taylor 	 * handle
6579e39c5baSBill Taylor 	 */
6589e39c5baSBill Taylor 	eqfunction = eq->eq_func;
6599e39c5baSBill Taylor 
6609e39c5baSBill Taylor 	/*
6619e39c5baSBill Taylor 	 * Keep pulling entries from the EQ until we find an entry owner by
6629e39c5baSBill Taylor 	 * the hardware.  As long as there the EQE's owned by SW, process
6639e39c5baSBill Taylor 	 * each entry by calling its handler function and updating the EQ
6649e39c5baSBill Taylor 	 * consumer index.
6659e39c5baSBill Taylor 	 */
6669e39c5baSBill Taylor 	do {
6679e39c5baSBill Taylor 		while (TAVOR_EQE_OWNER_IS_SW(eq, eqe)) {
6689e39c5baSBill Taylor 			/*
6699e39c5baSBill Taylor 			 * Call the EQ handler function.  But only call if we
6709e39c5baSBill Taylor 			 * are not in polled I/O mode (i.e. not processing
6719e39c5baSBill Taylor 			 * because of a system panic).  Note: We don't call
6729e39c5baSBill Taylor 			 * the EQ handling functions from a system panic
6739e39c5baSBill Taylor 			 * because we are primarily concerned only with
6749e39c5baSBill Taylor 			 * ensuring that the event queues do not overflow (or,
6759e39c5baSBill Taylor 			 * more specifically, the event queue associated with
6769e39c5baSBill Taylor 			 * the CQ that is being used in the sync/dump process).
6779e39c5baSBill Taylor 			 * Also, we don't want to make any upcalls (to the
6789e39c5baSBill Taylor 			 * IBTF) because we can't guarantee when/if those
6799e39c5baSBill Taylor 			 * calls would ever return.  And, if we're in panic,
6809e39c5baSBill Taylor 			 * then we reached here through a PollCQ() call (from
6819e39c5baSBill Taylor 			 * tavor_cq_poll()), and we need to ensure that we
6829e39c5baSBill Taylor 			 * successfully return any work completions to the
6839e39c5baSBill Taylor 			 * caller.
6849e39c5baSBill Taylor 			 */
6859e39c5baSBill Taylor 			if (ddi_in_panic() == 0) {
6869e39c5baSBill Taylor 				eqfunction(state, eq, eqe);
6879e39c5baSBill Taylor 			}
6889e39c5baSBill Taylor 
6899e39c5baSBill Taylor 			/* Reset entry to hardware ownership */
6909e39c5baSBill Taylor 			TAVOR_EQE_OWNER_SET_HW(eq, eqe);
6919e39c5baSBill Taylor 
6929e39c5baSBill Taylor 			/* Sync the current EQE for device */
6939e39c5baSBill Taylor 			tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORDEV,
6949e39c5baSBill Taylor 			    TAVOR_EQ_SYNC_NORMAL);
6959e39c5baSBill Taylor 
6969e39c5baSBill Taylor 			/* Increment the consumer index */
6979e39c5baSBill Taylor 			cons_indx = (cons_indx + 1) & wrap_around_mask;
6989e39c5baSBill Taylor 
6999e39c5baSBill Taylor 			/* Update the pointer to the next EQ entry */
7009e39c5baSBill Taylor 			eqe = &eq->eq_buf[cons_indx];
7019e39c5baSBill Taylor 
7029e39c5baSBill Taylor 			/* Sync the next EQE to read */
7039e39c5baSBill Taylor 			tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
7049e39c5baSBill Taylor 			    TAVOR_EQ_SYNC_NORMAL);
7059e39c5baSBill Taylor 		}
7069e39c5baSBill Taylor 
7079e39c5baSBill Taylor 		/*
7089e39c5baSBill Taylor 		 * Clear the ECR.  Specifically, clear the bit corresponding
7099e39c5baSBill Taylor 		 * to the event queue just processed.
7109e39c5baSBill Taylor 		 */
7119e39c5baSBill Taylor 		ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
7129e39c5baSBill Taylor 		ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
7139e39c5baSBill Taylor 
7149e39c5baSBill Taylor 		/* Write an EQ doorbell to update the consumer index */
7159e39c5baSBill Taylor 		eq->eq_consindx = cons_indx;
7169e39c5baSBill Taylor 		tavor_eq_doorbell(state, TAVOR_EQDB_SET_CONSINDX, eq->eq_eqnum,
7179e39c5baSBill Taylor 		    cons_indx);
7189e39c5baSBill Taylor 
7199e39c5baSBill Taylor 		/* Write another EQ doorbell to rearm */
7209e39c5baSBill Taylor 		tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, eq->eq_eqnum, 0);
7219e39c5baSBill Taylor 
7229e39c5baSBill Taylor 		/*
7239e39c5baSBill Taylor 		 * NOTE: Due to the nature of Mellanox hardware, we do not have
7249e39c5baSBill Taylor 		 * to do an explicit PIO read to ensure that the doorbell write
7259e39c5baSBill Taylor 		 * has been flushed to the hardware.  There is state encoded in
7269e39c5baSBill Taylor 		 * the doorbell information we write which makes this
7279e39c5baSBill Taylor 		 * unnecessary.  We can be assured that if an event needs to be
7289e39c5baSBill Taylor 		 * generated, the hardware will make sure that it is, solving
7299e39c5baSBill Taylor 		 * the possible race condition.
7309e39c5baSBill Taylor 		 */
7319e39c5baSBill Taylor 
7329e39c5baSBill Taylor 		/* Sync the next EQE to read */
7339e39c5baSBill Taylor 		tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
7349e39c5baSBill Taylor 		    TAVOR_EQ_SYNC_NORMAL);
7359e39c5baSBill Taylor 
7369e39c5baSBill Taylor 	} while (TAVOR_EQE_OWNER_IS_SW(eq, eqe));
7379e39c5baSBill Taylor }
7389e39c5baSBill Taylor 
7399e39c5baSBill Taylor 
7409e39c5baSBill Taylor /*
7419e39c5baSBill Taylor  * tavor_eq_catastrophic
7429e39c5baSBill Taylor  *    Context: Only called from interrupt context (and during panic)
7439e39c5baSBill Taylor  */
7449e39c5baSBill Taylor static void
tavor_eq_catastrophic(tavor_state_t * state)7459e39c5baSBill Taylor tavor_eq_catastrophic(tavor_state_t *state)
7469e39c5baSBill Taylor {
7479e39c5baSBill Taylor 	ibt_async_code_t	type;
7489e39c5baSBill Taylor 	ibc_async_event_t	event;
7499e39c5baSBill Taylor 	uint32_t		*base_addr;
7509e39c5baSBill Taylor 	uint32_t		buf_size;
7519e39c5baSBill Taylor 	uint32_t		word;
7529e39c5baSBill Taylor 	uint8_t			err_type;
7539e39c5baSBill Taylor 	uint32_t		err_buf;
7549e39c5baSBill Taylor 	int			i;
7559e39c5baSBill Taylor 
7569e39c5baSBill Taylor 	bzero(&event, sizeof (ibc_async_event_t));
7579e39c5baSBill Taylor 
7589e39c5baSBill Taylor 	base_addr = (uint32_t *)(uintptr_t)(
7599e39c5baSBill Taylor 	    (uintptr_t)state->ts_reg_cmd_baseaddr +
7609e39c5baSBill Taylor 	    state->ts_fw.error_buf_addr);
7619e39c5baSBill Taylor 	buf_size = state->ts_fw.error_buf_sz;
7629e39c5baSBill Taylor 
7639e39c5baSBill Taylor 	word = ddi_get32(state->ts_reg_cmdhdl, base_addr);
7649e39c5baSBill Taylor 
7659e39c5baSBill Taylor 	err_type = (word & 0xFF000000) >> 24;
7669e39c5baSBill Taylor 	type	 = IBT_ERROR_LOCAL_CATASTROPHIC;
7679e39c5baSBill Taylor 
7689e39c5baSBill Taylor 	switch (err_type) {
7699e39c5baSBill Taylor 	case TAVOR_CATASTROPHIC_INTERNAL_ERROR:
7709e39c5baSBill Taylor 		cmn_err(CE_WARN, "Catastrophic Internal Error: 0x%02x",
7719e39c5baSBill Taylor 		    err_type);
7729e39c5baSBill Taylor 
7739e39c5baSBill Taylor 		break;
7749e39c5baSBill Taylor 
7759e39c5baSBill Taylor 	case TAVOR_CATASTROPHIC_UPLINK_BUS_ERROR:
7769e39c5baSBill Taylor 		cmn_err(CE_WARN, "Catastrophic Uplink Bus Error: 0x%02x",
7779e39c5baSBill Taylor 		    err_type);
7789e39c5baSBill Taylor 
7799e39c5baSBill Taylor 		break;
7809e39c5baSBill Taylor 
7819e39c5baSBill Taylor 	case TAVOR_CATASTROPHIC_DDR_DATA_ERROR:
7829e39c5baSBill Taylor 		cmn_err(CE_WARN, "Catastrophic DDR Data Error: 0x%02x",
7839e39c5baSBill Taylor 		    err_type);
7849e39c5baSBill Taylor 
7859e39c5baSBill Taylor 		break;
7869e39c5baSBill Taylor 
7879e39c5baSBill Taylor 	case TAVOR_CATASTROPHIC_INTERNAL_PARITY_ERROR:
7889e39c5baSBill Taylor 		cmn_err(CE_WARN, "Catastrophic Internal Parity Error: 0x%02x",
7899e39c5baSBill Taylor 		    err_type);
7909e39c5baSBill Taylor 
7919e39c5baSBill Taylor 		break;
7929e39c5baSBill Taylor 
7939e39c5baSBill Taylor 	default:
7949e39c5baSBill Taylor 		/* Unknown type of Catastrophic error */
7959e39c5baSBill Taylor 		cmn_err(CE_WARN, "Catastrophic Unknown Error: 0x%02x",
7969e39c5baSBill Taylor 		    err_type);
7979e39c5baSBill Taylor 
7989e39c5baSBill Taylor 		break;
7999e39c5baSBill Taylor 	}
8009e39c5baSBill Taylor 
8019e39c5baSBill Taylor 	/*
8029e39c5baSBill Taylor 	 * Read in the catastrophic error buffer from the hardware, printing
8039e39c5baSBill Taylor 	 * only to the log file only
8049e39c5baSBill Taylor 	 */
8059e39c5baSBill Taylor 	for (i = 0; i < buf_size; i += 4) {
8069e39c5baSBill Taylor 		base_addr = (uint32_t *)((uintptr_t)(state->ts_reg_cmd_baseaddr
8079e39c5baSBill Taylor 		    + state->ts_fw.error_buf_addr + (i * 4)));
8089e39c5baSBill Taylor 		err_buf = ddi_get32(state->ts_reg_cmdhdl, base_addr);
8099e39c5baSBill Taylor 		cmn_err(CE_WARN, "catastrophic_error[%02x]: %08X", i, err_buf);
8109e39c5baSBill Taylor 	}
8119e39c5baSBill Taylor 
8129e39c5baSBill Taylor 	/*
8139e39c5baSBill Taylor 	 * We also call the IBTF here to inform it of the catastrophic error.
8149e39c5baSBill Taylor 	 * Note: Since no event information (i.e. QP handles, CQ handles,
8159e39c5baSBill Taylor 	 * etc.) is necessary, we pass a NULL pointer instead of a pointer to
8169e39c5baSBill Taylor 	 * an empty ibc_async_event_t struct.
8179e39c5baSBill Taylor 	 *
8189e39c5baSBill Taylor 	 * But we also check if "ts_ibtfpriv" is NULL.  If it is then it
8199e39c5baSBill Taylor 	 * means that we've have either received this event before we
8209e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
8219e39c5baSBill Taylor 	 * are in the process of detaching.
8229e39c5baSBill Taylor 	 */
8239e39c5baSBill Taylor 	if (state->ts_ibtfpriv != NULL) {
8249e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
8259e39c5baSBill Taylor 	}
8269e39c5baSBill Taylor }
8279e39c5baSBill Taylor 
8289e39c5baSBill Taylor 
8299e39c5baSBill Taylor /*
8309e39c5baSBill Taylor  * tavor_eq_alloc()
8319e39c5baSBill Taylor  *    Context: Only called from attach() path context
8329e39c5baSBill Taylor  */
8339e39c5baSBill Taylor static int
tavor_eq_alloc(tavor_state_t * state,uint32_t log_eq_size,uint_t intr,tavor_eqhdl_t * eqhdl)8349e39c5baSBill Taylor tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size, uint_t intr,
8359e39c5baSBill Taylor     tavor_eqhdl_t *eqhdl)
8369e39c5baSBill Taylor {
8379e39c5baSBill Taylor 	tavor_rsrc_t		*eqc, *rsrc;
8389e39c5baSBill Taylor 	tavor_hw_eqc_t		eqc_entry;
8399e39c5baSBill Taylor 	tavor_eqhdl_t		eq;
8409e39c5baSBill Taylor 	ibt_mr_attr_t		mr_attr;
8419e39c5baSBill Taylor 	tavor_mr_options_t	op;
8429e39c5baSBill Taylor 	tavor_pdhdl_t		pd;
8439e39c5baSBill Taylor 	tavor_mrhdl_t		mr;
8449e39c5baSBill Taylor 	tavor_hw_eqe_t		*buf;
8459e39c5baSBill Taylor 	uint64_t		addr;
8469e39c5baSBill Taylor 	uint32_t		lkey;
8479e39c5baSBill Taylor 	uint_t			dma_xfer_mode;
8489e39c5baSBill Taylor 	int			status, i;
8499e39c5baSBill Taylor 
8509e39c5baSBill Taylor 	/* Use the internal protection domain (PD) for setting up EQs */
8519e39c5baSBill Taylor 	pd = state->ts_pdhdl_internal;
8529e39c5baSBill Taylor 
8539e39c5baSBill Taylor 	/* Increment the reference count on the protection domain (PD) */
8549e39c5baSBill Taylor 	tavor_pd_refcnt_inc(pd);
8559e39c5baSBill Taylor 
8569e39c5baSBill Taylor 	/*
8579e39c5baSBill Taylor 	 * Allocate an EQ context entry.  This will be filled in with all
8589e39c5baSBill Taylor 	 * the necessary parameters to define the Event Queue.  And then
8599e39c5baSBill Taylor 	 * ownership will be passed to the hardware in the final step
8609e39c5baSBill Taylor 	 * below.  If we fail here, we must undo the protection domain
8619e39c5baSBill Taylor 	 * reference count.
8629e39c5baSBill Taylor 	 */
8639e39c5baSBill Taylor 	status = tavor_rsrc_alloc(state, TAVOR_EQC, 1, TAVOR_SLEEP, &eqc);
8649e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
8659e39c5baSBill Taylor 		goto eqalloc_fail1;
8669e39c5baSBill Taylor 	}
8679e39c5baSBill Taylor 
8689e39c5baSBill Taylor 	/*
8699e39c5baSBill Taylor 	 * Allocate the software structure for tracking the event queue (i.e.
8709e39c5baSBill Taylor 	 * the Tavor Event Queue handle).  If we fail here, we must undo the
8719e39c5baSBill Taylor 	 * protection domain reference count and the previous resource
8729e39c5baSBill Taylor 	 * allocation.
8739e39c5baSBill Taylor 	 */
8749e39c5baSBill Taylor 	status = tavor_rsrc_alloc(state, TAVOR_EQHDL, 1, TAVOR_SLEEP, &rsrc);
8759e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
8769e39c5baSBill Taylor 		goto eqalloc_fail2;
8779e39c5baSBill Taylor 	}
8789e39c5baSBill Taylor 	eq = (tavor_eqhdl_t)rsrc->tr_addr;
8799e39c5baSBill Taylor 
8809e39c5baSBill Taylor 	/*
8819e39c5baSBill Taylor 	 * Allocate the memory for Event Queue.  Note: Although we use the
8829e39c5baSBill Taylor 	 * common queue allocation routine, we always specify
8839e39c5baSBill Taylor 	 * TAVOR_QUEUE_LOCATION_NORMAL (i.e. EQ located in system memory)
8849e39c5baSBill Taylor 	 * because it would be inefficient to have EQs located in DDR memory.
8859e39c5baSBill Taylor 	 * This is primarily because EQs are read from (by software) more
8869e39c5baSBill Taylor 	 * than they are written to.  Also note that, unlike Tavor QP work
8879e39c5baSBill Taylor 	 * queues, event queues do not have the same strict alignment
8889e39c5baSBill Taylor 	 * requirements.  It is sufficient for the EQ memory to be both
8899e39c5baSBill Taylor 	 * aligned to and bound to addresses which are a multiple of EQE size.
8909e39c5baSBill Taylor 	 */
8919e39c5baSBill Taylor 	eq->eq_eqinfo.qa_size = (1 << log_eq_size) * sizeof (tavor_hw_eqe_t);
8929e39c5baSBill Taylor 	eq->eq_eqinfo.qa_alloc_align = sizeof (tavor_hw_eqe_t);
8939e39c5baSBill Taylor 	eq->eq_eqinfo.qa_bind_align  = sizeof (tavor_hw_eqe_t);
8949e39c5baSBill Taylor 	eq->eq_eqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
8959e39c5baSBill Taylor 	status = tavor_queue_alloc(state, &eq->eq_eqinfo, TAVOR_SLEEP);
8969e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
8979e39c5baSBill Taylor 		goto eqalloc_fail3;
8989e39c5baSBill Taylor 	}
8999e39c5baSBill Taylor 	buf = (tavor_hw_eqe_t *)eq->eq_eqinfo.qa_buf_aligned;
9009e39c5baSBill Taylor 
9019e39c5baSBill Taylor 	/*
9029e39c5baSBill Taylor 	 * Initialize each of the Event Queue Entries (EQE) by setting their
9039e39c5baSBill Taylor 	 * ownership to hardware ("owner" bit set to HW).  This is in
9049e39c5baSBill Taylor 	 * preparation for the final transfer of ownership (below) of the
9059e39c5baSBill Taylor 	 * EQ context itself.
9069e39c5baSBill Taylor 	 */
9079e39c5baSBill Taylor 	for (i = 0; i < (1 << log_eq_size); i++) {
9089e39c5baSBill Taylor 		TAVOR_EQE_OWNER_SET_HW(eq, &buf[i]);
9099e39c5baSBill Taylor 	}
9109e39c5baSBill Taylor 
9119e39c5baSBill Taylor 	/*
9129e39c5baSBill Taylor 	 * Register the memory for the EQ.  The memory for the EQ must
9139e39c5baSBill Taylor 	 * be registered in the Tavor TPT tables.  This gives us the LKey
9149e39c5baSBill Taylor 	 * to specify in the EQ context below.
9159e39c5baSBill Taylor 	 *
9169e39c5baSBill Taylor 	 * Because we are in the attach path we use NOSLEEP here so that we
9179e39c5baSBill Taylor 	 * SPIN in the HCR since the event queues are not setup yet, and we
9189e39c5baSBill Taylor 	 * cannot NOSPIN at this point in time.
9199e39c5baSBill Taylor 	 */
9209e39c5baSBill Taylor 	mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
9219e39c5baSBill Taylor 	mr_attr.mr_len	 = eq->eq_eqinfo.qa_size;
9229e39c5baSBill Taylor 	mr_attr.mr_as	 = NULL;
9239e39c5baSBill Taylor 	mr_attr.mr_flags = IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
9249e39c5baSBill Taylor 	dma_xfer_mode	 = state->ts_cfg_profile->cp_streaming_consistent;
9259e39c5baSBill Taylor 	if (dma_xfer_mode == DDI_DMA_STREAMING) {
9269e39c5baSBill Taylor 		mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
9279e39c5baSBill Taylor 	}
9289e39c5baSBill Taylor 	op.mro_bind_type   = state->ts_cfg_profile->cp_iommu_bypass;
9299e39c5baSBill Taylor 	op.mro_bind_dmahdl = eq->eq_eqinfo.qa_dmahdl;
9309e39c5baSBill Taylor 	op.mro_bind_override_addr = 0;
9319e39c5baSBill Taylor 	status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
9329e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
9339e39c5baSBill Taylor 		goto eqalloc_fail4;
9349e39c5baSBill Taylor 	}
9359e39c5baSBill Taylor 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
9369e39c5baSBill Taylor 	addr = mr->mr_bindinfo.bi_addr;
9379e39c5baSBill Taylor 	lkey = mr->mr_lkey;
9389e39c5baSBill Taylor 
9399e39c5baSBill Taylor 	/* Determine if later ddi_dma_sync will be necessary */
9409e39c5baSBill Taylor 	eq->eq_sync = TAVOR_EQ_IS_SYNC_REQ(state, eq->eq_eqinfo);
9419e39c5baSBill Taylor 
9429e39c5baSBill Taylor 	/* Sync entire EQ for use by the hardware (if necessary) */
9439e39c5baSBill Taylor 	if (eq->eq_sync) {
9449e39c5baSBill Taylor 		(void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
9459e39c5baSBill Taylor 		    eq->eq_eqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
9469e39c5baSBill Taylor 	}
9479e39c5baSBill Taylor 
9489e39c5baSBill Taylor 	/*
9499e39c5baSBill Taylor 	 * Fill in the EQC entry.  This is the final step before passing
9509e39c5baSBill Taylor 	 * ownership of the EQC entry to the Tavor hardware.  We use all of
9519e39c5baSBill Taylor 	 * the information collected/calculated above to fill in the
9529e39c5baSBill Taylor 	 * requisite portions of the EQC.  Note:  We create all EQs in the
9539e39c5baSBill Taylor 	 * "fired" state.  We will arm them later (after our interrupt
9549e39c5baSBill Taylor 	 * routine had been registered.)
9559e39c5baSBill Taylor 	 */
9569e39c5baSBill Taylor 	bzero(&eqc_entry, sizeof (tavor_hw_eqc_t));
9579e39c5baSBill Taylor 	eqc_entry.owner		= TAVOR_HW_OWNER;
9589e39c5baSBill Taylor 	eqc_entry.xlat		= TAVOR_VA2PA_XLAT_ENABLED;
9599e39c5baSBill Taylor 	eqc_entry.state		= TAVOR_EQ_FIRED;
9609e39c5baSBill Taylor 	eqc_entry.start_addr_h	= (addr >> 32);
9619e39c5baSBill Taylor 	eqc_entry.start_addr_l	= (addr & 0xFFFFFFFF);
9629e39c5baSBill Taylor 	eqc_entry.log_eq_sz	= log_eq_size;
9639e39c5baSBill Taylor 	eqc_entry.usr_page	= 0;
9649e39c5baSBill Taylor 	eqc_entry.pd		= pd->pd_pdnum;
9659e39c5baSBill Taylor 	eqc_entry.intr		= intr;
9669e39c5baSBill Taylor 	eqc_entry.lkey		= lkey;
9679e39c5baSBill Taylor 
9689e39c5baSBill Taylor 	/*
9699e39c5baSBill Taylor 	 * Write the EQC entry to hardware.  Lastly, we pass ownership of
9709e39c5baSBill Taylor 	 * the entry to the hardware (using the Tavor SW2HW_EQ firmware
9719e39c5baSBill Taylor 	 * command).  Note: in general, this operation shouldn't fail.  But
9729e39c5baSBill Taylor 	 * if it does, we have to undo everything we've done above before
9739e39c5baSBill Taylor 	 * returning error.
9749e39c5baSBill Taylor 	 */
9759e39c5baSBill Taylor 	status = tavor_cmn_ownership_cmd_post(state, SW2HW_EQ, &eqc_entry,
9769e39c5baSBill Taylor 	    sizeof (tavor_hw_eqc_t), eqc->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
9779e39c5baSBill Taylor 	if (status != TAVOR_CMD_SUCCESS) {
9789e39c5baSBill Taylor 		cmn_err(CE_CONT, "Tavor: SW2HW_EQ command failed: %08x\n",
9799e39c5baSBill Taylor 		    status);
9809e39c5baSBill Taylor 		goto eqalloc_fail5;
9819e39c5baSBill Taylor 	}
9829e39c5baSBill Taylor 
9839e39c5baSBill Taylor 	/*
9849e39c5baSBill Taylor 	 * Fill in the rest of the Tavor Event Queue handle.  Having
9859e39c5baSBill Taylor 	 * successfully transferred ownership of the EQC, we can update the
9869e39c5baSBill Taylor 	 * following fields for use in further operations on the EQ.
9879e39c5baSBill Taylor 	 */
9889e39c5baSBill Taylor 	eq->eq_eqcrsrcp	 = eqc;
9899e39c5baSBill Taylor 	eq->eq_rsrcp	 = rsrc;
9909e39c5baSBill Taylor 	eq->eq_consindx	 = 0;
9919e39c5baSBill Taylor 	eq->eq_eqnum	 = eqc->tr_indx;
9929e39c5baSBill Taylor 	eq->eq_buf	 = buf;
9939e39c5baSBill Taylor 	eq->eq_bufsz	 = (1 << log_eq_size);
9949e39c5baSBill Taylor 	eq->eq_mrhdl	 = mr;
9959e39c5baSBill Taylor 	*eqhdl		 = eq;
9969e39c5baSBill Taylor 
9979e39c5baSBill Taylor 	return (DDI_SUCCESS);
9989e39c5baSBill Taylor 
9999e39c5baSBill Taylor /*
10009e39c5baSBill Taylor  * The following is cleanup for all possible failure cases in this routine
10019e39c5baSBill Taylor  */
10029e39c5baSBill Taylor eqalloc_fail5:
10039e39c5baSBill Taylor 	if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
10049e39c5baSBill Taylor 	    TAVOR_NOSLEEP) != DDI_SUCCESS) {
10059e39c5baSBill Taylor 		TAVOR_WARNING(state, "failed to deregister EQ memory");
10069e39c5baSBill Taylor 	}
10079e39c5baSBill Taylor eqalloc_fail4:
10089e39c5baSBill Taylor 	tavor_queue_free(state, &eq->eq_eqinfo);
10099e39c5baSBill Taylor eqalloc_fail3:
10109e39c5baSBill Taylor 	tavor_rsrc_free(state, &rsrc);
10119e39c5baSBill Taylor eqalloc_fail2:
10129e39c5baSBill Taylor 	tavor_rsrc_free(state, &eqc);
10139e39c5baSBill Taylor eqalloc_fail1:
10149e39c5baSBill Taylor 	tavor_pd_refcnt_dec(pd);
10159e39c5baSBill Taylor eqalloc_fail:
10169e39c5baSBill Taylor 	return (status);
10179e39c5baSBill Taylor }
10189e39c5baSBill Taylor 
10199e39c5baSBill Taylor 
10209e39c5baSBill Taylor /*
10219e39c5baSBill Taylor  * tavor_eq_free()
10229e39c5baSBill Taylor  *    Context: Only called from attach() and/or detach() path contexts
10239e39c5baSBill Taylor  */
10249e39c5baSBill Taylor static int
tavor_eq_free(tavor_state_t * state,tavor_eqhdl_t * eqhdl)10259e39c5baSBill Taylor tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl)
10269e39c5baSBill Taylor {
10279e39c5baSBill Taylor 	tavor_rsrc_t		*eqc, *rsrc;
10289e39c5baSBill Taylor 	tavor_hw_eqc_t		eqc_entry;
10299e39c5baSBill Taylor 	tavor_pdhdl_t		pd;
10309e39c5baSBill Taylor 	tavor_mrhdl_t		mr;
10319e39c5baSBill Taylor 	tavor_eqhdl_t		eq;
10329e39c5baSBill Taylor 	uint32_t		eqnum;
10339e39c5baSBill Taylor 	int			status;
10349e39c5baSBill Taylor 
10359e39c5baSBill Taylor 	/*
10369e39c5baSBill Taylor 	 * Pull all the necessary information from the Tavor Event Queue
10379e39c5baSBill Taylor 	 * handle.  This is necessary here because the resource for the
10389e39c5baSBill Taylor 	 * EQ handle is going to be freed up as part of this operation.
10399e39c5baSBill Taylor 	 */
10409e39c5baSBill Taylor 	eq	= *eqhdl;
10419e39c5baSBill Taylor 	eqc	= eq->eq_eqcrsrcp;
10429e39c5baSBill Taylor 	rsrc	= eq->eq_rsrcp;
10439e39c5baSBill Taylor 	pd	= state->ts_pdhdl_internal;
10449e39c5baSBill Taylor 	mr	= eq->eq_mrhdl;
10459e39c5baSBill Taylor 	eqnum	= eq->eq_eqnum;
10469e39c5baSBill Taylor 
10479e39c5baSBill Taylor 	/*
10489e39c5baSBill Taylor 	 * Reclaim EQC entry from hardware (using the Tavor HW2SW_EQ
10499e39c5baSBill Taylor 	 * firmware command).  If the ownership transfer fails for any reason,
10509e39c5baSBill Taylor 	 * then it is an indication that something (either in HW or SW) has
10519e39c5baSBill Taylor 	 * gone seriously wrong.
10529e39c5baSBill Taylor 	 */
10539e39c5baSBill Taylor 	status = tavor_cmn_ownership_cmd_post(state, HW2SW_EQ, &eqc_entry,
10549e39c5baSBill Taylor 	    sizeof (tavor_hw_eqc_t), eqnum, TAVOR_CMD_NOSLEEP_SPIN);
10559e39c5baSBill Taylor 	if (status != TAVOR_CMD_SUCCESS) {
10569e39c5baSBill Taylor 		TAVOR_WARNING(state, "failed to reclaim EQC ownership");
10579e39c5baSBill Taylor 		cmn_err(CE_CONT, "Tavor: HW2SW_EQ command failed: %08x\n",
10589e39c5baSBill Taylor 		    status);
10599e39c5baSBill Taylor 		return (DDI_FAILURE);
10609e39c5baSBill Taylor 	}
10619e39c5baSBill Taylor 
10629e39c5baSBill Taylor 	/*
10639e39c5baSBill Taylor 	 * Deregister the memory for the Event Queue.  If this fails
10649e39c5baSBill Taylor 	 * for any reason, then it is an indication that something (either
10659e39c5baSBill Taylor 	 * in HW or SW) has gone seriously wrong.  So we print a warning
10669e39c5baSBill Taylor 	 * message and continue.
10679e39c5baSBill Taylor 	 */
10689e39c5baSBill Taylor 	status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
10699e39c5baSBill Taylor 	    TAVOR_NOSLEEP);
10709e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {
10719e39c5baSBill Taylor 		TAVOR_WARNING(state, "failed to deregister EQ memory");
10729e39c5baSBill Taylor 	}
10739e39c5baSBill Taylor 
10749e39c5baSBill Taylor 	/* Free the memory for the EQ */
10759e39c5baSBill Taylor 	tavor_queue_free(state, &eq->eq_eqinfo);
10769e39c5baSBill Taylor 
10779e39c5baSBill Taylor 	/* Free the Tavor Event Queue handle */
10789e39c5baSBill Taylor 	tavor_rsrc_free(state, &rsrc);
10799e39c5baSBill Taylor 
10809e39c5baSBill Taylor 	/* Free up the EQC entry resource */
10819e39c5baSBill Taylor 	tavor_rsrc_free(state, &eqc);
10829e39c5baSBill Taylor 
10839e39c5baSBill Taylor 	/* Decrement the reference count on the protection domain (PD) */
10849e39c5baSBill Taylor 	tavor_pd_refcnt_dec(pd);
10859e39c5baSBill Taylor 
10869e39c5baSBill Taylor 	/* Set the eqhdl pointer to NULL and return success */
10879e39c5baSBill Taylor 	*eqhdl = NULL;
10889e39c5baSBill Taylor 
10899e39c5baSBill Taylor 	return (DDI_SUCCESS);
10909e39c5baSBill Taylor }
10919e39c5baSBill Taylor 
10929e39c5baSBill Taylor 
10939e39c5baSBill Taylor /*
10949e39c5baSBill Taylor  * tavor_eq_handler_init
10959e39c5baSBill Taylor  *    Context: Only called from attach() path context
10969e39c5baSBill Taylor  */
10979e39c5baSBill Taylor static int
tavor_eq_handler_init(tavor_state_t * state,tavor_eqhdl_t eq,uint_t evt_type_mask,int (* eq_func)(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe))10989e39c5baSBill Taylor tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
10999e39c5baSBill Taylor     uint_t evt_type_mask, int (*eq_func)(tavor_state_t *state,
11009e39c5baSBill Taylor     tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe))
11019e39c5baSBill Taylor {
11029e39c5baSBill Taylor 	int		status;
11039e39c5baSBill Taylor 
11049e39c5baSBill Taylor 	/*
11059e39c5baSBill Taylor 	 * Save away the EQ handler function and the event type mask.  These
11069e39c5baSBill Taylor 	 * will be used later during interrupt and event queue processing.
11079e39c5baSBill Taylor 	 */
11089e39c5baSBill Taylor 	eq->eq_func	   = eq_func;
11099e39c5baSBill Taylor 	eq->eq_evttypemask = evt_type_mask;
11109e39c5baSBill Taylor 
11119e39c5baSBill Taylor 	/*
11129e39c5baSBill Taylor 	 * Map the EQ to a specific class of event (or events) depending
11139e39c5baSBill Taylor 	 * on the mask value passed in.  The TAVOR_EVT_NO_MASK means not
11149e39c5baSBill Taylor 	 * to attempt associating the EQ with any specific class of event.
11159e39c5baSBill Taylor 	 * This is particularly useful when initializing the events queues
11169e39c5baSBill Taylor 	 * used for CQ events.   The mapping is done using the Tavor MAP_EQ
11179e39c5baSBill Taylor 	 * firmware command.  Note: This command should not, in general, fail.
11189e39c5baSBill Taylor 	 * If it does, then something (probably HW related) has gone seriously
11199e39c5baSBill Taylor 	 * wrong.
11209e39c5baSBill Taylor 	 */
11219e39c5baSBill Taylor 	if (evt_type_mask != TAVOR_EVT_NO_MASK) {
11229e39c5baSBill Taylor 		status = tavor_map_eq_cmd_post(state,
11239e39c5baSBill Taylor 		    TAVOR_CMD_MAP_EQ_EVT_MAP, eq->eq_eqnum, evt_type_mask,
11249e39c5baSBill Taylor 		    TAVOR_CMD_NOSLEEP_SPIN);
11259e39c5baSBill Taylor 		if (status != TAVOR_CMD_SUCCESS) {
11269e39c5baSBill Taylor 			cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
11279e39c5baSBill Taylor 			    "%08x\n", status);
11289e39c5baSBill Taylor 			return (DDI_FAILURE);
11299e39c5baSBill Taylor 		}
11309e39c5baSBill Taylor 	}
11319e39c5baSBill Taylor 
11329e39c5baSBill Taylor 	return (DDI_SUCCESS);
11339e39c5baSBill Taylor }
11349e39c5baSBill Taylor 
11359e39c5baSBill Taylor 
11369e39c5baSBill Taylor /*
11379e39c5baSBill Taylor  * tavor_eq_handler_fini
11389e39c5baSBill Taylor  *    Context: Only called from attach() and/or detach() path contexts
11399e39c5baSBill Taylor  */
11409e39c5baSBill Taylor static int
tavor_eq_handler_fini(tavor_state_t * state,tavor_eqhdl_t eq)11419e39c5baSBill Taylor tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq)
11429e39c5baSBill Taylor {
11439e39c5baSBill Taylor 	int			status;
11449e39c5baSBill Taylor 
11459e39c5baSBill Taylor 	/*
11469e39c5baSBill Taylor 	 * Unmap the EQ from the event class to which it had been previously
11479e39c5baSBill Taylor 	 * mapped.  The unmapping is done using the Tavor MAP_EQ (in much
11489e39c5baSBill Taylor 	 * the same way that the initial mapping was done).  The difference,
11499e39c5baSBill Taylor 	 * however, is in the TAVOR_EQ_EVT_UNMAP flag that is passed to the
11509e39c5baSBill Taylor 	 * MAP_EQ firmware command.  The TAVOR_EVT_NO_MASK (which may have
11519e39c5baSBill Taylor 	 * been passed in at init time) still means that no association has
11529e39c5baSBill Taylor 	 * been made between the EQ and any specific class of event (and,
11539e39c5baSBill Taylor 	 * hence, no unmapping is necessary).  Note: This command should not,
11549e39c5baSBill Taylor 	 * in general, fail.  If it does, then something (probably HW related)
11559e39c5baSBill Taylor 	 * has gone seriously wrong.
11569e39c5baSBill Taylor 	 */
11579e39c5baSBill Taylor 	if (eq->eq_evttypemask != TAVOR_EVT_NO_MASK) {
11589e39c5baSBill Taylor 		status = tavor_map_eq_cmd_post(state,
11599e39c5baSBill Taylor 		    TAVOR_CMD_MAP_EQ_EVT_UNMAP, eq->eq_eqnum,
11609e39c5baSBill Taylor 		    eq->eq_evttypemask, TAVOR_CMD_NOSLEEP_SPIN);
11619e39c5baSBill Taylor 		if (status != TAVOR_CMD_SUCCESS) {
11629e39c5baSBill Taylor 			cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
11639e39c5baSBill Taylor 			    "%08x\n", status);
11649e39c5baSBill Taylor 			return (DDI_FAILURE);
11659e39c5baSBill Taylor 		}
11669e39c5baSBill Taylor 	}
11679e39c5baSBill Taylor 
11689e39c5baSBill Taylor 	return (DDI_SUCCESS);
11699e39c5baSBill Taylor }
11709e39c5baSBill Taylor 
11719e39c5baSBill Taylor 
11729e39c5baSBill Taylor /*
11739e39c5baSBill Taylor  * tavor_eqe_sync()
11749e39c5baSBill Taylor  *    Context: Can be called from interrupt or base context.
11759e39c5baSBill Taylor  *
11769e39c5baSBill Taylor  *    Typically, this routine does nothing unless the EQ memory is
11779e39c5baSBill Taylor  *    mapped as DDI_DMA_STREAMING.  However, there is a condition where
11789e39c5baSBill Taylor  *    ddi_dma_sync() is necessary even if the memory was mapped in
11799e39c5baSBill Taylor  *    consistent mode.  The "force_sync" parameter is used here to force
11809e39c5baSBill Taylor  *    the call to ddi_dma_sync() independent of how the EQ memory was
11819e39c5baSBill Taylor  *    mapped.
11829e39c5baSBill Taylor  */
11839e39c5baSBill Taylor static void
tavor_eqe_sync(tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe,uint_t flag,uint_t force_sync)11849e39c5baSBill Taylor tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
11859e39c5baSBill Taylor     uint_t force_sync)
11869e39c5baSBill Taylor {
11879e39c5baSBill Taylor 	ddi_dma_handle_t	dmahdl;
11889e39c5baSBill Taylor 	off_t			offset;
11899e39c5baSBill Taylor 
11909e39c5baSBill Taylor 	/* Determine if EQ needs to be synced or not */
11919e39c5baSBill Taylor 	if ((eq->eq_sync == 0) && (force_sync == TAVOR_EQ_SYNC_NORMAL)) {
11929e39c5baSBill Taylor 		return;
11939e39c5baSBill Taylor 	}
11949e39c5baSBill Taylor 
11959e39c5baSBill Taylor 	/* Get the DMA handle from EQ context */
11969e39c5baSBill Taylor 	dmahdl = eq->eq_mrhdl->mr_bindinfo.bi_dmahdl;
11979e39c5baSBill Taylor 
11989e39c5baSBill Taylor 	/* Calculate offset of next EQE */
11999e39c5baSBill Taylor 	offset = (off_t)((uintptr_t)eqe - (uintptr_t)&eq->eq_buf[0]);
1200*2570281cSToomas Soome 	(void) ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_eqe_t), flag);
12019e39c5baSBill Taylor }
12029e39c5baSBill Taylor 
12039e39c5baSBill Taylor 
12049e39c5baSBill Taylor /*
12059e39c5baSBill Taylor  * tavor_port_state_change_handler()
12069e39c5baSBill Taylor  *    Context: Only called from interrupt context
12079e39c5baSBill Taylor  */
12089e39c5baSBill Taylor static int
tavor_port_state_change_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)12099e39c5baSBill Taylor tavor_port_state_change_handler(tavor_state_t *state, tavor_eqhdl_t eq,
12109e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
12119e39c5baSBill Taylor {
12129e39c5baSBill Taylor 	ibc_async_event_t	event;
12139e39c5baSBill Taylor 	ibt_async_code_t	type;
12149e39c5baSBill Taylor 	uint_t			port, subtype;
12159e39c5baSBill Taylor 	uint_t			eqe_evttype;
12169e39c5baSBill Taylor 	char			link_msg[24];
12179e39c5baSBill Taylor 
12189e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
12199e39c5baSBill Taylor 
12209e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_PORT_STATE_CHANGE ||
12219e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
12229e39c5baSBill Taylor 
12239e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
12249e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
12259e39c5baSBill Taylor 
12269e39c5baSBill Taylor 		return (DDI_FAILURE);
12279e39c5baSBill Taylor 	}
12289e39c5baSBill Taylor 
12299e39c5baSBill Taylor 	/*
12309e39c5baSBill Taylor 	 * Depending on the type of Port State Change event, pass the
12319e39c5baSBill Taylor 	 * appropriate asynch event to the IBTF.
12329e39c5baSBill Taylor 	 */
12339e39c5baSBill Taylor 	port = TAVOR_EQE_PORTNUM_GET(eq, eqe);
12349e39c5baSBill Taylor 
12359e39c5baSBill Taylor 	/* Check for valid port number in event */
12369e39c5baSBill Taylor 	if ((port == 0) || (port > state->ts_cfg_profile->cp_num_ports)) {
12379e39c5baSBill Taylor 		TAVOR_WARNING(state, "Unexpected port number in port state "
12389e39c5baSBill Taylor 		    "change event");
12399e39c5baSBill Taylor 		cmn_err(CE_CONT, "  Port number: %02x\n", port);
12409e39c5baSBill Taylor 		return (DDI_FAILURE);
12419e39c5baSBill Taylor 	}
12429e39c5baSBill Taylor 
12439e39c5baSBill Taylor 	subtype = TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe);
12449e39c5baSBill Taylor 	if (subtype == TAVOR_PORT_LINK_ACTIVE) {
12459e39c5baSBill Taylor 		event.ev_port 	= port;
12469e39c5baSBill Taylor 		type		= IBT_EVENT_PORT_UP;
12479e39c5baSBill Taylor 
12489e39c5baSBill Taylor 		(void) snprintf(link_msg, 23, "port %d up", port);
12499e39c5baSBill Taylor 		ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_RESTORED,
12509e39c5baSBill Taylor 		    DDI_EXTERNAL_FAULT, link_msg);
12519e39c5baSBill Taylor 	} else if (subtype == TAVOR_PORT_LINK_DOWN) {
12529e39c5baSBill Taylor 		event.ev_port	= port;
12539e39c5baSBill Taylor 		type		= IBT_ERROR_PORT_DOWN;
12549e39c5baSBill Taylor 
12559e39c5baSBill Taylor 		(void) snprintf(link_msg, 23, "port %d down", port);
12569e39c5baSBill Taylor 		ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_LOST,
12579e39c5baSBill Taylor 		    DDI_EXTERNAL_FAULT, link_msg);
12589e39c5baSBill Taylor 	} else {
12599e39c5baSBill Taylor 		TAVOR_WARNING(state, "Unexpected subtype in port state change "
12609e39c5baSBill Taylor 		    "event");
12619e39c5baSBill Taylor 		cmn_err(CE_CONT, "  Event type: %02x, subtype: %02x\n",
12629e39c5baSBill Taylor 		    TAVOR_EQE_EVTTYPE_GET(eq, eqe), subtype);
12639e39c5baSBill Taylor 		return (DDI_FAILURE);
12649e39c5baSBill Taylor 	}
12659e39c5baSBill Taylor 
12669e39c5baSBill Taylor 	/*
12679e39c5baSBill Taylor 	 * Deliver the event to the IBTF.  Note: If "ts_ibtfpriv" is NULL,
12689e39c5baSBill Taylor 	 * then we have either received this event before we finished
12699e39c5baSBill Taylor 	 * attaching to the IBTF or we've received it while we are in the
12709e39c5baSBill Taylor 	 * process of detaching.
12719e39c5baSBill Taylor 	 */
12729e39c5baSBill Taylor 	if (state->ts_ibtfpriv != NULL) {
12739e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
12749e39c5baSBill Taylor 	}
12759e39c5baSBill Taylor 
12769e39c5baSBill Taylor 	return (DDI_SUCCESS);
12779e39c5baSBill Taylor }
12789e39c5baSBill Taylor 
12799e39c5baSBill Taylor 
12809e39c5baSBill Taylor /*
12819e39c5baSBill Taylor  * tavor_comm_estbl_handler()
12829e39c5baSBill Taylor  *    Context: Only called from interrupt context
12839e39c5baSBill Taylor  */
12849e39c5baSBill Taylor static int
tavor_comm_estbl_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)12859e39c5baSBill Taylor tavor_comm_estbl_handler(tavor_state_t *state, tavor_eqhdl_t eq,
12869e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
12879e39c5baSBill Taylor {
12889e39c5baSBill Taylor 	tavor_qphdl_t		qp;
12899e39c5baSBill Taylor 	uint_t			qpnum;
12909e39c5baSBill Taylor 	ibc_async_event_t	event;
12919e39c5baSBill Taylor 	ibt_async_code_t	type;
12929e39c5baSBill Taylor 	uint_t			eqe_evttype;
12939e39c5baSBill Taylor 
12949e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
12959e39c5baSBill Taylor 
12969e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_COMM_ESTABLISHED ||
12979e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
12989e39c5baSBill Taylor 
12999e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
13009e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
13019e39c5baSBill Taylor 
13029e39c5baSBill Taylor 		return (DDI_FAILURE);
13039e39c5baSBill Taylor 	}
13049e39c5baSBill Taylor 
13059e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
13069e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
13079e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
13089e39c5baSBill Taylor 
13099e39c5baSBill Taylor 	/*
13109e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
13119e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
13129e39c5baSBill Taylor 	 * should not deliver this event.
13139e39c5baSBill Taylor 	 *
13149e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
13159e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
13169e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
13179e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
13189e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
13199e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
13209e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
13219e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
13229e39c5baSBill Taylor 	 * handler.
13239e39c5baSBill Taylor 	 *
13249e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
13259e39c5baSBill Taylor 	 * means that we've have either received this event before we
13269e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
13279e39c5baSBill Taylor 	 * are in the process of detaching.
13289e39c5baSBill Taylor 	 */
13299e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
13309e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
13319e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
13329e39c5baSBill Taylor 		type		= IBT_EVENT_COM_EST_QP;
13339e39c5baSBill Taylor 
13349e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
13359e39c5baSBill Taylor 	}
13369e39c5baSBill Taylor 
13379e39c5baSBill Taylor 	return (DDI_SUCCESS);
13389e39c5baSBill Taylor }
13399e39c5baSBill Taylor 
13409e39c5baSBill Taylor 
13419e39c5baSBill Taylor /*
13429e39c5baSBill Taylor  * tavor_local_wq_cat_err_handler()
13439e39c5baSBill Taylor  *    Context: Only called from interrupt context
13449e39c5baSBill Taylor  */
13459e39c5baSBill Taylor static int
tavor_local_wq_cat_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)13469e39c5baSBill Taylor tavor_local_wq_cat_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
13479e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
13489e39c5baSBill Taylor {
13499e39c5baSBill Taylor 	tavor_qphdl_t		qp;
13509e39c5baSBill Taylor 	uint_t			qpnum;
13519e39c5baSBill Taylor 	ibc_async_event_t	event;
13529e39c5baSBill Taylor 	ibt_async_code_t	type;
13539e39c5baSBill Taylor 	uint_t			eqe_evttype;
13549e39c5baSBill Taylor 
13559e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
13569e39c5baSBill Taylor 
13579e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_WQ_CAT_ERROR ||
13589e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
13599e39c5baSBill Taylor 
13609e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
13619e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
13629e39c5baSBill Taylor 
13639e39c5baSBill Taylor 		return (DDI_FAILURE);
13649e39c5baSBill Taylor 	}
13659e39c5baSBill Taylor 
13669e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
13679e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
13689e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
13699e39c5baSBill Taylor 
13709e39c5baSBill Taylor 	/*
13719e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
13729e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
13739e39c5baSBill Taylor 	 * should not deliver this event.
13749e39c5baSBill Taylor 	 *
13759e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
13769e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
13779e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
13789e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
13799e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
13809e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
13819e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
13829e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
13839e39c5baSBill Taylor 	 * handler.
13849e39c5baSBill Taylor 	 *
13859e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
13869e39c5baSBill Taylor 	 * means that we've have either received this event before we
13879e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
13889e39c5baSBill Taylor 	 * are in the process of detaching.
13899e39c5baSBill Taylor 	 */
13909e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
13919e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
13929e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
13939e39c5baSBill Taylor 		type		= IBT_ERROR_CATASTROPHIC_QP;
13949e39c5baSBill Taylor 
13959e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
13969e39c5baSBill Taylor 	}
13979e39c5baSBill Taylor 
13989e39c5baSBill Taylor 	return (DDI_SUCCESS);
13999e39c5baSBill Taylor }
14009e39c5baSBill Taylor 
14019e39c5baSBill Taylor 
14029e39c5baSBill Taylor /*
14039e39c5baSBill Taylor  * tavor_invreq_local_wq_err_handler()
14049e39c5baSBill Taylor  *    Context: Only called from interrupt context
14059e39c5baSBill Taylor  */
14069e39c5baSBill Taylor static int
tavor_invreq_local_wq_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)14079e39c5baSBill Taylor tavor_invreq_local_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
14089e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
14099e39c5baSBill Taylor {
14109e39c5baSBill Taylor 	tavor_qphdl_t		qp;
14119e39c5baSBill Taylor 	uint_t			qpnum;
14129e39c5baSBill Taylor 	ibc_async_event_t	event;
14139e39c5baSBill Taylor 	ibt_async_code_t	type;
14149e39c5baSBill Taylor 	uint_t			eqe_evttype;
14159e39c5baSBill Taylor 
14169e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
14179e39c5baSBill Taylor 
14189e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_INV_REQ_LOCAL_WQ_ERROR ||
14199e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
14209e39c5baSBill Taylor 
14219e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
14229e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
14239e39c5baSBill Taylor 
14249e39c5baSBill Taylor 		return (DDI_FAILURE);
14259e39c5baSBill Taylor 	}
14269e39c5baSBill Taylor 
14279e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
14289e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
14299e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
14309e39c5baSBill Taylor 
14319e39c5baSBill Taylor 	/*
14329e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
14339e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
14349e39c5baSBill Taylor 	 * should not deliver this event.
14359e39c5baSBill Taylor 	 *
14369e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
14379e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
14389e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
14399e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
14409e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
14419e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
14429e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
14439e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
14449e39c5baSBill Taylor 	 * handler.
14459e39c5baSBill Taylor 	 *
14469e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
14479e39c5baSBill Taylor 	 * means that we've have either received this event before we
14489e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
14499e39c5baSBill Taylor 	 * are in the process of detaching.
14509e39c5baSBill Taylor 	 */
14519e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
14529e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
14539e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
14549e39c5baSBill Taylor 		type		= IBT_ERROR_INVALID_REQUEST_QP;
14559e39c5baSBill Taylor 
14569e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
14579e39c5baSBill Taylor 	}
14589e39c5baSBill Taylor 
14599e39c5baSBill Taylor 	return (DDI_SUCCESS);
14609e39c5baSBill Taylor }
14619e39c5baSBill Taylor 
14629e39c5baSBill Taylor 
14639e39c5baSBill Taylor /*
14649e39c5baSBill Taylor  * tavor_local_acc_vio_wq_err_handler()
14659e39c5baSBill Taylor  *    Context: Only called from interrupt context
14669e39c5baSBill Taylor  */
14679e39c5baSBill Taylor static int
tavor_local_acc_vio_wq_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)14689e39c5baSBill Taylor tavor_local_acc_vio_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
14699e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
14709e39c5baSBill Taylor {
14719e39c5baSBill Taylor 	tavor_qphdl_t		qp;
14729e39c5baSBill Taylor 	uint_t			qpnum;
14739e39c5baSBill Taylor 	ibc_async_event_t	event;
14749e39c5baSBill Taylor 	ibt_async_code_t	type;
14759e39c5baSBill Taylor 	uint_t			eqe_evttype;
14769e39c5baSBill Taylor 
14779e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
14789e39c5baSBill Taylor 
14799e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_ACC_VIO_WQ_ERROR ||
14809e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
14819e39c5baSBill Taylor 
14829e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
14839e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
14849e39c5baSBill Taylor 
14859e39c5baSBill Taylor 		return (DDI_FAILURE);
14869e39c5baSBill Taylor 	}
14879e39c5baSBill Taylor 
14889e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
14899e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
14909e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
14919e39c5baSBill Taylor 
14929e39c5baSBill Taylor 	/*
14939e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
14949e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
14959e39c5baSBill Taylor 	 * should not deliver this event.
14969e39c5baSBill Taylor 	 *
14979e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
14989e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
14999e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
15009e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
15019e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
15029e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
15039e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
15049e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
15059e39c5baSBill Taylor 	 * handler.
15069e39c5baSBill Taylor 	 *
15079e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
15089e39c5baSBill Taylor 	 * means that we've have either received this event before we
15099e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
15109e39c5baSBill Taylor 	 * are in the process of detaching.
15119e39c5baSBill Taylor 	 */
15129e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
15139e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
15149e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
15159e39c5baSBill Taylor 		type		= IBT_ERROR_ACCESS_VIOLATION_QP;
15169e39c5baSBill Taylor 
15179e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
15189e39c5baSBill Taylor 	}
15199e39c5baSBill Taylor 
15209e39c5baSBill Taylor 	return (DDI_SUCCESS);
15219e39c5baSBill Taylor }
15229e39c5baSBill Taylor 
15239e39c5baSBill Taylor 
15249e39c5baSBill Taylor /*
15259e39c5baSBill Taylor  * tavor_sendq_drained_handler()
15269e39c5baSBill Taylor  *    Context: Only called from interrupt context
15279e39c5baSBill Taylor  */
15289e39c5baSBill Taylor static int
tavor_sendq_drained_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)15299e39c5baSBill Taylor tavor_sendq_drained_handler(tavor_state_t *state, tavor_eqhdl_t eq,
15309e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
15319e39c5baSBill Taylor {
15329e39c5baSBill Taylor 	tavor_qphdl_t		qp;
15339e39c5baSBill Taylor 	uint_t			qpnum;
15349e39c5baSBill Taylor 	ibc_async_event_t	event;
15359e39c5baSBill Taylor 	uint_t			forward_sqd_event;
15369e39c5baSBill Taylor 	ibt_async_code_t	type;
15379e39c5baSBill Taylor 	uint_t			eqe_evttype;
15389e39c5baSBill Taylor 
15399e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
15409e39c5baSBill Taylor 
15419e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_SEND_QUEUE_DRAINED ||
15429e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
15439e39c5baSBill Taylor 
15449e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
15459e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
15469e39c5baSBill Taylor 
15479e39c5baSBill Taylor 		return (DDI_FAILURE);
15489e39c5baSBill Taylor 	}
15499e39c5baSBill Taylor 
15509e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
15519e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
15529e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
15539e39c5baSBill Taylor 
15549e39c5baSBill Taylor 	/*
15559e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
15569e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
15579e39c5baSBill Taylor 	 * should not deliver this event.
15589e39c5baSBill Taylor 	 *
15599e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
15609e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
15619e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
15629e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
15639e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
15649e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
15659e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
15669e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
15679e39c5baSBill Taylor 	 * handler.
15689e39c5baSBill Taylor 	 *
15699e39c5baSBill Taylor 	 * And then we check if "ts_ibtfpriv" is NULL.  If it is then it
15709e39c5baSBill Taylor 	 * means that we've have either received this event before we
15719e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
15729e39c5baSBill Taylor 	 * are in the process of detaching.
15739e39c5baSBill Taylor 	 */
15749e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
15759e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
15769e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
15779e39c5baSBill Taylor 		type		= IBT_EVENT_SQD;
15789e39c5baSBill Taylor 
15799e39c5baSBill Taylor 		/*
15809e39c5baSBill Taylor 		 * Grab the QP lock and update the QP state to reflect that
15819e39c5baSBill Taylor 		 * the Send Queue Drained event has arrived.  Also determine
15829e39c5baSBill Taylor 		 * whether the event is intended to be forwarded on to the
15839e39c5baSBill Taylor 		 * consumer or not.  This information is used below in
15849e39c5baSBill Taylor 		 * determining whether or not to call the IBTF.
15859e39c5baSBill Taylor 		 */
15869e39c5baSBill Taylor 		mutex_enter(&qp->qp_lock);
15879e39c5baSBill Taylor 		forward_sqd_event = qp->qp_forward_sqd_event;
15889e39c5baSBill Taylor 		qp->qp_forward_sqd_event  = 0;
15899e39c5baSBill Taylor 		qp->qp_sqd_still_draining = 0;
15909e39c5baSBill Taylor 		mutex_exit(&qp->qp_lock);
15919e39c5baSBill Taylor 
15929e39c5baSBill Taylor 		if (forward_sqd_event != 0) {
15939e39c5baSBill Taylor 			TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
15949e39c5baSBill Taylor 		}
15959e39c5baSBill Taylor 	}
15969e39c5baSBill Taylor 
15979e39c5baSBill Taylor 	return (DDI_SUCCESS);
15989e39c5baSBill Taylor }
15999e39c5baSBill Taylor 
16009e39c5baSBill Taylor 
16019e39c5baSBill Taylor /*
16029e39c5baSBill Taylor  * tavor_path_mig_handler()
16039e39c5baSBill Taylor  *    Context: Only called from interrupt context
16049e39c5baSBill Taylor  */
16059e39c5baSBill Taylor static int
tavor_path_mig_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)16069e39c5baSBill Taylor tavor_path_mig_handler(tavor_state_t *state, tavor_eqhdl_t eq,
16079e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
16089e39c5baSBill Taylor {
16099e39c5baSBill Taylor 	tavor_qphdl_t		qp;
16109e39c5baSBill Taylor 	uint_t			qpnum;
16119e39c5baSBill Taylor 	ibc_async_event_t	event;
16129e39c5baSBill Taylor 	ibt_async_code_t	type;
16139e39c5baSBill Taylor 	uint_t			eqe_evttype;
16149e39c5baSBill Taylor 
16159e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
16169e39c5baSBill Taylor 
16179e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATED ||
16189e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
16199e39c5baSBill Taylor 
16209e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
16219e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
16229e39c5baSBill Taylor 
16239e39c5baSBill Taylor 		return (DDI_FAILURE);
16249e39c5baSBill Taylor 	}
16259e39c5baSBill Taylor 
16269e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
16279e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
16289e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
16299e39c5baSBill Taylor 
16309e39c5baSBill Taylor 	/*
16319e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
16329e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
16339e39c5baSBill Taylor 	 * should not deliver this event.
16349e39c5baSBill Taylor 	 *
16359e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
16369e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
16379e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
16389e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
16399e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
16409e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
16419e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
16429e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
16439e39c5baSBill Taylor 	 * handler.
16449e39c5baSBill Taylor 	 *
16459e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
16469e39c5baSBill Taylor 	 * means that we've have either received this event before we
16479e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
16489e39c5baSBill Taylor 	 * are in the process of detaching.
16499e39c5baSBill Taylor 	 */
16509e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
16519e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
16529e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
16539e39c5baSBill Taylor 		type		= IBT_EVENT_PATH_MIGRATED_QP;
16549e39c5baSBill Taylor 
16559e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
16569e39c5baSBill Taylor 	}
16579e39c5baSBill Taylor 
16589e39c5baSBill Taylor 	return (DDI_SUCCESS);
16599e39c5baSBill Taylor }
16609e39c5baSBill Taylor 
16619e39c5baSBill Taylor 
16629e39c5baSBill Taylor /*
16639e39c5baSBill Taylor  * tavor_path_mig_err_handler()
16649e39c5baSBill Taylor  *    Context: Only called from interrupt context
16659e39c5baSBill Taylor  */
16669e39c5baSBill Taylor static int
tavor_path_mig_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)16679e39c5baSBill Taylor tavor_path_mig_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
16689e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
16699e39c5baSBill Taylor {
16709e39c5baSBill Taylor 	tavor_qphdl_t		qp;
16719e39c5baSBill Taylor 	uint_t			qpnum;
16729e39c5baSBill Taylor 	ibc_async_event_t	event;
16739e39c5baSBill Taylor 	ibt_async_code_t	type;
16749e39c5baSBill Taylor 	uint_t			eqe_evttype;
16759e39c5baSBill Taylor 
16769e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
16779e39c5baSBill Taylor 
16789e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATE_FAILED ||
16799e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
16809e39c5baSBill Taylor 
16819e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
16829e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
16839e39c5baSBill Taylor 
16849e39c5baSBill Taylor 		return (DDI_FAILURE);
16859e39c5baSBill Taylor 	}
16869e39c5baSBill Taylor 
16879e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
16889e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
16899e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
16909e39c5baSBill Taylor 
16919e39c5baSBill Taylor 	/*
16929e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
16939e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
16949e39c5baSBill Taylor 	 * should not deliver this event.
16959e39c5baSBill Taylor 	 *
16969e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
16979e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
16989e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
16999e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
17009e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
17019e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
17029e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
17039e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
17049e39c5baSBill Taylor 	 * handler.
17059e39c5baSBill Taylor 	 *
17069e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
17079e39c5baSBill Taylor 	 * means that we've have either received this event before we
17089e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
17099e39c5baSBill Taylor 	 * are in the process of detaching.
17109e39c5baSBill Taylor 	 */
17119e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
17129e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
17139e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
17149e39c5baSBill Taylor 		type		= IBT_ERROR_PATH_MIGRATE_REQ_QP;
17159e39c5baSBill Taylor 
17169e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
17179e39c5baSBill Taylor 	}
17189e39c5baSBill Taylor 
17199e39c5baSBill Taylor 	return (DDI_SUCCESS);
17209e39c5baSBill Taylor }
17219e39c5baSBill Taylor 
17229e39c5baSBill Taylor 
17239e39c5baSBill Taylor /*
17249e39c5baSBill Taylor  * tavor_srq_catastrophic_handler()
17259e39c5baSBill Taylor  *    Context: Only called from interrupt context
17269e39c5baSBill Taylor  */
17279e39c5baSBill Taylor static int
tavor_srq_catastrophic_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)17289e39c5baSBill Taylor tavor_srq_catastrophic_handler(tavor_state_t *state, tavor_eqhdl_t eq,
17299e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
17309e39c5baSBill Taylor {
17319e39c5baSBill Taylor 	tavor_qphdl_t		qp;
17329e39c5baSBill Taylor 	uint_t			qpnum;
17339e39c5baSBill Taylor 	ibc_async_event_t	event;
17349e39c5baSBill Taylor 	ibt_async_code_t	type;
17359e39c5baSBill Taylor 	uint_t			eqe_evttype;
17369e39c5baSBill Taylor 
17379e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
17389e39c5baSBill Taylor 
17399e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_SRQ_CATASTROPHIC_ERROR ||
17409e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
17419e39c5baSBill Taylor 
17429e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
17439e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
17449e39c5baSBill Taylor 
17459e39c5baSBill Taylor 		return (DDI_FAILURE);
17469e39c5baSBill Taylor 	}
17479e39c5baSBill Taylor 
17489e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
17499e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
17509e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
17519e39c5baSBill Taylor 
17529e39c5baSBill Taylor 	/*
17539e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
17549e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
17559e39c5baSBill Taylor 	 * should not deliver this event.
17569e39c5baSBill Taylor 	 *
17579e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
17589e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
17599e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
17609e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
17619e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
17629e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
17639e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
17649e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
17659e39c5baSBill Taylor 	 * handler.
17669e39c5baSBill Taylor 	 *
17679e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
17689e39c5baSBill Taylor 	 * means that we've have either received this event before we
17699e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
17709e39c5baSBill Taylor 	 * are in the process of detaching.
17719e39c5baSBill Taylor 	 */
17729e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
17739e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
17749e39c5baSBill Taylor 		event.ev_srq_hdl = (ibt_srq_hdl_t)qp->qp_srqhdl->srq_hdlrarg;
17759e39c5baSBill Taylor 		type		= IBT_ERROR_CATASTROPHIC_SRQ;
17769e39c5baSBill Taylor 
17779e39c5baSBill Taylor 		mutex_enter(&qp->qp_srqhdl->srq_lock);
17789e39c5baSBill Taylor 		qp->qp_srqhdl->srq_state = TAVOR_SRQ_STATE_ERROR;
17799e39c5baSBill Taylor 		mutex_exit(&qp->qp_srqhdl->srq_lock);
17809e39c5baSBill Taylor 
17819e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
17829e39c5baSBill Taylor 	}
17839e39c5baSBill Taylor 
17849e39c5baSBill Taylor 	return (DDI_SUCCESS);
17859e39c5baSBill Taylor }
17869e39c5baSBill Taylor 
17879e39c5baSBill Taylor 
17889e39c5baSBill Taylor /*
17899e39c5baSBill Taylor  * tavor_srq_last_wqe_reached_handler()
17909e39c5baSBill Taylor  *    Context: Only called from interrupt context
17919e39c5baSBill Taylor  */
17929e39c5baSBill Taylor static int
tavor_srq_last_wqe_reached_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)17939e39c5baSBill Taylor tavor_srq_last_wqe_reached_handler(tavor_state_t *state, tavor_eqhdl_t eq,
17949e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
17959e39c5baSBill Taylor {
17969e39c5baSBill Taylor 	tavor_qphdl_t		qp;
17979e39c5baSBill Taylor 	uint_t			qpnum;
17989e39c5baSBill Taylor 	ibc_async_event_t	event;
17999e39c5baSBill Taylor 	ibt_async_code_t	type;
18009e39c5baSBill Taylor 	uint_t			eqe_evttype;
18019e39c5baSBill Taylor 
18029e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
18039e39c5baSBill Taylor 
18049e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_SRQ_LAST_WQE_REACHED ||
18059e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
18069e39c5baSBill Taylor 
18079e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
18089e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
18099e39c5baSBill Taylor 
18109e39c5baSBill Taylor 		return (DDI_FAILURE);
18119e39c5baSBill Taylor 	}
18129e39c5baSBill Taylor 
18139e39c5baSBill Taylor 	/* Get the QP handle from QP number in event descriptor */
18149e39c5baSBill Taylor 	qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
18159e39c5baSBill Taylor 	qp = tavor_qphdl_from_qpnum(state, qpnum);
18169e39c5baSBill Taylor 
18179e39c5baSBill Taylor 	/*
18189e39c5baSBill Taylor 	 * If the QP handle is NULL, this is probably an indication
18199e39c5baSBill Taylor 	 * that the QP has been freed already.  In which case, we
18209e39c5baSBill Taylor 	 * should not deliver this event.
18219e39c5baSBill Taylor 	 *
18229e39c5baSBill Taylor 	 * We also check that the QP number in the handle is the
18239e39c5baSBill Taylor 	 * same as the QP number in the event queue entry.  This
18249e39c5baSBill Taylor 	 * extra check allows us to handle the case where a QP was
18259e39c5baSBill Taylor 	 * freed and then allocated again in the time it took to
18269e39c5baSBill Taylor 	 * handle the event queue processing.  By constantly incrementing
18279e39c5baSBill Taylor 	 * the non-constrained portion of the QP number every time
18289e39c5baSBill Taylor 	 * a new QP is allocated, we mitigate (somewhat) the chance
18299e39c5baSBill Taylor 	 * that a stale event could be passed to the client's QP
18309e39c5baSBill Taylor 	 * handler.
18319e39c5baSBill Taylor 	 *
18329e39c5baSBill Taylor 	 * Lastly, we check if "ts_ibtfpriv" is NULL.  If it is then it
18339e39c5baSBill Taylor 	 * means that we've have either received this event before we
18349e39c5baSBill Taylor 	 * finished attaching to the IBTF or we've received it while we
18359e39c5baSBill Taylor 	 * are in the process of detaching.
18369e39c5baSBill Taylor 	 */
18379e39c5baSBill Taylor 	if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
18389e39c5baSBill Taylor 	    (state->ts_ibtfpriv != NULL)) {
18399e39c5baSBill Taylor 		event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
18409e39c5baSBill Taylor 		type		= IBT_EVENT_EMPTY_CHAN;
18419e39c5baSBill Taylor 
18429e39c5baSBill Taylor 		TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
18439e39c5baSBill Taylor 	}
18449e39c5baSBill Taylor 
18459e39c5baSBill Taylor 	return (DDI_SUCCESS);
18469e39c5baSBill Taylor }
18479e39c5baSBill Taylor 
18489e39c5baSBill Taylor 
18499e39c5baSBill Taylor /*
18509e39c5baSBill Taylor  * tavor_ecc_detection_handler()
18519e39c5baSBill Taylor  *    Context: Only called from interrupt context
18529e39c5baSBill Taylor  */
18539e39c5baSBill Taylor static int
tavor_ecc_detection_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)18549e39c5baSBill Taylor tavor_ecc_detection_handler(tavor_state_t *state, tavor_eqhdl_t eq,
18559e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
18569e39c5baSBill Taylor {
18579e39c5baSBill Taylor 	uint_t			eqe_evttype;
18589e39c5baSBill Taylor 	uint_t			data;
18599e39c5baSBill Taylor 	int			i;
18609e39c5baSBill Taylor 
18619e39c5baSBill Taylor 	eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
18629e39c5baSBill Taylor 
18639e39c5baSBill Taylor 	ASSERT(eqe_evttype == TAVOR_EVT_ECC_DETECTION ||
18649e39c5baSBill Taylor 	    eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
18659e39c5baSBill Taylor 
18669e39c5baSBill Taylor 	if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
18679e39c5baSBill Taylor 		tavor_eq_overflow_handler(state, eq, eqe);
18689e39c5baSBill Taylor 
18699e39c5baSBill Taylor 		return (DDI_FAILURE);
18709e39c5baSBill Taylor 	}
18719e39c5baSBill Taylor 
18729e39c5baSBill Taylor 	/*
18739e39c5baSBill Taylor 	 * The "ECC Detection Event" indicates that a correctable single-bit
18749e39c5baSBill Taylor 	 * has occurred with the attached DDR.  The EQE provides some
18759e39c5baSBill Taylor 	 * additional information about the errored EQ.  So we print a warning
18769e39c5baSBill Taylor 	 * message here along with that additional information.
18779e39c5baSBill Taylor 	 */
18789e39c5baSBill Taylor 	TAVOR_WARNING(state, "ECC Correctable Error Event Detected");
18799e39c5baSBill Taylor 	for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
18809e39c5baSBill Taylor 		data = ((uint_t *)eqe)[i];
18819e39c5baSBill Taylor 		cmn_err(CE_CONT, "!  EQE[%02x]: %08x\n", i, data);
18829e39c5baSBill Taylor 	}
18839e39c5baSBill Taylor 
18849e39c5baSBill Taylor 	return (DDI_SUCCESS);
18859e39c5baSBill Taylor }
18869e39c5baSBill Taylor 
18879e39c5baSBill Taylor 
18889e39c5baSBill Taylor /*
18899e39c5baSBill Taylor  * tavor_eq_overflow_handler()
18909e39c5baSBill Taylor  *    Context: Only called from interrupt context
18919e39c5baSBill Taylor  */
18929e39c5baSBill Taylor void
tavor_eq_overflow_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)18939e39c5baSBill Taylor tavor_eq_overflow_handler(tavor_state_t *state, tavor_eqhdl_t eq,
18949e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
18959e39c5baSBill Taylor {
18969e39c5baSBill Taylor 	uint_t		error_type, data;
18979e39c5baSBill Taylor 
18989e39c5baSBill Taylor 	ASSERT(TAVOR_EQE_EVTTYPE_GET(eq, eqe) == TAVOR_EVT_EQ_OVERFLOW);
18999e39c5baSBill Taylor 
19009e39c5baSBill Taylor 	/*
19019e39c5baSBill Taylor 	 * The "Event Queue Overflow Event" indicates that something has
19029e39c5baSBill Taylor 	 * probably gone seriously wrong with some hardware (or, perhaps,
19039e39c5baSBill Taylor 	 * with the software... though it's unlikely in this case).  The EQE
19049e39c5baSBill Taylor 	 * provides some additional information about the errored EQ.  So we
19059e39c5baSBill Taylor 	 * print a warning message here along with that additional information.
19069e39c5baSBill Taylor 	 */
19079e39c5baSBill Taylor 	error_type = TAVOR_EQE_OPERRTYPE_GET(eq, eqe);
19089e39c5baSBill Taylor 	data	   = TAVOR_EQE_OPERRDATA_GET(eq, eqe);
19099e39c5baSBill Taylor 
19109e39c5baSBill Taylor 	TAVOR_WARNING(state, "Event Queue overflow");
19119e39c5baSBill Taylor 	cmn_err(CE_CONT, "  Error type: %02x, data: %08x\n", error_type, data);
19129e39c5baSBill Taylor }
19139e39c5baSBill Taylor 
19149e39c5baSBill Taylor 
19159e39c5baSBill Taylor /*
19169e39c5baSBill Taylor  * tavor_no_eqhandler
19179e39c5baSBill Taylor  *    Context: Only called from interrupt context
19189e39c5baSBill Taylor  */
19199e39c5baSBill Taylor /* ARGSUSED */
19209e39c5baSBill Taylor static int
tavor_no_eqhandler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)19219e39c5baSBill Taylor tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
19229e39c5baSBill Taylor     tavor_hw_eqe_t *eqe)
19239e39c5baSBill Taylor {
19249e39c5baSBill Taylor 	uint_t		data;
19259e39c5baSBill Taylor 	int		i;
19269e39c5baSBill Taylor 
19279e39c5baSBill Taylor 	/*
19289e39c5baSBill Taylor 	 * This "unexpected event" handler (or "catch-all" handler) will
19299e39c5baSBill Taylor 	 * receive all events for which no other handler has been registered.
19309e39c5baSBill Taylor 	 * If we end up here, then something has probably gone seriously wrong
19319e39c5baSBill Taylor 	 * with the Tavor hardware (or, perhaps, with the software... though
19329e39c5baSBill Taylor 	 * it's unlikely in this case).  The EQE provides all the information
19339e39c5baSBill Taylor 	 * about the event.  So we print a warning message here along with
19349e39c5baSBill Taylor 	 * the contents of the EQE.
19359e39c5baSBill Taylor 	 */
19369e39c5baSBill Taylor 	TAVOR_WARNING(state, "Unexpected Event handler");
19379e39c5baSBill Taylor 	cmn_err(CE_CONT, "  Event type: %02x, subtype: %02x\n",
19389e39c5baSBill Taylor 	    TAVOR_EQE_EVTTYPE_GET(eq, eqe), TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe));
19399e39c5baSBill Taylor 	for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
19409e39c5baSBill Taylor 		data = ((uint_t *)eqe)[i];
19419e39c5baSBill Taylor 		cmn_err(CE_CONT, "  EQE[%02x]: %08x\n", i, data);
19429e39c5baSBill Taylor 	}
19439e39c5baSBill Taylor 
19449e39c5baSBill Taylor 	return (DDI_SUCCESS);
19459e39c5baSBill Taylor }
1946