19e39c5baSBill Taylor /*
29e39c5baSBill Taylor * CDDL HEADER START
39e39c5baSBill Taylor *
49e39c5baSBill Taylor * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor *
89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor * See the License for the specific language governing permissions
119e39c5baSBill Taylor * and limitations under the License.
129e39c5baSBill Taylor *
139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor *
199e39c5baSBill Taylor * CDDL HEADER END
209e39c5baSBill Taylor */
219e39c5baSBill Taylor
229e39c5baSBill Taylor /*
239e39c5baSBill Taylor * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
249e39c5baSBill Taylor * Use is subject to license terms.
259e39c5baSBill Taylor */
269e39c5baSBill Taylor
279e39c5baSBill Taylor /*
289e39c5baSBill Taylor * tavor_event.c
299e39c5baSBill Taylor * Tavor Interrupt and Event Processing Routines
309e39c5baSBill Taylor *
319e39c5baSBill Taylor * Implements all the routines necessary for allocating, freeing, and
329e39c5baSBill Taylor * handling all of the various event types that the Tavor hardware can
339e39c5baSBill Taylor * generate.
349e39c5baSBill Taylor * These routines include the main Tavor interrupt service routine
359e39c5baSBill Taylor * (tavor_isr()) as well as all the code necessary to setup and handle
369e39c5baSBill Taylor * events from each of the many event queues used by the Tavor device.
379e39c5baSBill Taylor */
389e39c5baSBill Taylor
399e39c5baSBill Taylor #include <sys/types.h>
409e39c5baSBill Taylor #include <sys/conf.h>
419e39c5baSBill Taylor #include <sys/ddi.h>
429e39c5baSBill Taylor #include <sys/sunddi.h>
439e39c5baSBill Taylor #include <sys/modctl.h>
449e39c5baSBill Taylor
459e39c5baSBill Taylor #include <sys/ib/adapters/tavor/tavor.h>
469e39c5baSBill Taylor
479e39c5baSBill Taylor static void tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq);
489e39c5baSBill Taylor static void tavor_eq_catastrophic(tavor_state_t *state);
499e39c5baSBill Taylor static int tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size,
509e39c5baSBill Taylor uint_t intr, tavor_eqhdl_t *eqhdl);
519e39c5baSBill Taylor static int tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl);
529e39c5baSBill Taylor static int tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
539e39c5baSBill Taylor uint_t evt_type_mask, int (*eqfunc)(tavor_state_t *state,
549e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe));
559e39c5baSBill Taylor static int tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq);
569e39c5baSBill Taylor static void tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
579e39c5baSBill Taylor uint_t force_sync);
589e39c5baSBill Taylor static int tavor_port_state_change_handler(tavor_state_t *state,
599e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
609e39c5baSBill Taylor static int tavor_comm_estbl_handler(tavor_state_t *state,
619e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
629e39c5baSBill Taylor static int tavor_local_wq_cat_err_handler(tavor_state_t *state,
639e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
649e39c5baSBill Taylor static int tavor_invreq_local_wq_err_handler(tavor_state_t *state,
659e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
669e39c5baSBill Taylor static int tavor_local_acc_vio_wq_err_handler(tavor_state_t *state,
679e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
689e39c5baSBill Taylor static int tavor_sendq_drained_handler(tavor_state_t *state,
699e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
709e39c5baSBill Taylor static int tavor_path_mig_handler(tavor_state_t *state,
719e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
729e39c5baSBill Taylor static int tavor_path_mig_err_handler(tavor_state_t *state,
739e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
749e39c5baSBill Taylor static int tavor_srq_catastrophic_handler(tavor_state_t *state,
759e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
769e39c5baSBill Taylor static int tavor_srq_last_wqe_reached_handler(tavor_state_t *state,
779e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
789e39c5baSBill Taylor static int tavor_ecc_detection_handler(tavor_state_t *state,
799e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
809e39c5baSBill Taylor static int tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
819e39c5baSBill Taylor tavor_hw_eqe_t *eqe);
829e39c5baSBill Taylor
839e39c5baSBill Taylor
849e39c5baSBill Taylor /*
859e39c5baSBill Taylor * tavor_eq_init_all
869e39c5baSBill Taylor * Context: Only called from attach() path context
879e39c5baSBill Taylor */
889e39c5baSBill Taylor int
tavor_eq_init_all(tavor_state_t * state)899e39c5baSBill Taylor tavor_eq_init_all(tavor_state_t *state)
909e39c5baSBill Taylor {
919e39c5baSBill Taylor uint_t log_eq_size, intr_num;
929e39c5baSBill Taylor uint_t num_eq, num_eq_init, num_eq_unmap;
939e39c5baSBill Taylor int status, i;
949e39c5baSBill Taylor
959e39c5baSBill Taylor /*
969e39c5baSBill Taylor * For now, all Event Queues default to the same size (pulled from
979e39c5baSBill Taylor * the current configuration profile) and are all assigned to the
989e39c5baSBill Taylor * same interrupt or MSI. In the future we may support assigning
999e39c5baSBill Taylor * EQs to specific interrupts or MSIs XXX
1009e39c5baSBill Taylor */
1019e39c5baSBill Taylor log_eq_size = state->ts_cfg_profile->cp_log_default_eq_sz;
1029e39c5baSBill Taylor
1039e39c5baSBill Taylor /*
1049e39c5baSBill Taylor * If MSI is to be used, then set intr_num to the MSI number
1059e39c5baSBill Taylor * (currently zero because we're using only one) or'd with the
1069e39c5baSBill Taylor * MSI enable flag. Otherwise, for regular (i.e. 'legacy') interrupt,
1079e39c5baSBill Taylor * use the 'inta_pin' value returned by QUERY_ADAPTER.
1089e39c5baSBill Taylor */
1099e39c5baSBill Taylor if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
1109e39c5baSBill Taylor intr_num = TAVOR_EQ_MSI_ENABLE_FLAG | 0;
1119e39c5baSBill Taylor } else {
1129e39c5baSBill Taylor intr_num = state->ts_adapter.inta_pin;
1139e39c5baSBill Taylor }
1149e39c5baSBill Taylor
1159e39c5baSBill Taylor /*
1169e39c5baSBill Taylor * Total number of supported EQs is hardcoded. Tavor hardware
1179e39c5baSBill Taylor * supports up to 64 EQs. We are currently using only 45 of them
1189e39c5baSBill Taylor * We will set aside the first 32 for use with Completion Queues (CQ)
1199e39c5baSBill Taylor * and reserve a few of the other 32 for each specific class of event
1209e39c5baSBill Taylor * (see below for more details).
1219e39c5baSBill Taylor */
1229e39c5baSBill Taylor num_eq = TAVOR_NUM_EQ_USED;
1239e39c5baSBill Taylor
1249e39c5baSBill Taylor /*
1259e39c5baSBill Taylor * The "num_eq_unmap" variable is used in any possible failure
1269e39c5baSBill Taylor * cleanup (below) to indicate which events queues might require
1279e39c5baSBill Taylor * possible event class unmapping.
1289e39c5baSBill Taylor */
1299e39c5baSBill Taylor num_eq_unmap = 0;
1309e39c5baSBill Taylor
1319e39c5baSBill Taylor /*
1329e39c5baSBill Taylor * Allocate and initialize all the Event Queues. If any of these
1339e39c5baSBill Taylor * EQ allocations fail then jump to the end, cleanup what had been
1349e39c5baSBill Taylor * successfully initialized, and return an error.
1359e39c5baSBill Taylor */
1369e39c5baSBill Taylor for (i = 0; i < num_eq; i++) {
1379e39c5baSBill Taylor status = tavor_eq_alloc(state, log_eq_size, intr_num,
1389e39c5baSBill Taylor &state->ts_eqhdl[i]);
1399e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1409e39c5baSBill Taylor num_eq_init = i;
1419e39c5baSBill Taylor goto all_eq_init_fail;
1429e39c5baSBill Taylor }
1439e39c5baSBill Taylor }
1449e39c5baSBill Taylor num_eq_init = num_eq;
1459e39c5baSBill Taylor
1469e39c5baSBill Taylor /*
1479e39c5baSBill Taylor * Setup EQ0-EQ31 for use with Completion Queues. Note: We can
1489e39c5baSBill Taylor * cast the return value to void here because, when we use the
1499e39c5baSBill Taylor * TAVOR_EVT_NO_MASK flag, it is not possible for
1509e39c5baSBill Taylor * tavor_eq_handler_init() to return an error.
1519e39c5baSBill Taylor */
1529e39c5baSBill Taylor for (i = 0; i < 32; i++) {
1539e39c5baSBill Taylor (void) tavor_eq_handler_init(state, state->ts_eqhdl[i],
1549e39c5baSBill Taylor TAVOR_EVT_NO_MASK, tavor_cq_handler);
1559e39c5baSBill Taylor }
1569e39c5baSBill Taylor num_eq_unmap = 32;
1579e39c5baSBill Taylor
1589e39c5baSBill Taylor /*
1599e39c5baSBill Taylor * Setup EQ32 for handling Completion Queue Error Events.
1609e39c5baSBill Taylor *
1619e39c5baSBill Taylor * These events include things like CQ overflow or CQ access
1629e39c5baSBill Taylor * violation errors. If this setup fails for any reason (which, in
1639e39c5baSBill Taylor * general, it really never should), then jump to the end, cleanup
1649e39c5baSBill Taylor * everything that has been successfully initialized, and return an
1659e39c5baSBill Taylor * error.
1669e39c5baSBill Taylor */
1679e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[32],
1689e39c5baSBill Taylor TAVOR_EVT_MSK_CQ_ERRORS, tavor_cq_err_handler);
1699e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1709e39c5baSBill Taylor goto all_eq_init_fail;
1719e39c5baSBill Taylor }
1729e39c5baSBill Taylor num_eq_unmap = 33;
1739e39c5baSBill Taylor
1749e39c5baSBill Taylor /*
1759e39c5baSBill Taylor * Setup EQ33 for handling Port State Change Events
1769e39c5baSBill Taylor *
1779e39c5baSBill Taylor * These events include things like Port Up and Port Down events.
1789e39c5baSBill Taylor * If this setup fails for any reason (which, in general, it really
1799e39c5baSBill Taylor * never should), then undo all previous EQ mapping, jump to the end,
1809e39c5baSBill Taylor * cleanup everything that has been successfully initialized, and
1819e39c5baSBill Taylor * return an error.
1829e39c5baSBill Taylor */
1839e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[33],
1849e39c5baSBill Taylor TAVOR_EVT_MSK_PORT_STATE_CHANGE, tavor_port_state_change_handler);
1859e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1869e39c5baSBill Taylor goto all_eq_init_fail;
1879e39c5baSBill Taylor }
1889e39c5baSBill Taylor num_eq_unmap = 34;
1899e39c5baSBill Taylor
1909e39c5baSBill Taylor /*
1919e39c5baSBill Taylor * Setup EQ34 for handling Communication Established Events
1929e39c5baSBill Taylor *
1939e39c5baSBill Taylor * These events correspond to the IB affiliated asynchronous events
1949e39c5baSBill Taylor * that are used for connection management. If this setup fails for
1959e39c5baSBill Taylor * any reason (which, in general, it really never should), then undo
1969e39c5baSBill Taylor * all previous EQ mapping, jump to the end, cleanup everything that
1979e39c5baSBill Taylor * has been successfully initialized, and return an error.
1989e39c5baSBill Taylor */
1999e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[34],
2009e39c5baSBill Taylor TAVOR_EVT_MSK_COMM_ESTABLISHED, tavor_comm_estbl_handler);
2019e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2029e39c5baSBill Taylor goto all_eq_init_fail;
2039e39c5baSBill Taylor }
2049e39c5baSBill Taylor num_eq_unmap = 35;
2059e39c5baSBill Taylor
2069e39c5baSBill Taylor /*
2079e39c5baSBill Taylor * Setup EQ35 for handling Command Completion Events
2089e39c5baSBill Taylor *
2099e39c5baSBill Taylor * These events correspond to the Tavor generated events that are used
2109e39c5baSBill Taylor * to indicate Tavor firmware command completion. These events are
2119e39c5baSBill Taylor * only generated when Tavor firmware commands are posted using the
2129e39c5baSBill Taylor * asynchronous completion mechanism. If this setup fails for any
2139e39c5baSBill Taylor * reason (which, in general, it really never should), then undo all
2149e39c5baSBill Taylor * previous EQ mapping, jump to the end, cleanup everything that has
2159e39c5baSBill Taylor * been successfully initialized, and return an error.
2169e39c5baSBill Taylor */
2179e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[35],
2189e39c5baSBill Taylor TAVOR_EVT_MSK_COMMAND_INTF_COMP, tavor_cmd_complete_handler);
2199e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2209e39c5baSBill Taylor goto all_eq_init_fail;
2219e39c5baSBill Taylor }
2229e39c5baSBill Taylor num_eq_unmap = 36;
2239e39c5baSBill Taylor
2249e39c5baSBill Taylor /*
2259e39c5baSBill Taylor * Setup EQ36 for handling Local WQ Catastrophic Error Events
2269e39c5baSBill Taylor *
2279e39c5baSBill Taylor * These events correspond to the similarly-named IB affiliated
2289e39c5baSBill Taylor * asynchronous error type. If this setup fails for any reason
2299e39c5baSBill Taylor * (which, in general, it really never should), then undo all previous
2309e39c5baSBill Taylor * EQ mapping, jump to the end, cleanup everything that has been
2319e39c5baSBill Taylor * successfully initialized, and return an error.
2329e39c5baSBill Taylor */
2339e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[36],
2349e39c5baSBill Taylor TAVOR_EVT_MSK_LOCAL_WQ_CAT_ERROR, tavor_local_wq_cat_err_handler);
2359e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2369e39c5baSBill Taylor goto all_eq_init_fail;
2379e39c5baSBill Taylor }
2389e39c5baSBill Taylor num_eq_unmap = 37;
2399e39c5baSBill Taylor
2409e39c5baSBill Taylor /*
2419e39c5baSBill Taylor * Setup EQ37 for handling Invalid Req Local WQ Error Events
2429e39c5baSBill Taylor *
2439e39c5baSBill Taylor * These events also correspond to the similarly-named IB affiliated
2449e39c5baSBill Taylor * asynchronous error type. If this setup fails for any reason
2459e39c5baSBill Taylor * (which, in general, it really never should), then undo all previous
2469e39c5baSBill Taylor * EQ mapping, jump to the end, cleanup everything that has been
2479e39c5baSBill Taylor * successfully initialized, and return an error.
2489e39c5baSBill Taylor */
2499e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[37],
2509e39c5baSBill Taylor TAVOR_EVT_MSK_INV_REQ_LOCAL_WQ_ERROR,
2519e39c5baSBill Taylor tavor_invreq_local_wq_err_handler);
2529e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2539e39c5baSBill Taylor goto all_eq_init_fail;
2549e39c5baSBill Taylor }
2559e39c5baSBill Taylor num_eq_unmap = 38;
2569e39c5baSBill Taylor
2579e39c5baSBill Taylor /*
2589e39c5baSBill Taylor * Setup EQ38 for handling Local Access Violation WQ Error Events
2599e39c5baSBill Taylor *
2609e39c5baSBill Taylor * These events also correspond to the similarly-named IB affiliated
2619e39c5baSBill Taylor * asynchronous error type. If this setup fails for any reason
2629e39c5baSBill Taylor * (which, in general, it really never should), then undo all previous
2639e39c5baSBill Taylor * EQ mapping, jump to the end, cleanup everything that has been
2649e39c5baSBill Taylor * successfully initialized, and return an error.
2659e39c5baSBill Taylor */
2669e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[38],
2679e39c5baSBill Taylor TAVOR_EVT_MSK_LOCAL_ACC_VIO_WQ_ERROR,
2689e39c5baSBill Taylor tavor_local_acc_vio_wq_err_handler);
2699e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2709e39c5baSBill Taylor goto all_eq_init_fail;
2719e39c5baSBill Taylor }
2729e39c5baSBill Taylor num_eq_unmap = 39;
2739e39c5baSBill Taylor
2749e39c5baSBill Taylor /*
2759e39c5baSBill Taylor * Setup EQ39 for handling Send Queue Drained Events
2769e39c5baSBill Taylor *
2779e39c5baSBill Taylor * These events correspond to the IB affiliated asynchronous events
2789e39c5baSBill Taylor * that are used to indicate completion of a Send Queue Drained QP
2799e39c5baSBill Taylor * state transition. If this setup fails for any reason (which, in
2809e39c5baSBill Taylor * general, it really never should), then undo all previous EQ
2819e39c5baSBill Taylor * mapping, jump to the end, cleanup everything that has been
2829e39c5baSBill Taylor * successfully initialized, and return an error.
2839e39c5baSBill Taylor */
2849e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[39],
2859e39c5baSBill Taylor TAVOR_EVT_MSK_SEND_QUEUE_DRAINED, tavor_sendq_drained_handler);
2869e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2879e39c5baSBill Taylor goto all_eq_init_fail;
2889e39c5baSBill Taylor }
2899e39c5baSBill Taylor num_eq_unmap = 40;
2909e39c5baSBill Taylor
2919e39c5baSBill Taylor /*
2929e39c5baSBill Taylor * Setup EQ40 for handling Path Migration Succeeded Events
2939e39c5baSBill Taylor *
2949e39c5baSBill Taylor * These events correspond to the IB affiliated asynchronous events
2959e39c5baSBill Taylor * that are used to indicate successful completion of a path
2969e39c5baSBill Taylor * migration. If this setup fails for any reason (which, in general,
2979e39c5baSBill Taylor * it really never should), then undo all previous EQ mapping, jump
2989e39c5baSBill Taylor * to the end, cleanup everything that has been successfully
2999e39c5baSBill Taylor * initialized, and return an error.
3009e39c5baSBill Taylor */
3019e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[40],
3029e39c5baSBill Taylor TAVOR_EVT_MSK_PATH_MIGRATED, tavor_path_mig_handler);
3039e39c5baSBill Taylor if (status != DDI_SUCCESS) {
3049e39c5baSBill Taylor goto all_eq_init_fail;
3059e39c5baSBill Taylor }
3069e39c5baSBill Taylor num_eq_unmap = 41;
3079e39c5baSBill Taylor
3089e39c5baSBill Taylor /*
3099e39c5baSBill Taylor * Setup EQ41 for handling Path Migration Failed Events
3109e39c5baSBill Taylor *
3119e39c5baSBill Taylor * These events correspond to the IB affiliated asynchronous events
3129e39c5baSBill Taylor * that are used to indicate that path migration was not successful.
3139e39c5baSBill Taylor * If this setup fails for any reason (which, in general, it really
3149e39c5baSBill Taylor * never should), then undo all previous EQ mapping, jump to the end,
3159e39c5baSBill Taylor * cleanup everything that has been successfully initialized, and
3169e39c5baSBill Taylor * return an error.
3179e39c5baSBill Taylor */
3189e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[41],
3199e39c5baSBill Taylor TAVOR_EVT_MSK_PATH_MIGRATE_FAILED, tavor_path_mig_err_handler);
3209e39c5baSBill Taylor if (status != DDI_SUCCESS) {
3219e39c5baSBill Taylor goto all_eq_init_fail;
3229e39c5baSBill Taylor }
3239e39c5baSBill Taylor num_eq_unmap = 42;
3249e39c5baSBill Taylor
3259e39c5baSBill Taylor /*
3269e39c5baSBill Taylor * Setup EQ42 for handling Local Catastrophic Error Events
3279e39c5baSBill Taylor *
3289e39c5baSBill Taylor * These events correspond to the similarly-named IB unaffiliated
3299e39c5baSBill Taylor * asynchronous error type. If this setup fails for any reason
3309e39c5baSBill Taylor * (which, in general, it really never should), then undo all previous
3319e39c5baSBill Taylor * EQ mapping, jump to the end, cleanup everything that has been
3329e39c5baSBill Taylor * successfully initialized, and return an error.
3339e39c5baSBill Taylor *
3349e39c5baSBill Taylor * This error is unique, in that an EQE is not generated if this event
3359e39c5baSBill Taylor * occurs. Instead, an interrupt is called and we must poll the
3369e39c5baSBill Taylor * Catastrophic Error buffer in CR-Space. This mapping is setup simply
3379e39c5baSBill Taylor * to enable this error reporting. We pass in a NULL handler since it
3389e39c5baSBill Taylor * will never be called.
3399e39c5baSBill Taylor */
3409e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[42],
3419e39c5baSBill Taylor TAVOR_EVT_MSK_LOCAL_CAT_ERROR, NULL);
3429e39c5baSBill Taylor if (status != DDI_SUCCESS) {
3439e39c5baSBill Taylor goto all_eq_init_fail;
3449e39c5baSBill Taylor }
3459e39c5baSBill Taylor num_eq_unmap = 43;
3469e39c5baSBill Taylor
3479e39c5baSBill Taylor /*
3489e39c5baSBill Taylor * Setup EQ43 for handling SRQ Catastrophic Error Events
3499e39c5baSBill Taylor *
3509e39c5baSBill Taylor * These events correspond to the similarly-named IB affiliated
3519e39c5baSBill Taylor * asynchronous error type. If this setup fails for any reason
3529e39c5baSBill Taylor * (which, in general, it really never should), then undo all previous
3539e39c5baSBill Taylor * EQ mapping, jump to the end, cleanup everything that has been
3549e39c5baSBill Taylor * successfully initialized, and return an error.
3559e39c5baSBill Taylor */
3569e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[43],
3579e39c5baSBill Taylor TAVOR_EVT_MSK_SRQ_CATASTROPHIC_ERROR,
3589e39c5baSBill Taylor tavor_srq_catastrophic_handler);
3599e39c5baSBill Taylor if (status != DDI_SUCCESS) {
3609e39c5baSBill Taylor goto all_eq_init_fail;
3619e39c5baSBill Taylor }
3629e39c5baSBill Taylor num_eq_unmap = 44;
3639e39c5baSBill Taylor
3649e39c5baSBill Taylor /*
3659e39c5baSBill Taylor * Setup EQ44 for handling SRQ Last WQE Reached Events
3669e39c5baSBill Taylor *
3679e39c5baSBill Taylor * These events correspond to the similarly-named IB affiliated
3689e39c5baSBill Taylor * asynchronous event type. If this setup fails for any reason
3699e39c5baSBill Taylor * (which, in general, it really never should), then undo all previous
3709e39c5baSBill Taylor * EQ mapping, jump to the end, cleanup everything that has been
3719e39c5baSBill Taylor * successfully initialized, and return an error.
3729e39c5baSBill Taylor */
3739e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[44],
3749e39c5baSBill Taylor TAVOR_EVT_MSK_SRQ_LAST_WQE_REACHED,
3759e39c5baSBill Taylor tavor_srq_last_wqe_reached_handler);
3769e39c5baSBill Taylor if (status != DDI_SUCCESS) {
3779e39c5baSBill Taylor goto all_eq_init_fail;
3789e39c5baSBill Taylor }
3799e39c5baSBill Taylor num_eq_unmap = 45;
3809e39c5baSBill Taylor
3819e39c5baSBill Taylor /*
3829e39c5baSBill Taylor * Setup EQ45 for handling ECC error detection events
3839e39c5baSBill Taylor *
3849e39c5baSBill Taylor * These events correspond to the similarly-named IB affiliated
3859e39c5baSBill Taylor * asynchronous event type. If this setup fails for any reason
3869e39c5baSBill Taylor * (which, in general, it really never should), then undo all previous
3879e39c5baSBill Taylor * EQ mapping, jump to the end, cleanup everything that has been
3889e39c5baSBill Taylor * successfully initialized, and return an error.
3899e39c5baSBill Taylor */
3909e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[45],
3919e39c5baSBill Taylor TAVOR_EVT_MSK_ECC_DETECTION,
3929e39c5baSBill Taylor tavor_ecc_detection_handler);
3939e39c5baSBill Taylor if (status != DDI_SUCCESS) {
3949e39c5baSBill Taylor goto all_eq_init_fail;
3959e39c5baSBill Taylor }
3969e39c5baSBill Taylor num_eq_unmap = 46;
3979e39c5baSBill Taylor
3989e39c5baSBill Taylor /*
3999e39c5baSBill Taylor * Setup EQ46 to catch all other types of events. Specifically, we
4009e39c5baSBill Taylor * do not catch the "Local EEC Catastrophic Error Event" because we
4019e39c5baSBill Taylor * should have no EEC (the Tavor driver does not support RD). We also
4029e39c5baSBill Taylor * choose not to handle any of the address translation page fault
4039e39c5baSBill Taylor * event types. Since we are not doing any page fault handling (and
4049e39c5baSBill Taylor * since the Tavor firmware does not currently support any such
4059e39c5baSBill Taylor * handling), we allow these events to go to the catch-all handler.
4069e39c5baSBill Taylor */
4079e39c5baSBill Taylor status = tavor_eq_handler_init(state, state->ts_eqhdl[46],
4089e39c5baSBill Taylor TAVOR_EVT_CATCHALL_MASK, tavor_no_eqhandler);
4099e39c5baSBill Taylor if (status != DDI_SUCCESS) {
4109e39c5baSBill Taylor goto all_eq_init_fail;
4119e39c5baSBill Taylor }
4129e39c5baSBill Taylor
4139e39c5baSBill Taylor return (DDI_SUCCESS);
4149e39c5baSBill Taylor
4159e39c5baSBill Taylor all_eq_init_fail:
4169e39c5baSBill Taylor /* Unmap any of the partially mapped EQs from above */
4179e39c5baSBill Taylor for (i = 0; i < num_eq_unmap; i++) {
4189e39c5baSBill Taylor (void) tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
4199e39c5baSBill Taylor }
4209e39c5baSBill Taylor
4219e39c5baSBill Taylor /* Free up any of the partially allocated EQs from above */
4229e39c5baSBill Taylor for (i = 0; i < num_eq_init; i++) {
4239e39c5baSBill Taylor (void) tavor_eq_free(state, &state->ts_eqhdl[i]);
4249e39c5baSBill Taylor }
4259e39c5baSBill Taylor return (status);
4269e39c5baSBill Taylor }
4279e39c5baSBill Taylor
4289e39c5baSBill Taylor
4299e39c5baSBill Taylor /*
4309e39c5baSBill Taylor * tavor_eq_fini_all
4319e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts
4329e39c5baSBill Taylor */
4339e39c5baSBill Taylor int
tavor_eq_fini_all(tavor_state_t * state)4349e39c5baSBill Taylor tavor_eq_fini_all(tavor_state_t *state)
4359e39c5baSBill Taylor {
4369e39c5baSBill Taylor uint_t num_eq;
4379e39c5baSBill Taylor int status, i;
4389e39c5baSBill Taylor
4399e39c5baSBill Taylor /*
4409e39c5baSBill Taylor * Grab the total number of supported EQs again. This is the same
4419e39c5baSBill Taylor * hardcoded value that was used above (during the event queue
4429e39c5baSBill Taylor * initialization.)
4439e39c5baSBill Taylor */
4449e39c5baSBill Taylor num_eq = TAVOR_NUM_EQ_USED;
4459e39c5baSBill Taylor
4469e39c5baSBill Taylor /*
4479e39c5baSBill Taylor * For each of the event queues that we initialized and mapped
4489e39c5baSBill Taylor * earlier, attempt to unmap the events from the EQ.
4499e39c5baSBill Taylor */
4509e39c5baSBill Taylor for (i = 0; i < num_eq; i++) {
4519e39c5baSBill Taylor status = tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
4529e39c5baSBill Taylor if (status != DDI_SUCCESS) {
4539e39c5baSBill Taylor return (DDI_FAILURE);
4549e39c5baSBill Taylor }
4559e39c5baSBill Taylor }
4569e39c5baSBill Taylor
4579e39c5baSBill Taylor /*
4589e39c5baSBill Taylor * Teardown and free up all the Event Queues that were allocated
4599e39c5baSBill Taylor * earlier.
4609e39c5baSBill Taylor */
4619e39c5baSBill Taylor for (i = 0; i < num_eq; i++) {
4629e39c5baSBill Taylor status = tavor_eq_free(state, &state->ts_eqhdl[i]);
4639e39c5baSBill Taylor if (status != DDI_SUCCESS) {
4649e39c5baSBill Taylor return (DDI_FAILURE);
4659e39c5baSBill Taylor }
4669e39c5baSBill Taylor }
4679e39c5baSBill Taylor
4689e39c5baSBill Taylor return (DDI_SUCCESS);
4699e39c5baSBill Taylor }
4709e39c5baSBill Taylor
4719e39c5baSBill Taylor
4729e39c5baSBill Taylor /*
4739e39c5baSBill Taylor * tavor_eq_arm_all
4749e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts
4759e39c5baSBill Taylor */
4769e39c5baSBill Taylor void
tavor_eq_arm_all(tavor_state_t * state)4779e39c5baSBill Taylor tavor_eq_arm_all(tavor_state_t *state)
4789e39c5baSBill Taylor {
4799e39c5baSBill Taylor uint_t num_eq;
4809e39c5baSBill Taylor int i;
4819e39c5baSBill Taylor
4829e39c5baSBill Taylor /*
4839e39c5baSBill Taylor * Grab the total number of supported EQs again. This is the same
4849e39c5baSBill Taylor * hardcoded value that was used above (during the event queue
4859e39c5baSBill Taylor * initialization.)
4869e39c5baSBill Taylor */
4879e39c5baSBill Taylor num_eq = TAVOR_NUM_EQ_USED;
4889e39c5baSBill Taylor
4899e39c5baSBill Taylor /*
4909e39c5baSBill Taylor * For each of the event queues that we initialized and mapped
4919e39c5baSBill Taylor * earlier, attempt to arm it for event generation.
4929e39c5baSBill Taylor */
4939e39c5baSBill Taylor for (i = 0; i < num_eq; i++) {
4949e39c5baSBill Taylor tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, i, 0);
4959e39c5baSBill Taylor }
4969e39c5baSBill Taylor }
4979e39c5baSBill Taylor
4989e39c5baSBill Taylor
4999e39c5baSBill Taylor /*
5009e39c5baSBill Taylor * tavor_isr()
5019e39c5baSBill Taylor * Context: Only called from interrupt context (and during panic)
5029e39c5baSBill Taylor */
5039e39c5baSBill Taylor /* ARGSUSED */
5049e39c5baSBill Taylor uint_t
tavor_isr(caddr_t arg1,caddr_t arg2)5059e39c5baSBill Taylor tavor_isr(caddr_t arg1, caddr_t arg2)
5069e39c5baSBill Taylor {
5079e39c5baSBill Taylor tavor_state_t *state;
5089e39c5baSBill Taylor uint64_t *ecr, *clr_int;
5099e39c5baSBill Taylor uint64_t ecrreg, int_mask;
5109e39c5baSBill Taylor uint_t status;
5119e39c5baSBill Taylor int i;
5129e39c5baSBill Taylor
5139e39c5baSBill Taylor /*
5149e39c5baSBill Taylor * Grab the Tavor softstate pointer from the input parameter
5159e39c5baSBill Taylor */
5169e39c5baSBill Taylor state = (tavor_state_t *)arg1;
5179e39c5baSBill Taylor
5189e39c5baSBill Taylor /*
5199e39c5baSBill Taylor * Find the pointers to the ECR and clr_INT registers
5209e39c5baSBill Taylor */
5219e39c5baSBill Taylor ecr = state->ts_cmd_regs.ecr;
5229e39c5baSBill Taylor clr_int = state->ts_cmd_regs.clr_int;
5239e39c5baSBill Taylor
5249e39c5baSBill Taylor /*
5259e39c5baSBill Taylor * Read the ECR register. Each of the 64 bits in the ECR register
5269e39c5baSBill Taylor * corresponds to an event queue. If a bit is set, then the
5279e39c5baSBill Taylor * corresponding event queue has fired.
5289e39c5baSBill Taylor */
5299e39c5baSBill Taylor ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
5309e39c5baSBill Taylor
5319e39c5baSBill Taylor /*
5329e39c5baSBill Taylor * As long as there are bits set (i.e. as long as there are still
5339e39c5baSBill Taylor * EQs in the "fired" state), call tavor_eq_poll() to process each
5349e39c5baSBill Taylor * fired EQ. If no ECR bits are set, do not claim the interrupt.
5359e39c5baSBill Taylor */
5369e39c5baSBill Taylor status = DDI_INTR_UNCLAIMED;
5379e39c5baSBill Taylor do {
5389e39c5baSBill Taylor i = 0;
5399e39c5baSBill Taylor while (ecrreg != 0x0) {
5409e39c5baSBill Taylor if (ecrreg & 0x1) {
5419e39c5baSBill Taylor tavor_eq_poll(state, state->ts_eqhdl[i]);
5429e39c5baSBill Taylor status = DDI_INTR_CLAIMED;
5439e39c5baSBill Taylor }
5449e39c5baSBill Taylor ecrreg = ecrreg >> 1;
5459e39c5baSBill Taylor i++;
5469e39c5baSBill Taylor }
5479e39c5baSBill Taylor
5489e39c5baSBill Taylor /*
5499e39c5baSBill Taylor * Clear the interrupt. Note: Depending on the type of
5509e39c5baSBill Taylor * event (interrupt or MSI), we need to use a different
5519e39c5baSBill Taylor * mask to clear the event. In the case of MSI, the bit
5529e39c5baSBill Taylor * to clear corresponds to the MSI number, and for legacy
5539e39c5baSBill Taylor * interrupts the bit corresponds to the value in 'inta_pin'.
5549e39c5baSBill Taylor */
5559e39c5baSBill Taylor if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
5569e39c5baSBill Taylor int_mask = ((uint64_t)1 << 0);
5579e39c5baSBill Taylor } else {
5589e39c5baSBill Taylor int_mask = ((uint64_t)1 << state->ts_adapter.inta_pin);
5599e39c5baSBill Taylor }
5609e39c5baSBill Taylor ddi_put64(state->ts_reg_cmdhdl, clr_int, int_mask);
5619e39c5baSBill Taylor
5629e39c5baSBill Taylor /* Reread the ECR register */
5639e39c5baSBill Taylor ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
5649e39c5baSBill Taylor
5659e39c5baSBill Taylor } while (ecrreg != 0x0);
5669e39c5baSBill Taylor
5679e39c5baSBill Taylor return (status);
5689e39c5baSBill Taylor }
5699e39c5baSBill Taylor
5709e39c5baSBill Taylor
5719e39c5baSBill Taylor /*
5729e39c5baSBill Taylor * tavor_eq_doorbell
5739e39c5baSBill Taylor * Context: Only called from interrupt context
5749e39c5baSBill Taylor */
5759e39c5baSBill Taylor void
tavor_eq_doorbell(tavor_state_t * state,uint32_t eq_cmd,uint32_t eqn,uint32_t eq_param)5769e39c5baSBill Taylor tavor_eq_doorbell(tavor_state_t *state, uint32_t eq_cmd, uint32_t eqn,
5779e39c5baSBill Taylor uint32_t eq_param)
5789e39c5baSBill Taylor {
5799e39c5baSBill Taylor uint64_t doorbell = 0;
5809e39c5baSBill Taylor
5819e39c5baSBill Taylor /* Build the doorbell from the parameters */
5829e39c5baSBill Taylor doorbell = ((uint64_t)eq_cmd << TAVOR_EQDB_CMD_SHIFT) |
5839e39c5baSBill Taylor ((uint64_t)eqn << TAVOR_EQDB_EQN_SHIFT) | eq_param;
5849e39c5baSBill Taylor
5859e39c5baSBill Taylor /* Write the doorbell to UAR */
5869e39c5baSBill Taylor TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->eq,
5879e39c5baSBill Taylor doorbell);
5889e39c5baSBill Taylor }
5899e39c5baSBill Taylor
5909e39c5baSBill Taylor /*
5919e39c5baSBill Taylor * tavor_eq_poll
5929e39c5baSBill Taylor * Context: Only called from interrupt context (and during panic)
5939e39c5baSBill Taylor */
5949e39c5baSBill Taylor static void
tavor_eq_poll(tavor_state_t * state,tavor_eqhdl_t eq)5959e39c5baSBill Taylor tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq)
5969e39c5baSBill Taylor {
5979e39c5baSBill Taylor uint64_t *clr_ecr;
5989e39c5baSBill Taylor tavor_hw_eqe_t *eqe;
5999e39c5baSBill Taylor uint64_t ecr_mask;
6009e39c5baSBill Taylor uint32_t cons_indx, wrap_around_mask;
6019e39c5baSBill Taylor int (*eqfunction)(tavor_state_t *state, tavor_eqhdl_t eq,
6029e39c5baSBill Taylor tavor_hw_eqe_t *eqe);
6039e39c5baSBill Taylor
6049e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eq))
6059e39c5baSBill Taylor
6069e39c5baSBill Taylor /* Find the pointer to the clr_ECR register */
6079e39c5baSBill Taylor clr_ecr = state->ts_cmd_regs.clr_ecr;
6089e39c5baSBill Taylor
6099e39c5baSBill Taylor /*
6109e39c5baSBill Taylor * Check for Local Catastrophic Error If we have this kind of error,
6119e39c5baSBill Taylor * then we don't need to do anything else here, as this kind of
6129e39c5baSBill Taylor * catastrophic error is handled separately. So we call the
6139e39c5baSBill Taylor * catastrophic handler, clear the ECR and then return.
6149e39c5baSBill Taylor */
6159e39c5baSBill Taylor if (eq->eq_evttypemask == TAVOR_EVT_MSK_LOCAL_CAT_ERROR) {
6169e39c5baSBill Taylor /*
6179e39c5baSBill Taylor * Call Catastrophic Error handler
6189e39c5baSBill Taylor */
6199e39c5baSBill Taylor tavor_eq_catastrophic(state);
6209e39c5baSBill Taylor
6219e39c5baSBill Taylor /*
6229e39c5baSBill Taylor * Clear the ECR. Specifically, clear the bit corresponding
6239e39c5baSBill Taylor * to the event queue just processed.
6249e39c5baSBill Taylor */
6259e39c5baSBill Taylor ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
6269e39c5baSBill Taylor ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
6279e39c5baSBill Taylor
6289e39c5baSBill Taylor return;
6299e39c5baSBill Taylor }
6309e39c5baSBill Taylor
6319e39c5baSBill Taylor /* Get the consumer pointer index */
6329e39c5baSBill Taylor cons_indx = eq->eq_consindx;
6339e39c5baSBill Taylor
6349e39c5baSBill Taylor /*
6359e39c5baSBill Taylor * Calculate the wrap around mask. Note: This operation only works
6369e39c5baSBill Taylor * because all Tavor event queues have power-of-2 sizes
6379e39c5baSBill Taylor */
6389e39c5baSBill Taylor wrap_around_mask = (eq->eq_bufsz - 1);
6399e39c5baSBill Taylor
6409e39c5baSBill Taylor /* Calculate the pointer to the first EQ entry */
6419e39c5baSBill Taylor eqe = &eq->eq_buf[cons_indx];
6429e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eqe))
6439e39c5baSBill Taylor
6449e39c5baSBill Taylor /*
6459e39c5baSBill Taylor * Sync the current EQE to read
6469e39c5baSBill Taylor * We need to force a ddi_dma_sync() here (independent of how the
6479e39c5baSBill Taylor * EQ was mapped) because it is possible for us to receive the
6489e39c5baSBill Taylor * interrupt, do a read of the ECR, and have each of these
6499e39c5baSBill Taylor * operations complete successfully even though the hardware's DMA
6509e39c5baSBill Taylor * to the EQ has not yet completed.
6519e39c5baSBill Taylor */
6529e39c5baSBill Taylor tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU, TAVOR_EQ_SYNC_FORCE);
6539e39c5baSBill Taylor
6549e39c5baSBill Taylor /*
6559e39c5baSBill Taylor * Pull the handler function for this EQ from the Tavor Event Queue
6569e39c5baSBill Taylor * handle
6579e39c5baSBill Taylor */
6589e39c5baSBill Taylor eqfunction = eq->eq_func;
6599e39c5baSBill Taylor
6609e39c5baSBill Taylor /*
6619e39c5baSBill Taylor * Keep pulling entries from the EQ until we find an entry owner by
6629e39c5baSBill Taylor * the hardware. As long as there the EQE's owned by SW, process
6639e39c5baSBill Taylor * each entry by calling its handler function and updating the EQ
6649e39c5baSBill Taylor * consumer index.
6659e39c5baSBill Taylor */
6669e39c5baSBill Taylor do {
6679e39c5baSBill Taylor while (TAVOR_EQE_OWNER_IS_SW(eq, eqe)) {
6689e39c5baSBill Taylor /*
6699e39c5baSBill Taylor * Call the EQ handler function. But only call if we
6709e39c5baSBill Taylor * are not in polled I/O mode (i.e. not processing
6719e39c5baSBill Taylor * because of a system panic). Note: We don't call
6729e39c5baSBill Taylor * the EQ handling functions from a system panic
6739e39c5baSBill Taylor * because we are primarily concerned only with
6749e39c5baSBill Taylor * ensuring that the event queues do not overflow (or,
6759e39c5baSBill Taylor * more specifically, the event queue associated with
6769e39c5baSBill Taylor * the CQ that is being used in the sync/dump process).
6779e39c5baSBill Taylor * Also, we don't want to make any upcalls (to the
6789e39c5baSBill Taylor * IBTF) because we can't guarantee when/if those
6799e39c5baSBill Taylor * calls would ever return. And, if we're in panic,
6809e39c5baSBill Taylor * then we reached here through a PollCQ() call (from
6819e39c5baSBill Taylor * tavor_cq_poll()), and we need to ensure that we
6829e39c5baSBill Taylor * successfully return any work completions to the
6839e39c5baSBill Taylor * caller.
6849e39c5baSBill Taylor */
6859e39c5baSBill Taylor if (ddi_in_panic() == 0) {
6869e39c5baSBill Taylor eqfunction(state, eq, eqe);
6879e39c5baSBill Taylor }
6889e39c5baSBill Taylor
6899e39c5baSBill Taylor /* Reset entry to hardware ownership */
6909e39c5baSBill Taylor TAVOR_EQE_OWNER_SET_HW(eq, eqe);
6919e39c5baSBill Taylor
6929e39c5baSBill Taylor /* Sync the current EQE for device */
6939e39c5baSBill Taylor tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORDEV,
6949e39c5baSBill Taylor TAVOR_EQ_SYNC_NORMAL);
6959e39c5baSBill Taylor
6969e39c5baSBill Taylor /* Increment the consumer index */
6979e39c5baSBill Taylor cons_indx = (cons_indx + 1) & wrap_around_mask;
6989e39c5baSBill Taylor
6999e39c5baSBill Taylor /* Update the pointer to the next EQ entry */
7009e39c5baSBill Taylor eqe = &eq->eq_buf[cons_indx];
7019e39c5baSBill Taylor
7029e39c5baSBill Taylor /* Sync the next EQE to read */
7039e39c5baSBill Taylor tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
7049e39c5baSBill Taylor TAVOR_EQ_SYNC_NORMAL);
7059e39c5baSBill Taylor }
7069e39c5baSBill Taylor
7079e39c5baSBill Taylor /*
7089e39c5baSBill Taylor * Clear the ECR. Specifically, clear the bit corresponding
7099e39c5baSBill Taylor * to the event queue just processed.
7109e39c5baSBill Taylor */
7119e39c5baSBill Taylor ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
7129e39c5baSBill Taylor ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
7139e39c5baSBill Taylor
7149e39c5baSBill Taylor /* Write an EQ doorbell to update the consumer index */
7159e39c5baSBill Taylor eq->eq_consindx = cons_indx;
7169e39c5baSBill Taylor tavor_eq_doorbell(state, TAVOR_EQDB_SET_CONSINDX, eq->eq_eqnum,
7179e39c5baSBill Taylor cons_indx);
7189e39c5baSBill Taylor
7199e39c5baSBill Taylor /* Write another EQ doorbell to rearm */
7209e39c5baSBill Taylor tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, eq->eq_eqnum, 0);
7219e39c5baSBill Taylor
7229e39c5baSBill Taylor /*
7239e39c5baSBill Taylor * NOTE: Due to the nature of Mellanox hardware, we do not have
7249e39c5baSBill Taylor * to do an explicit PIO read to ensure that the doorbell write
7259e39c5baSBill Taylor * has been flushed to the hardware. There is state encoded in
7269e39c5baSBill Taylor * the doorbell information we write which makes this
7279e39c5baSBill Taylor * unnecessary. We can be assured that if an event needs to be
7289e39c5baSBill Taylor * generated, the hardware will make sure that it is, solving
7299e39c5baSBill Taylor * the possible race condition.
7309e39c5baSBill Taylor */
7319e39c5baSBill Taylor
7329e39c5baSBill Taylor /* Sync the next EQE to read */
7339e39c5baSBill Taylor tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
7349e39c5baSBill Taylor TAVOR_EQ_SYNC_NORMAL);
7359e39c5baSBill Taylor
7369e39c5baSBill Taylor } while (TAVOR_EQE_OWNER_IS_SW(eq, eqe));
7379e39c5baSBill Taylor }
7389e39c5baSBill Taylor
7399e39c5baSBill Taylor
7409e39c5baSBill Taylor /*
7419e39c5baSBill Taylor * tavor_eq_catastrophic
7429e39c5baSBill Taylor * Context: Only called from interrupt context (and during panic)
7439e39c5baSBill Taylor */
7449e39c5baSBill Taylor static void
tavor_eq_catastrophic(tavor_state_t * state)7459e39c5baSBill Taylor tavor_eq_catastrophic(tavor_state_t *state)
7469e39c5baSBill Taylor {
7479e39c5baSBill Taylor ibt_async_code_t type;
7489e39c5baSBill Taylor ibc_async_event_t event;
7499e39c5baSBill Taylor uint32_t *base_addr;
7509e39c5baSBill Taylor uint32_t buf_size;
7519e39c5baSBill Taylor uint32_t word;
7529e39c5baSBill Taylor uint8_t err_type;
7539e39c5baSBill Taylor uint32_t err_buf;
7549e39c5baSBill Taylor int i;
7559e39c5baSBill Taylor
7569e39c5baSBill Taylor bzero(&event, sizeof (ibc_async_event_t));
7579e39c5baSBill Taylor
7589e39c5baSBill Taylor base_addr = (uint32_t *)(uintptr_t)(
7599e39c5baSBill Taylor (uintptr_t)state->ts_reg_cmd_baseaddr +
7609e39c5baSBill Taylor state->ts_fw.error_buf_addr);
7619e39c5baSBill Taylor buf_size = state->ts_fw.error_buf_sz;
7629e39c5baSBill Taylor
7639e39c5baSBill Taylor word = ddi_get32(state->ts_reg_cmdhdl, base_addr);
7649e39c5baSBill Taylor
7659e39c5baSBill Taylor err_type = (word & 0xFF000000) >> 24;
7669e39c5baSBill Taylor type = IBT_ERROR_LOCAL_CATASTROPHIC;
7679e39c5baSBill Taylor
7689e39c5baSBill Taylor switch (err_type) {
7699e39c5baSBill Taylor case TAVOR_CATASTROPHIC_INTERNAL_ERROR:
7709e39c5baSBill Taylor cmn_err(CE_WARN, "Catastrophic Internal Error: 0x%02x",
7719e39c5baSBill Taylor err_type);
7729e39c5baSBill Taylor
7739e39c5baSBill Taylor break;
7749e39c5baSBill Taylor
7759e39c5baSBill Taylor case TAVOR_CATASTROPHIC_UPLINK_BUS_ERROR:
7769e39c5baSBill Taylor cmn_err(CE_WARN, "Catastrophic Uplink Bus Error: 0x%02x",
7779e39c5baSBill Taylor err_type);
7789e39c5baSBill Taylor
7799e39c5baSBill Taylor break;
7809e39c5baSBill Taylor
7819e39c5baSBill Taylor case TAVOR_CATASTROPHIC_DDR_DATA_ERROR:
7829e39c5baSBill Taylor cmn_err(CE_WARN, "Catastrophic DDR Data Error: 0x%02x",
7839e39c5baSBill Taylor err_type);
7849e39c5baSBill Taylor
7859e39c5baSBill Taylor break;
7869e39c5baSBill Taylor
7879e39c5baSBill Taylor case TAVOR_CATASTROPHIC_INTERNAL_PARITY_ERROR:
7889e39c5baSBill Taylor cmn_err(CE_WARN, "Catastrophic Internal Parity Error: 0x%02x",
7899e39c5baSBill Taylor err_type);
7909e39c5baSBill Taylor
7919e39c5baSBill Taylor break;
7929e39c5baSBill Taylor
7939e39c5baSBill Taylor default:
7949e39c5baSBill Taylor /* Unknown type of Catastrophic error */
7959e39c5baSBill Taylor cmn_err(CE_WARN, "Catastrophic Unknown Error: 0x%02x",
7969e39c5baSBill Taylor err_type);
7979e39c5baSBill Taylor
7989e39c5baSBill Taylor break;
7999e39c5baSBill Taylor }
8009e39c5baSBill Taylor
8019e39c5baSBill Taylor /*
8029e39c5baSBill Taylor * Read in the catastrophic error buffer from the hardware, printing
8039e39c5baSBill Taylor * only to the log file only
8049e39c5baSBill Taylor */
8059e39c5baSBill Taylor for (i = 0; i < buf_size; i += 4) {
8069e39c5baSBill Taylor base_addr = (uint32_t *)((uintptr_t)(state->ts_reg_cmd_baseaddr
8079e39c5baSBill Taylor + state->ts_fw.error_buf_addr + (i * 4)));
8089e39c5baSBill Taylor err_buf = ddi_get32(state->ts_reg_cmdhdl, base_addr);
8099e39c5baSBill Taylor cmn_err(CE_WARN, "catastrophic_error[%02x]: %08X", i, err_buf);
8109e39c5baSBill Taylor }
8119e39c5baSBill Taylor
8129e39c5baSBill Taylor /*
8139e39c5baSBill Taylor * We also call the IBTF here to inform it of the catastrophic error.
8149e39c5baSBill Taylor * Note: Since no event information (i.e. QP handles, CQ handles,
8159e39c5baSBill Taylor * etc.) is necessary, we pass a NULL pointer instead of a pointer to
8169e39c5baSBill Taylor * an empty ibc_async_event_t struct.
8179e39c5baSBill Taylor *
8189e39c5baSBill Taylor * But we also check if "ts_ibtfpriv" is NULL. If it is then it
8199e39c5baSBill Taylor * means that we've have either received this event before we
8209e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
8219e39c5baSBill Taylor * are in the process of detaching.
8229e39c5baSBill Taylor */
8239e39c5baSBill Taylor if (state->ts_ibtfpriv != NULL) {
8249e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
8259e39c5baSBill Taylor }
8269e39c5baSBill Taylor }
8279e39c5baSBill Taylor
8289e39c5baSBill Taylor
8299e39c5baSBill Taylor /*
8309e39c5baSBill Taylor * tavor_eq_alloc()
8319e39c5baSBill Taylor * Context: Only called from attach() path context
8329e39c5baSBill Taylor */
8339e39c5baSBill Taylor static int
tavor_eq_alloc(tavor_state_t * state,uint32_t log_eq_size,uint_t intr,tavor_eqhdl_t * eqhdl)8349e39c5baSBill Taylor tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size, uint_t intr,
8359e39c5baSBill Taylor tavor_eqhdl_t *eqhdl)
8369e39c5baSBill Taylor {
8379e39c5baSBill Taylor tavor_rsrc_t *eqc, *rsrc;
8389e39c5baSBill Taylor tavor_hw_eqc_t eqc_entry;
8399e39c5baSBill Taylor tavor_eqhdl_t eq;
8409e39c5baSBill Taylor ibt_mr_attr_t mr_attr;
8419e39c5baSBill Taylor tavor_mr_options_t op;
8429e39c5baSBill Taylor tavor_pdhdl_t pd;
8439e39c5baSBill Taylor tavor_mrhdl_t mr;
8449e39c5baSBill Taylor tavor_hw_eqe_t *buf;
8459e39c5baSBill Taylor uint64_t addr;
8469e39c5baSBill Taylor uint32_t lkey;
8479e39c5baSBill Taylor uint_t dma_xfer_mode;
8489e39c5baSBill Taylor int status, i;
8499e39c5baSBill Taylor
8509e39c5baSBill Taylor /* Use the internal protection domain (PD) for setting up EQs */
8519e39c5baSBill Taylor pd = state->ts_pdhdl_internal;
8529e39c5baSBill Taylor
8539e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */
8549e39c5baSBill Taylor tavor_pd_refcnt_inc(pd);
8559e39c5baSBill Taylor
8569e39c5baSBill Taylor /*
8579e39c5baSBill Taylor * Allocate an EQ context entry. This will be filled in with all
8589e39c5baSBill Taylor * the necessary parameters to define the Event Queue. And then
8599e39c5baSBill Taylor * ownership will be passed to the hardware in the final step
8609e39c5baSBill Taylor * below. If we fail here, we must undo the protection domain
8619e39c5baSBill Taylor * reference count.
8629e39c5baSBill Taylor */
8639e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_EQC, 1, TAVOR_SLEEP, &eqc);
8649e39c5baSBill Taylor if (status != DDI_SUCCESS) {
8659e39c5baSBill Taylor goto eqalloc_fail1;
8669e39c5baSBill Taylor }
8679e39c5baSBill Taylor
8689e39c5baSBill Taylor /*
8699e39c5baSBill Taylor * Allocate the software structure for tracking the event queue (i.e.
8709e39c5baSBill Taylor * the Tavor Event Queue handle). If we fail here, we must undo the
8719e39c5baSBill Taylor * protection domain reference count and the previous resource
8729e39c5baSBill Taylor * allocation.
8739e39c5baSBill Taylor */
8749e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_EQHDL, 1, TAVOR_SLEEP, &rsrc);
8759e39c5baSBill Taylor if (status != DDI_SUCCESS) {
8769e39c5baSBill Taylor goto eqalloc_fail2;
8779e39c5baSBill Taylor }
8789e39c5baSBill Taylor eq = (tavor_eqhdl_t)rsrc->tr_addr;
8799e39c5baSBill Taylor
8809e39c5baSBill Taylor /*
8819e39c5baSBill Taylor * Allocate the memory for Event Queue. Note: Although we use the
8829e39c5baSBill Taylor * common queue allocation routine, we always specify
8839e39c5baSBill Taylor * TAVOR_QUEUE_LOCATION_NORMAL (i.e. EQ located in system memory)
8849e39c5baSBill Taylor * because it would be inefficient to have EQs located in DDR memory.
8859e39c5baSBill Taylor * This is primarily because EQs are read from (by software) more
8869e39c5baSBill Taylor * than they are written to. Also note that, unlike Tavor QP work
8879e39c5baSBill Taylor * queues, event queues do not have the same strict alignment
8889e39c5baSBill Taylor * requirements. It is sufficient for the EQ memory to be both
8899e39c5baSBill Taylor * aligned to and bound to addresses which are a multiple of EQE size.
8909e39c5baSBill Taylor */
8919e39c5baSBill Taylor eq->eq_eqinfo.qa_size = (1 << log_eq_size) * sizeof (tavor_hw_eqe_t);
8929e39c5baSBill Taylor eq->eq_eqinfo.qa_alloc_align = sizeof (tavor_hw_eqe_t);
8939e39c5baSBill Taylor eq->eq_eqinfo.qa_bind_align = sizeof (tavor_hw_eqe_t);
8949e39c5baSBill Taylor eq->eq_eqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
8959e39c5baSBill Taylor status = tavor_queue_alloc(state, &eq->eq_eqinfo, TAVOR_SLEEP);
8969e39c5baSBill Taylor if (status != DDI_SUCCESS) {
8979e39c5baSBill Taylor goto eqalloc_fail3;
8989e39c5baSBill Taylor }
8999e39c5baSBill Taylor buf = (tavor_hw_eqe_t *)eq->eq_eqinfo.qa_buf_aligned;
9009e39c5baSBill Taylor
9019e39c5baSBill Taylor /*
9029e39c5baSBill Taylor * Initialize each of the Event Queue Entries (EQE) by setting their
9039e39c5baSBill Taylor * ownership to hardware ("owner" bit set to HW). This is in
9049e39c5baSBill Taylor * preparation for the final transfer of ownership (below) of the
9059e39c5baSBill Taylor * EQ context itself.
9069e39c5baSBill Taylor */
9079e39c5baSBill Taylor for (i = 0; i < (1 << log_eq_size); i++) {
9089e39c5baSBill Taylor TAVOR_EQE_OWNER_SET_HW(eq, &buf[i]);
9099e39c5baSBill Taylor }
9109e39c5baSBill Taylor
9119e39c5baSBill Taylor /*
9129e39c5baSBill Taylor * Register the memory for the EQ. The memory for the EQ must
9139e39c5baSBill Taylor * be registered in the Tavor TPT tables. This gives us the LKey
9149e39c5baSBill Taylor * to specify in the EQ context below.
9159e39c5baSBill Taylor *
9169e39c5baSBill Taylor * Because we are in the attach path we use NOSLEEP here so that we
9179e39c5baSBill Taylor * SPIN in the HCR since the event queues are not setup yet, and we
9189e39c5baSBill Taylor * cannot NOSPIN at this point in time.
9199e39c5baSBill Taylor */
9209e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
9219e39c5baSBill Taylor mr_attr.mr_len = eq->eq_eqinfo.qa_size;
9229e39c5baSBill Taylor mr_attr.mr_as = NULL;
9239e39c5baSBill Taylor mr_attr.mr_flags = IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
9249e39c5baSBill Taylor dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
9259e39c5baSBill Taylor if (dma_xfer_mode == DDI_DMA_STREAMING) {
9269e39c5baSBill Taylor mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
9279e39c5baSBill Taylor }
9289e39c5baSBill Taylor op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
9299e39c5baSBill Taylor op.mro_bind_dmahdl = eq->eq_eqinfo.qa_dmahdl;
9309e39c5baSBill Taylor op.mro_bind_override_addr = 0;
9319e39c5baSBill Taylor status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
9329e39c5baSBill Taylor if (status != DDI_SUCCESS) {
9339e39c5baSBill Taylor goto eqalloc_fail4;
9349e39c5baSBill Taylor }
9359e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
9369e39c5baSBill Taylor addr = mr->mr_bindinfo.bi_addr;
9379e39c5baSBill Taylor lkey = mr->mr_lkey;
9389e39c5baSBill Taylor
9399e39c5baSBill Taylor /* Determine if later ddi_dma_sync will be necessary */
9409e39c5baSBill Taylor eq->eq_sync = TAVOR_EQ_IS_SYNC_REQ(state, eq->eq_eqinfo);
9419e39c5baSBill Taylor
9429e39c5baSBill Taylor /* Sync entire EQ for use by the hardware (if necessary) */
9439e39c5baSBill Taylor if (eq->eq_sync) {
9449e39c5baSBill Taylor (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
9459e39c5baSBill Taylor eq->eq_eqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
9469e39c5baSBill Taylor }
9479e39c5baSBill Taylor
9489e39c5baSBill Taylor /*
9499e39c5baSBill Taylor * Fill in the EQC entry. This is the final step before passing
9509e39c5baSBill Taylor * ownership of the EQC entry to the Tavor hardware. We use all of
9519e39c5baSBill Taylor * the information collected/calculated above to fill in the
9529e39c5baSBill Taylor * requisite portions of the EQC. Note: We create all EQs in the
9539e39c5baSBill Taylor * "fired" state. We will arm them later (after our interrupt
9549e39c5baSBill Taylor * routine had been registered.)
9559e39c5baSBill Taylor */
9569e39c5baSBill Taylor bzero(&eqc_entry, sizeof (tavor_hw_eqc_t));
9579e39c5baSBill Taylor eqc_entry.owner = TAVOR_HW_OWNER;
9589e39c5baSBill Taylor eqc_entry.xlat = TAVOR_VA2PA_XLAT_ENABLED;
9599e39c5baSBill Taylor eqc_entry.state = TAVOR_EQ_FIRED;
9609e39c5baSBill Taylor eqc_entry.start_addr_h = (addr >> 32);
9619e39c5baSBill Taylor eqc_entry.start_addr_l = (addr & 0xFFFFFFFF);
9629e39c5baSBill Taylor eqc_entry.log_eq_sz = log_eq_size;
9639e39c5baSBill Taylor eqc_entry.usr_page = 0;
9649e39c5baSBill Taylor eqc_entry.pd = pd->pd_pdnum;
9659e39c5baSBill Taylor eqc_entry.intr = intr;
9669e39c5baSBill Taylor eqc_entry.lkey = lkey;
9679e39c5baSBill Taylor
9689e39c5baSBill Taylor /*
9699e39c5baSBill Taylor * Write the EQC entry to hardware. Lastly, we pass ownership of
9709e39c5baSBill Taylor * the entry to the hardware (using the Tavor SW2HW_EQ firmware
9719e39c5baSBill Taylor * command). Note: in general, this operation shouldn't fail. But
9729e39c5baSBill Taylor * if it does, we have to undo everything we've done above before
9739e39c5baSBill Taylor * returning error.
9749e39c5baSBill Taylor */
9759e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_EQ, &eqc_entry,
9769e39c5baSBill Taylor sizeof (tavor_hw_eqc_t), eqc->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
9779e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
9789e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_EQ command failed: %08x\n",
9799e39c5baSBill Taylor status);
9809e39c5baSBill Taylor goto eqalloc_fail5;
9819e39c5baSBill Taylor }
9829e39c5baSBill Taylor
9839e39c5baSBill Taylor /*
9849e39c5baSBill Taylor * Fill in the rest of the Tavor Event Queue handle. Having
9859e39c5baSBill Taylor * successfully transferred ownership of the EQC, we can update the
9869e39c5baSBill Taylor * following fields for use in further operations on the EQ.
9879e39c5baSBill Taylor */
9889e39c5baSBill Taylor eq->eq_eqcrsrcp = eqc;
9899e39c5baSBill Taylor eq->eq_rsrcp = rsrc;
9909e39c5baSBill Taylor eq->eq_consindx = 0;
9919e39c5baSBill Taylor eq->eq_eqnum = eqc->tr_indx;
9929e39c5baSBill Taylor eq->eq_buf = buf;
9939e39c5baSBill Taylor eq->eq_bufsz = (1 << log_eq_size);
9949e39c5baSBill Taylor eq->eq_mrhdl = mr;
9959e39c5baSBill Taylor *eqhdl = eq;
9969e39c5baSBill Taylor
9979e39c5baSBill Taylor return (DDI_SUCCESS);
9989e39c5baSBill Taylor
9999e39c5baSBill Taylor /*
10009e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine
10019e39c5baSBill Taylor */
10029e39c5baSBill Taylor eqalloc_fail5:
10039e39c5baSBill Taylor if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
10049e39c5baSBill Taylor TAVOR_NOSLEEP) != DDI_SUCCESS) {
10059e39c5baSBill Taylor TAVOR_WARNING(state, "failed to deregister EQ memory");
10069e39c5baSBill Taylor }
10079e39c5baSBill Taylor eqalloc_fail4:
10089e39c5baSBill Taylor tavor_queue_free(state, &eq->eq_eqinfo);
10099e39c5baSBill Taylor eqalloc_fail3:
10109e39c5baSBill Taylor tavor_rsrc_free(state, &rsrc);
10119e39c5baSBill Taylor eqalloc_fail2:
10129e39c5baSBill Taylor tavor_rsrc_free(state, &eqc);
10139e39c5baSBill Taylor eqalloc_fail1:
10149e39c5baSBill Taylor tavor_pd_refcnt_dec(pd);
10159e39c5baSBill Taylor eqalloc_fail:
10169e39c5baSBill Taylor return (status);
10179e39c5baSBill Taylor }
10189e39c5baSBill Taylor
10199e39c5baSBill Taylor
10209e39c5baSBill Taylor /*
10219e39c5baSBill Taylor * tavor_eq_free()
10229e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts
10239e39c5baSBill Taylor */
10249e39c5baSBill Taylor static int
tavor_eq_free(tavor_state_t * state,tavor_eqhdl_t * eqhdl)10259e39c5baSBill Taylor tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl)
10269e39c5baSBill Taylor {
10279e39c5baSBill Taylor tavor_rsrc_t *eqc, *rsrc;
10289e39c5baSBill Taylor tavor_hw_eqc_t eqc_entry;
10299e39c5baSBill Taylor tavor_pdhdl_t pd;
10309e39c5baSBill Taylor tavor_mrhdl_t mr;
10319e39c5baSBill Taylor tavor_eqhdl_t eq;
10329e39c5baSBill Taylor uint32_t eqnum;
10339e39c5baSBill Taylor int status;
10349e39c5baSBill Taylor
10359e39c5baSBill Taylor /*
10369e39c5baSBill Taylor * Pull all the necessary information from the Tavor Event Queue
10379e39c5baSBill Taylor * handle. This is necessary here because the resource for the
10389e39c5baSBill Taylor * EQ handle is going to be freed up as part of this operation.
10399e39c5baSBill Taylor */
10409e39c5baSBill Taylor eq = *eqhdl;
10419e39c5baSBill Taylor eqc = eq->eq_eqcrsrcp;
10429e39c5baSBill Taylor rsrc = eq->eq_rsrcp;
10439e39c5baSBill Taylor pd = state->ts_pdhdl_internal;
10449e39c5baSBill Taylor mr = eq->eq_mrhdl;
10459e39c5baSBill Taylor eqnum = eq->eq_eqnum;
10469e39c5baSBill Taylor
10479e39c5baSBill Taylor /*
10489e39c5baSBill Taylor * Reclaim EQC entry from hardware (using the Tavor HW2SW_EQ
10499e39c5baSBill Taylor * firmware command). If the ownership transfer fails for any reason,
10509e39c5baSBill Taylor * then it is an indication that something (either in HW or SW) has
10519e39c5baSBill Taylor * gone seriously wrong.
10529e39c5baSBill Taylor */
10539e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_EQ, &eqc_entry,
10549e39c5baSBill Taylor sizeof (tavor_hw_eqc_t), eqnum, TAVOR_CMD_NOSLEEP_SPIN);
10559e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
10569e39c5baSBill Taylor TAVOR_WARNING(state, "failed to reclaim EQC ownership");
10579e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_EQ command failed: %08x\n",
10589e39c5baSBill Taylor status);
10599e39c5baSBill Taylor return (DDI_FAILURE);
10609e39c5baSBill Taylor }
10619e39c5baSBill Taylor
10629e39c5baSBill Taylor /*
10639e39c5baSBill Taylor * Deregister the memory for the Event Queue. If this fails
10649e39c5baSBill Taylor * for any reason, then it is an indication that something (either
10659e39c5baSBill Taylor * in HW or SW) has gone seriously wrong. So we print a warning
10669e39c5baSBill Taylor * message and continue.
10679e39c5baSBill Taylor */
10689e39c5baSBill Taylor status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
10699e39c5baSBill Taylor TAVOR_NOSLEEP);
10709e39c5baSBill Taylor if (status != DDI_SUCCESS) {
10719e39c5baSBill Taylor TAVOR_WARNING(state, "failed to deregister EQ memory");
10729e39c5baSBill Taylor }
10739e39c5baSBill Taylor
10749e39c5baSBill Taylor /* Free the memory for the EQ */
10759e39c5baSBill Taylor tavor_queue_free(state, &eq->eq_eqinfo);
10769e39c5baSBill Taylor
10779e39c5baSBill Taylor /* Free the Tavor Event Queue handle */
10789e39c5baSBill Taylor tavor_rsrc_free(state, &rsrc);
10799e39c5baSBill Taylor
10809e39c5baSBill Taylor /* Free up the EQC entry resource */
10819e39c5baSBill Taylor tavor_rsrc_free(state, &eqc);
10829e39c5baSBill Taylor
10839e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */
10849e39c5baSBill Taylor tavor_pd_refcnt_dec(pd);
10859e39c5baSBill Taylor
10869e39c5baSBill Taylor /* Set the eqhdl pointer to NULL and return success */
10879e39c5baSBill Taylor *eqhdl = NULL;
10889e39c5baSBill Taylor
10899e39c5baSBill Taylor return (DDI_SUCCESS);
10909e39c5baSBill Taylor }
10919e39c5baSBill Taylor
10929e39c5baSBill Taylor
10939e39c5baSBill Taylor /*
10949e39c5baSBill Taylor * tavor_eq_handler_init
10959e39c5baSBill Taylor * Context: Only called from attach() path context
10969e39c5baSBill Taylor */
10979e39c5baSBill Taylor static int
tavor_eq_handler_init(tavor_state_t * state,tavor_eqhdl_t eq,uint_t evt_type_mask,int (* eq_func)(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe))10989e39c5baSBill Taylor tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
10999e39c5baSBill Taylor uint_t evt_type_mask, int (*eq_func)(tavor_state_t *state,
11009e39c5baSBill Taylor tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe))
11019e39c5baSBill Taylor {
11029e39c5baSBill Taylor int status;
11039e39c5baSBill Taylor
11049e39c5baSBill Taylor /*
11059e39c5baSBill Taylor * Save away the EQ handler function and the event type mask. These
11069e39c5baSBill Taylor * will be used later during interrupt and event queue processing.
11079e39c5baSBill Taylor */
11089e39c5baSBill Taylor eq->eq_func = eq_func;
11099e39c5baSBill Taylor eq->eq_evttypemask = evt_type_mask;
11109e39c5baSBill Taylor
11119e39c5baSBill Taylor /*
11129e39c5baSBill Taylor * Map the EQ to a specific class of event (or events) depending
11139e39c5baSBill Taylor * on the mask value passed in. The TAVOR_EVT_NO_MASK means not
11149e39c5baSBill Taylor * to attempt associating the EQ with any specific class of event.
11159e39c5baSBill Taylor * This is particularly useful when initializing the events queues
11169e39c5baSBill Taylor * used for CQ events. The mapping is done using the Tavor MAP_EQ
11179e39c5baSBill Taylor * firmware command. Note: This command should not, in general, fail.
11189e39c5baSBill Taylor * If it does, then something (probably HW related) has gone seriously
11199e39c5baSBill Taylor * wrong.
11209e39c5baSBill Taylor */
11219e39c5baSBill Taylor if (evt_type_mask != TAVOR_EVT_NO_MASK) {
11229e39c5baSBill Taylor status = tavor_map_eq_cmd_post(state,
11239e39c5baSBill Taylor TAVOR_CMD_MAP_EQ_EVT_MAP, eq->eq_eqnum, evt_type_mask,
11249e39c5baSBill Taylor TAVOR_CMD_NOSLEEP_SPIN);
11259e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
11269e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
11279e39c5baSBill Taylor "%08x\n", status);
11289e39c5baSBill Taylor return (DDI_FAILURE);
11299e39c5baSBill Taylor }
11309e39c5baSBill Taylor }
11319e39c5baSBill Taylor
11329e39c5baSBill Taylor return (DDI_SUCCESS);
11339e39c5baSBill Taylor }
11349e39c5baSBill Taylor
11359e39c5baSBill Taylor
11369e39c5baSBill Taylor /*
11379e39c5baSBill Taylor * tavor_eq_handler_fini
11389e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts
11399e39c5baSBill Taylor */
11409e39c5baSBill Taylor static int
tavor_eq_handler_fini(tavor_state_t * state,tavor_eqhdl_t eq)11419e39c5baSBill Taylor tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq)
11429e39c5baSBill Taylor {
11439e39c5baSBill Taylor int status;
11449e39c5baSBill Taylor
11459e39c5baSBill Taylor /*
11469e39c5baSBill Taylor * Unmap the EQ from the event class to which it had been previously
11479e39c5baSBill Taylor * mapped. The unmapping is done using the Tavor MAP_EQ (in much
11489e39c5baSBill Taylor * the same way that the initial mapping was done). The difference,
11499e39c5baSBill Taylor * however, is in the TAVOR_EQ_EVT_UNMAP flag that is passed to the
11509e39c5baSBill Taylor * MAP_EQ firmware command. The TAVOR_EVT_NO_MASK (which may have
11519e39c5baSBill Taylor * been passed in at init time) still means that no association has
11529e39c5baSBill Taylor * been made between the EQ and any specific class of event (and,
11539e39c5baSBill Taylor * hence, no unmapping is necessary). Note: This command should not,
11549e39c5baSBill Taylor * in general, fail. If it does, then something (probably HW related)
11559e39c5baSBill Taylor * has gone seriously wrong.
11569e39c5baSBill Taylor */
11579e39c5baSBill Taylor if (eq->eq_evttypemask != TAVOR_EVT_NO_MASK) {
11589e39c5baSBill Taylor status = tavor_map_eq_cmd_post(state,
11599e39c5baSBill Taylor TAVOR_CMD_MAP_EQ_EVT_UNMAP, eq->eq_eqnum,
11609e39c5baSBill Taylor eq->eq_evttypemask, TAVOR_CMD_NOSLEEP_SPIN);
11619e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
11629e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
11639e39c5baSBill Taylor "%08x\n", status);
11649e39c5baSBill Taylor return (DDI_FAILURE);
11659e39c5baSBill Taylor }
11669e39c5baSBill Taylor }
11679e39c5baSBill Taylor
11689e39c5baSBill Taylor return (DDI_SUCCESS);
11699e39c5baSBill Taylor }
11709e39c5baSBill Taylor
11719e39c5baSBill Taylor
11729e39c5baSBill Taylor /*
11739e39c5baSBill Taylor * tavor_eqe_sync()
11749e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
11759e39c5baSBill Taylor *
11769e39c5baSBill Taylor * Typically, this routine does nothing unless the EQ memory is
11779e39c5baSBill Taylor * mapped as DDI_DMA_STREAMING. However, there is a condition where
11789e39c5baSBill Taylor * ddi_dma_sync() is necessary even if the memory was mapped in
11799e39c5baSBill Taylor * consistent mode. The "force_sync" parameter is used here to force
11809e39c5baSBill Taylor * the call to ddi_dma_sync() independent of how the EQ memory was
11819e39c5baSBill Taylor * mapped.
11829e39c5baSBill Taylor */
11839e39c5baSBill Taylor static void
tavor_eqe_sync(tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe,uint_t flag,uint_t force_sync)11849e39c5baSBill Taylor tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
11859e39c5baSBill Taylor uint_t force_sync)
11869e39c5baSBill Taylor {
11879e39c5baSBill Taylor ddi_dma_handle_t dmahdl;
11889e39c5baSBill Taylor off_t offset;
11899e39c5baSBill Taylor
11909e39c5baSBill Taylor /* Determine if EQ needs to be synced or not */
11919e39c5baSBill Taylor if ((eq->eq_sync == 0) && (force_sync == TAVOR_EQ_SYNC_NORMAL)) {
11929e39c5baSBill Taylor return;
11939e39c5baSBill Taylor }
11949e39c5baSBill Taylor
11959e39c5baSBill Taylor /* Get the DMA handle from EQ context */
11969e39c5baSBill Taylor dmahdl = eq->eq_mrhdl->mr_bindinfo.bi_dmahdl;
11979e39c5baSBill Taylor
11989e39c5baSBill Taylor /* Calculate offset of next EQE */
11999e39c5baSBill Taylor offset = (off_t)((uintptr_t)eqe - (uintptr_t)&eq->eq_buf[0]);
1200*2570281cSToomas Soome (void) ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_eqe_t), flag);
12019e39c5baSBill Taylor }
12029e39c5baSBill Taylor
12039e39c5baSBill Taylor
12049e39c5baSBill Taylor /*
12059e39c5baSBill Taylor * tavor_port_state_change_handler()
12069e39c5baSBill Taylor * Context: Only called from interrupt context
12079e39c5baSBill Taylor */
12089e39c5baSBill Taylor static int
tavor_port_state_change_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)12099e39c5baSBill Taylor tavor_port_state_change_handler(tavor_state_t *state, tavor_eqhdl_t eq,
12109e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
12119e39c5baSBill Taylor {
12129e39c5baSBill Taylor ibc_async_event_t event;
12139e39c5baSBill Taylor ibt_async_code_t type;
12149e39c5baSBill Taylor uint_t port, subtype;
12159e39c5baSBill Taylor uint_t eqe_evttype;
12169e39c5baSBill Taylor char link_msg[24];
12179e39c5baSBill Taylor
12189e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
12199e39c5baSBill Taylor
12209e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_PORT_STATE_CHANGE ||
12219e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
12229e39c5baSBill Taylor
12239e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
12249e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
12259e39c5baSBill Taylor
12269e39c5baSBill Taylor return (DDI_FAILURE);
12279e39c5baSBill Taylor }
12289e39c5baSBill Taylor
12299e39c5baSBill Taylor /*
12309e39c5baSBill Taylor * Depending on the type of Port State Change event, pass the
12319e39c5baSBill Taylor * appropriate asynch event to the IBTF.
12329e39c5baSBill Taylor */
12339e39c5baSBill Taylor port = TAVOR_EQE_PORTNUM_GET(eq, eqe);
12349e39c5baSBill Taylor
12359e39c5baSBill Taylor /* Check for valid port number in event */
12369e39c5baSBill Taylor if ((port == 0) || (port > state->ts_cfg_profile->cp_num_ports)) {
12379e39c5baSBill Taylor TAVOR_WARNING(state, "Unexpected port number in port state "
12389e39c5baSBill Taylor "change event");
12399e39c5baSBill Taylor cmn_err(CE_CONT, " Port number: %02x\n", port);
12409e39c5baSBill Taylor return (DDI_FAILURE);
12419e39c5baSBill Taylor }
12429e39c5baSBill Taylor
12439e39c5baSBill Taylor subtype = TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe);
12449e39c5baSBill Taylor if (subtype == TAVOR_PORT_LINK_ACTIVE) {
12459e39c5baSBill Taylor event.ev_port = port;
12469e39c5baSBill Taylor type = IBT_EVENT_PORT_UP;
12479e39c5baSBill Taylor
12489e39c5baSBill Taylor (void) snprintf(link_msg, 23, "port %d up", port);
12499e39c5baSBill Taylor ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_RESTORED,
12509e39c5baSBill Taylor DDI_EXTERNAL_FAULT, link_msg);
12519e39c5baSBill Taylor } else if (subtype == TAVOR_PORT_LINK_DOWN) {
12529e39c5baSBill Taylor event.ev_port = port;
12539e39c5baSBill Taylor type = IBT_ERROR_PORT_DOWN;
12549e39c5baSBill Taylor
12559e39c5baSBill Taylor (void) snprintf(link_msg, 23, "port %d down", port);
12569e39c5baSBill Taylor ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_LOST,
12579e39c5baSBill Taylor DDI_EXTERNAL_FAULT, link_msg);
12589e39c5baSBill Taylor } else {
12599e39c5baSBill Taylor TAVOR_WARNING(state, "Unexpected subtype in port state change "
12609e39c5baSBill Taylor "event");
12619e39c5baSBill Taylor cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n",
12629e39c5baSBill Taylor TAVOR_EQE_EVTTYPE_GET(eq, eqe), subtype);
12639e39c5baSBill Taylor return (DDI_FAILURE);
12649e39c5baSBill Taylor }
12659e39c5baSBill Taylor
12669e39c5baSBill Taylor /*
12679e39c5baSBill Taylor * Deliver the event to the IBTF. Note: If "ts_ibtfpriv" is NULL,
12689e39c5baSBill Taylor * then we have either received this event before we finished
12699e39c5baSBill Taylor * attaching to the IBTF or we've received it while we are in the
12709e39c5baSBill Taylor * process of detaching.
12719e39c5baSBill Taylor */
12729e39c5baSBill Taylor if (state->ts_ibtfpriv != NULL) {
12739e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
12749e39c5baSBill Taylor }
12759e39c5baSBill Taylor
12769e39c5baSBill Taylor return (DDI_SUCCESS);
12779e39c5baSBill Taylor }
12789e39c5baSBill Taylor
12799e39c5baSBill Taylor
12809e39c5baSBill Taylor /*
12819e39c5baSBill Taylor * tavor_comm_estbl_handler()
12829e39c5baSBill Taylor * Context: Only called from interrupt context
12839e39c5baSBill Taylor */
12849e39c5baSBill Taylor static int
tavor_comm_estbl_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)12859e39c5baSBill Taylor tavor_comm_estbl_handler(tavor_state_t *state, tavor_eqhdl_t eq,
12869e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
12879e39c5baSBill Taylor {
12889e39c5baSBill Taylor tavor_qphdl_t qp;
12899e39c5baSBill Taylor uint_t qpnum;
12909e39c5baSBill Taylor ibc_async_event_t event;
12919e39c5baSBill Taylor ibt_async_code_t type;
12929e39c5baSBill Taylor uint_t eqe_evttype;
12939e39c5baSBill Taylor
12949e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
12959e39c5baSBill Taylor
12969e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_COMM_ESTABLISHED ||
12979e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
12989e39c5baSBill Taylor
12999e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
13009e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
13019e39c5baSBill Taylor
13029e39c5baSBill Taylor return (DDI_FAILURE);
13039e39c5baSBill Taylor }
13049e39c5baSBill Taylor
13059e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
13069e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
13079e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
13089e39c5baSBill Taylor
13099e39c5baSBill Taylor /*
13109e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
13119e39c5baSBill Taylor * that the QP has been freed already. In which case, we
13129e39c5baSBill Taylor * should not deliver this event.
13139e39c5baSBill Taylor *
13149e39c5baSBill Taylor * We also check that the QP number in the handle is the
13159e39c5baSBill Taylor * same as the QP number in the event queue entry. This
13169e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
13179e39c5baSBill Taylor * freed and then allocated again in the time it took to
13189e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
13199e39c5baSBill Taylor * the non-constrained portion of the QP number every time
13209e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
13219e39c5baSBill Taylor * that a stale event could be passed to the client's QP
13229e39c5baSBill Taylor * handler.
13239e39c5baSBill Taylor *
13249e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
13259e39c5baSBill Taylor * means that we've have either received this event before we
13269e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
13279e39c5baSBill Taylor * are in the process of detaching.
13289e39c5baSBill Taylor */
13299e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
13309e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
13319e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
13329e39c5baSBill Taylor type = IBT_EVENT_COM_EST_QP;
13339e39c5baSBill Taylor
13349e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
13359e39c5baSBill Taylor }
13369e39c5baSBill Taylor
13379e39c5baSBill Taylor return (DDI_SUCCESS);
13389e39c5baSBill Taylor }
13399e39c5baSBill Taylor
13409e39c5baSBill Taylor
13419e39c5baSBill Taylor /*
13429e39c5baSBill Taylor * tavor_local_wq_cat_err_handler()
13439e39c5baSBill Taylor * Context: Only called from interrupt context
13449e39c5baSBill Taylor */
13459e39c5baSBill Taylor static int
tavor_local_wq_cat_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)13469e39c5baSBill Taylor tavor_local_wq_cat_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
13479e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
13489e39c5baSBill Taylor {
13499e39c5baSBill Taylor tavor_qphdl_t qp;
13509e39c5baSBill Taylor uint_t qpnum;
13519e39c5baSBill Taylor ibc_async_event_t event;
13529e39c5baSBill Taylor ibt_async_code_t type;
13539e39c5baSBill Taylor uint_t eqe_evttype;
13549e39c5baSBill Taylor
13559e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
13569e39c5baSBill Taylor
13579e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_WQ_CAT_ERROR ||
13589e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
13599e39c5baSBill Taylor
13609e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
13619e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
13629e39c5baSBill Taylor
13639e39c5baSBill Taylor return (DDI_FAILURE);
13649e39c5baSBill Taylor }
13659e39c5baSBill Taylor
13669e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
13679e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
13689e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
13699e39c5baSBill Taylor
13709e39c5baSBill Taylor /*
13719e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
13729e39c5baSBill Taylor * that the QP has been freed already. In which case, we
13739e39c5baSBill Taylor * should not deliver this event.
13749e39c5baSBill Taylor *
13759e39c5baSBill Taylor * We also check that the QP number in the handle is the
13769e39c5baSBill Taylor * same as the QP number in the event queue entry. This
13779e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
13789e39c5baSBill Taylor * freed and then allocated again in the time it took to
13799e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
13809e39c5baSBill Taylor * the non-constrained portion of the QP number every time
13819e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
13829e39c5baSBill Taylor * that a stale event could be passed to the client's QP
13839e39c5baSBill Taylor * handler.
13849e39c5baSBill Taylor *
13859e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
13869e39c5baSBill Taylor * means that we've have either received this event before we
13879e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
13889e39c5baSBill Taylor * are in the process of detaching.
13899e39c5baSBill Taylor */
13909e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
13919e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
13929e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
13939e39c5baSBill Taylor type = IBT_ERROR_CATASTROPHIC_QP;
13949e39c5baSBill Taylor
13959e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
13969e39c5baSBill Taylor }
13979e39c5baSBill Taylor
13989e39c5baSBill Taylor return (DDI_SUCCESS);
13999e39c5baSBill Taylor }
14009e39c5baSBill Taylor
14019e39c5baSBill Taylor
14029e39c5baSBill Taylor /*
14039e39c5baSBill Taylor * tavor_invreq_local_wq_err_handler()
14049e39c5baSBill Taylor * Context: Only called from interrupt context
14059e39c5baSBill Taylor */
14069e39c5baSBill Taylor static int
tavor_invreq_local_wq_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)14079e39c5baSBill Taylor tavor_invreq_local_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
14089e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
14099e39c5baSBill Taylor {
14109e39c5baSBill Taylor tavor_qphdl_t qp;
14119e39c5baSBill Taylor uint_t qpnum;
14129e39c5baSBill Taylor ibc_async_event_t event;
14139e39c5baSBill Taylor ibt_async_code_t type;
14149e39c5baSBill Taylor uint_t eqe_evttype;
14159e39c5baSBill Taylor
14169e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
14179e39c5baSBill Taylor
14189e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_INV_REQ_LOCAL_WQ_ERROR ||
14199e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
14209e39c5baSBill Taylor
14219e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
14229e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
14239e39c5baSBill Taylor
14249e39c5baSBill Taylor return (DDI_FAILURE);
14259e39c5baSBill Taylor }
14269e39c5baSBill Taylor
14279e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
14289e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
14299e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
14309e39c5baSBill Taylor
14319e39c5baSBill Taylor /*
14329e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
14339e39c5baSBill Taylor * that the QP has been freed already. In which case, we
14349e39c5baSBill Taylor * should not deliver this event.
14359e39c5baSBill Taylor *
14369e39c5baSBill Taylor * We also check that the QP number in the handle is the
14379e39c5baSBill Taylor * same as the QP number in the event queue entry. This
14389e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
14399e39c5baSBill Taylor * freed and then allocated again in the time it took to
14409e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
14419e39c5baSBill Taylor * the non-constrained portion of the QP number every time
14429e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
14439e39c5baSBill Taylor * that a stale event could be passed to the client's QP
14449e39c5baSBill Taylor * handler.
14459e39c5baSBill Taylor *
14469e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
14479e39c5baSBill Taylor * means that we've have either received this event before we
14489e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
14499e39c5baSBill Taylor * are in the process of detaching.
14509e39c5baSBill Taylor */
14519e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
14529e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
14539e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
14549e39c5baSBill Taylor type = IBT_ERROR_INVALID_REQUEST_QP;
14559e39c5baSBill Taylor
14569e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
14579e39c5baSBill Taylor }
14589e39c5baSBill Taylor
14599e39c5baSBill Taylor return (DDI_SUCCESS);
14609e39c5baSBill Taylor }
14619e39c5baSBill Taylor
14629e39c5baSBill Taylor
14639e39c5baSBill Taylor /*
14649e39c5baSBill Taylor * tavor_local_acc_vio_wq_err_handler()
14659e39c5baSBill Taylor * Context: Only called from interrupt context
14669e39c5baSBill Taylor */
14679e39c5baSBill Taylor static int
tavor_local_acc_vio_wq_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)14689e39c5baSBill Taylor tavor_local_acc_vio_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
14699e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
14709e39c5baSBill Taylor {
14719e39c5baSBill Taylor tavor_qphdl_t qp;
14729e39c5baSBill Taylor uint_t qpnum;
14739e39c5baSBill Taylor ibc_async_event_t event;
14749e39c5baSBill Taylor ibt_async_code_t type;
14759e39c5baSBill Taylor uint_t eqe_evttype;
14769e39c5baSBill Taylor
14779e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
14789e39c5baSBill Taylor
14799e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_ACC_VIO_WQ_ERROR ||
14809e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
14819e39c5baSBill Taylor
14829e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
14839e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
14849e39c5baSBill Taylor
14859e39c5baSBill Taylor return (DDI_FAILURE);
14869e39c5baSBill Taylor }
14879e39c5baSBill Taylor
14889e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
14899e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
14909e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
14919e39c5baSBill Taylor
14929e39c5baSBill Taylor /*
14939e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
14949e39c5baSBill Taylor * that the QP has been freed already. In which case, we
14959e39c5baSBill Taylor * should not deliver this event.
14969e39c5baSBill Taylor *
14979e39c5baSBill Taylor * We also check that the QP number in the handle is the
14989e39c5baSBill Taylor * same as the QP number in the event queue entry. This
14999e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
15009e39c5baSBill Taylor * freed and then allocated again in the time it took to
15019e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
15029e39c5baSBill Taylor * the non-constrained portion of the QP number every time
15039e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
15049e39c5baSBill Taylor * that a stale event could be passed to the client's QP
15059e39c5baSBill Taylor * handler.
15069e39c5baSBill Taylor *
15079e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
15089e39c5baSBill Taylor * means that we've have either received this event before we
15099e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
15109e39c5baSBill Taylor * are in the process of detaching.
15119e39c5baSBill Taylor */
15129e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
15139e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
15149e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
15159e39c5baSBill Taylor type = IBT_ERROR_ACCESS_VIOLATION_QP;
15169e39c5baSBill Taylor
15179e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
15189e39c5baSBill Taylor }
15199e39c5baSBill Taylor
15209e39c5baSBill Taylor return (DDI_SUCCESS);
15219e39c5baSBill Taylor }
15229e39c5baSBill Taylor
15239e39c5baSBill Taylor
15249e39c5baSBill Taylor /*
15259e39c5baSBill Taylor * tavor_sendq_drained_handler()
15269e39c5baSBill Taylor * Context: Only called from interrupt context
15279e39c5baSBill Taylor */
15289e39c5baSBill Taylor static int
tavor_sendq_drained_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)15299e39c5baSBill Taylor tavor_sendq_drained_handler(tavor_state_t *state, tavor_eqhdl_t eq,
15309e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
15319e39c5baSBill Taylor {
15329e39c5baSBill Taylor tavor_qphdl_t qp;
15339e39c5baSBill Taylor uint_t qpnum;
15349e39c5baSBill Taylor ibc_async_event_t event;
15359e39c5baSBill Taylor uint_t forward_sqd_event;
15369e39c5baSBill Taylor ibt_async_code_t type;
15379e39c5baSBill Taylor uint_t eqe_evttype;
15389e39c5baSBill Taylor
15399e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
15409e39c5baSBill Taylor
15419e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_SEND_QUEUE_DRAINED ||
15429e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
15439e39c5baSBill Taylor
15449e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
15459e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
15469e39c5baSBill Taylor
15479e39c5baSBill Taylor return (DDI_FAILURE);
15489e39c5baSBill Taylor }
15499e39c5baSBill Taylor
15509e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
15519e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
15529e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
15539e39c5baSBill Taylor
15549e39c5baSBill Taylor /*
15559e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
15569e39c5baSBill Taylor * that the QP has been freed already. In which case, we
15579e39c5baSBill Taylor * should not deliver this event.
15589e39c5baSBill Taylor *
15599e39c5baSBill Taylor * We also check that the QP number in the handle is the
15609e39c5baSBill Taylor * same as the QP number in the event queue entry. This
15619e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
15629e39c5baSBill Taylor * freed and then allocated again in the time it took to
15639e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
15649e39c5baSBill Taylor * the non-constrained portion of the QP number every time
15659e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
15669e39c5baSBill Taylor * that a stale event could be passed to the client's QP
15679e39c5baSBill Taylor * handler.
15689e39c5baSBill Taylor *
15699e39c5baSBill Taylor * And then we check if "ts_ibtfpriv" is NULL. If it is then it
15709e39c5baSBill Taylor * means that we've have either received this event before we
15719e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
15729e39c5baSBill Taylor * are in the process of detaching.
15739e39c5baSBill Taylor */
15749e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
15759e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
15769e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
15779e39c5baSBill Taylor type = IBT_EVENT_SQD;
15789e39c5baSBill Taylor
15799e39c5baSBill Taylor /*
15809e39c5baSBill Taylor * Grab the QP lock and update the QP state to reflect that
15819e39c5baSBill Taylor * the Send Queue Drained event has arrived. Also determine
15829e39c5baSBill Taylor * whether the event is intended to be forwarded on to the
15839e39c5baSBill Taylor * consumer or not. This information is used below in
15849e39c5baSBill Taylor * determining whether or not to call the IBTF.
15859e39c5baSBill Taylor */
15869e39c5baSBill Taylor mutex_enter(&qp->qp_lock);
15879e39c5baSBill Taylor forward_sqd_event = qp->qp_forward_sqd_event;
15889e39c5baSBill Taylor qp->qp_forward_sqd_event = 0;
15899e39c5baSBill Taylor qp->qp_sqd_still_draining = 0;
15909e39c5baSBill Taylor mutex_exit(&qp->qp_lock);
15919e39c5baSBill Taylor
15929e39c5baSBill Taylor if (forward_sqd_event != 0) {
15939e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
15949e39c5baSBill Taylor }
15959e39c5baSBill Taylor }
15969e39c5baSBill Taylor
15979e39c5baSBill Taylor return (DDI_SUCCESS);
15989e39c5baSBill Taylor }
15999e39c5baSBill Taylor
16009e39c5baSBill Taylor
16019e39c5baSBill Taylor /*
16029e39c5baSBill Taylor * tavor_path_mig_handler()
16039e39c5baSBill Taylor * Context: Only called from interrupt context
16049e39c5baSBill Taylor */
16059e39c5baSBill Taylor static int
tavor_path_mig_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)16069e39c5baSBill Taylor tavor_path_mig_handler(tavor_state_t *state, tavor_eqhdl_t eq,
16079e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
16089e39c5baSBill Taylor {
16099e39c5baSBill Taylor tavor_qphdl_t qp;
16109e39c5baSBill Taylor uint_t qpnum;
16119e39c5baSBill Taylor ibc_async_event_t event;
16129e39c5baSBill Taylor ibt_async_code_t type;
16139e39c5baSBill Taylor uint_t eqe_evttype;
16149e39c5baSBill Taylor
16159e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
16169e39c5baSBill Taylor
16179e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATED ||
16189e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
16199e39c5baSBill Taylor
16209e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
16219e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
16229e39c5baSBill Taylor
16239e39c5baSBill Taylor return (DDI_FAILURE);
16249e39c5baSBill Taylor }
16259e39c5baSBill Taylor
16269e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
16279e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
16289e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
16299e39c5baSBill Taylor
16309e39c5baSBill Taylor /*
16319e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
16329e39c5baSBill Taylor * that the QP has been freed already. In which case, we
16339e39c5baSBill Taylor * should not deliver this event.
16349e39c5baSBill Taylor *
16359e39c5baSBill Taylor * We also check that the QP number in the handle is the
16369e39c5baSBill Taylor * same as the QP number in the event queue entry. This
16379e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
16389e39c5baSBill Taylor * freed and then allocated again in the time it took to
16399e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
16409e39c5baSBill Taylor * the non-constrained portion of the QP number every time
16419e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
16429e39c5baSBill Taylor * that a stale event could be passed to the client's QP
16439e39c5baSBill Taylor * handler.
16449e39c5baSBill Taylor *
16459e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
16469e39c5baSBill Taylor * means that we've have either received this event before we
16479e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
16489e39c5baSBill Taylor * are in the process of detaching.
16499e39c5baSBill Taylor */
16509e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
16519e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
16529e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
16539e39c5baSBill Taylor type = IBT_EVENT_PATH_MIGRATED_QP;
16549e39c5baSBill Taylor
16559e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
16569e39c5baSBill Taylor }
16579e39c5baSBill Taylor
16589e39c5baSBill Taylor return (DDI_SUCCESS);
16599e39c5baSBill Taylor }
16609e39c5baSBill Taylor
16619e39c5baSBill Taylor
16629e39c5baSBill Taylor /*
16639e39c5baSBill Taylor * tavor_path_mig_err_handler()
16649e39c5baSBill Taylor * Context: Only called from interrupt context
16659e39c5baSBill Taylor */
16669e39c5baSBill Taylor static int
tavor_path_mig_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)16679e39c5baSBill Taylor tavor_path_mig_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
16689e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
16699e39c5baSBill Taylor {
16709e39c5baSBill Taylor tavor_qphdl_t qp;
16719e39c5baSBill Taylor uint_t qpnum;
16729e39c5baSBill Taylor ibc_async_event_t event;
16739e39c5baSBill Taylor ibt_async_code_t type;
16749e39c5baSBill Taylor uint_t eqe_evttype;
16759e39c5baSBill Taylor
16769e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
16779e39c5baSBill Taylor
16789e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATE_FAILED ||
16799e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
16809e39c5baSBill Taylor
16819e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
16829e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
16839e39c5baSBill Taylor
16849e39c5baSBill Taylor return (DDI_FAILURE);
16859e39c5baSBill Taylor }
16869e39c5baSBill Taylor
16879e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
16889e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
16899e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
16909e39c5baSBill Taylor
16919e39c5baSBill Taylor /*
16929e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
16939e39c5baSBill Taylor * that the QP has been freed already. In which case, we
16949e39c5baSBill Taylor * should not deliver this event.
16959e39c5baSBill Taylor *
16969e39c5baSBill Taylor * We also check that the QP number in the handle is the
16979e39c5baSBill Taylor * same as the QP number in the event queue entry. This
16989e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
16999e39c5baSBill Taylor * freed and then allocated again in the time it took to
17009e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
17019e39c5baSBill Taylor * the non-constrained portion of the QP number every time
17029e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
17039e39c5baSBill Taylor * that a stale event could be passed to the client's QP
17049e39c5baSBill Taylor * handler.
17059e39c5baSBill Taylor *
17069e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
17079e39c5baSBill Taylor * means that we've have either received this event before we
17089e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
17099e39c5baSBill Taylor * are in the process of detaching.
17109e39c5baSBill Taylor */
17119e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
17129e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
17139e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
17149e39c5baSBill Taylor type = IBT_ERROR_PATH_MIGRATE_REQ_QP;
17159e39c5baSBill Taylor
17169e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
17179e39c5baSBill Taylor }
17189e39c5baSBill Taylor
17199e39c5baSBill Taylor return (DDI_SUCCESS);
17209e39c5baSBill Taylor }
17219e39c5baSBill Taylor
17229e39c5baSBill Taylor
17239e39c5baSBill Taylor /*
17249e39c5baSBill Taylor * tavor_srq_catastrophic_handler()
17259e39c5baSBill Taylor * Context: Only called from interrupt context
17269e39c5baSBill Taylor */
17279e39c5baSBill Taylor static int
tavor_srq_catastrophic_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)17289e39c5baSBill Taylor tavor_srq_catastrophic_handler(tavor_state_t *state, tavor_eqhdl_t eq,
17299e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
17309e39c5baSBill Taylor {
17319e39c5baSBill Taylor tavor_qphdl_t qp;
17329e39c5baSBill Taylor uint_t qpnum;
17339e39c5baSBill Taylor ibc_async_event_t event;
17349e39c5baSBill Taylor ibt_async_code_t type;
17359e39c5baSBill Taylor uint_t eqe_evttype;
17369e39c5baSBill Taylor
17379e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
17389e39c5baSBill Taylor
17399e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_SRQ_CATASTROPHIC_ERROR ||
17409e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
17419e39c5baSBill Taylor
17429e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
17439e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
17449e39c5baSBill Taylor
17459e39c5baSBill Taylor return (DDI_FAILURE);
17469e39c5baSBill Taylor }
17479e39c5baSBill Taylor
17489e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
17499e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
17509e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
17519e39c5baSBill Taylor
17529e39c5baSBill Taylor /*
17539e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
17549e39c5baSBill Taylor * that the QP has been freed already. In which case, we
17559e39c5baSBill Taylor * should not deliver this event.
17569e39c5baSBill Taylor *
17579e39c5baSBill Taylor * We also check that the QP number in the handle is the
17589e39c5baSBill Taylor * same as the QP number in the event queue entry. This
17599e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
17609e39c5baSBill Taylor * freed and then allocated again in the time it took to
17619e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
17629e39c5baSBill Taylor * the non-constrained portion of the QP number every time
17639e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
17649e39c5baSBill Taylor * that a stale event could be passed to the client's QP
17659e39c5baSBill Taylor * handler.
17669e39c5baSBill Taylor *
17679e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
17689e39c5baSBill Taylor * means that we've have either received this event before we
17699e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
17709e39c5baSBill Taylor * are in the process of detaching.
17719e39c5baSBill Taylor */
17729e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
17739e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
17749e39c5baSBill Taylor event.ev_srq_hdl = (ibt_srq_hdl_t)qp->qp_srqhdl->srq_hdlrarg;
17759e39c5baSBill Taylor type = IBT_ERROR_CATASTROPHIC_SRQ;
17769e39c5baSBill Taylor
17779e39c5baSBill Taylor mutex_enter(&qp->qp_srqhdl->srq_lock);
17789e39c5baSBill Taylor qp->qp_srqhdl->srq_state = TAVOR_SRQ_STATE_ERROR;
17799e39c5baSBill Taylor mutex_exit(&qp->qp_srqhdl->srq_lock);
17809e39c5baSBill Taylor
17819e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
17829e39c5baSBill Taylor }
17839e39c5baSBill Taylor
17849e39c5baSBill Taylor return (DDI_SUCCESS);
17859e39c5baSBill Taylor }
17869e39c5baSBill Taylor
17879e39c5baSBill Taylor
17889e39c5baSBill Taylor /*
17899e39c5baSBill Taylor * tavor_srq_last_wqe_reached_handler()
17909e39c5baSBill Taylor * Context: Only called from interrupt context
17919e39c5baSBill Taylor */
17929e39c5baSBill Taylor static int
tavor_srq_last_wqe_reached_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)17939e39c5baSBill Taylor tavor_srq_last_wqe_reached_handler(tavor_state_t *state, tavor_eqhdl_t eq,
17949e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
17959e39c5baSBill Taylor {
17969e39c5baSBill Taylor tavor_qphdl_t qp;
17979e39c5baSBill Taylor uint_t qpnum;
17989e39c5baSBill Taylor ibc_async_event_t event;
17999e39c5baSBill Taylor ibt_async_code_t type;
18009e39c5baSBill Taylor uint_t eqe_evttype;
18019e39c5baSBill Taylor
18029e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
18039e39c5baSBill Taylor
18049e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_SRQ_LAST_WQE_REACHED ||
18059e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
18069e39c5baSBill Taylor
18079e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
18089e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
18099e39c5baSBill Taylor
18109e39c5baSBill Taylor return (DDI_FAILURE);
18119e39c5baSBill Taylor }
18129e39c5baSBill Taylor
18139e39c5baSBill Taylor /* Get the QP handle from QP number in event descriptor */
18149e39c5baSBill Taylor qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
18159e39c5baSBill Taylor qp = tavor_qphdl_from_qpnum(state, qpnum);
18169e39c5baSBill Taylor
18179e39c5baSBill Taylor /*
18189e39c5baSBill Taylor * If the QP handle is NULL, this is probably an indication
18199e39c5baSBill Taylor * that the QP has been freed already. In which case, we
18209e39c5baSBill Taylor * should not deliver this event.
18219e39c5baSBill Taylor *
18229e39c5baSBill Taylor * We also check that the QP number in the handle is the
18239e39c5baSBill Taylor * same as the QP number in the event queue entry. This
18249e39c5baSBill Taylor * extra check allows us to handle the case where a QP was
18259e39c5baSBill Taylor * freed and then allocated again in the time it took to
18269e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
18279e39c5baSBill Taylor * the non-constrained portion of the QP number every time
18289e39c5baSBill Taylor * a new QP is allocated, we mitigate (somewhat) the chance
18299e39c5baSBill Taylor * that a stale event could be passed to the client's QP
18309e39c5baSBill Taylor * handler.
18319e39c5baSBill Taylor *
18329e39c5baSBill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
18339e39c5baSBill Taylor * means that we've have either received this event before we
18349e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
18359e39c5baSBill Taylor * are in the process of detaching.
18369e39c5baSBill Taylor */
18379e39c5baSBill Taylor if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
18389e39c5baSBill Taylor (state->ts_ibtfpriv != NULL)) {
18399e39c5baSBill Taylor event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
18409e39c5baSBill Taylor type = IBT_EVENT_EMPTY_CHAN;
18419e39c5baSBill Taylor
18429e39c5baSBill Taylor TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
18439e39c5baSBill Taylor }
18449e39c5baSBill Taylor
18459e39c5baSBill Taylor return (DDI_SUCCESS);
18469e39c5baSBill Taylor }
18479e39c5baSBill Taylor
18489e39c5baSBill Taylor
18499e39c5baSBill Taylor /*
18509e39c5baSBill Taylor * tavor_ecc_detection_handler()
18519e39c5baSBill Taylor * Context: Only called from interrupt context
18529e39c5baSBill Taylor */
18539e39c5baSBill Taylor static int
tavor_ecc_detection_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)18549e39c5baSBill Taylor tavor_ecc_detection_handler(tavor_state_t *state, tavor_eqhdl_t eq,
18559e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
18569e39c5baSBill Taylor {
18579e39c5baSBill Taylor uint_t eqe_evttype;
18589e39c5baSBill Taylor uint_t data;
18599e39c5baSBill Taylor int i;
18609e39c5baSBill Taylor
18619e39c5baSBill Taylor eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
18629e39c5baSBill Taylor
18639e39c5baSBill Taylor ASSERT(eqe_evttype == TAVOR_EVT_ECC_DETECTION ||
18649e39c5baSBill Taylor eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
18659e39c5baSBill Taylor
18669e39c5baSBill Taylor if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
18679e39c5baSBill Taylor tavor_eq_overflow_handler(state, eq, eqe);
18689e39c5baSBill Taylor
18699e39c5baSBill Taylor return (DDI_FAILURE);
18709e39c5baSBill Taylor }
18719e39c5baSBill Taylor
18729e39c5baSBill Taylor /*
18739e39c5baSBill Taylor * The "ECC Detection Event" indicates that a correctable single-bit
18749e39c5baSBill Taylor * has occurred with the attached DDR. The EQE provides some
18759e39c5baSBill Taylor * additional information about the errored EQ. So we print a warning
18769e39c5baSBill Taylor * message here along with that additional information.
18779e39c5baSBill Taylor */
18789e39c5baSBill Taylor TAVOR_WARNING(state, "ECC Correctable Error Event Detected");
18799e39c5baSBill Taylor for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
18809e39c5baSBill Taylor data = ((uint_t *)eqe)[i];
18819e39c5baSBill Taylor cmn_err(CE_CONT, "! EQE[%02x]: %08x\n", i, data);
18829e39c5baSBill Taylor }
18839e39c5baSBill Taylor
18849e39c5baSBill Taylor return (DDI_SUCCESS);
18859e39c5baSBill Taylor }
18869e39c5baSBill Taylor
18879e39c5baSBill Taylor
18889e39c5baSBill Taylor /*
18899e39c5baSBill Taylor * tavor_eq_overflow_handler()
18909e39c5baSBill Taylor * Context: Only called from interrupt context
18919e39c5baSBill Taylor */
18929e39c5baSBill Taylor void
tavor_eq_overflow_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)18939e39c5baSBill Taylor tavor_eq_overflow_handler(tavor_state_t *state, tavor_eqhdl_t eq,
18949e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
18959e39c5baSBill Taylor {
18969e39c5baSBill Taylor uint_t error_type, data;
18979e39c5baSBill Taylor
18989e39c5baSBill Taylor ASSERT(TAVOR_EQE_EVTTYPE_GET(eq, eqe) == TAVOR_EVT_EQ_OVERFLOW);
18999e39c5baSBill Taylor
19009e39c5baSBill Taylor /*
19019e39c5baSBill Taylor * The "Event Queue Overflow Event" indicates that something has
19029e39c5baSBill Taylor * probably gone seriously wrong with some hardware (or, perhaps,
19039e39c5baSBill Taylor * with the software... though it's unlikely in this case). The EQE
19049e39c5baSBill Taylor * provides some additional information about the errored EQ. So we
19059e39c5baSBill Taylor * print a warning message here along with that additional information.
19069e39c5baSBill Taylor */
19079e39c5baSBill Taylor error_type = TAVOR_EQE_OPERRTYPE_GET(eq, eqe);
19089e39c5baSBill Taylor data = TAVOR_EQE_OPERRDATA_GET(eq, eqe);
19099e39c5baSBill Taylor
19109e39c5baSBill Taylor TAVOR_WARNING(state, "Event Queue overflow");
19119e39c5baSBill Taylor cmn_err(CE_CONT, " Error type: %02x, data: %08x\n", error_type, data);
19129e39c5baSBill Taylor }
19139e39c5baSBill Taylor
19149e39c5baSBill Taylor
19159e39c5baSBill Taylor /*
19169e39c5baSBill Taylor * tavor_no_eqhandler
19179e39c5baSBill Taylor * Context: Only called from interrupt context
19189e39c5baSBill Taylor */
19199e39c5baSBill Taylor /* ARGSUSED */
19209e39c5baSBill Taylor static int
tavor_no_eqhandler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)19219e39c5baSBill Taylor tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
19229e39c5baSBill Taylor tavor_hw_eqe_t *eqe)
19239e39c5baSBill Taylor {
19249e39c5baSBill Taylor uint_t data;
19259e39c5baSBill Taylor int i;
19269e39c5baSBill Taylor
19279e39c5baSBill Taylor /*
19289e39c5baSBill Taylor * This "unexpected event" handler (or "catch-all" handler) will
19299e39c5baSBill Taylor * receive all events for which no other handler has been registered.
19309e39c5baSBill Taylor * If we end up here, then something has probably gone seriously wrong
19319e39c5baSBill Taylor * with the Tavor hardware (or, perhaps, with the software... though
19329e39c5baSBill Taylor * it's unlikely in this case). The EQE provides all the information
19339e39c5baSBill Taylor * about the event. So we print a warning message here along with
19349e39c5baSBill Taylor * the contents of the EQE.
19359e39c5baSBill Taylor */
19369e39c5baSBill Taylor TAVOR_WARNING(state, "Unexpected Event handler");
19379e39c5baSBill Taylor cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n",
19389e39c5baSBill Taylor TAVOR_EQE_EVTTYPE_GET(eq, eqe), TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe));
19399e39c5baSBill Taylor for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
19409e39c5baSBill Taylor data = ((uint_t *)eqe)[i];
19419e39c5baSBill Taylor cmn_err(CE_CONT, " EQE[%02x]: %08x\n", i, data);
19429e39c5baSBill Taylor }
19439e39c5baSBill Taylor
19449e39c5baSBill Taylor return (DDI_SUCCESS);
19459e39c5baSBill Taylor }
1946