/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * tavor.c * Tavor (InfiniBand) HCA Driver attach/detach Routines * * Implements all the routines necessary for the attach, setup, * initialization (and subsequent possible teardown and detach) of the * Tavor InfiniBand HCA driver. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Tavor HCA State Pointer */ void *tavor_statep; /* * The Tavor "userland resource database" is common to instances of the * Tavor HCA driver. This structure "tavor_userland_rsrc_db" contains all * the necessary information to maintain it. */ tavor_umap_db_t tavor_userland_rsrc_db; static int tavor_attach(dev_info_t *, ddi_attach_cmd_t); static int tavor_detach(dev_info_t *, ddi_detach_cmd_t); static int tavor_open(dev_t *, int, int, cred_t *); static int tavor_close(dev_t, int, int, cred_t *); static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance); static void tavor_drv_fini(tavor_state_t *state); static void tavor_drv_fini2(tavor_state_t *state); static int tavor_isr_init(tavor_state_t *state); static void tavor_isr_fini(tavor_state_t *state); static int tavor_hw_init(tavor_state_t *state); static void tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup); static int tavor_soft_state_init(tavor_state_t *state); static void tavor_soft_state_fini(tavor_state_t *state); static int tavor_hca_port_init(tavor_state_t *state); static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init); static void tavor_hca_config_setup(tavor_state_t *state, tavor_hw_initqueryhca_t *inithca); static int tavor_internal_uarpgs_init(tavor_state_t *state); static void tavor_internal_uarpgs_fini(tavor_state_t *state); static int tavor_special_qp_contexts_reserve(tavor_state_t *state); static void tavor_special_qp_contexts_unreserve(tavor_state_t *state); static int tavor_sw_reset(tavor_state_t *state); static int tavor_mcg_init(tavor_state_t *state); static void tavor_mcg_fini(tavor_state_t *state); static int tavor_fw_version_check(tavor_state_t *state); static void tavor_device_info_report(tavor_state_t *state); static void tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl); static void tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl, uint_t offset); static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr, uint32_t *data); static void tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl, uint_t offset); static int tavor_intr_or_msi_init(tavor_state_t *state); static int tavor_add_intrs(tavor_state_t *state, int intr_type); static int tavor_intr_or_msi_fini(tavor_state_t *state); /* X86 fastreboot support */ static int tavor_intr_disable(tavor_state_t *); static int tavor_quiesce(dev_info_t *); /* Character/Block Operations */ static struct cb_ops tavor_cb_ops = { tavor_open, /* open */ tavor_close, /* close */ nodev, /* strategy (block) */ nodev, /* print (block) */ nodev, /* dump (block) */ nodev, /* read */ nodev, /* write */ tavor_ioctl, /* ioctl */ tavor_devmap, /* devmap */ NULL, /* mmap */ nodev, /* segmap */ nochpoll, /* chpoll */ ddi_prop_op, /* prop_op */ NULL, /* streams */ D_NEW | D_MP | D_64BIT | D_HOTPLUG | D_DEVMAP, /* flags */ CB_REV /* rev */ }; /* Driver Operations */ static struct dev_ops tavor_ops = { DEVO_REV, /* struct rev */ 0, /* refcnt */ tavor_getinfo, /* getinfo */ nulldev, /* identify */ nulldev, /* probe */ tavor_attach, /* attach */ tavor_detach, /* detach */ nodev, /* reset */ &tavor_cb_ops, /* cb_ops */ NULL, /* bus_ops */ nodev, /* power */ tavor_quiesce, /* devo_quiesce */ }; /* Module Driver Info */ static struct modldrv tavor_modldrv = { &mod_driverops, "Tavor InfiniBand HCA Driver", &tavor_ops }; /* Module Linkage */ static struct modlinkage tavor_modlinkage = { MODREV_1, &tavor_modldrv, NULL }; /* * This extern refers to the ibc_operations_t function vector that is defined * in the tavor_ci.c file. */ extern ibc_operations_t tavor_ibc_ops; #ifndef NPROBE extern int tnf_mod_load(void); extern int tnf_mod_unload(struct modlinkage *mlp); #endif /* * _init() */ int _init() { int status; #ifndef NPROBE (void) tnf_mod_load(); #endif TAVOR_TNF_ENTER(tavor_init); status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t), (size_t)TAVOR_INITIAL_STATES); if (status != 0) { TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_init); #ifndef NPROBE (void) tnf_mod_unload(&tavor_modlinkage); #endif return (status); } status = ibc_init(&tavor_modlinkage); if (status != 0) { TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, ""); ddi_soft_state_fini(&tavor_statep); TAVOR_TNF_EXIT(tavor_init); #ifndef NPROBE (void) tnf_mod_unload(&tavor_modlinkage); #endif return (status); } status = mod_install(&tavor_modlinkage); if (status != 0) { TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, ""); ibc_fini(&tavor_modlinkage); ddi_soft_state_fini(&tavor_statep); TAVOR_TNF_EXIT(tavor_init); #ifndef NPROBE (void) tnf_mod_unload(&tavor_modlinkage); #endif return (status); } /* Initialize the Tavor "userland resources database" */ tavor_umap_db_init(); TAVOR_TNF_EXIT(tavor_init); return (status); } /* * _info() */ int _info(struct modinfo *modinfop) { int status; TAVOR_TNF_ENTER(tavor_info); status = mod_info(&tavor_modlinkage, modinfop); TAVOR_TNF_EXIT(tavor_info); return (status); } /* * _fini() */ int _fini() { int status; TAVOR_TNF_ENTER(tavor_fini); status = mod_remove(&tavor_modlinkage); if (status != 0) { TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_fini); return (status); } /* Destroy the Tavor "userland resources database" */ tavor_umap_db_fini(); ibc_fini(&tavor_modlinkage); ddi_soft_state_fini(&tavor_statep); #ifndef NPROBE (void) tnf_mod_unload(&tavor_modlinkage); #endif TAVOR_TNF_EXIT(tavor_fini); return (status); } /* * tavor_getinfo() */ /* ARGSUSED */ static int tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) { dev_t dev; tavor_state_t *state; minor_t instance; TAVOR_TNF_ENTER(tavor_getinfo); switch (cmd) { case DDI_INFO_DEVT2DEVINFO: dev = (dev_t)arg; instance = TAVOR_DEV_INSTANCE(dev); state = ddi_get_soft_state(tavor_statep, instance); if (state == NULL) { TNF_PROBE_0(tavor_getinfo_gss_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_getinfo); return (DDI_FAILURE); } *result = (void *)state->ts_dip; return (DDI_SUCCESS); case DDI_INFO_DEVT2INSTANCE: dev = (dev_t)arg; instance = TAVOR_DEV_INSTANCE(dev); *result = (void *)(uintptr_t)instance; return (DDI_SUCCESS); default: TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, ""); break; } TAVOR_TNF_EXIT(tavor_getinfo); return (DDI_FAILURE); } /* * tavor_open() */ /* ARGSUSED */ static int tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp) { tavor_state_t *state; tavor_rsrc_t *rsrcp; tavor_umap_db_entry_t *umapdb, *umapdb2; minor_t instance; uint64_t key, value; uint_t tr_indx; dev_t dev; int status; TAVOR_TNF_ENTER(tavor_open); instance = TAVOR_DEV_INSTANCE(*devp); state = ddi_get_soft_state(tavor_statep, instance); if (state == NULL) { TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_open); return (ENXIO); } /* * Only allow driver to be opened for character access, and verify * whether exclusive access is allowed. */ if ((otyp != OTYP_CHR) || ((flag & FEXCL) && secpolicy_excl_open(credp) != 0)) { TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_open); return (EINVAL); } /* * Search for the current process PID in the "userland resources * database". If it is not found, then attempt to allocate a UAR * page and add the ("key", "value") pair to the database. * Note: As a last step we always return a devp appropriate for * the open. Either we return a new minor number (based on the * instance and the UAR page index) or we return the current minor * number for the given client process. * * We also add an entry to the database to allow for lookup from * "dev_t" to the current process PID. This is necessary because, * under certain circumstance, the process PID that calls the Tavor * close() entry point may not be the same as the one who called * open(). Specifically, this can happen if a child process calls * the Tavor's open() entry point, gets a UAR page, maps it out (using * mmap()), and then exits without calling munmap(). Because mmap() * adds a reference to the file descriptor, at the exit of the child * process the file descriptor is "inherited" by the parent (and will * be close()'d by the parent's PID only when it exits). * * Note: We use the tavor_umap_db_find_nolock() and * tavor_umap_db_add_nolock() database access routines below (with * an explicit mutex_enter of the database lock - "tdl_umapdb_lock") * to ensure that the multiple accesses (in this case searching for, * and then adding _two_ database entries) can be done atomically. */ key = ddi_get_pid(); mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_UARPG_RSRC, &value, 0, NULL); if (status != DDI_SUCCESS) { /* * If we are in 'maintenance mode', we cannot alloc a UAR page. * But we still need some rsrcp value, and a mostly unique * tr_indx value. So we set rsrcp to NULL for maintenance * mode, and use a rolling count for tr_indx. The field * 'ts_open_tr_indx' is used only in this maintenance mode * condition. * * Otherwise, if we are in operational mode then we allocate * the UAR page as normal, and use the rsrcp value and tr_indx * value from that allocation. */ if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { rsrcp = NULL; tr_indx = state->ts_open_tr_indx++; } else { /* Allocate a new UAR page for this process */ status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_NOSLEEP, &rsrcp); if (status != DDI_SUCCESS) { mutex_exit( &tavor_userland_rsrc_db.tdl_umapdb_lock); TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_open); return (EAGAIN); } tr_indx = rsrcp->tr_indx; } /* * Allocate an entry to track the UAR page resource in the * "userland resources database". */ umapdb = tavor_umap_db_alloc(instance, key, MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp); if (umapdb == NULL) { mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); /* If in "maintenance mode", don't free the rsrc */ if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { tavor_rsrc_free(state, &rsrcp); } TNF_PROBE_0(tavor_open_umap_db_alloc_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_open); return (EAGAIN); } /* * Create a new device number. Minor number is a function of * the UAR page index (15 bits) and the device instance number * (3 bits). */ dev = makedevice(getmajor(*devp), (tr_indx << TAVOR_MINORNUM_SHIFT) | instance); /* * Allocate another entry in the "userland resources database" * to track the association of the device number (above) to * the current process ID (in "key"). */ umapdb2 = tavor_umap_db_alloc(instance, dev, MLNX_UMAP_PID_RSRC, (uint64_t)key); if (umapdb2 == NULL) { mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); tavor_umap_db_free(umapdb); /* If in "maintenance mode", don't free the rsrc */ if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { tavor_rsrc_free(state, &rsrcp); } TNF_PROBE_0(tavor_open_umap_db_alloc_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_open); return (EAGAIN); } /* Add the entries to the database */ tavor_umap_db_add_nolock(umapdb); tavor_umap_db_add_nolock(umapdb2); } else { /* * Return the same device number as on the original open() * call. This was calculated as a function of the UAR page * index (top 16 bits) and the device instance number */ rsrcp = (tavor_rsrc_t *)(uintptr_t)value; dev = makedevice(getmajor(*devp), (rsrcp->tr_indx << TAVOR_MINORNUM_SHIFT) | instance); } mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); *devp = dev; TAVOR_TNF_EXIT(tavor_open); return (0); } /* * tavor_close() */ /* ARGSUSED */ static int tavor_close(dev_t dev, int flag, int otyp, cred_t *credp) { tavor_state_t *state; tavor_rsrc_t *rsrcp; tavor_umap_db_entry_t *umapdb; tavor_umap_db_priv_t *priv; minor_t instance; uint64_t key, value; int status; TAVOR_TNF_ENTER(tavor_close); instance = TAVOR_DEV_INSTANCE(dev); state = ddi_get_soft_state(tavor_statep, instance); if (state == NULL) { TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_close); return (ENXIO); } /* * Search for "dev_t" in the "userland resources database". As * explained above in tavor_open(), we can't depend on using the * current process ID here to do the lookup because the process * that ultimately closes may not be the same one who opened * (because of inheritance). * So we lookup the "dev_t" (which points to the PID of the process * that opened), and we remove the entry from the database (and free * it up). Then we do another query based on the PID value. And when * we find that database entry, we free it up too and then free the * Tavor UAR page resource. * * Note: We use the tavor_umap_db_find_nolock() database access * routine below (with an explicit mutex_enter of the database lock) * to ensure that the multiple accesses (which attempt to remove the * two database entries) can be done atomically. * * This works the same in both maintenance mode and HCA mode, except * for the call to tavor_rsrc_free(). In the case of maintenance mode, * this call is not needed, as it was not allocated in tavor_open() * above. */ key = dev; mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC, &value, TAVOR_UMAP_DB_REMOVE, &umapdb); if (status == DDI_SUCCESS) { /* * If the "tdb_priv" field is non-NULL, it indicates that * some "on close" handling is still necessary. Call * tavor_umap_db_handle_onclose_cb() to do the handling (i.e. * to invoke all the registered callbacks). Then free up * the resources associated with "tdb_priv" and continue * closing. */ priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv; if (priv != NULL) { tavor_umap_db_handle_onclose_cb(priv); kmem_free(priv, sizeof (tavor_umap_db_priv_t)); umapdb->tdbe_common.tdb_priv = (void *)NULL; } tavor_umap_db_free(umapdb); /* * Now do another lookup using PID as the key (copy it from * "value"). When this lookup is complete, the "value" field * will contain the tavor_rsrc_t pointer for the UAR page * resource. */ key = value; status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE, &umapdb); if (status == DDI_SUCCESS) { tavor_umap_db_free(umapdb); /* If in "maintenance mode", don't free the rsrc */ if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { rsrcp = (tavor_rsrc_t *)(uintptr_t)value; tavor_rsrc_free(state, &rsrcp); } } } mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); TAVOR_TNF_EXIT(tavor_close); return (0); } /* * tavor_attach() * Context: Only called from attach() path context */ static int tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { tavor_state_t *state; ibc_clnt_hdl_t tmp_ibtfpriv; ibc_status_t ibc_status; int instance; int status; TAVOR_TNF_ENTER(tavor_attach); #ifdef __lock_lint (void) tavor_quiesce(dip); #endif switch (cmd) { case DDI_ATTACH: instance = ddi_get_instance(dip); status = ddi_soft_state_zalloc(tavor_statep, instance); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, ""); cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " "attach_ssz_fail", instance); goto fail_attach_nomsg; } state = ddi_get_soft_state(tavor_statep, instance); if (state == NULL) { ddi_soft_state_free(tavor_statep, instance); TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, ""); cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " "attach_gss_fail", instance); goto fail_attach_nomsg; } /* clear the attach error buffer */ TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf); /* * Initialize Tavor driver and hardware. * * Note: If this initialization fails we may still wish to * create a device node and remain operational so that Tavor * firmware can be updated/flashed (i.e. "maintenance mode"). * If this is the case, then "ts_operational_mode" will be * equal to TAVOR_MAINTENANCE_MODE. We will not attempt to * attach to the IBTF or register with the IBMF (i.e. no * InfiniBand interfaces will be enabled). */ status = tavor_drv_init(state, dip, instance); if ((status != DDI_SUCCESS) && (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { TNF_PROBE_0(tavor_attach_drvinit_fail, TAVOR_TNF_ERROR, ""); goto fail_attach; } /* Create the minor node for device */ status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance, DDI_PSEUDO, 0); if (status != DDI_SUCCESS) { tavor_drv_fini(state); TAVOR_ATTACH_MSG(state->ts_attach_buf, "attach_create_mn_fail"); TNF_PROBE_0(tavor_attach_create_mn_fail, TAVOR_TNF_ERROR, ""); goto fail_attach; } /* * If we are in "maintenance mode", then we don't want to * register with the IBTF. All InfiniBand interfaces are * uninitialized, and the device is only capable of handling * requests to update/flash firmware (or test/debug requests). */ if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { /* Attach to InfiniBand Transport Framework (IBTF) */ ibc_status = ibc_attach(&tmp_ibtfpriv, &state->ts_ibtfinfo); if (ibc_status != IBC_SUCCESS) { ddi_remove_minor_node(dip, "devctl"); tavor_drv_fini(state); TNF_PROBE_0(tavor_attach_ibcattach_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "attach_ibcattach_fail"); goto fail_attach; } /* * Now that we've successfully attached to the IBTF, * we enable all appropriate asynch and CQ events to * be forwarded to the IBTF. */ TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv); ibc_post_attach(state->ts_ibtfpriv); /* Register agents with IB Mgmt Framework (IBMF) */ status = tavor_agent_handlers_init(state); if (status != DDI_SUCCESS) { (void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH); TAVOR_QUIESCE_IBTF_CALLB(state); if (state->ts_in_evcallb != 0) { TAVOR_WARNING(state, "unable to " "quiesce Tavor IBTF callbacks"); } ibc_detach(tmp_ibtfpriv); ddi_remove_minor_node(dip, "devctl"); tavor_drv_fini(state); TNF_PROBE_0(tavor_attach_agentinit_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "attach_agentinit_fail"); goto fail_attach; } } /* Report that driver was loaded */ ddi_report_dev(dip); /* Send device information to log file */ tavor_device_info_report(state); /* Report attach in maintenance mode, if appropriate */ if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { cmn_err(CE_NOTE, "tavor%d: driver attached " "(for maintenance mode only)", state->ts_instance); } TAVOR_TNF_EXIT(tavor_attach); return (DDI_SUCCESS); case DDI_RESUME: /* Add code here for DDI_RESUME XXX */ TAVOR_TNF_EXIT(tavor_attach); return (DDI_FAILURE); default: TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, ""); break; } fail_attach: cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance, state->ts_attach_buf); tavor_drv_fini2(state); ddi_soft_state_free(tavor_statep, instance); fail_attach_nomsg: TAVOR_TNF_EXIT(tavor_attach); return (DDI_FAILURE); } /* * tavor_detach() * Context: Only called from detach() path context */ static int tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { tavor_state_t *state; ibc_clnt_hdl_t tmp_ibtfpriv; ibc_status_t ibc_status; int instance, status; TAVOR_TNF_ENTER(tavor_detach); instance = ddi_get_instance(dip); state = ddi_get_soft_state(tavor_statep, instance); if (state == NULL) { TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_detach); return (DDI_FAILURE); } switch (cmd) { case DDI_DETACH: /* * If we are in "maintenance mode", then we do not want to * do teardown for any of the InfiniBand interfaces. * Specifically, this means not detaching from IBTF (we never * attached to begin with) and not deregistering from IBMF. */ if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { /* Unregister agents from IB Mgmt Framework (IBMF) */ status = tavor_agent_handlers_fini(state); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_detach_agentfini_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_detach); return (DDI_FAILURE); } /* * Attempt the "pre-detach" from InfiniBand Transport * Framework (IBTF). At this point the IBTF is still * capable of handling incoming asynch and completion * events. This "pre-detach" is primarily a mechanism * to notify the appropriate IBTF clients that the * HCA is being removed/offlined. */ ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd); if (ibc_status != IBC_SUCCESS) { status = tavor_agent_handlers_init(state); if (status != DDI_SUCCESS) { TAVOR_WARNING(state, "failed to " "restart Tavor agents"); } TNF_PROBE_0(tavor_detach_ibcpredetach_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_detach); return (DDI_FAILURE); } /* * Before we can fully detach from the IBTF we need to * ensure that we have handled all outstanding event * callbacks. This is accomplished by quiescing the * event callback mechanism. Note: if we are unable * to successfully quiesce the callbacks, then this is * an indication that something has probably gone * seriously wrong. We print out a warning, but * continue. */ tmp_ibtfpriv = state->ts_ibtfpriv; TAVOR_QUIESCE_IBTF_CALLB(state); if (state->ts_in_evcallb != 0) { TAVOR_WARNING(state, "unable to quiesce Tavor " "IBTF callbacks"); } /* Complete the detach from the IBTF */ ibc_detach(tmp_ibtfpriv); } /* Remove the minor node for device */ ddi_remove_minor_node(dip, "devctl"); /* * Only call tavor_drv_fini() if we are in Tavor HCA mode. * (Because if we are in "maintenance mode", then we never * successfully finished init.) Only report successful * detach for normal HCA mode. */ if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { /* Cleanup driver resources and shutdown hardware */ tavor_drv_fini(state); cmn_err(CE_CONT, "Tavor driver successfully " "detached\n"); } tavor_drv_fini2(state); ddi_soft_state_free(tavor_statep, instance); TAVOR_TNF_EXIT(tavor_detach); return (DDI_SUCCESS); case DDI_SUSPEND: /* Add code here for DDI_SUSPEND XXX */ TAVOR_TNF_EXIT(tavor_detach); return (DDI_FAILURE); default: TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, ""); break; } TAVOR_TNF_EXIT(tavor_detach); return (DDI_FAILURE); } /* * tavor_drv_init() * Context: Only called from attach() path context */ static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance) { int status; TAVOR_TNF_ENTER(tavor_drv_init); /* Save away devinfo and instance */ state->ts_dip = dip; state->ts_instance = instance; /* * Check and set the operational mode of the device. If the driver is * bound to the Tavor device in "maintenance mode", then this generally * means that either the device has been specifically jumpered to * start in this mode or the firmware boot process has failed to * successfully load either the primary or the secondary firmware * image. */ if (TAVOR_IS_HCA_MODE(state->ts_dip)) { state->ts_operational_mode = TAVOR_HCA_MODE; } else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) { state->ts_operational_mode = TAVOR_COMPAT_MODE; } else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) { state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; return (DDI_FAILURE); } else { state->ts_operational_mode = 0; /* invalid operational mode */ TAVOR_WARNING(state, "unexpected device type detected"); TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* * Initialize the Tavor hardware. * Note: If this routine returns an error, it is often an reasonably * good indication that something Tavor firmware-related has caused * the failure. In order to give the user an opportunity (if desired) * to update or reflash the Tavor firmware image, we set * "ts_operational_mode" flag (described above) to indicate that we * wish to enter maintenance mode. */ status = tavor_hw_init(state); if (status != DDI_SUCCESS) { state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance, state->ts_attach_buf); TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_drv_init); return (DDI_FAILURE); } /* Setup Tavor interrupt handler */ status = tavor_isr_init(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_drv_init); return (DDI_FAILURE); } /* Initialize Tavor softstate */ status = tavor_soft_state_init(state); if (status != DDI_SUCCESS) { tavor_isr_fini(state); tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_drv_init); return (DDI_FAILURE); } TAVOR_TNF_EXIT(tavor_drv_init); return (DDI_SUCCESS); } /* * tavor_drv_fini() * Context: Only called from attach() and/or detach() path contexts */ static void tavor_drv_fini(tavor_state_t *state) { TAVOR_TNF_ENTER(tavor_drv_fini); /* Cleanup Tavor softstate */ tavor_soft_state_fini(state); /* Teardown Tavor interrupts */ tavor_isr_fini(state); /* Cleanup Tavor resources and shutdown hardware */ tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); TAVOR_TNF_EXIT(tavor_drv_fini); } /* * tavor_drv_fini2() * Context: Only called from attach() and/or detach() path contexts */ static void tavor_drv_fini2(tavor_state_t *state) { TAVOR_TNF_ENTER(tavor_drv_fini2); /* TAVOR_DRV_CLEANUP_LEVEL1 */ if (state->ts_reg_cmdhdl) { ddi_regs_map_free(&state->ts_reg_cmdhdl); state->ts_reg_cmdhdl = NULL; } /* TAVOR_DRV_CLEANUP_LEVEL0 */ if (state->ts_pci_cfghdl) { pci_config_teardown(&state->ts_pci_cfghdl); state->ts_pci_cfghdl = NULL; } TAVOR_TNF_EXIT(tavor_drv_fini2); } /* * tavor_isr_init() * Context: Only called from attach() path context */ static int tavor_isr_init(tavor_state_t *state) { int status; TAVOR_TNF_ENTER(tavor_isr_init); /* * Add a handler for the interrupt or MSI */ status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr, (caddr_t)state, NULL); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_isr_init); return (DDI_FAILURE); } /* * Enable the software interrupt. Note: Even though we are only * using one (1) interrupt/MSI, depending on the value returned in * the capability flag, we have to call either ddi_intr_block_enable() * or ddi_intr_enable(). */ if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_isr_init_blockenable_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_isr_init); return (DDI_FAILURE); } } else { status = ddi_intr_enable(state->ts_intrmsi_hdl); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_isr_init_intrenable_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_isr_init); return (DDI_FAILURE); } } /* * Now that the ISR has been setup, arm all the EQs for event * generation. */ tavor_eq_arm_all(state); TAVOR_TNF_EXIT(tavor_isr_init); return (DDI_SUCCESS); } /* * tavor_isr_fini() * Context: Only called from attach() and/or detach() path contexts */ static void tavor_isr_fini(tavor_state_t *state) { TAVOR_TNF_ENTER(tavor_isr_fini); /* Disable the software interrupt */ if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { (void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1); } else { (void) ddi_intr_disable(state->ts_intrmsi_hdl); } /* * Remove the software handler for the interrupt or MSI */ (void) ddi_intr_remove_handler(state->ts_intrmsi_hdl); TAVOR_TNF_EXIT(tavor_isr_fini); } /* * tavor_fix_error_buf() * Context: Only called from attach(). * * The error_buf_addr returned from QUERY_FW is a PCI address. * We need to convert it to an offset from the base address, * which is stored in the assigned-addresses property. */ static int tavor_fix_error_buf(tavor_state_t *state) { int assigned_addr_len; pci_regspec_t *assigned_addr; if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip, DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr, (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS) return (DDI_FAILURE); state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low + ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32); ddi_prop_free(assigned_addr); return (DDI_SUCCESS); } /* * tavor_hw_init() * Context: Only called from attach() path context */ static int tavor_hw_init(tavor_state_t *state) { tavor_drv_cleanup_level_t cleanup; sm_nodeinfo_t nodeinfo; uint64_t errorcode; off_t ddr_size; int status; int retries; TAVOR_TNF_ENTER(tavor_hw_init); /* This is where driver initialization begins */ cleanup = TAVOR_DRV_CLEANUP_LEVEL0; /* Setup device access attributes */ state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0; state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC; state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC; /* Setup for PCI config read/write of HCA device */ status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_PCI_config_space_regmap_fail"); /* This case is not the degraded one */ return (DDI_FAILURE); } /* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */ status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR, &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr, &state->ts_reg_cmdhdl); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_CMD_ddirms_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL1; status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR, &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr, &state->ts_reg_uarhdl); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_UAR_ddirms_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL2; status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size); if (status != DDI_SUCCESS) { cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed " "(check HCA-attached DIMM memory?)\n"); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_DDR_ddi_regsize_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } #if !defined(_ELF64) && !defined(__sparc) /* * For 32 bit x86/x64 kernels, where there is limited kernel virtual * memory available, define a minimal memory footprint. This is * specified in order to not take up too much resources, thus starving * out others. Only specified if the HCA DIMM is equal to or greater * than 256MB. * * Note: x86/x64 install and safemode boot are both 32bit. */ ddr_size = TAVOR_DDR_SIZE_MIN; #endif /* !(_ELF64) && !(__sparc) */ state->ts_cfg_profile_setting = ddr_size; status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr, &state->ts_reg_ddrhdl); /* * On 32-bit platform testing (primarily x86), it was seen that the * ddi_regs_map_setup() call would fail because there wasn't enough * kernel virtual address space available to map in the entire 256MB * DDR. So we add this check in here, so that if the 256 (or other * larger value of DDR) map in fails, that we fallback to try the lower * size of 128MB. * * Note: If we only have 128MB of DDR in the system in the first place, * we don't try another ddi_regs_map_setup(), and just skip over this * check and return failures. */ if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) { /* Try falling back to 128MB DDR mapping */ status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128, &state->ts_reg_accattr, &state->ts_reg_ddrhdl); /* * 128MB DDR mapping worked. * Set the updated config profile setting here. */ if (status == DDI_SUCCESS) { TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success, TAVOR_TNF_TRACE, ""); state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128; } } if (status != DDI_SUCCESS) { if (status == DDI_ME_RNUMBER_RANGE) { cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed " "(check HCA-attached DIMM memory?)\n"); } tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_DDR_ddirms_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL3; /* Setup Tavor Host Command Register (HCR) */ state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *) ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET); /* Setup Tavor Event Cause Register (ecr and clr_ecr) */ state->ts_cmd_regs.ecr = (uint64_t *) ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET); state->ts_cmd_regs.clr_ecr = (uint64_t *) ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET); /* Setup Tavor Software Reset register (sw_reset) */ state->ts_cmd_regs.sw_reset = (uint32_t *) ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET); /* Setup Tavor Clear Interrupt register (clr_int) */ state->ts_cmd_regs.clr_int = (uint64_t *) ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET); /* Initialize the Phase1 Tavor configuration profile */ status = tavor_cfg_profile_init_phase1(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL4; /* Do a software reset of the Tavor HW to ensure proper state */ status = tavor_sw_reset(state); if (status != TAVOR_CMD_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* Post the SYS_EN command to start the hardware */ status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL, &errorcode, TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { if ((status == TAVOR_CMD_BAD_NVMEM) || (status == TAVOR_CMD_DDR_MEM_ERR)) { cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " "0x%" PRIx64 " (invalid firmware image?)\n", status, errorcode); } else { cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " "0x%" PRIx64 "\n", status, errorcode); } tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sys_en_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL5; /* First phase of init for Tavor configuration/resources */ status = tavor_rsrc_init_phase1(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_rsrcinit1_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL6; /* Query the DDR properties (e.g. total DDR size) */ status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0, &state->ts_ddr, sizeof (tavor_hw_queryddr_t), TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_query_ddr_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* Figure out how big the firmware image (in DDR) is */ status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw, sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_query_fw_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } if (tavor_fix_error_buf(state) != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_fixerrorbuf_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* Validate that the FW version is appropriate */ status = tavor_fw_version_check(state); if (status != DDI_SUCCESS) { if (state->ts_operational_mode == TAVOR_HCA_MODE) { cmn_err(CE_CONT, "Unsupported Tavor FW version: " "expected: %04d.%04d.%04d, " "actual: %04d.%04d.%04d\n", TAVOR_FW_VER_MAJOR, TAVOR_FW_VER_MINOR, TAVOR_FW_VER_SUBMINOR, state->ts_fw.fw_rev_major, state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor); } else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) { cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: " "expected: %04d.%04d.%04d, " "actual: %04d.%04d.%04d\n", TAVOR_COMPAT_FW_VER_MAJOR, TAVOR_COMPAT_FW_VER_MINOR, TAVOR_COMPAT_FW_VER_SUBMINOR, state->ts_fw.fw_rev_major, state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor); } else { cmn_err(CE_CONT, "Unsupported FW version: " "%04d.%04d.%04d\n", state->ts_fw.fw_rev_major, state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor); } tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_checkfwver_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_checkfwver_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } drv_usecwait(10); retries = 1000; /* retry up to 1 second before giving up */ retry: /* Call MOD_STAT_CFG to setup SRQ support (or disable) */ status = tavor_mod_stat_cfg_cmd_post(state); if (status != DDI_SUCCESS) { if (retries > 0) { drv_usecwait(1000); retries--; goto retry; } cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mod_stat_cfg_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* Figure out Tavor device limits */ status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0, &state->ts_devlim, sizeof (tavor_hw_querydevlim_t), TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_query_devlim_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* Initialize the Phase2 Tavor configuration profile */ status = tavor_cfg_profile_init_phase2(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* Second phase of init for Tavor configuration/resources */ status = tavor_rsrc_init_phase2(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_rsrcinit2_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL7; /* Miscellaneous query information */ status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0, &state->ts_adapter, sizeof (tavor_hw_queryadapter_t), TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_query_adapter_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* Prepare configuration for Tavor INIT_HCA command */ tavor_hca_config_setup(state, &state->ts_hcaparams); /* Post command to init Tavor HCA */ status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams, TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_init_hca_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL8; /* Allocate protection domain (PD) for Tavor internal use */ status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_internal_pd_alloc_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL9; /* Setup Tavor internal UAR pages (0 and 1) */ status = tavor_internal_uarpgs_init(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_internal_uarpgs_alloc_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL10; /* Query and initialize the Tavor interrupt/MSI information */ status = tavor_intr_or_msi_init(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_intr_or_msi_init_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "intr_or_msi_init_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL11; /* Setup all of the Tavor EQs */ status = tavor_eq_init_all(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_eqinitall_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL12; /* Set aside contexts for QP0 and QP1 */ status = tavor_special_qp_contexts_reserve(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_reserve_special_qp_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL13; /* Initialize for multicast group handling */ status = tavor_mcg_init(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_LEVEL14; /* Initialize the Tavor IB port(s) */ status = tavor_hca_port_init(state); if (status != DDI_SUCCESS) { tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_hca_port_init_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_hca_port_init_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } cleanup = TAVOR_DRV_CLEANUP_ALL; /* Determine NodeGUID and SystemImageGUID */ status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, &nodeinfo); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_getnodeinfo_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } /* * If the NodeGUID value was set in OBP properties, then we use that * value. But we still print a message if the value we queried from * firmware does not match this value. * * Otherwise if OBP value is not set then we use the value from * firmware unconditionally. */ if (state->ts_cfg_profile->cp_nodeguid) { state->ts_nodeguid = state->ts_cfg_profile->cp_nodeguid; } else { state->ts_nodeguid = nodeinfo.NodeGUID; } if (state->ts_nodeguid != nodeinfo.NodeGUID) { cmn_err(CE_NOTE, "!NodeGUID value queried from firmware " "does not match value set by device property"); } /* * If the SystemImageGUID value was set in OBP properties, then we use * that value. But we still print a message if the value we queried * from firmware does not match this value. * * Otherwise if OBP value is not set then we use the value from * firmware unconditionally. */ if (state->ts_cfg_profile->cp_sysimgguid) { state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid; } else { state->ts_sysimgguid = nodeinfo.SystemImageGUID; } if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) { cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware " "does not match value set by device property"); } /* Get NodeDescription */ status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, (sm_nodedesc_t *)&state->ts_nodedesc); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n", status); tavor_hw_fini(state, cleanup); TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_getnodedesc_cmd_fail"); TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_FAILURE); } TAVOR_TNF_EXIT(tavor_hw_init); return (DDI_SUCCESS); } /* * tavor_hw_fini() * Context: Only called from attach() and/or detach() path contexts */ static void tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup) { uint_t num_ports; int status; TAVOR_TNF_ENTER(tavor_hw_fini); switch (cleanup) { /* * If we add more driver initialization steps that should be cleaned * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the * first entry (i.e. corresponds to the last init step). */ case TAVOR_DRV_CLEANUP_ALL: /* Shutdown the Tavor IB port(s) */ num_ports = state->ts_cfg_profile->cp_num_ports; (void) tavor_hca_ports_shutdown(state, num_ports); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL14: /* Teardown resources used for multicast group handling */ tavor_mcg_fini(state); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL13: /* Unreserve the special QP contexts */ tavor_special_qp_contexts_unreserve(state); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL12: /* * Attempt to teardown all event queues (EQ). If we fail * here then print a warning message and return. Something * (either in HW or SW) has gone seriously wrong. */ status = tavor_eq_fini_all(state); if (status != DDI_SUCCESS) { TAVOR_WARNING(state, "failed to teardown EQs"); TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_hw_fini); return; } /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL11: status = tavor_intr_or_msi_fini(state); if (status != DDI_SUCCESS) { TAVOR_WARNING(state, "failed to free intr/MSI"); TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_hw_fini); return; } /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL10: /* Free the resources for the Tavor internal UAR pages */ tavor_internal_uarpgs_fini(state); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL9: /* * Free the PD that was used internally by Tavor software. If * we fail here then print a warning and return. Something * (probably software-related, but perhaps HW) has gone wrong. */ status = tavor_pd_free(state, &state->ts_pdhdl_internal); if (status != DDI_SUCCESS) { TAVOR_WARNING(state, "failed to free internal PD"); TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_hw_fini); return; } /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL8: /* * Post the CLOSE_HCA command to Tavor firmware. If we fail * here then print a warning and return. Something (either in * HW or SW) has gone seriously wrong. */ status = tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { TAVOR_WARNING(state, "failed to shutdown HCA"); TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_hw_fini); return; } /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL7: /* Cleanup all the phase2 resources first */ tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL6: /* Then cleanup the phase1 resources */ tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL5: /* * Post the SYS_DIS command to Tavor firmware to shut * everything down again. If we fail here then print a * warning and return. Something (probably in HW, but maybe * in SW) has gone seriously wrong. */ status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { TAVOR_WARNING(state, "failed to shutdown hardware"); TNF_PROBE_0(tavor_hw_fini_sys_dis_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_hw_fini); return; } /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL4: /* Teardown any resources allocated for the config profile */ tavor_cfg_profile_fini(state); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL3: ddi_regs_map_free(&state->ts_reg_ddrhdl); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL2: ddi_regs_map_free(&state->ts_reg_uarhdl); /* FALLTHROUGH */ case TAVOR_DRV_CLEANUP_LEVEL1: case TAVOR_DRV_CLEANUP_LEVEL0: /* * LEVEL1 and LEVEL0 resources are freed in * tavor_drv_fini2(). */ break; default: TAVOR_WARNING(state, "unexpected driver cleanup level"); TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_hw_fini); return; } TAVOR_TNF_EXIT(tavor_hw_fini); } /* * tavor_soft_state_init() * Context: Only called from attach() path context */ static int tavor_soft_state_init(tavor_state_t *state) { ibt_hca_attr_t *hca_attr; uint64_t maxval, val; ibt_hca_flags_t caps = IBT_HCA_NO_FLAGS; int status; TAVOR_TNF_ENTER(tavor_soft_state_init); /* * The ibc_hca_info_t struct is passed to the IBTF. This is the * routine where we initialize it. Many of the init values come from * either configuration variables or successful queries of the Tavor * hardware abilities */ state->ts_ibtfinfo.hca_ci_vers = IBCI_V4; state->ts_ibtfinfo.hca_handle = (ibc_hca_hdl_t)state; state->ts_ibtfinfo.hca_ops = &tavor_ibc_ops; hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP); state->ts_ibtfinfo.hca_attr = hca_attr; hca_attr->hca_dip = state->ts_dip; hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major; hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor; hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor; /* * Determine HCA capabilities: * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST, * IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE, * or IBT_HCA_SHUTDOWN_PORT * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID, * IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always * supported * All other features are conditionally supported, depending on the * status return by the Tavor HCA (in QUERY_DEV_LIM) */ if (state->ts_devlim.ud_multi) { caps |= IBT_HCA_UD_MULTICAST; } if (state->ts_devlim.atomic) { caps |= IBT_HCA_ATOMICS_HCA; } if (state->ts_devlim.apm) { caps |= IBT_HCA_AUTO_PATH_MIG; } if (state->ts_devlim.pkey_v) { caps |= IBT_HCA_PKEY_CNTR; } if (state->ts_devlim.qkey_v) { caps |= IBT_HCA_QKEY_CNTR; } if (state->ts_cfg_profile->cp_srq_enable) { caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ; } caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT | IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE | IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE); hca_attr->hca_flags = caps; hca_attr->hca_flags2 = IBT_HCA2_DMA_MR; /* Determine VendorID, DeviceID, and revision ID */ hca_attr->hca_vendor_id = state->ts_adapter.vendor_id; hca_attr->hca_device_id = state->ts_adapter.device_id; hca_attr->hca_version_id = state->ts_adapter.rev_id; /* * Determine number of available QPs and max QP size. Number of * available QPs is determined by subtracting the number of * "reserved QPs" (i.e. reserved for firmware use) from the * total number configured. */ val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); hca_attr->hca_max_qp = val - ((uint64_t)1 << state->ts_devlim.log_rsvd_qp); maxval = ((uint64_t)1 << state->ts_devlim.log_max_qp_sz); val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz); if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size " "exceeds device maximum", tnf_uint, maxsz, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_maxqpsz_toobig_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_qp_sz = val; /* Determine max scatter-gather size in WQEs */ maxval = state->ts_devlim.max_sg; val = state->ts_cfg_profile->cp_wqe_max_sgl; if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl " "exceeds device maximum", tnf_uint, maxsgl, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_toomanysgl_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } /* If the rounded value for max SGL is too large, cap it */ if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) { state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval; val = maxval; } else { val = state->ts_cfg_profile->cp_wqe_real_max_sgl; } hca_attr->hca_max_sgl = val; hca_attr->hca_max_rd_sgl = 0; /* zero because RD is unsupported */ /* * Determine number of available CQs and max CQ size. Number of * available CQs is determined by subtracting the number of * "reserved CQs" (i.e. reserved for firmware use) from the * total number configured. */ val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq); hca_attr->hca_max_cq = val - ((uint64_t)1 << state->ts_devlim.log_rsvd_cq); maxval = ((uint64_t)1 << state->ts_devlim.log_max_cq_sz); val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1; if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size " "exceeds device maximum", tnf_uint, maxsz, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_maxcqsz_toobig_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_cq_sz = val; /* * Determine number of available SRQs and max SRQ size. Number of * available SRQs is determined by subtracting the number of * "reserved SRQs" (i.e. reserved for firmware use) from the * total number configured. */ val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq); hca_attr->hca_max_srqs = val - ((uint64_t)1 << state->ts_devlim.log_rsvd_srq); maxval = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz); val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz); if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size " "exceeds device maximum", tnf_uint, maxsz, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_maxsrqsz_toobig_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_srqs_sz = val; val = state->ts_cfg_profile->cp_srq_max_sgl; maxval = state->ts_devlim.max_sg; if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq " "sgl exceeds device maximum", tnf_uint, maxsgl, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_toomanysrqsgl_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_srq_sgl = val; /* * Determine supported HCA page sizes * XXX * For now we simply return the system pagesize as the only supported * pagesize */ hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K : IBT_PAGE_4K); /* * Determine number of available MemReg, MemWin, and their max size. * Number of available MRs and MWs is determined by subtracting * the number of "reserved MPTs" (i.e. reserved for firmware use) * from the total number configured for each. */ val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt); hca_attr->hca_max_memr = val - ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt); hca_attr->hca_max_mem_win = val - ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt); maxval = state->ts_devlim.log_max_mrw_sz; val = state->ts_cfg_profile->cp_log_max_mrw_sz; if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size " "exceeds device maximum", tnf_uint, maxsz, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_maxmrwsz_toobig_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_memr_len = ((uint64_t)1 << val); /* Determine RDMA/Atomic properties */ val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb); hca_attr->hca_max_rsc = val; val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp; hca_attr->hca_max_rdma_in_qp = val; val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp; hca_attr->hca_max_rdma_out_qp = val; hca_attr->hca_max_rdma_in_ee = 0; hca_attr->hca_max_rdma_out_ee = 0; /* * Determine maximum number of raw IPv6 and Ether QPs. Set to 0 * because neither type of raw QP is supported */ hca_attr->hca_max_ipv6_qp = 0; hca_attr->hca_max_ether_qp = 0; /* Determine max number of MCGs and max QP-per-MCG */ val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); hca_attr->hca_max_mcg_qps = val; val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg); hca_attr->hca_max_mcg = val; val = state->ts_cfg_profile->cp_num_qp_per_mcg; hca_attr->hca_max_qp_per_mcg = val; /* Determine max number partitions (i.e. PKeys) */ maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); val = ((uint64_t)state->ts_cfg_profile->cp_num_ports << state->ts_cfg_profile->cp_log_max_pkeytbl); if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys " "exceeds device maximum", tnf_uint, maxpkey, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_toomanypkey_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_partitions = val; /* Determine number of ports */ maxval = state->ts_devlim.num_ports; val = state->ts_cfg_profile->cp_num_ports; if ((val > maxval) || (val == 0)) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports " "exceeds device maximum", tnf_uint, maxports, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_toomanyports_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_nports = val; /* Copy NodeGUID and SystemImageGUID from softstate */ hca_attr->hca_node_guid = state->ts_nodeguid; hca_attr->hca_si_guid = state->ts_sysimgguid; /* * Determine local ACK delay. Use the value suggested by the Tavor * hardware (from the QUERY_DEV_LIM command) */ hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay; /* Determine max SGID table and PKey table sizes */ val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl); hca_attr->hca_max_port_sgid_tbl_sz = val; val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl); hca_attr->hca_max_port_pkey_tbl_sz = val; /* Determine max number of PDs */ maxval = ((uint64_t)1 << state->ts_devlim.log_max_pd); val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd); if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD " "exceeds device maximum", tnf_uint, maxpd, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_toomanypd_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_pd = val; /* Determine max number of Address Handles */ maxval = ((uint64_t)1 << state->ts_devlim.log_max_av); val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah); if (val > maxval) { kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH " "exceeds device maximum", tnf_uint, maxah, maxval); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_toomanyah_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } hca_attr->hca_max_ah = val; /* No RDDs or EECs (since Reliable Datagram is not supported) */ hca_attr->hca_max_rdd = 0; hca_attr->hca_max_eec = 0; /* Initialize lock for reserved UAR page access */ mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(state->ts_intrmsi_pri)); /* Initialize the flash fields */ state->ts_fw_flashstarted = 0; mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(state->ts_intrmsi_pri)); /* Initialize the lock for the info ioctl */ mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(state->ts_intrmsi_pri)); /* Initialize the AVL tree for QP number support */ tavor_qpn_avl_init(state); /* Initialize the kstat info structure */ status = tavor_kstat_init(state); if (status != DDI_SUCCESS) { tavor_qpn_avl_fini(state); mutex_destroy(&state->ts_info_lock); mutex_destroy(&state->ts_fw_flashlock); mutex_destroy(&state->ts_uar_lock); kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail, TAVOR_TNF_ERROR, ""); TAVOR_ATTACH_MSG(state->ts_attach_buf, "soft_state_init_kstatinit_fail"); TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_FAILURE); } TAVOR_TNF_EXIT(tavor_soft_state_init); return (DDI_SUCCESS); } /* * tavor_soft_state_fini() * Context: Called only from detach() path context */ static void tavor_soft_state_fini(tavor_state_t *state) { TAVOR_TNF_ENTER(tavor_soft_state_fini); /* Teardown the kstat info */ tavor_kstat_fini(state); /* Teardown the AVL tree for QP number support */ tavor_qpn_avl_fini(state); /* Free up info ioctl mutex */ mutex_destroy(&state->ts_info_lock); /* Free up flash mutex */ mutex_destroy(&state->ts_fw_flashlock); /* Free up the UAR page access mutex */ mutex_destroy(&state->ts_uar_lock); /* Free up the hca_attr struct */ kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t)); TAVOR_TNF_EXIT(tavor_soft_state_fini); } /* * tavor_hca_config_setup() * Context: Only called from attach() path context */ static void tavor_hca_config_setup(tavor_state_t *state, tavor_hw_initqueryhca_t *inithca) { tavor_rsrc_pool_info_t *rsrc_pool; uint64_t ddr_baseaddr, ddr_base_map_addr; uint64_t offset, addr; uint_t mcg_size; TAVOR_TNF_ENTER(tavor_hca_config_setup); /* Set "host endianness". Default is big endian */ #ifdef _LITTLE_ENDIAN inithca->big_endian = 0; #else inithca->big_endian = 1; #endif /* No Address Vector Protection, but Port Checking on by default */ inithca->udav_chk = TAVOR_UDAV_PROTECT_DISABLED; inithca->udav_port_chk = TAVOR_UDAV_PORTCHK_ENABLED; ddr_baseaddr = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr; ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr; /* Setup QPC table */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->context.qpc_baseaddr_h = (addr >> 32); inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7; inithca->context.log_num_qp = state->ts_cfg_profile->cp_log_num_qp; /* Setup EEC table (initialize to zero - RD unsupported) */ inithca->context.eec_baseaddr_h = 0; inithca->context.eec_baseaddr_l = 0; inithca->context.log_num_ee = 0; /* Setup CQC table */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->context.cqc_baseaddr_h = (addr >> 32); inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; inithca->context.log_num_cq = state->ts_cfg_profile->cp_log_num_cq; /* Setup SRQC table */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->context.srqc_baseaddr_h = (addr >> 32); inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; inithca->context.log_num_srq = state->ts_cfg_profile->cp_log_num_srq; /* Setup EQPC table */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->context.eqpc_baseaddr = addr; /* Setup EEEC table (initialize to zero - RD unsupported) */ inithca->context.eeec_baseaddr = 0; /* Setup EQC table */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->context.eqc_baseaddr_h = (addr >> 32); inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; inithca->context.log_num_eq = TAVOR_NUM_EQ_SHIFT; /* Setup RDB table */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->context.rdb_baseaddr_h = (addr >> 32); inithca->context.rdb_baseaddr_l = 0; /* Setup Multicast */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->multi.mc_baseaddr = addr; mcg_size = TAVOR_MCGMEM_SZ(state); inithca->multi.log_mc_tbl_ent = highbit(mcg_size) - 1; inithca->multi.mc_tbl_hash_sz = (1 << state->ts_cfg_profile->cp_log_num_mcg_hash); inithca->multi.mc_hash_fn = TAVOR_MCG_DEFAULT_HASH_FN; inithca->multi.log_mc_tbl_sz = state->ts_cfg_profile->cp_log_num_mcg; /* Setup TPT */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->tpt.mpt_baseaddr = addr; inithca->tpt.mttseg_sz = TAVOR_MTTSEG_SIZE_SHIFT; inithca->tpt.log_mpt_sz = state->ts_cfg_profile->cp_log_num_mpt; inithca->tpt.mtt_version = TAVOR_MTT_PG_WALK_VER; rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->tpt.mtt_baseaddr = addr; /* Setup UAR */ rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; addr = ddr_base_map_addr + offset; inithca->uar.uarscr_baseaddr = addr; inithca->uar.uar_pg_sz = PAGESHIFT - 0xC; TAVOR_TNF_EXIT(tavor_hca_config_setup); } /* * tavor_hca_port_init() * Context: Only called from attach() path context */ static int tavor_hca_port_init(tavor_state_t *state) { tavor_hw_initib_t *portinits, *initib; tavor_cfg_profile_t *cfgprof; uint_t num_ports; int i, status; uint64_t maxval, val; uint64_t sysimgguid, nodeguid, portguid; TAVOR_TNF_ENTER(tavor_hca_port_init); cfgprof = state->ts_cfg_profile; /* Get number of HCA ports */ num_ports = cfgprof->cp_num_ports; /* Allocate space for Tavor port init struct(s) */ portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports * sizeof (tavor_hw_initib_t), KM_SLEEP); /* Post command to initialize Tavor HCA port */ for (i = 0; i < num_ports; i++) { initib = &portinits[i]; /* * Determine whether we need to override the firmware's * default SystemImageGUID setting. */ sysimgguid = cfgprof->cp_sysimgguid; if (sysimgguid != 0) { initib->set_sysimg_guid = 1; initib->sysimg_guid = sysimgguid; } /* * Determine whether we need to override the firmware's * default NodeGUID setting. */ nodeguid = cfgprof->cp_nodeguid; if (nodeguid != 0) { initib->set_node_guid = 1; initib->node_guid = nodeguid; } /* * Determine whether we need to override the firmware's * default PortGUID setting. */ portguid = cfgprof->cp_portguid[i]; if (portguid != 0) { initib->set_port_guid0 = 1; initib->guid0 = portguid; } /* Validate max MTU size */ maxval = state->ts_devlim.max_mtu; val = cfgprof->cp_max_mtu; if (val > maxval) { TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " "MTU size exceeds device maximum", tnf_uint, maxmtu, maxval); TAVOR_TNF_EXIT(tavor_hca_port_init); goto init_ports_fail; } initib->mtu_cap = val; /* Validate the max port width */ maxval = state->ts_devlim.max_port_width; val = cfgprof->cp_max_port_width; if (val > maxval) { TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " "port width exceeds device maximum", tnf_uint, maxportwidth, maxval); TAVOR_TNF_EXIT(tavor_hca_port_init); goto init_ports_fail; } initib->port_width_cap = val; /* Validate max VL cap size */ maxval = state->ts_devlim.max_vl; val = cfgprof->cp_max_vlcap; if (val > maxval) { TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " "VLcap size exceeds device maximum", tnf_uint, maxvlcap, maxval); TAVOR_TNF_EXIT(tavor_hca_port_init); goto init_ports_fail; } initib->vl_cap = val; /* Validate max GID table size */ maxval = ((uint64_t)1 << state->ts_devlim.log_max_gid); val = ((uint64_t)1 << cfgprof->cp_log_max_gidtbl); if (val > maxval) { TNF_PROBE_2(tavor_hca_port_init_gidtable_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " "GID table size exceeds device maximum", tnf_uint, maxgidtbl, maxval); TAVOR_TNF_EXIT(tavor_hca_port_init); goto init_ports_fail; } initib->max_gid = val; /* Validate max PKey table size */ maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); val = ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl); if (val > maxval) { TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail, TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " "PKey table size exceeds device maximum", tnf_uint, maxpkeytbl, maxval); TAVOR_TNF_EXIT(tavor_hca_port_init); goto init_ports_fail; } initib->max_pkey = val; /* * Post the INIT_IB command to Tavor firmware. When this * command completes, the corresponding Tavor port will be * physically "Up" and initialized. */ status = tavor_init_ib_cmd_post(state, initib, i + 1, TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command " "failed: %08x\n", i + 1, status); TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail, TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, tnf_uint, port, i + 1); TAVOR_TNF_EXIT(tavor_hca_port_init); goto init_ports_fail; } } /* Free up the memory for Tavor port init struct(s), return success */ kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); TAVOR_TNF_EXIT(tavor_hca_port_init); return (DDI_SUCCESS); init_ports_fail: /* * Free up the memory for Tavor port init struct(s), shutdown any * successfully initialized ports, and return failure */ kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); (void) tavor_hca_ports_shutdown(state, i); TAVOR_TNF_EXIT(tavor_hca_port_init); return (DDI_FAILURE); } /* * tavor_hca_ports_shutdown() * Context: Only called from attach() and/or detach() path contexts */ static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init) { int i, status; TAVOR_TNF_ENTER(tavor_hca_ports_shutdown); /* * Post commands to shutdown all init'd Tavor HCA ports. Note: if * any of these commands fail for any reason, it would be entirely * unexpected and probably indicative a serious problem (HW or SW). * Although we do return void from this function, this type of failure * should not go unreported. That is why we have the warning message * and the detailed TNF information. */ for (i = 0; i < num_init; i++) { status = tavor_close_ib_cmd_post(state, i + 1, TAVOR_CMD_NOSLEEP_SPIN); if (status != TAVOR_CMD_SUCCESS) { TAVOR_WARNING(state, "failed to shutdown HCA port"); TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail, TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, tnf_uint, port, i + 1); TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); return (status); } } TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); return (TAVOR_CMD_SUCCESS); } /* * tavor_internal_uarpgs_init * Context: Only called from attach() path context */ static int tavor_internal_uarpgs_init(tavor_state_t *state) { int status; TAVOR_TNF_ENTER(tavor_internal_uarpgs_init); /* * Save away reserved Tavor UAR page #0. This UAR page is not to * be used by software. */ status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, &state->ts_uarpg0_rsrc_rsrvd); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); return (DDI_FAILURE); } /* * Save away Tavor UAR page #1 (for internal use). This UAR page is * the privileged UAR page through which all kernel generated * doorbells will be rung. */ status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, &state->ts_uarpg1_rsrc); if (status != DDI_SUCCESS) { tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); return (DDI_FAILURE); } /* Setup pointer to UAR page #1 doorbells */ state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr; TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); return (DDI_SUCCESS); } /* * tavor_internal_uarpgs_fini * Context: Only called from attach() and/or detach() path contexts */ static void tavor_internal_uarpgs_fini(tavor_state_t *state) { TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini); /* Free up Tavor UAR page #1 (kernel driver doorbells) */ tavor_rsrc_free(state, &state->ts_uarpg1_rsrc); /* Free up Tavor UAR page #0 (reserved) */ tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini); } /* * tavor_special_qp_contexts_reserve() * Context: Only called from attach() path context */ static int tavor_special_qp_contexts_reserve(tavor_state_t *state) { tavor_rsrc_t *qp0_rsrc, *qp1_rsrc; int status; TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve); /* Initialize the lock used for special QP rsrc management */ mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(state->ts_intrmsi_pri)); /* * Reserve contexts for QP0. These QP contexts will be setup to * act as aliases for the real QP0. Note: We are required to grab * two QPs (one per port) even if we are operating in single-port * mode. */ status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc); if (status != DDI_SUCCESS) { mutex_destroy(&state->ts_spec_qplock); TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); return (DDI_FAILURE); } state->ts_spec_qp0 = qp0_rsrc; /* * Reserve contexts for QP1. These QP contexts will be setup to * act as aliases for the real QP1. Note: We are required to grab * two QPs (one per port) even if we are operating in single-port * mode. */ status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc); if (status != DDI_SUCCESS) { tavor_rsrc_free(state, &qp0_rsrc); mutex_destroy(&state->ts_spec_qplock); TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); return (DDI_FAILURE); } state->ts_spec_qp1 = qp1_rsrc; TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); return (DDI_SUCCESS); } /* * tavor_special_qp_contexts_unreserve() * Context: Only called from attach() and/or detach() path contexts */ static void tavor_special_qp_contexts_unreserve(tavor_state_t *state) { TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve); /* Unreserve contexts for QP1 */ tavor_rsrc_free(state, &state->ts_spec_qp1); /* Unreserve contexts for QP0 */ tavor_rsrc_free(state, &state->ts_spec_qp0); /* Destroy the lock used for special QP rsrc management */ mutex_destroy(&state->ts_spec_qplock); TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve); } /* * tavor_sw_reset() * Context: Currently called only from attach() path context */ static int tavor_sw_reset(tavor_state_t *state) { dev_info_t *dip, *pdip; ddi_acc_handle_t hdl = state->ts_pci_cfghdl, phdl; uint32_t reset_delay; int status, i; TAVOR_TNF_ENTER(tavor_sw_reset); /* * If the configured software reset delay is set to zero, then we * will not attempt a software reset of the Tavor device. */ reset_delay = state->ts_cfg_profile->cp_sw_reset_delay; if (reset_delay == 0) { TAVOR_TNF_EXIT(tavor_sw_reset); return (DDI_SUCCESS); } /* * Get dip for HCA device _and_ parent device as well. Parent access * is necessary here because software reset of the Tavor hardware * will reinitialize both the config registers of the PCI bridge * (parent, if it exists) and the IB HCA (self) */ dip = state->ts_dip; pdip = ddi_get_parent(dip); /* Query the PCI capabilities of the HCA device */ tavor_pci_capability_list(state, hdl); /* * Read all PCI config info (reg0...reg63). Note: According to the * Tavor software reset application note, we should not read or * restore the values in reg22 and reg23. */ for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { if ((i != TAVOR_SW_RESET_REG22_RSVD) && (i != TAVOR_SW_RESET_REG23_RSVD)) { state->ts_cfg_data[i] = pci_config_get32(hdl, i << 2); } } if (TAVOR_PARENT_IS_BRIDGE(pdip)) { /* * Setup for PCI config read/write of bridge device */ status = pci_config_setup(pdip, &phdl); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_sw_reset); return (DDI_FAILURE); } /* * Read all PCI config info (reg0...reg63). Note: According to * the Tavor software reset application note, we should not * read or restore the values in reg22 and reg23. */ for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { if ((i != TAVOR_SW_RESET_REG22_RSVD) && (i != TAVOR_SW_RESET_REG23_RSVD)) { state->ts_cfg_pdata[i] = pci_config_get32(phdl, i << 2); } } } /* * Perform the software reset (by writing 1 at offset 0xF0010) */ ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset, TAVOR_SW_RESET_START); drv_usecwait(reset_delay); if (TAVOR_PARENT_IS_BRIDGE(pdip)) { /* * Bridge exists, so wait for the bridge to become ready. * * The above delay is necessary to avoid system panic from * Master Abort. If the device is accessed before this delay, * device will not respond to config cycles and they will be * terminate with a Master Abort which will panic the system. * Below is the loop we use to poll status from the device to * determine if it is OK to proceed. */ i = 0; while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) { drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); } /* * Write all the PCI config registers back into each device * (except for reg22 and reg23 - see above) */ for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { if ((i != TAVOR_SW_RESET_REG22_RSVD) && (i != TAVOR_SW_RESET_REG23_RSVD)) { pci_config_put32(phdl, i << 2, state->ts_cfg_pdata[i]); } } /* * Tear down the config setup (for bridge device) */ pci_config_teardown(&phdl); /* No Bridge Device */ } else { /* * Bridge does not exist, so instead wait for the device itself * to become ready. * * The above delay is necessary to avoid system panic from * Master Abort. If the device is accessed before this delay, * device will not respond to config cycles and they will be * terminate with a Master Abort which will panic the system. * Below is the loop we use to poll status from the device to * determine if it is OK to proceed. */ i = 0; while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) { drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); } } for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { if ((i != TAVOR_SW_RESET_REG22_RSVD) && (i != TAVOR_SW_RESET_REG23_RSVD)) { pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]); } } TAVOR_TNF_EXIT(tavor_sw_reset); return (DDI_SUCCESS); } /* * tavor_mcg_init() * Context: Only called from attach() path context */ static int tavor_mcg_init(tavor_state_t *state) { uint_t mcg_tmp_sz; TAVOR_TNF_ENTER(tavor_mcg_init); /* * Allocate space for the MCG temporary copy buffer. This is * used by the Attach/Detach Multicast Group code */ mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP); /* * Initialize the multicast group mutex. This ensures atomic * access to add, modify, and remove entries in the multicast * group hash lists. */ mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(state->ts_intrmsi_pri)); TAVOR_TNF_EXIT(tavor_mcg_init); return (DDI_SUCCESS); } /* * tavor_mcg_fini() * Context: Only called from attach() and/or detach() path contexts */ static void tavor_mcg_fini(tavor_state_t *state) { uint_t mcg_tmp_sz; TAVOR_TNF_ENTER(tavor_mcg_fini); /* Free up the space used for the MCG temporary copy buffer */ mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); kmem_free(state->ts_mcgtmp, mcg_tmp_sz); /* Destroy the multicast group mutex */ mutex_destroy(&state->ts_mcglock); TAVOR_TNF_EXIT(tavor_mcg_fini); } /* * tavor_fw_version_check() * Context: Only called from attach() path context */ static int tavor_fw_version_check(tavor_state_t *state) { uint_t tavor_fw_ver_major; uint_t tavor_fw_ver_minor; uint_t tavor_fw_ver_subminor; /* * Depending on which version of driver we have attached, the firmware * version checks will be different. We set up the comparison values * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware * running in tavor mode). */ switch (state->ts_operational_mode) { case TAVOR_HCA_MODE: tavor_fw_ver_major = TAVOR_FW_VER_MAJOR; tavor_fw_ver_minor = TAVOR_FW_VER_MINOR; tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR; break; case TAVOR_COMPAT_MODE: tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR; tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR; tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR; break; default: return (DDI_FAILURE); } /* * If FW revision major number is less than acceptable, * return failure, else if greater return success. If * the major numbers are equal than check the minor number */ if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) { return (DDI_FAILURE); } else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) { return (DDI_SUCCESS); } /* * Do the same check as above, except for minor revision numbers * If the minor numbers are equal than check the subminor number */ if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) { return (DDI_FAILURE); } else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) { return (DDI_SUCCESS); } /* * Once again we do the same check as above, except for the subminor * revision number. If the subminor numbers are equal here, then * these are the same firmware version, return success */ if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) { return (DDI_FAILURE); } else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) { return (DDI_SUCCESS); } return (DDI_SUCCESS); } /* * tavor_device_info_report() * Context: Only called from attach() path context */ static void tavor_device_info_report(tavor_state_t *state) { cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, " "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major, state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor, state->ts_adapter.rev_id); cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n", state->ts_instance, state->ts_nodedesc, state->ts_nodeguid); } /* * tavor_pci_capability_list() * Context: Only called from attach() path context */ static void tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl) { uint_t offset, data; TAVOR_TNF_ENTER(tavor_pci_capability_list); /* * Check for the "PCI Capabilities" bit in the "Status Register". * Bit 4 in this register indicates the presence of a "PCI * Capabilities" list. */ data = pci_config_get16(hdl, 0x6); if ((data & 0x10) == 0) { TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_pci_capability_list); return; } /* * Starting from offset 0x34 in PCI config space, find the * head of "PCI capabilities" list, and walk the list. If * capabilities of a known type are encountered (e.g. * "PCI-X Capability"), then call the appropriate handler * function. */ offset = pci_config_get8(hdl, 0x34); while (offset != 0x0) { data = pci_config_get8(hdl, offset); /* * Check for known capability types. Tavor has the * following: * o VPD Capability (0x03) * o PCI-X Capability (0x07) * o MSI Capability (0x05) * o MSIX Capability (0x11) */ switch (data) { case 0x03: tavor_pci_capability_vpd(state, hdl, offset); break; case 0x07: tavor_pci_capability_pcix(state, hdl, offset); break; case 0x05: break; default: break; } /* Get offset of next entry in list */ offset = pci_config_get8(hdl, offset + 1); } TAVOR_TNF_EXIT(tavor_pci_capability_list); } /* * tavor_pci_read_vpd() * Context: Only called from attach() path context * utility routine for tavor_pci_capability_vpd() */ static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr, uint32_t *data) { int retry = 4; /* retry counter for EEPROM poll */ uint32_t val; int vpd_addr = offset + 2; int vpd_data = offset + 4; TAVOR_TNF_ENTER(tavor_pci_read_vpd); /* * In order to read a 32-bit value from VPD, we are to write down * the address (offset in the VPD itself) to the address register. * To signal the read, we also clear bit 31. We then poll on bit 31 * and when it is set, we can then read our 4 bytes from the data * register. */ (void) pci_config_put32(hdl, offset, addr << 16); do { drv_usecwait(1000); val = pci_config_get16(hdl, vpd_addr); if ((val >> 15) & 0x01) { *data = pci_config_get32(hdl, vpd_data); TAVOR_TNF_EXIT(tavor_pci_read_vpd); return (DDI_SUCCESS); } } while (--retry); TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_pci_read_vpd); return (DDI_FAILURE); } /* * tavor_pci_capability_vpd() * Context: Only called from attach() path context */ static void tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl, uint_t offset) { uint8_t name_length; uint8_t pn_length; int i, err = 0; int vpd_str_id = 0; int vpd_ro_desc; int vpd_ro_pn_desc; #ifndef _LITTLE_ENDIAN uint32_t data32; #endif /* _LITTLE_ENDIAN */ union { uint32_t vpd_int[TAVOR_VPD_HDR_DWSIZE]; uchar_t vpd_char[TAVOR_VPD_HDR_BSIZE]; } vpd; TAVOR_TNF_ENTER(tavor_pci_capability_vpd); /* * Read Vital Product Data (VPD) from PCI-X capability. */ for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]); if (err != DDI_SUCCESS) { cmn_err(CE_NOTE, "!VPD read failed\n"); goto out; } } #ifndef _LITTLE_ENDIAN /* * Need to swap bytes for big endian. */ for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { data32 = vpd.vpd_int[i]; vpd.vpd_char[(i << 2) + 3] = (uchar_t)((data32 & 0xFF000000) >> 24); vpd.vpd_char[(i << 2) + 2] = (uchar_t)((data32 & 0x00FF0000) >> 16); vpd.vpd_char[(i << 2) + 1] = (uchar_t)((data32 & 0x0000FF00) >> 8); vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF); } #endif /* _LITTLE_ENDIAN */ /* Check for VPD String ID Tag */ if (vpd.vpd_char[vpd_str_id] == 0x82) { /* get the product name */ name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1]; if (name_length > sizeof (state->ts_hca_name)) { cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n", name_length); goto out; } (void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3], name_length); state->ts_hca_name[name_length] = 0; /* get the part number */ vpd_ro_desc = name_length + 3; /* read-only tag location */ vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */ /* * Verify read-only tag and Part Number keyword. */ if (vpd.vpd_char[vpd_ro_desc] != 0x90 || (vpd.vpd_char[vpd_ro_pn_desc] != 'P' && vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) { cmn_err(CE_NOTE, "!VPD Part Number not found\n"); goto out; } pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2]; if (pn_length > sizeof (state->ts_hca_pn)) { cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n", name_length); goto out; } (void) memcpy(state->ts_hca_pn, &vpd.vpd_char[vpd_ro_pn_desc + 3], pn_length); state->ts_hca_pn[pn_length] = 0; state->ts_hca_pn_len = pn_length; } else { /* Wrong VPD String ID Tag */ cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n", vpd.vpd_char[0]); goto out; } TAVOR_TNF_EXIT(tavor_pci_capability_vpd); return; out: state->ts_hca_pn_len = 0; TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_pci_capability_vpd); } /* * tavor_pci_capability_pcix() * Context: Only called from attach() path context */ static void tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl, uint_t offset) { uint_t command, status; int max_out_splt_trans, max_mem_rd_byte_cnt; int designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt; TAVOR_TNF_ENTER(tavor_pci_capability_pcix); /* * Query the current values for the PCI-X Command Register and * the PCI-X Status Register. */ command = pci_config_get16(hdl, offset + 2); status = pci_config_get32(hdl, offset + 4); /* * Check for config property specifying "maximum outstanding * split transactions". If the property is defined and valid * (i.e. no larger than the so-called "designed maximum"), * then use the specified value to update the PCI-X Command Register. * Otherwise, extract the value from the Tavor config profile. */ designed_max_out_splt_trans = ((status >> 23) & 7); max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1); if ((max_out_splt_trans != -1) && ((max_out_splt_trans < 0) || (max_out_splt_trans > designed_max_out_splt_trans))) { cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-" "split-trans\" (%d) invalid or exceeds device maximum" " (%d), using default value (%d)\n", state->ts_instance, max_out_splt_trans, designed_max_out_splt_trans, state->ts_cfg_profile->cp_max_out_splt_trans); max_out_splt_trans = state->ts_cfg_profile->cp_max_out_splt_trans; } else if (max_out_splt_trans == -1) { max_out_splt_trans = state->ts_cfg_profile->cp_max_out_splt_trans; } /* * The config profile setting for max_out_splt_trans is determined * based on arch. Check tavor_cfg.c for more information. A value of * '-1' in the patchable variable means "do not change". A value of * '0' means 1 outstanding splt trans and other values as defined by * PCI. So we do one more check here, that if 'max_out_splt_trans' is * -1 (ie: < 0) we do not set the PCI command and leave it at the * default. */ if (max_out_splt_trans >= 0) { command = ((command & 0xFF8F) | max_out_splt_trans << 4); } /* * Check for config property specifying "maximum memory read * byte count. If the property is defined and valid * (i.e. no larger than the so-called "designed maximum"), * then use the specified value to update the PCI-X Command Register. * Otherwise, extract the value from the Tavor config profile. */ designed_max_mem_rd_byte_cnt = ((status >> 21) & 3); max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1); if ((max_mem_rd_byte_cnt != -1) && ((max_mem_rd_byte_cnt < 0) || (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) { cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-" "count\" (%d) invalid or exceeds device maximum" " (%d), using default value (%d)\n", state->ts_instance, max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt, state->ts_cfg_profile->cp_max_mem_rd_byte_cnt); max_mem_rd_byte_cnt = state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; } else if (max_mem_rd_byte_cnt == -1) { max_mem_rd_byte_cnt = state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; } /* * The config profile setting for max_mem_rd_byte_cnt is determined * based on arch. Check tavor_cfg.c for more information. A value of * '-1' in the patchable variable means "do not change". A value of * '0' means minimum (512B) read, and other values as defined by * PCI. So we do one more check here, that if 'max_mem_rd_byte_cnt' is * -1 (ie: < 0) we do not set the PCI command and leave it at the * default. */ if (max_mem_rd_byte_cnt >= 0) { command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2); } /* * Update the PCI-X Command Register with the newly configured * values. */ pci_config_put16(hdl, offset + 2, command); TAVOR_TNF_EXIT(tavor_pci_capability_pcix); } /* * tavor_intr_or_msi_init() * Context: Only called from attach() path context */ static int tavor_intr_or_msi_init(tavor_state_t *state) { int status; TAVOR_TNF_ENTER(tavor_intr_or_msi_init); /* Query for the list of supported interrupt event types */ status = ddi_intr_get_supported_types(state->ts_dip, &state->ts_intr_types_avail); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_intr_or_msi_init); return (DDI_FAILURE); } /* * If Tavor/Arbel supports MSI in this system (and, if it * hasn't been overridden by a configuration variable), then * the default behavior is to use a single MSI. Otherwise, * fallback to using legacy interrupts. Also, if MSI allocatis chosen, * but fails for whatever reasons, then fallback to using legacy * interrupts. */ if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI); if (status == DDI_SUCCESS) { state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI; TAVOR_TNF_EXIT(tavor_intr_or_msi_init); return (DDI_SUCCESS); } } /* * MSI interrupt allocation failed, or was not available. Fallback to * legacy interrupt support. */ if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) { status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED); if (status == DDI_SUCCESS) { state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED; TAVOR_TNF_EXIT(tavor_intr_or_msi_init); return (DDI_SUCCESS); } } /* * Neither MSI or legacy interrupts were successful. return failure. */ TAVOR_TNF_EXIT(tavor_intr_or_msi_setup); return (DDI_FAILURE); } /* * tavor_add_intrs() * Context: Only called from attach() patch context */ static int tavor_add_intrs(tavor_state_t *state, int intr_type) { int status; TAVOR_TNF_ENTER(tavor_add_intrs); /* Get number of interrupts/MSI supported */ status = ddi_intr_get_nintrs(state->ts_dip, intr_type, &state->ts_intrmsi_count); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_add_intrs_getnintrs_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } /* Get number of available interrupts/MSI */ status = ddi_intr_get_navail(state->ts_dip, intr_type, &state->ts_intrmsi_avail); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_add_intrs_getnavail_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } /* Ensure that we have at least one (1) usable MSI or interrupt */ if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) { TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } /* Attempt to allocate a single interrupt/MSI handle */ status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl, intr_type, 0, 1, &state->ts_intrmsi_allocd, DDI_INTR_ALLOC_STRICT); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_add_intrs_intralloc_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } /* Ensure that we have allocated at least one (1) MSI or interrupt */ if (state->ts_intrmsi_allocd < 1) { TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } /* * Extract the priority for the allocated interrupt/MSI. This * will be used later when initializing certain mutexes. */ status = ddi_intr_get_pri(state->ts_intrmsi_hdl, &state->ts_intrmsi_pri); if (status != DDI_SUCCESS) { /* Free the allocated interrupt/MSI handle */ (void) ddi_intr_free(state->ts_intrmsi_hdl); TNF_PROBE_0(tavor_add_intrs_getpri_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } /* Make sure the interrupt/MSI priority is below 'high level' */ if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) { /* Free the allocated interrupt/MSI handle */ (void) ddi_intr_free(state->ts_intrmsi_hdl); TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } /* Get add'l capability information regarding interrupt/MSI */ status = ddi_intr_get_cap(state->ts_intrmsi_hdl, &state->ts_intrmsi_cap); if (status != DDI_SUCCESS) { /* Free the allocated interrupt/MSI handle */ (void) ddi_intr_free(state->ts_intrmsi_hdl); TNF_PROBE_0(tavor_add_intrs_getcap_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_FAILURE); } TAVOR_TNF_EXIT(tavor_add_intrs); return (DDI_SUCCESS); } /* * tavor_intr_or_msi_fini() * Context: Only called from attach() and/or detach() path contexts */ static int tavor_intr_or_msi_fini(tavor_state_t *state) { int status; TAVOR_TNF_ENTER(tavor_intr_or_msi_fini); /* Free the allocated interrupt/MSI handle */ status = ddi_intr_free(state->ts_intrmsi_hdl); if (status != DDI_SUCCESS) { TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail, TAVOR_TNF_ERROR, ""); TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); return (DDI_FAILURE); } TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); return (DDI_SUCCESS); } /* Disable Tavor interrupts */ static int tavor_intr_disable(tavor_state_t *state) { ushort_t msi_ctrl = 0, caps_ctrl = 0; ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl; ASSERT(pci_cfg_hdl != NULL); ASSERT(state->ts_intr_types_avail & (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI)); /* * Check if MSI interrupts are used. If so, disable MSI interupts. * If not, since Tavor doesn't support MSI-X interrupts, assuming the * legacy interrupt is used instead, disable the legacy interrupt. */ if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI, &caps_ctrl) == DDI_SUCCESS)) { if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, 0, caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16) return (DDI_FAILURE); } ASSERT(msi_ctrl != 0); if (!(msi_ctrl & PCI_MSI_ENABLE_BIT)) return (DDI_SUCCESS); if (msi_ctrl & PCI_MSI_PVM_MASK) { int offset = (msi_ctrl & PCI_MSI_64BIT_MASK) ? PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK; /* Clear all inums in MSI */ PCI_CAP_PUT32(pci_cfg_hdl, 0, caps_ctrl, offset, 0); } /* Disable MSI interrupts */ msi_ctrl &= ~PCI_MSI_ENABLE_BIT; PCI_CAP_PUT16(pci_cfg_hdl, 0, caps_ctrl, PCI_MSI_CTRL, msi_ctrl); } else { uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM); ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED); /* Disable the legacy interrupts */ cmdreg |= PCI_COMM_INTX_DISABLE; pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg); } return (DDI_SUCCESS); } /* Tavor quiesce(9F) entry */ static int tavor_quiesce(dev_info_t *dip) { tavor_state_t *state = ddi_get_soft_state(tavor_statep, DEVI(dip)->devi_instance); ASSERT(state != NULL); /* start fastreboot */ state->ts_quiescing = B_TRUE; /* If it's in maintenance mode, do nothing but return with SUCCESS */ if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { return (DDI_SUCCESS); } /* Shutdown HCA ports */ if (tavor_hca_ports_shutdown(state, state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) { state->ts_quiescing = B_FALSE; return (DDI_FAILURE); } /* Close HCA */ if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != TAVOR_CMD_SUCCESS) { state->ts_quiescing = B_FALSE; return (DDI_FAILURE); } /* Shutdown FW */ if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != TAVOR_CMD_SUCCESS) { state->ts_quiescing = B_FALSE; return (DDI_FAILURE); } /* Disable interrupts */ if (tavor_intr_disable(state) != DDI_SUCCESS) { state->ts_quiescing = B_FALSE; return (DDI_FAILURE); } /* SW-reset */ if (tavor_sw_reset(state) != DDI_SUCCESS) { state->ts_quiescing = B_FALSE; return (DDI_FAILURE); } return (DDI_SUCCESS); }