19e39c5baSBill Taylor /* 29e39c5baSBill Taylor * CDDL HEADER START 39e39c5baSBill Taylor * 49e39c5baSBill Taylor * The contents of this file are subject to the terms of the 59e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 69e39c5baSBill Taylor * You may not use this file except in compliance with the License. 79e39c5baSBill Taylor * 89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 109e39c5baSBill Taylor * See the License for the specific language governing permissions 119e39c5baSBill Taylor * and limitations under the License. 129e39c5baSBill Taylor * 139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 189e39c5baSBill Taylor * 199e39c5baSBill Taylor * CDDL HEADER END 209e39c5baSBill Taylor */ 219e39c5baSBill Taylor 229e39c5baSBill Taylor /* 23*17a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 249e39c5baSBill Taylor */ 259e39c5baSBill Taylor 269e39c5baSBill Taylor /* 279e39c5baSBill Taylor * hermon_cfg.c 289e39c5baSBill Taylor * Hermon Configuration Profile Routines 299e39c5baSBill Taylor * 309e39c5baSBill Taylor * Implements the routines necessary for initializing and (later) tearing 319e39c5baSBill Taylor * down the list of Hermon configuration information. 329e39c5baSBill Taylor */ 339e39c5baSBill Taylor 349e39c5baSBill Taylor #include <sys/types.h> 359e39c5baSBill Taylor #include <sys/conf.h> 369e39c5baSBill Taylor #include <sys/ddi.h> 379e39c5baSBill Taylor #include <sys/sunddi.h> 389e39c5baSBill Taylor #include <sys/modctl.h> 399e39c5baSBill Taylor #include <sys/bitmap.h> 409e39c5baSBill Taylor 419e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 429e39c5baSBill Taylor 439e39c5baSBill Taylor /* 449e39c5baSBill Taylor * Below are the elements that make up the Hermon configuration profile. 459e39c5baSBill Taylor * For advanced users who wish to alter these values, this can be done via 469e39c5baSBill Taylor * the /etc/system file. By default, values are assigned to the number of 479e39c5baSBill Taylor * supported resources, either from the HCA's reported capacities or by 489e39c5baSBill Taylor * a by-design limit in the driver. 499e39c5baSBill Taylor */ 509e39c5baSBill Taylor 519e39c5baSBill Taylor /* Number of supported QPs, CQs and SRQs */ 529e39c5baSBill Taylor uint32_t hermon_log_num_qp = HERMON_NUM_QP_SHIFT; 539e39c5baSBill Taylor uint32_t hermon_log_num_cq = HERMON_NUM_CQ_SHIFT; 549e39c5baSBill Taylor uint32_t hermon_log_num_srq = HERMON_NUM_SRQ_SHIFT; 559e39c5baSBill Taylor 569e39c5baSBill Taylor /* Number of supported SGL per WQE for SQ/RQ, and for SRQ */ 579e39c5baSBill Taylor /* XXX use the same for all queues if limitation in srq.h is resolved */ 589e39c5baSBill Taylor uint32_t hermon_wqe_max_sgl = HERMON_NUM_SGL_PER_WQE; 599e39c5baSBill Taylor uint32_t hermon_srq_max_sgl = HERMON_SRQ_MAX_SGL; 609e39c5baSBill Taylor 619e39c5baSBill Taylor /* Maximum "responder resources" (in) and "initiator depth" (out) per QP */ 629e39c5baSBill Taylor uint32_t hermon_log_num_rdb_per_qp = HERMON_LOG_NUM_RDB_PER_QP; 639e39c5baSBill Taylor 649e39c5baSBill Taylor /* 659e39c5baSBill Taylor * Number of multicast groups (MCGs), number of QP per MCG, and the number 669e39c5baSBill Taylor * of entries (from the total number) in the multicast group "hash table" 679e39c5baSBill Taylor */ 689e39c5baSBill Taylor uint32_t hermon_log_num_mcg = HERMON_NUM_MCG_SHIFT; 699e39c5baSBill Taylor uint32_t hermon_num_qp_per_mcg = HERMON_NUM_QP_PER_MCG; 709fa01fafSagiri uint32_t hermon_log_num_mcg_hash = HERMON_NUM_MCG_HASH_SHIFT; 719e39c5baSBill Taylor 729e39c5baSBill Taylor /* Number of UD AVs */ 739e39c5baSBill Taylor uint32_t hermon_log_num_ah = HERMON_NUM_AH_SHIFT; 749e39c5baSBill Taylor 759e39c5baSBill Taylor /* Number of EQs and their default size */ 769e39c5baSBill Taylor uint32_t hermon_log_num_eq = HERMON_NUM_EQ_SHIFT; 779e39c5baSBill Taylor uint32_t hermon_log_eq_sz = HERMON_DEFAULT_EQ_SZ_SHIFT; 789e39c5baSBill Taylor 799e39c5baSBill Taylor /* 809e39c5baSBill Taylor * Number of supported MPTs, MTTs and also the maximum MPT size. 819e39c5baSBill Taylor */ 829e39c5baSBill Taylor uint32_t hermon_log_num_mtt = HERMON_NUM_MTT_SHIFT; 839e39c5baSBill Taylor uint32_t hermon_log_num_dmpt = HERMON_NUM_DMPT_SHIFT; 849e39c5baSBill Taylor uint32_t hermon_log_max_mrw_sz = HERMON_MAX_MEM_MPT_SHIFT; 859e39c5baSBill Taylor 869e39c5baSBill Taylor /* 879e39c5baSBill Taylor * Number of supported UAR (User Access Regions) for this HCA. 889e39c5baSBill Taylor * We could in the future read in uar_sz from devlim, and thus 899e39c5baSBill Taylor * derive the number of UAR. Since this is derived from PAGESIZE, 909e39c5baSBill Taylor * however, this means that x86 systems would have twice as many 919e39c5baSBill Taylor * UARs as SPARC systems. Therefore for consistency's sake, we will 929e39c5baSBill Taylor * just use 1024 pages, which is the maximum on SPARC systems. 939e39c5baSBill Taylor */ 949e39c5baSBill Taylor uint32_t hermon_log_num_uar = HERMON_NUM_UAR_SHIFT; 959e39c5baSBill Taylor 969e39c5baSBill Taylor /* 979e39c5baSBill Taylor * Number of remaps allowed for FMR before a sync is required. This value 989e39c5baSBill Taylor * determines how many times we can fmr_deregister() before the underlying fmr 999e39c5baSBill Taylor * framework places the region to wait for an MTT_SYNC operation, cleaning up 1009e39c5baSBill Taylor * the old mappings. 1019e39c5baSBill Taylor */ 1029e39c5baSBill Taylor uint32_t hermon_fmr_num_remaps = HERMON_FMR_MAX_REMAPS; 1039e39c5baSBill Taylor 1049e39c5baSBill Taylor /* 1059e39c5baSBill Taylor * Number of supported Hermon mailboxes ("In" and "Out") and their maximum 1069e39c5baSBill Taylor * sizes, respectively 1079e39c5baSBill Taylor */ 1089e39c5baSBill Taylor uint32_t hermon_log_num_inmbox = HERMON_NUM_MAILBOXES_SHIFT; 1099e39c5baSBill Taylor uint32_t hermon_log_num_outmbox = HERMON_NUM_MAILBOXES_SHIFT; 1109e39c5baSBill Taylor uint32_t hermon_log_inmbox_size = HERMON_MBOX_SIZE_SHIFT; 1119e39c5baSBill Taylor uint32_t hermon_log_outmbox_size = HERMON_MBOX_SIZE_SHIFT; 1129e39c5baSBill Taylor uint32_t hermon_log_num_intr_inmbox = HERMON_NUM_INTR_MAILBOXES_SHIFT; 1139e39c5baSBill Taylor uint32_t hermon_log_num_intr_outmbox = HERMON_NUM_INTR_MAILBOXES_SHIFT; 1149e39c5baSBill Taylor 1159e39c5baSBill Taylor /* Number of supported Protection Domains (PD) */ 1169e39c5baSBill Taylor uint32_t hermon_log_num_pd = HERMON_NUM_PD_SHIFT; 1179e39c5baSBill Taylor 1189e39c5baSBill Taylor /* 1199e39c5baSBill Taylor * Number of total supported PKeys per PKey table (i.e. 1209e39c5baSBill Taylor * per port). Also the number of SGID per GID table. 1219e39c5baSBill Taylor */ 1229e39c5baSBill Taylor uint32_t hermon_log_max_pkeytbl = HERMON_NUM_PKEYTBL_SHIFT; 1239e39c5baSBill Taylor uint32_t hermon_log_max_gidtbl = HERMON_NUM_GIDTBL_SHIFT; 1249e39c5baSBill Taylor 1259e39c5baSBill Taylor /* Maximum supported MTU and portwidth */ 1269e39c5baSBill Taylor uint32_t hermon_max_mtu = HERMON_MAX_MTU; 1279e39c5baSBill Taylor uint32_t hermon_max_port_width = HERMON_MAX_PORT_WIDTH; 1289e39c5baSBill Taylor 1299e39c5baSBill Taylor /* Number of supported Virtual Lanes (VL) */ 1309e39c5baSBill Taylor uint32_t hermon_max_vlcap = HERMON_MAX_VLCAP; 1319e39c5baSBill Taylor 1329e39c5baSBill Taylor /* 1339e39c5baSBill Taylor * Whether or not to use the built-in (i.e. in firmware) agents for QP0 and 1349e39c5baSBill Taylor * QP1, respectively. 1359e39c5baSBill Taylor */ 13676c04273SRajkumar Sivaprakasam uint32_t hermon_qp0_agents_in_fw = 0; 1379e39c5baSBill Taylor uint32_t hermon_qp1_agents_in_fw = 0; 1389e39c5baSBill Taylor 1399e39c5baSBill Taylor /* 1409e39c5baSBill Taylor * Whether DMA mappings should bypass the PCI IOMMU or not. 1419e39c5baSBill Taylor * hermon_iommu_bypass is a global setting for all memory addresses. 1429e39c5baSBill Taylor */ 1439e39c5baSBill Taylor uint32_t hermon_iommu_bypass = 1; 1449e39c5baSBill Taylor 1459e39c5baSBill Taylor /* 1469e39c5baSBill Taylor * Whether *DATA* buffers should be bound w/ Relaxed Ordering (RO) turned on 1479e39c5baSBill Taylor * via the SW workaround (HCAs don't support RO in HW). Defaulted on, 1489e39c5baSBill Taylor * though care must be taken w/ some Userland clients that *MAY* have 1499e39c5baSBill Taylor * peeked in the data to understand when data xfer was done - MPI does 1509e39c5baSBill Taylor * as an efficiency 1519e39c5baSBill Taylor */ 1529e39c5baSBill Taylor 1539e39c5baSBill Taylor uint32_t hermon_kernel_data_ro = HERMON_RO_ENABLED; /* default */ 1549e39c5baSBill Taylor uint32_t hermon_user_data_ro = HERMON_RO_ENABLED; /* default */ 1559e39c5baSBill Taylor 1569e39c5baSBill Taylor /* 1579e39c5baSBill Taylor * Whether Hermon should use MSI (Message Signaled Interrupts), if available. 1589e39c5baSBill Taylor * Note: 0 indicates 'legacy interrupt', 1 indicates MSI (if available) 1599e39c5baSBill Taylor */ 1609e39c5baSBill Taylor uint32_t hermon_use_msi_if_avail = 1; 1619e39c5baSBill Taylor 1629e39c5baSBill Taylor /* 1639e39c5baSBill Taylor * This is a patchable variable that determines the time we will wait after 1649e39c5baSBill Taylor * initiating SW reset before we do our first read from Hermon config space. 1659e39c5baSBill Taylor * If this value is set too small (less than the default 100ms), it is 1669e39c5baSBill Taylor * possible for Hermon hardware to be unready to respond to the config cycle 1679e39c5baSBill Taylor * reads. This could cause master abort on the PCI bridge. Note: If 1689e39c5baSBill Taylor * "hermon_sw_reset_delay" is set to zero, then no software reset of the Hermon 1699e39c5baSBill Taylor * device will be attempted. 1709e39c5baSBill Taylor */ 1719e39c5baSBill Taylor uint32_t hermon_sw_reset_delay = HERMON_SW_RESET_DELAY; 1729e39c5baSBill Taylor 1739e39c5baSBill Taylor /* 1749e39c5baSBill Taylor * These are patchable variables for hermon command polling. The poll_delay is 1759e39c5baSBill Taylor * the number of usec to wait in-between calls to poll the 'go' bit. The 1769e39c5baSBill Taylor * poll_max is the total number of usec to loop in waiting for the 'go' bit to 1779e39c5baSBill Taylor * clear. 1789e39c5baSBill Taylor */ 1799e39c5baSBill Taylor uint32_t hermon_cmd_poll_delay = HERMON_CMD_POLL_DELAY; 1809e39c5baSBill Taylor uint32_t hermon_cmd_poll_max = HERMON_CMD_POLL_MAX; 1819e39c5baSBill Taylor 1829e39c5baSBill Taylor /* 1839e39c5baSBill Taylor * This is a patchable variable that determines the frequency with which 1849e39c5baSBill Taylor * the AckReq bit will be set in outgoing RC packets. The AckReq bit will be 1859e39c5baSBill Taylor * set in at least every 2^hermon_qp_ackreq_freq packets (but at least once 1869e39c5baSBill Taylor * per message, i.e. in the last packet). Tuning this value can increase 1879e39c5baSBill Taylor * IB fabric utilization by cutting down on the number of unnecessary ACKs. 1889e39c5baSBill Taylor */ 1899e39c5baSBill Taylor uint32_t hermon_qp_ackreq_freq = HERMON_QP_ACKREQ_FREQ; 1909e39c5baSBill Taylor 1919e39c5baSBill Taylor static void hermon_cfg_wqe_sizes(hermon_state_t *state, 1929e39c5baSBill Taylor hermon_cfg_profile_t *cp); 1939e39c5baSBill Taylor #ifdef __sparc 1949e39c5baSBill Taylor static void hermon_check_iommu_bypass(hermon_state_t *state, 1959e39c5baSBill Taylor hermon_cfg_profile_t *cp); 1969e39c5baSBill Taylor #endif 1979e39c5baSBill Taylor 1989e39c5baSBill Taylor /* 1999e39c5baSBill Taylor * hermon_cfg_profile_init_phase1() 2009e39c5baSBill Taylor * Context: Only called from attach() path context 2019e39c5baSBill Taylor */ 2029e39c5baSBill Taylor int 2039e39c5baSBill Taylor hermon_cfg_profile_init_phase1(hermon_state_t *state) 2049e39c5baSBill Taylor { 2059e39c5baSBill Taylor hermon_cfg_profile_t *cp; 2069e39c5baSBill Taylor 2079e39c5baSBill Taylor /* 2089e39c5baSBill Taylor * Allocate space for the configuration profile structure 2099e39c5baSBill Taylor */ 2109e39c5baSBill Taylor cp = (hermon_cfg_profile_t *)kmem_zalloc(sizeof (hermon_cfg_profile_t), 2119e39c5baSBill Taylor KM_SLEEP); 2129e39c5baSBill Taylor 2139e39c5baSBill Taylor /* 2149e39c5baSBill Taylor * Common to all profiles. 2159e39c5baSBill Taylor */ 2169e39c5baSBill Taylor cp->cp_qp0_agents_in_fw = hermon_qp0_agents_in_fw; 2179e39c5baSBill Taylor cp->cp_qp1_agents_in_fw = hermon_qp1_agents_in_fw; 2189e39c5baSBill Taylor cp->cp_sw_reset_delay = hermon_sw_reset_delay; 2199e39c5baSBill Taylor cp->cp_cmd_poll_delay = hermon_cmd_poll_delay; 2209e39c5baSBill Taylor cp->cp_cmd_poll_max = hermon_cmd_poll_max; 2219e39c5baSBill Taylor cp->cp_ackreq_freq = hermon_qp_ackreq_freq; 2229e39c5baSBill Taylor cp->cp_fmr_max_remaps = hermon_fmr_num_remaps; 2239e39c5baSBill Taylor 2249e39c5baSBill Taylor /* 2259e39c5baSBill Taylor * Although most of the configuration is enabled in "phase2" of the 2269e39c5baSBill Taylor * cfg_profile_init, we have to setup the OUT mailboxes soon, since 2279e39c5baSBill Taylor * they are used immediately after this "phase1" completes, to run the 2289e39c5baSBill Taylor * firmware and get the device limits, which we'll need for 'phase2'. 2299e39c5baSBill Taylor * That's done in rsrc_init_phase1, called shortly after we do this 2309e39c5baSBill Taylor * and the sw reset - see hermon.c 2319e39c5baSBill Taylor */ 2329e39c5baSBill Taylor if (state->hs_cfg_profile_setting == HERMON_CFG_MEMFREE) { 2339e39c5baSBill Taylor cp->cp_log_num_outmbox = hermon_log_num_outmbox; 2349e39c5baSBill Taylor cp->cp_log_outmbox_size = hermon_log_outmbox_size; 2359e39c5baSBill Taylor cp->cp_log_num_inmbox = hermon_log_num_inmbox; 2369e39c5baSBill Taylor cp->cp_log_inmbox_size = hermon_log_inmbox_size; 2379e39c5baSBill Taylor cp->cp_log_num_intr_inmbox = hermon_log_num_intr_inmbox; 2389e39c5baSBill Taylor cp->cp_log_num_intr_outmbox = hermon_log_num_intr_outmbox; 2399e39c5baSBill Taylor 2409e39c5baSBill Taylor } else { 2419e39c5baSBill Taylor return (DDI_FAILURE); 2429e39c5baSBill Taylor } 2439e39c5baSBill Taylor 244c7facc54SBill Taylor /* 245c7facc54SBill Taylor * Set IOMMU bypass or not. Ensure consistency of flags with 246c7facc54SBill Taylor * architecture type. 247c7facc54SBill Taylor */ 248c7facc54SBill Taylor #ifdef __sparc 249c7facc54SBill Taylor if (hermon_iommu_bypass == 1) { 250c7facc54SBill Taylor hermon_check_iommu_bypass(state, cp); 251c7facc54SBill Taylor } else { 252c7facc54SBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL; 253c7facc54SBill Taylor } 254c7facc54SBill Taylor #else 255c7facc54SBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL; 256c7facc54SBill Taylor #endif 257c7facc54SBill Taylor 2589e39c5baSBill Taylor /* Attach the configuration profile to Hermon softstate */ 2599e39c5baSBill Taylor state->hs_cfg_profile = cp; 2609e39c5baSBill Taylor 2619e39c5baSBill Taylor return (DDI_SUCCESS); 2629e39c5baSBill Taylor } 2639e39c5baSBill Taylor 2649e39c5baSBill Taylor /* 2659e39c5baSBill Taylor * hermon_cfg_profile_init_phase2() 2669e39c5baSBill Taylor * Context: Only called from attach() path context 2679e39c5baSBill Taylor */ 2689e39c5baSBill Taylor int 2699e39c5baSBill Taylor hermon_cfg_profile_init_phase2(hermon_state_t *state) 2709e39c5baSBill Taylor { 2719e39c5baSBill Taylor hermon_cfg_profile_t *cp; 2729e39c5baSBill Taylor hermon_hw_querydevlim_t *devlim; 2739e39c5baSBill Taylor hermon_hw_query_port_t *port; 2749e39c5baSBill Taylor uint32_t num, size; 2759e39c5baSBill Taylor int i; 2769e39c5baSBill Taylor 2779e39c5baSBill Taylor /* Read in the device limits */ 2789e39c5baSBill Taylor devlim = &state->hs_devlim; 2799e39c5baSBill Taylor /* and the port information */ 2809e39c5baSBill Taylor port = &state->hs_queryport; 2819e39c5baSBill Taylor 2829e39c5baSBill Taylor /* Read the configuration profile */ 2839e39c5baSBill Taylor cp = state->hs_cfg_profile; 2849e39c5baSBill Taylor 2859e39c5baSBill Taylor /* 2869e39c5baSBill Taylor * We configure all Hermon HCAs with the same profile, which 2879e39c5baSBill Taylor * is based upon the default value assignments above. If we want to 2889e39c5baSBill Taylor * add additional profiles in the future, they can be added here. 2899e39c5baSBill Taylor * Note the reference to "Memfree" is a holdover from Arbel/Sinai 2909e39c5baSBill Taylor */ 2919e39c5baSBill Taylor if (state->hs_cfg_profile_setting != HERMON_CFG_MEMFREE) { 2929e39c5baSBill Taylor return (DDI_FAILURE); 2939e39c5baSBill Taylor } 2949e39c5baSBill Taylor 2959e39c5baSBill Taylor /* 2969e39c5baSBill Taylor * Note for most configuration parameters, we use the lesser of our 2979e39c5baSBill Taylor * desired configuration value or the device-defined maximum value. 2989e39c5baSBill Taylor */ 2999e39c5baSBill Taylor cp->cp_log_num_mtt = min(hermon_log_num_mtt, devlim->log_max_mtt); 3009e39c5baSBill Taylor cp->cp_log_num_dmpt = min(hermon_log_num_dmpt, devlim->log_max_dmpt); 3019e39c5baSBill Taylor cp->cp_log_num_cmpt = HERMON_LOG_CMPT_PER_TYPE + 2; /* times 4, */ 3029e39c5baSBill Taylor /* per PRM */ 3039e39c5baSBill Taylor cp->cp_log_max_mrw_sz = min(hermon_log_max_mrw_sz, 3049e39c5baSBill Taylor devlim->log_max_mrw_sz); 3059e39c5baSBill Taylor cp->cp_log_num_pd = min(hermon_log_num_pd, devlim->log_max_pd); 3069e39c5baSBill Taylor cp->cp_log_num_qp = min(hermon_log_num_qp, devlim->log_max_qp); 3079e39c5baSBill Taylor cp->cp_log_num_cq = min(hermon_log_num_cq, devlim->log_max_cq); 3089e39c5baSBill Taylor cp->cp_log_num_srq = min(hermon_log_num_srq, devlim->log_max_srq); 3099e39c5baSBill Taylor cp->cp_log_num_eq = min(hermon_log_num_eq, devlim->log_max_eq); 3109e39c5baSBill Taylor cp->cp_log_eq_sz = min(hermon_log_eq_sz, devlim->log_max_eq_sz); 3119e39c5baSBill Taylor cp->cp_log_num_rdb = cp->cp_log_num_qp + 3129e39c5baSBill Taylor min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp); 3139e39c5baSBill Taylor cp->cp_hca_max_rdma_in_qp = cp->cp_hca_max_rdma_out_qp = 3149e39c5baSBill Taylor 1 << min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp); 3159fa01fafSagiri cp->cp_num_qp_per_mcg = max(hermon_num_qp_per_mcg, 3169fa01fafSagiri HERMON_NUM_QP_PER_MCG_MIN); 3179fa01fafSagiri cp->cp_num_qp_per_mcg = min(cp->cp_num_qp_per_mcg, 3189fa01fafSagiri (1 << devlim->log_max_qp_mcg) - 8); 3199fa01fafSagiri cp->cp_num_qp_per_mcg = (1 << highbit(cp->cp_num_qp_per_mcg + 7)) - 8; 3209e39c5baSBill Taylor cp->cp_log_num_mcg = min(hermon_log_num_mcg, devlim->log_max_mcg); 3219e39c5baSBill Taylor cp->cp_log_num_mcg_hash = hermon_log_num_mcg_hash; 3229e39c5baSBill Taylor 3239e39c5baSBill Taylor /* until srq_resize is debugged, disable it */ 3249e39c5baSBill Taylor cp->cp_srq_resize_enabled = 0; 3259e39c5baSBill Taylor 3269e39c5baSBill Taylor /* cp->cp_log_num_uar = hermon_log_num_uar; */ 3279e39c5baSBill Taylor /* 3289e39c5baSBill Taylor * now, we HAVE to calculate the number of UAR pages, so that we can 3299e39c5baSBill Taylor * get the blueflame stuff correct as well 3309e39c5baSBill Taylor */ 3319e39c5baSBill Taylor 3329e39c5baSBill Taylor size = devlim->log_max_uar_sz; 3339e39c5baSBill Taylor /* 1MB (2^^20) times size (2^^size) / sparc_pg (2^^13) */ 3349e39c5baSBill Taylor num = (20 + size) - 13; /* XXX - consider using PAGESHIFT */ 3359e39c5baSBill Taylor if (devlim->blu_flm) 3369e39c5baSBill Taylor num -= 1; /* if blueflame, only half the size for UARs */ 3379e39c5baSBill Taylor cp->cp_log_num_uar = min(hermon_log_num_uar, num); 3389e39c5baSBill Taylor 3399e39c5baSBill Taylor 3409e39c5baSBill Taylor /* while we're at it, calculate the index of the kernel uar page */ 3419e39c5baSBill Taylor /* either the reserved uar's or 128, whichever is smaller */ 3429e39c5baSBill Taylor state->hs_kernel_uar_index = (devlim->num_rsvd_uar > 128) ? 3439e39c5baSBill Taylor devlim->num_rsvd_uar : 128; 3449e39c5baSBill Taylor 3459e39c5baSBill Taylor cp->cp_log_max_pkeytbl = port->log_max_pkey; 3469e39c5baSBill Taylor 3479e39c5baSBill Taylor cp->cp_log_max_qp_sz = devlim->log_max_qp_sz; 3489e39c5baSBill Taylor cp->cp_log_max_cq_sz = devlim->log_max_cq_sz; 3499e39c5baSBill Taylor cp->cp_log_max_srq_sz = devlim->log_max_srq_sz; 3509e39c5baSBill Taylor cp->cp_log_max_gidtbl = port->log_max_gid; 3519e39c5baSBill Taylor cp->cp_max_mtu = port->ib_mtu; /* XXX now from query_port */ 3529e39c5baSBill Taylor cp->cp_max_port_width = port->ib_port_wid; /* now from query_port */ 3539e39c5baSBill Taylor cp->cp_max_vlcap = port->max_vl; 3549e39c5baSBill Taylor cp->cp_log_num_ah = hermon_log_num_ah; 3559e39c5baSBill Taylor 356*17a2b317SBill Taylor /* Paranoia, ensure no arrays indexed by port_num are out of bounds */ 357*17a2b317SBill Taylor cp->cp_num_ports = devlim->num_ports; 358*17a2b317SBill Taylor if (cp->cp_num_ports > HERMON_MAX_PORTS) { 359*17a2b317SBill Taylor cmn_err(CE_CONT, "device has more ports (%d) than are " 360*17a2b317SBill Taylor "supported; Using %d ports\n", 361*17a2b317SBill Taylor cp->cp_num_ports, HERMON_MAX_PORTS); 362*17a2b317SBill Taylor cp->cp_num_ports = HERMON_MAX_PORTS; 363*17a2b317SBill Taylor }; 364*17a2b317SBill Taylor 3659e39c5baSBill Taylor /* allocate variable sized arrays */ 3669e39c5baSBill Taylor for (i = 0; i < HERMON_MAX_PORTS; i++) { 3679e39c5baSBill Taylor state->hs_pkey[i] = kmem_zalloc((1 << cp->cp_log_max_pkeytbl) * 3689e39c5baSBill Taylor sizeof (ib_pkey_t), KM_SLEEP); 3699e39c5baSBill Taylor state->hs_guid[i] = kmem_zalloc((1 << cp->cp_log_max_gidtbl) * 3709e39c5baSBill Taylor sizeof (ib_guid_t), KM_SLEEP); 3719e39c5baSBill Taylor } 3729e39c5baSBill Taylor 3739e39c5baSBill Taylor /* Determine WQE sizes from requested max SGLs */ 3749e39c5baSBill Taylor hermon_cfg_wqe_sizes(state, cp); 3759e39c5baSBill Taylor 3769e39c5baSBill Taylor /* Set whether to use MSIs or not */ 3779e39c5baSBill Taylor cp->cp_use_msi_if_avail = hermon_use_msi_if_avail; 3789e39c5baSBill Taylor 3799e39c5baSBill Taylor return (DDI_SUCCESS); 3809e39c5baSBill Taylor } 3819e39c5baSBill Taylor 3829e39c5baSBill Taylor 3839e39c5baSBill Taylor /* 3849e39c5baSBill Taylor * hermon_cfg_profile_fini() 3859e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts 3869e39c5baSBill Taylor */ 3879e39c5baSBill Taylor void 3889e39c5baSBill Taylor hermon_cfg_profile_fini(hermon_state_t *state) 3899e39c5baSBill Taylor { 3909e39c5baSBill Taylor /* 3919e39c5baSBill Taylor * Free up the space for configuration profile 3929e39c5baSBill Taylor */ 3939e39c5baSBill Taylor kmem_free(state->hs_cfg_profile, sizeof (hermon_cfg_profile_t)); 3949e39c5baSBill Taylor } 3959e39c5baSBill Taylor 3969e39c5baSBill Taylor 3979e39c5baSBill Taylor /* 3989e39c5baSBill Taylor * hermon_cfg_wqe_sizes() 3999e39c5baSBill Taylor * Context: Only called from attach() path context 4009e39c5baSBill Taylor */ 4019e39c5baSBill Taylor static void 4029e39c5baSBill Taylor hermon_cfg_wqe_sizes(hermon_state_t *state, hermon_cfg_profile_t *cp) 4039e39c5baSBill Taylor { 4049e39c5baSBill Taylor uint_t max_size, log2; 4059e39c5baSBill Taylor uint_t max_sgl, real_max_sgl; 4069e39c5baSBill Taylor 4079e39c5baSBill Taylor /* 4089e39c5baSBill Taylor * Get the requested maximum number SGL per WQE from the Hermon 4099e39c5baSBill Taylor * patchable variable 4109e39c5baSBill Taylor */ 4119e39c5baSBill Taylor max_sgl = hermon_wqe_max_sgl; 4129e39c5baSBill Taylor 4139e39c5baSBill Taylor /* 4149e39c5baSBill Taylor * Use requested maximum number of SGL to calculate the max descriptor 4159e39c5baSBill Taylor * size (while guaranteeing that the descriptor size is a power-of-2 4169e39c5baSBill Taylor * cachelines). We have to use the calculation for QP1 MLX transport 4179e39c5baSBill Taylor * because the possibility that we might need to inline a GRH, along 4189e39c5baSBill Taylor * with all the other headers and alignment restrictions, sets the 4199e39c5baSBill Taylor * maximum for the number of SGLs that we can advertise support for. 4209e39c5baSBill Taylor */ 4219e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (max_sgl << 4)); 4229e39c5baSBill Taylor log2 = highbit(max_size); 4239e39c5baSBill Taylor if ((max_size & (max_size - 1)) == 0) { 4249e39c5baSBill Taylor log2 = log2 - 1; 4259e39c5baSBill Taylor } 4269e39c5baSBill Taylor max_size = (1 << log2); 4279e39c5baSBill Taylor 4289e39c5baSBill Taylor max_size = min(max_size, state->hs_devlim.max_desc_sz_sq); 4299e39c5baSBill Taylor 4309e39c5baSBill Taylor /* 4319e39c5baSBill Taylor * Then use the calculated max descriptor size to determine the "real" 4329e39c5baSBill Taylor * maximum SGL (the number beyond which we would roll over to the next 4339e39c5baSBill Taylor * power-of-2). 4349e39c5baSBill Taylor */ 4359e39c5baSBill Taylor real_max_sgl = (max_size - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4; 4369e39c5baSBill Taylor 4379e39c5baSBill Taylor /* Then save away this configuration information */ 4389e39c5baSBill Taylor cp->cp_wqe_max_sgl = max_sgl; 4399e39c5baSBill Taylor cp->cp_wqe_real_max_sgl = real_max_sgl; 4409e39c5baSBill Taylor 4419e39c5baSBill Taylor /* SRQ SGL gets set to it's own patchable variable value */ 4429e39c5baSBill Taylor cp->cp_srq_max_sgl = hermon_srq_max_sgl; 4439e39c5baSBill Taylor } 4449e39c5baSBill Taylor 4459e39c5baSBill Taylor #ifdef __sparc 4469e39c5baSBill Taylor /* 4479e39c5baSBill Taylor * hermon_check_iommu_bypass() 4489e39c5baSBill Taylor * Context: Only called from attach() path context 4499e39c5baSBill Taylor * XXX This is a DMA allocation routine outside the normal 4509e39c5baSBill Taylor * path. FMA hardening will not like this. 4519e39c5baSBill Taylor */ 4529e39c5baSBill Taylor static void 4539e39c5baSBill Taylor hermon_check_iommu_bypass(hermon_state_t *state, hermon_cfg_profile_t *cp) 4549e39c5baSBill Taylor { 4559e39c5baSBill Taylor ddi_dma_handle_t dmahdl; 4569e39c5baSBill Taylor ddi_dma_attr_t dma_attr; 4579e39c5baSBill Taylor int status; 4589e39c5baSBill Taylor ddi_acc_handle_t acc_hdl; 4599e39c5baSBill Taylor caddr_t kaddr; 4609e39c5baSBill Taylor size_t actual_len; 4619e39c5baSBill Taylor ddi_dma_cookie_t cookie; 4629e39c5baSBill Taylor uint_t cookiecnt; 4639e39c5baSBill Taylor 4649e39c5baSBill Taylor hermon_dma_attr_init(state, &dma_attr); 4659e39c5baSBill Taylor 4669e39c5baSBill Taylor /* Try mapping for IOMMU bypass (Force Physical) */ 4679e39c5baSBill Taylor dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL | 4689e39c5baSBill Taylor DDI_DMA_RELAXED_ORDERING; 4699e39c5baSBill Taylor 4709e39c5baSBill Taylor /* 4719e39c5baSBill Taylor * Call ddi_dma_alloc_handle(). If this returns DDI_DMA_BADATTR then 4729e39c5baSBill Taylor * it is not possible to use IOMMU bypass with our PCI bridge parent. 4739e39c5baSBill Taylor * Since the function we are in can only be called if iommu bypass was 4749e39c5baSBill Taylor * requested in the config profile, we configure for bypass if the 4759e39c5baSBill Taylor * ddi_dma_alloc_handle() was successful. Otherwise, we configure 4769e39c5baSBill Taylor * for non-bypass (ie: normal) mapping. 4779e39c5baSBill Taylor */ 4789e39c5baSBill Taylor status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, 4799e39c5baSBill Taylor DDI_DMA_SLEEP, NULL, &dmahdl); 4809e39c5baSBill Taylor if (status == DDI_DMA_BADATTR) { 4819e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL; 4829e39c5baSBill Taylor return; 4839e39c5baSBill Taylor } else if (status != DDI_SUCCESS) { /* failed somehow */ 4849e39c5baSBill Taylor hermon_kernel_data_ro = HERMON_RO_DISABLED; 4859e39c5baSBill Taylor hermon_user_data_ro = HERMON_RO_DISABLED; 4869e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_BYPASS; 4879e39c5baSBill Taylor return; 4889e39c5baSBill Taylor } else { 4899e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_BYPASS; 4909e39c5baSBill Taylor } 4919e39c5baSBill Taylor 4929e39c5baSBill Taylor status = ddi_dma_mem_alloc(dmahdl, 256, 4939e39c5baSBill Taylor &state->hs_reg_accattr, DDI_DMA_CONSISTENT, 4949e39c5baSBill Taylor DDI_DMA_SLEEP, NULL, (caddr_t *)&kaddr, &actual_len, &acc_hdl); 4959e39c5baSBill Taylor 4969e39c5baSBill Taylor if (status != DDI_SUCCESS) { /* failed somehow */ 4979e39c5baSBill Taylor hermon_kernel_data_ro = HERMON_RO_DISABLED; 4989e39c5baSBill Taylor hermon_user_data_ro = HERMON_RO_DISABLED; 4999e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl); 5009e39c5baSBill Taylor return; 5019e39c5baSBill Taylor } 5029e39c5baSBill Taylor 5039e39c5baSBill Taylor status = ddi_dma_addr_bind_handle(dmahdl, NULL, kaddr, actual_len, 5049e39c5baSBill Taylor DDI_DMA_RDWR, DDI_DMA_SLEEP, NULL, &cookie, &cookiecnt); 5059e39c5baSBill Taylor 5069e39c5baSBill Taylor if (status == DDI_DMA_MAPPED) { 5079e39c5baSBill Taylor (void) ddi_dma_unbind_handle(dmahdl); 5089e39c5baSBill Taylor } else { 5099e39c5baSBill Taylor hermon_kernel_data_ro = HERMON_RO_DISABLED; 5109e39c5baSBill Taylor hermon_user_data_ro = HERMON_RO_DISABLED; 5119e39c5baSBill Taylor } 5129e39c5baSBill Taylor 5139e39c5baSBill Taylor ddi_dma_mem_free(&acc_hdl); 5149e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl); 5159e39c5baSBill Taylor } 5169e39c5baSBill Taylor #endif 517