19e39c5baSBill Taylor /* 29e39c5baSBill Taylor * CDDL HEADER START 39e39c5baSBill Taylor * 49e39c5baSBill Taylor * The contents of this file are subject to the terms of the 59e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 69e39c5baSBill Taylor * You may not use this file except in compliance with the License. 79e39c5baSBill Taylor * 89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 109e39c5baSBill Taylor * See the License for the specific language governing permissions 119e39c5baSBill Taylor * and limitations under the License. 129e39c5baSBill Taylor * 139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 189e39c5baSBill Taylor * 199e39c5baSBill Taylor * CDDL HEADER END 209e39c5baSBill Taylor */ 219e39c5baSBill Taylor 229e39c5baSBill Taylor /* 239e39c5baSBill Taylor * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 249e39c5baSBill Taylor * Use is subject to license terms. 259e39c5baSBill Taylor */ 269e39c5baSBill Taylor 279e39c5baSBill Taylor /* 289e39c5baSBill Taylor * hermon_cfg.c 299e39c5baSBill Taylor * Hermon Configuration Profile Routines 309e39c5baSBill Taylor * 319e39c5baSBill Taylor * Implements the routines necessary for initializing and (later) tearing 329e39c5baSBill Taylor * down the list of Hermon configuration information. 339e39c5baSBill Taylor */ 349e39c5baSBill Taylor 359e39c5baSBill Taylor #include <sys/types.h> 369e39c5baSBill Taylor #include <sys/conf.h> 379e39c5baSBill Taylor #include <sys/ddi.h> 389e39c5baSBill Taylor #include <sys/sunddi.h> 399e39c5baSBill Taylor #include <sys/modctl.h> 409e39c5baSBill Taylor #include <sys/bitmap.h> 419e39c5baSBill Taylor 429e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 439e39c5baSBill Taylor 449e39c5baSBill Taylor /* 459e39c5baSBill Taylor * Below are the elements that make up the Hermon configuration profile. 469e39c5baSBill Taylor * For advanced users who wish to alter these values, this can be done via 479e39c5baSBill Taylor * the /etc/system file. By default, values are assigned to the number of 489e39c5baSBill Taylor * supported resources, either from the HCA's reported capacities or by 499e39c5baSBill Taylor * a by-design limit in the driver. 509e39c5baSBill Taylor */ 519e39c5baSBill Taylor 529e39c5baSBill Taylor /* Number of supported QPs, CQs and SRQs */ 539e39c5baSBill Taylor uint32_t hermon_log_num_qp = HERMON_NUM_QP_SHIFT; 549e39c5baSBill Taylor uint32_t hermon_log_num_cq = HERMON_NUM_CQ_SHIFT; 559e39c5baSBill Taylor uint32_t hermon_log_num_srq = HERMON_NUM_SRQ_SHIFT; 569e39c5baSBill Taylor 579e39c5baSBill Taylor /* Number of supported SGL per WQE for SQ/RQ, and for SRQ */ 589e39c5baSBill Taylor /* XXX use the same for all queues if limitation in srq.h is resolved */ 599e39c5baSBill Taylor uint32_t hermon_wqe_max_sgl = HERMON_NUM_SGL_PER_WQE; 609e39c5baSBill Taylor uint32_t hermon_srq_max_sgl = HERMON_SRQ_MAX_SGL; 619e39c5baSBill Taylor 629e39c5baSBill Taylor /* Maximum "responder resources" (in) and "initiator depth" (out) per QP */ 639e39c5baSBill Taylor uint32_t hermon_log_num_rdb_per_qp = HERMON_LOG_NUM_RDB_PER_QP; 649e39c5baSBill Taylor 659e39c5baSBill Taylor /* 669e39c5baSBill Taylor * Number of multicast groups (MCGs), number of QP per MCG, and the number 679e39c5baSBill Taylor * of entries (from the total number) in the multicast group "hash table" 689e39c5baSBill Taylor */ 699e39c5baSBill Taylor uint32_t hermon_log_num_mcg = HERMON_NUM_MCG_SHIFT; 709e39c5baSBill Taylor uint32_t hermon_num_qp_per_mcg = HERMON_NUM_QP_PER_MCG; 71*9fa01fafSagiri uint32_t hermon_log_num_mcg_hash = HERMON_NUM_MCG_HASH_SHIFT; 729e39c5baSBill Taylor 739e39c5baSBill Taylor /* Number of UD AVs */ 749e39c5baSBill Taylor uint32_t hermon_log_num_ah = HERMON_NUM_AH_SHIFT; 759e39c5baSBill Taylor 769e39c5baSBill Taylor /* Number of EQs and their default size */ 779e39c5baSBill Taylor uint32_t hermon_log_num_eq = HERMON_NUM_EQ_SHIFT; 789e39c5baSBill Taylor uint32_t hermon_log_eq_sz = HERMON_DEFAULT_EQ_SZ_SHIFT; 799e39c5baSBill Taylor 809e39c5baSBill Taylor /* 819e39c5baSBill Taylor * Number of supported MPTs, MTTs and also the maximum MPT size. 829e39c5baSBill Taylor */ 839e39c5baSBill Taylor uint32_t hermon_log_num_mtt = HERMON_NUM_MTT_SHIFT; 849e39c5baSBill Taylor uint32_t hermon_log_num_dmpt = HERMON_NUM_DMPT_SHIFT; 859e39c5baSBill Taylor uint32_t hermon_log_max_mrw_sz = HERMON_MAX_MEM_MPT_SHIFT; 869e39c5baSBill Taylor 879e39c5baSBill Taylor /* 889e39c5baSBill Taylor * Number of supported UAR (User Access Regions) for this HCA. 899e39c5baSBill Taylor * We could in the future read in uar_sz from devlim, and thus 909e39c5baSBill Taylor * derive the number of UAR. Since this is derived from PAGESIZE, 919e39c5baSBill Taylor * however, this means that x86 systems would have twice as many 929e39c5baSBill Taylor * UARs as SPARC systems. Therefore for consistency's sake, we will 939e39c5baSBill Taylor * just use 1024 pages, which is the maximum on SPARC systems. 949e39c5baSBill Taylor */ 959e39c5baSBill Taylor uint32_t hermon_log_num_uar = HERMON_NUM_UAR_SHIFT; 969e39c5baSBill Taylor 979e39c5baSBill Taylor /* 989e39c5baSBill Taylor * Number of remaps allowed for FMR before a sync is required. This value 999e39c5baSBill Taylor * determines how many times we can fmr_deregister() before the underlying fmr 1009e39c5baSBill Taylor * framework places the region to wait for an MTT_SYNC operation, cleaning up 1019e39c5baSBill Taylor * the old mappings. 1029e39c5baSBill Taylor */ 1039e39c5baSBill Taylor uint32_t hermon_fmr_num_remaps = HERMON_FMR_MAX_REMAPS; 1049e39c5baSBill Taylor 1059e39c5baSBill Taylor /* 1069e39c5baSBill Taylor * Number of supported Hermon mailboxes ("In" and "Out") and their maximum 1079e39c5baSBill Taylor * sizes, respectively 1089e39c5baSBill Taylor */ 1099e39c5baSBill Taylor uint32_t hermon_log_num_inmbox = HERMON_NUM_MAILBOXES_SHIFT; 1109e39c5baSBill Taylor uint32_t hermon_log_num_outmbox = HERMON_NUM_MAILBOXES_SHIFT; 1119e39c5baSBill Taylor uint32_t hermon_log_inmbox_size = HERMON_MBOX_SIZE_SHIFT; 1129e39c5baSBill Taylor uint32_t hermon_log_outmbox_size = HERMON_MBOX_SIZE_SHIFT; 1139e39c5baSBill Taylor uint32_t hermon_log_num_intr_inmbox = HERMON_NUM_INTR_MAILBOXES_SHIFT; 1149e39c5baSBill Taylor uint32_t hermon_log_num_intr_outmbox = HERMON_NUM_INTR_MAILBOXES_SHIFT; 1159e39c5baSBill Taylor 1169e39c5baSBill Taylor /* Number of supported Protection Domains (PD) */ 1179e39c5baSBill Taylor uint32_t hermon_log_num_pd = HERMON_NUM_PD_SHIFT; 1189e39c5baSBill Taylor 1199e39c5baSBill Taylor /* 1209e39c5baSBill Taylor * Number of total supported PKeys per PKey table (i.e. 1219e39c5baSBill Taylor * per port). Also the number of SGID per GID table. 1229e39c5baSBill Taylor */ 1239e39c5baSBill Taylor uint32_t hermon_log_max_pkeytbl = HERMON_NUM_PKEYTBL_SHIFT; 1249e39c5baSBill Taylor uint32_t hermon_log_max_gidtbl = HERMON_NUM_GIDTBL_SHIFT; 1259e39c5baSBill Taylor 1269e39c5baSBill Taylor /* Maximum supported MTU and portwidth */ 1279e39c5baSBill Taylor uint32_t hermon_max_mtu = HERMON_MAX_MTU; 1289e39c5baSBill Taylor uint32_t hermon_max_port_width = HERMON_MAX_PORT_WIDTH; 1299e39c5baSBill Taylor 1309e39c5baSBill Taylor /* Number of supported Virtual Lanes (VL) */ 1319e39c5baSBill Taylor uint32_t hermon_max_vlcap = HERMON_MAX_VLCAP; 1329e39c5baSBill Taylor 1339e39c5baSBill Taylor /* 1349e39c5baSBill Taylor * Whether or not to use the built-in (i.e. in firmware) agents for QP0 and 1359e39c5baSBill Taylor * QP1, respectively. 1369e39c5baSBill Taylor */ 13776c04273SRajkumar Sivaprakasam uint32_t hermon_qp0_agents_in_fw = 0; 1389e39c5baSBill Taylor uint32_t hermon_qp1_agents_in_fw = 0; 1399e39c5baSBill Taylor 1409e39c5baSBill Taylor /* 1419e39c5baSBill Taylor * Whether DMA mappings should bypass the PCI IOMMU or not. 1429e39c5baSBill Taylor * hermon_iommu_bypass is a global setting for all memory addresses. 1439e39c5baSBill Taylor */ 1449e39c5baSBill Taylor uint32_t hermon_iommu_bypass = 1; 1459e39c5baSBill Taylor 1469e39c5baSBill Taylor /* 1479e39c5baSBill Taylor * Whether *DATA* buffers should be bound w/ Relaxed Ordering (RO) turned on 1489e39c5baSBill Taylor * via the SW workaround (HCAs don't support RO in HW). Defaulted on, 1499e39c5baSBill Taylor * though care must be taken w/ some Userland clients that *MAY* have 1509e39c5baSBill Taylor * peeked in the data to understand when data xfer was done - MPI does 1519e39c5baSBill Taylor * as an efficiency 1529e39c5baSBill Taylor */ 1539e39c5baSBill Taylor 1549e39c5baSBill Taylor uint32_t hermon_kernel_data_ro = HERMON_RO_ENABLED; /* default */ 1559e39c5baSBill Taylor uint32_t hermon_user_data_ro = HERMON_RO_ENABLED; /* default */ 1569e39c5baSBill Taylor 1579e39c5baSBill Taylor /* 1589e39c5baSBill Taylor * Whether Hermon should use MSI (Message Signaled Interrupts), if available. 1599e39c5baSBill Taylor * Note: 0 indicates 'legacy interrupt', 1 indicates MSI (if available) 1609e39c5baSBill Taylor */ 1619e39c5baSBill Taylor uint32_t hermon_use_msi_if_avail = 1; 1629e39c5baSBill Taylor 1639e39c5baSBill Taylor /* 1649e39c5baSBill Taylor * This is a patchable variable that determines the time we will wait after 1659e39c5baSBill Taylor * initiating SW reset before we do our first read from Hermon config space. 1669e39c5baSBill Taylor * If this value is set too small (less than the default 100ms), it is 1679e39c5baSBill Taylor * possible for Hermon hardware to be unready to respond to the config cycle 1689e39c5baSBill Taylor * reads. This could cause master abort on the PCI bridge. Note: If 1699e39c5baSBill Taylor * "hermon_sw_reset_delay" is set to zero, then no software reset of the Hermon 1709e39c5baSBill Taylor * device will be attempted. 1719e39c5baSBill Taylor */ 1729e39c5baSBill Taylor uint32_t hermon_sw_reset_delay = HERMON_SW_RESET_DELAY; 1739e39c5baSBill Taylor 1749e39c5baSBill Taylor /* 1759e39c5baSBill Taylor * These are patchable variables for hermon command polling. The poll_delay is 1769e39c5baSBill Taylor * the number of usec to wait in-between calls to poll the 'go' bit. The 1779e39c5baSBill Taylor * poll_max is the total number of usec to loop in waiting for the 'go' bit to 1789e39c5baSBill Taylor * clear. 1799e39c5baSBill Taylor */ 1809e39c5baSBill Taylor uint32_t hermon_cmd_poll_delay = HERMON_CMD_POLL_DELAY; 1819e39c5baSBill Taylor uint32_t hermon_cmd_poll_max = HERMON_CMD_POLL_MAX; 1829e39c5baSBill Taylor 1839e39c5baSBill Taylor /* 1849e39c5baSBill Taylor * This is a patchable variable that determines the frequency with which 1859e39c5baSBill Taylor * the AckReq bit will be set in outgoing RC packets. The AckReq bit will be 1869e39c5baSBill Taylor * set in at least every 2^hermon_qp_ackreq_freq packets (but at least once 1879e39c5baSBill Taylor * per message, i.e. in the last packet). Tuning this value can increase 1889e39c5baSBill Taylor * IB fabric utilization by cutting down on the number of unnecessary ACKs. 1899e39c5baSBill Taylor */ 1909e39c5baSBill Taylor uint32_t hermon_qp_ackreq_freq = HERMON_QP_ACKREQ_FREQ; 1919e39c5baSBill Taylor 1929e39c5baSBill Taylor static void hermon_cfg_wqe_sizes(hermon_state_t *state, 1939e39c5baSBill Taylor hermon_cfg_profile_t *cp); 1949e39c5baSBill Taylor #ifdef __sparc 1959e39c5baSBill Taylor static void hermon_check_iommu_bypass(hermon_state_t *state, 1969e39c5baSBill Taylor hermon_cfg_profile_t *cp); 1979e39c5baSBill Taylor #endif 1989e39c5baSBill Taylor 1999e39c5baSBill Taylor /* 2009e39c5baSBill Taylor * hermon_cfg_profile_init_phase1() 2019e39c5baSBill Taylor * Context: Only called from attach() path context 2029e39c5baSBill Taylor */ 2039e39c5baSBill Taylor int 2049e39c5baSBill Taylor hermon_cfg_profile_init_phase1(hermon_state_t *state) 2059e39c5baSBill Taylor { 2069e39c5baSBill Taylor hermon_cfg_profile_t *cp; 2079e39c5baSBill Taylor 2089e39c5baSBill Taylor /* 2099e39c5baSBill Taylor * Allocate space for the configuration profile structure 2109e39c5baSBill Taylor */ 2119e39c5baSBill Taylor cp = (hermon_cfg_profile_t *)kmem_zalloc(sizeof (hermon_cfg_profile_t), 2129e39c5baSBill Taylor KM_SLEEP); 2139e39c5baSBill Taylor 2149e39c5baSBill Taylor /* 2159e39c5baSBill Taylor * Common to all profiles. 2169e39c5baSBill Taylor */ 2179e39c5baSBill Taylor cp->cp_qp0_agents_in_fw = hermon_qp0_agents_in_fw; 2189e39c5baSBill Taylor cp->cp_qp1_agents_in_fw = hermon_qp1_agents_in_fw; 2199e39c5baSBill Taylor cp->cp_sw_reset_delay = hermon_sw_reset_delay; 2209e39c5baSBill Taylor cp->cp_cmd_poll_delay = hermon_cmd_poll_delay; 2219e39c5baSBill Taylor cp->cp_cmd_poll_max = hermon_cmd_poll_max; 2229e39c5baSBill Taylor cp->cp_ackreq_freq = hermon_qp_ackreq_freq; 2239e39c5baSBill Taylor cp->cp_fmr_max_remaps = hermon_fmr_num_remaps; 2249e39c5baSBill Taylor 2259e39c5baSBill Taylor /* 2269e39c5baSBill Taylor * Although most of the configuration is enabled in "phase2" of the 2279e39c5baSBill Taylor * cfg_profile_init, we have to setup the OUT mailboxes soon, since 2289e39c5baSBill Taylor * they are used immediately after this "phase1" completes, to run the 2299e39c5baSBill Taylor * firmware and get the device limits, which we'll need for 'phase2'. 2309e39c5baSBill Taylor * That's done in rsrc_init_phase1, called shortly after we do this 2319e39c5baSBill Taylor * and the sw reset - see hermon.c 2329e39c5baSBill Taylor */ 2339e39c5baSBill Taylor if (state->hs_cfg_profile_setting == HERMON_CFG_MEMFREE) { 2349e39c5baSBill Taylor cp->cp_log_num_outmbox = hermon_log_num_outmbox; 2359e39c5baSBill Taylor cp->cp_log_outmbox_size = hermon_log_outmbox_size; 2369e39c5baSBill Taylor cp->cp_log_num_inmbox = hermon_log_num_inmbox; 2379e39c5baSBill Taylor cp->cp_log_inmbox_size = hermon_log_inmbox_size; 2389e39c5baSBill Taylor cp->cp_log_num_intr_inmbox = hermon_log_num_intr_inmbox; 2399e39c5baSBill Taylor cp->cp_log_num_intr_outmbox = hermon_log_num_intr_outmbox; 2409e39c5baSBill Taylor 2419e39c5baSBill Taylor } else { 2429e39c5baSBill Taylor return (DDI_FAILURE); 2439e39c5baSBill Taylor } 2449e39c5baSBill Taylor 2459e39c5baSBill Taylor /* Attach the configuration profile to Hermon softstate */ 2469e39c5baSBill Taylor state->hs_cfg_profile = cp; 2479e39c5baSBill Taylor 2489e39c5baSBill Taylor return (DDI_SUCCESS); 2499e39c5baSBill Taylor } 2509e39c5baSBill Taylor 2519e39c5baSBill Taylor /* 2529e39c5baSBill Taylor * hermon_cfg_profile_init_phase2() 2539e39c5baSBill Taylor * Context: Only called from attach() path context 2549e39c5baSBill Taylor */ 2559e39c5baSBill Taylor int 2569e39c5baSBill Taylor hermon_cfg_profile_init_phase2(hermon_state_t *state) 2579e39c5baSBill Taylor { 2589e39c5baSBill Taylor hermon_cfg_profile_t *cp; 2599e39c5baSBill Taylor hermon_hw_querydevlim_t *devlim; 2609e39c5baSBill Taylor hermon_hw_query_port_t *port; 2619e39c5baSBill Taylor uint32_t num, size; 2629e39c5baSBill Taylor int i; 2639e39c5baSBill Taylor 2649e39c5baSBill Taylor /* Read in the device limits */ 2659e39c5baSBill Taylor devlim = &state->hs_devlim; 2669e39c5baSBill Taylor /* and the port information */ 2679e39c5baSBill Taylor port = &state->hs_queryport; 2689e39c5baSBill Taylor 2699e39c5baSBill Taylor /* Read the configuration profile */ 2709e39c5baSBill Taylor cp = state->hs_cfg_profile; 2719e39c5baSBill Taylor 2729e39c5baSBill Taylor /* 2739e39c5baSBill Taylor * We configure all Hermon HCAs with the same profile, which 2749e39c5baSBill Taylor * is based upon the default value assignments above. If we want to 2759e39c5baSBill Taylor * add additional profiles in the future, they can be added here. 2769e39c5baSBill Taylor * Note the reference to "Memfree" is a holdover from Arbel/Sinai 2779e39c5baSBill Taylor */ 2789e39c5baSBill Taylor if (state->hs_cfg_profile_setting != HERMON_CFG_MEMFREE) { 2799e39c5baSBill Taylor return (DDI_FAILURE); 2809e39c5baSBill Taylor } 2819e39c5baSBill Taylor 2829e39c5baSBill Taylor /* 2839e39c5baSBill Taylor * Note for most configuration parameters, we use the lesser of our 2849e39c5baSBill Taylor * desired configuration value or the device-defined maximum value. 2859e39c5baSBill Taylor */ 2869e39c5baSBill Taylor cp->cp_log_num_mtt = min(hermon_log_num_mtt, devlim->log_max_mtt); 2879e39c5baSBill Taylor cp->cp_log_num_dmpt = min(hermon_log_num_dmpt, devlim->log_max_dmpt); 2889e39c5baSBill Taylor cp->cp_log_num_cmpt = HERMON_LOG_CMPT_PER_TYPE + 2; /* times 4, */ 2899e39c5baSBill Taylor /* per PRM */ 2909e39c5baSBill Taylor cp->cp_log_max_mrw_sz = min(hermon_log_max_mrw_sz, 2919e39c5baSBill Taylor devlim->log_max_mrw_sz); 2929e39c5baSBill Taylor cp->cp_log_num_pd = min(hermon_log_num_pd, devlim->log_max_pd); 2939e39c5baSBill Taylor cp->cp_log_num_qp = min(hermon_log_num_qp, devlim->log_max_qp); 2949e39c5baSBill Taylor cp->cp_log_num_cq = min(hermon_log_num_cq, devlim->log_max_cq); 2959e39c5baSBill Taylor cp->cp_log_num_srq = min(hermon_log_num_srq, devlim->log_max_srq); 2969e39c5baSBill Taylor cp->cp_log_num_eq = min(hermon_log_num_eq, devlim->log_max_eq); 2979e39c5baSBill Taylor cp->cp_log_eq_sz = min(hermon_log_eq_sz, devlim->log_max_eq_sz); 2989e39c5baSBill Taylor cp->cp_log_num_rdb = cp->cp_log_num_qp + 2999e39c5baSBill Taylor min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp); 3009e39c5baSBill Taylor cp->cp_hca_max_rdma_in_qp = cp->cp_hca_max_rdma_out_qp = 3019e39c5baSBill Taylor 1 << min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp); 302*9fa01fafSagiri cp->cp_num_qp_per_mcg = max(hermon_num_qp_per_mcg, 303*9fa01fafSagiri HERMON_NUM_QP_PER_MCG_MIN); 304*9fa01fafSagiri cp->cp_num_qp_per_mcg = min(cp->cp_num_qp_per_mcg, 305*9fa01fafSagiri (1 << devlim->log_max_qp_mcg) - 8); 306*9fa01fafSagiri cp->cp_num_qp_per_mcg = (1 << highbit(cp->cp_num_qp_per_mcg + 7)) - 8; 3079e39c5baSBill Taylor cp->cp_log_num_mcg = min(hermon_log_num_mcg, devlim->log_max_mcg); 3089e39c5baSBill Taylor cp->cp_log_num_mcg_hash = hermon_log_num_mcg_hash; 3099e39c5baSBill Taylor 3109e39c5baSBill Taylor /* until srq_resize is debugged, disable it */ 3119e39c5baSBill Taylor cp->cp_srq_resize_enabled = 0; 3129e39c5baSBill Taylor 3139e39c5baSBill Taylor /* cp->cp_log_num_uar = hermon_log_num_uar; */ 3149e39c5baSBill Taylor /* 3159e39c5baSBill Taylor * now, we HAVE to calculate the number of UAR pages, so that we can 3169e39c5baSBill Taylor * get the blueflame stuff correct as well 3179e39c5baSBill Taylor */ 3189e39c5baSBill Taylor 3199e39c5baSBill Taylor size = devlim->log_max_uar_sz; 3209e39c5baSBill Taylor /* 1MB (2^^20) times size (2^^size) / sparc_pg (2^^13) */ 3219e39c5baSBill Taylor num = (20 + size) - 13; /* XXX - consider using PAGESHIFT */ 3229e39c5baSBill Taylor if (devlim->blu_flm) 3239e39c5baSBill Taylor num -= 1; /* if blueflame, only half the size for UARs */ 3249e39c5baSBill Taylor cp->cp_log_num_uar = min(hermon_log_num_uar, num); 3259e39c5baSBill Taylor 3269e39c5baSBill Taylor 3279e39c5baSBill Taylor /* while we're at it, calculate the index of the kernel uar page */ 3289e39c5baSBill Taylor /* either the reserved uar's or 128, whichever is smaller */ 3299e39c5baSBill Taylor state->hs_kernel_uar_index = (devlim->num_rsvd_uar > 128) ? 3309e39c5baSBill Taylor devlim->num_rsvd_uar : 128; 3319e39c5baSBill Taylor 3329e39c5baSBill Taylor cp->cp_log_max_pkeytbl = port->log_max_pkey; 3339e39c5baSBill Taylor 3349e39c5baSBill Taylor cp->cp_log_max_qp_sz = devlim->log_max_qp_sz; 3359e39c5baSBill Taylor cp->cp_log_max_cq_sz = devlim->log_max_cq_sz; 3369e39c5baSBill Taylor cp->cp_log_max_srq_sz = devlim->log_max_srq_sz; 3379e39c5baSBill Taylor cp->cp_log_max_gidtbl = port->log_max_gid; 3389e39c5baSBill Taylor cp->cp_max_mtu = port->ib_mtu; /* XXX now from query_port */ 3399e39c5baSBill Taylor cp->cp_max_port_width = port->ib_port_wid; /* now from query_port */ 3409e39c5baSBill Taylor cp->cp_max_vlcap = port->max_vl; 3419e39c5baSBill Taylor cp->cp_num_ports = devlim->num_ports; 3429e39c5baSBill Taylor cp->cp_log_num_ah = hermon_log_num_ah; 3439e39c5baSBill Taylor 3449e39c5baSBill Taylor /* allocate variable sized arrays */ 3459e39c5baSBill Taylor for (i = 0; i < HERMON_MAX_PORTS; i++) { 3469e39c5baSBill Taylor state->hs_pkey[i] = kmem_zalloc((1 << cp->cp_log_max_pkeytbl) * 3479e39c5baSBill Taylor sizeof (ib_pkey_t), KM_SLEEP); 3489e39c5baSBill Taylor state->hs_guid[i] = kmem_zalloc((1 << cp->cp_log_max_gidtbl) * 3499e39c5baSBill Taylor sizeof (ib_guid_t), KM_SLEEP); 3509e39c5baSBill Taylor } 3519e39c5baSBill Taylor 3529e39c5baSBill Taylor /* Determine WQE sizes from requested max SGLs */ 3539e39c5baSBill Taylor hermon_cfg_wqe_sizes(state, cp); 3549e39c5baSBill Taylor 3559e39c5baSBill Taylor /* 3569e39c5baSBill Taylor * Set IOMMU bypass or not. Ensure consistency of flags with 3579e39c5baSBill Taylor * architecture type. 3589e39c5baSBill Taylor */ 3599e39c5baSBill Taylor #ifdef __sparc 3609e39c5baSBill Taylor if (hermon_iommu_bypass == 1) { 3619e39c5baSBill Taylor hermon_check_iommu_bypass(state, cp); 3629e39c5baSBill Taylor } else { 3639e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL; 3649e39c5baSBill Taylor } 3659e39c5baSBill Taylor #else 3669e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL; 3679e39c5baSBill Taylor #endif 3689e39c5baSBill Taylor 3699e39c5baSBill Taylor /* Set whether to use MSIs or not */ 3709e39c5baSBill Taylor cp->cp_use_msi_if_avail = hermon_use_msi_if_avail; 3719e39c5baSBill Taylor 3729e39c5baSBill Taylor return (DDI_SUCCESS); 3739e39c5baSBill Taylor } 3749e39c5baSBill Taylor 3759e39c5baSBill Taylor 3769e39c5baSBill Taylor /* 3779e39c5baSBill Taylor * hermon_cfg_profile_fini() 3789e39c5baSBill Taylor * Context: Only called from attach() and/or detach() path contexts 3799e39c5baSBill Taylor */ 3809e39c5baSBill Taylor void 3819e39c5baSBill Taylor hermon_cfg_profile_fini(hermon_state_t *state) 3829e39c5baSBill Taylor { 3839e39c5baSBill Taylor /* 3849e39c5baSBill Taylor * Free up the space for configuration profile 3859e39c5baSBill Taylor */ 3869e39c5baSBill Taylor kmem_free(state->hs_cfg_profile, sizeof (hermon_cfg_profile_t)); 3879e39c5baSBill Taylor } 3889e39c5baSBill Taylor 3899e39c5baSBill Taylor 3909e39c5baSBill Taylor /* 3919e39c5baSBill Taylor * hermon_cfg_wqe_sizes() 3929e39c5baSBill Taylor * Context: Only called from attach() path context 3939e39c5baSBill Taylor */ 3949e39c5baSBill Taylor static void 3959e39c5baSBill Taylor hermon_cfg_wqe_sizes(hermon_state_t *state, hermon_cfg_profile_t *cp) 3969e39c5baSBill Taylor { 3979e39c5baSBill Taylor uint_t max_size, log2; 3989e39c5baSBill Taylor uint_t max_sgl, real_max_sgl; 3999e39c5baSBill Taylor 4009e39c5baSBill Taylor /* 4019e39c5baSBill Taylor * Get the requested maximum number SGL per WQE from the Hermon 4029e39c5baSBill Taylor * patchable variable 4039e39c5baSBill Taylor */ 4049e39c5baSBill Taylor max_sgl = hermon_wqe_max_sgl; 4059e39c5baSBill Taylor 4069e39c5baSBill Taylor /* 4079e39c5baSBill Taylor * Use requested maximum number of SGL to calculate the max descriptor 4089e39c5baSBill Taylor * size (while guaranteeing that the descriptor size is a power-of-2 4099e39c5baSBill Taylor * cachelines). We have to use the calculation for QP1 MLX transport 4109e39c5baSBill Taylor * because the possibility that we might need to inline a GRH, along 4119e39c5baSBill Taylor * with all the other headers and alignment restrictions, sets the 4129e39c5baSBill Taylor * maximum for the number of SGLs that we can advertise support for. 4139e39c5baSBill Taylor */ 4149e39c5baSBill Taylor max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (max_sgl << 4)); 4159e39c5baSBill Taylor log2 = highbit(max_size); 4169e39c5baSBill Taylor if ((max_size & (max_size - 1)) == 0) { 4179e39c5baSBill Taylor log2 = log2 - 1; 4189e39c5baSBill Taylor } 4199e39c5baSBill Taylor max_size = (1 << log2); 4209e39c5baSBill Taylor 4219e39c5baSBill Taylor max_size = min(max_size, state->hs_devlim.max_desc_sz_sq); 4229e39c5baSBill Taylor 4239e39c5baSBill Taylor /* 4249e39c5baSBill Taylor * Then use the calculated max descriptor size to determine the "real" 4259e39c5baSBill Taylor * maximum SGL (the number beyond which we would roll over to the next 4269e39c5baSBill Taylor * power-of-2). 4279e39c5baSBill Taylor */ 4289e39c5baSBill Taylor real_max_sgl = (max_size - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4; 4299e39c5baSBill Taylor 4309e39c5baSBill Taylor /* Then save away this configuration information */ 4319e39c5baSBill Taylor cp->cp_wqe_max_sgl = max_sgl; 4329e39c5baSBill Taylor cp->cp_wqe_real_max_sgl = real_max_sgl; 4339e39c5baSBill Taylor 4349e39c5baSBill Taylor /* SRQ SGL gets set to it's own patchable variable value */ 4359e39c5baSBill Taylor cp->cp_srq_max_sgl = hermon_srq_max_sgl; 4369e39c5baSBill Taylor } 4379e39c5baSBill Taylor 4389e39c5baSBill Taylor #ifdef __sparc 4399e39c5baSBill Taylor /* 4409e39c5baSBill Taylor * hermon_check_iommu_bypass() 4419e39c5baSBill Taylor * Context: Only called from attach() path context 4429e39c5baSBill Taylor * XXX This is a DMA allocation routine outside the normal 4439e39c5baSBill Taylor * path. FMA hardening will not like this. 4449e39c5baSBill Taylor */ 4459e39c5baSBill Taylor static void 4469e39c5baSBill Taylor hermon_check_iommu_bypass(hermon_state_t *state, hermon_cfg_profile_t *cp) 4479e39c5baSBill Taylor { 4489e39c5baSBill Taylor ddi_dma_handle_t dmahdl; 4499e39c5baSBill Taylor ddi_dma_attr_t dma_attr; 4509e39c5baSBill Taylor int status; 4519e39c5baSBill Taylor ddi_acc_handle_t acc_hdl; 4529e39c5baSBill Taylor caddr_t kaddr; 4539e39c5baSBill Taylor size_t actual_len; 4549e39c5baSBill Taylor ddi_dma_cookie_t cookie; 4559e39c5baSBill Taylor uint_t cookiecnt; 4569e39c5baSBill Taylor 4579e39c5baSBill Taylor hermon_dma_attr_init(state, &dma_attr); 4589e39c5baSBill Taylor 4599e39c5baSBill Taylor /* Try mapping for IOMMU bypass (Force Physical) */ 4609e39c5baSBill Taylor dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL | 4619e39c5baSBill Taylor DDI_DMA_RELAXED_ORDERING; 4629e39c5baSBill Taylor 4639e39c5baSBill Taylor /* 4649e39c5baSBill Taylor * Call ddi_dma_alloc_handle(). If this returns DDI_DMA_BADATTR then 4659e39c5baSBill Taylor * it is not possible to use IOMMU bypass with our PCI bridge parent. 4669e39c5baSBill Taylor * Since the function we are in can only be called if iommu bypass was 4679e39c5baSBill Taylor * requested in the config profile, we configure for bypass if the 4689e39c5baSBill Taylor * ddi_dma_alloc_handle() was successful. Otherwise, we configure 4699e39c5baSBill Taylor * for non-bypass (ie: normal) mapping. 4709e39c5baSBill Taylor */ 4719e39c5baSBill Taylor status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, 4729e39c5baSBill Taylor DDI_DMA_SLEEP, NULL, &dmahdl); 4739e39c5baSBill Taylor if (status == DDI_DMA_BADATTR) { 4749e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL; 4759e39c5baSBill Taylor return; 4769e39c5baSBill Taylor } else if (status != DDI_SUCCESS) { /* failed somehow */ 4779e39c5baSBill Taylor hermon_kernel_data_ro = HERMON_RO_DISABLED; 4789e39c5baSBill Taylor hermon_user_data_ro = HERMON_RO_DISABLED; 4799e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_BYPASS; 4809e39c5baSBill Taylor return; 4819e39c5baSBill Taylor } else { 4829e39c5baSBill Taylor cp->cp_iommu_bypass = HERMON_BINDMEM_BYPASS; 4839e39c5baSBill Taylor } 4849e39c5baSBill Taylor 4859e39c5baSBill Taylor status = ddi_dma_mem_alloc(dmahdl, 256, 4869e39c5baSBill Taylor &state->hs_reg_accattr, DDI_DMA_CONSISTENT, 4879e39c5baSBill Taylor DDI_DMA_SLEEP, NULL, (caddr_t *)&kaddr, &actual_len, &acc_hdl); 4889e39c5baSBill Taylor 4899e39c5baSBill Taylor if (status != DDI_SUCCESS) { /* failed somehow */ 4909e39c5baSBill Taylor hermon_kernel_data_ro = HERMON_RO_DISABLED; 4919e39c5baSBill Taylor hermon_user_data_ro = HERMON_RO_DISABLED; 4929e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl); 4939e39c5baSBill Taylor return; 4949e39c5baSBill Taylor } 4959e39c5baSBill Taylor 4969e39c5baSBill Taylor status = ddi_dma_addr_bind_handle(dmahdl, NULL, kaddr, actual_len, 4979e39c5baSBill Taylor DDI_DMA_RDWR, DDI_DMA_SLEEP, NULL, &cookie, &cookiecnt); 4989e39c5baSBill Taylor 4999e39c5baSBill Taylor if (status == DDI_DMA_MAPPED) { 5009e39c5baSBill Taylor (void) ddi_dma_unbind_handle(dmahdl); 5019e39c5baSBill Taylor } else { 5029e39c5baSBill Taylor hermon_kernel_data_ro = HERMON_RO_DISABLED; 5039e39c5baSBill Taylor hermon_user_data_ro = HERMON_RO_DISABLED; 5049e39c5baSBill Taylor } 5059e39c5baSBill Taylor 5069e39c5baSBill Taylor ddi_dma_mem_free(&acc_hdl); 5079e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl); 5089e39c5baSBill Taylor } 5099e39c5baSBill Taylor #endif 510