19e39c5baSBill Taylor /*
29e39c5baSBill Taylor  * CDDL HEADER START
39e39c5baSBill Taylor  *
49e39c5baSBill Taylor  * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor  * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor  * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor  *
89e39c5baSBill Taylor  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor  * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor  * See the License for the specific language governing permissions
119e39c5baSBill Taylor  * and limitations under the License.
129e39c5baSBill Taylor  *
139e39c5baSBill Taylor  * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor  * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor  * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor  * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor  *
199e39c5baSBill Taylor  * CDDL HEADER END
209e39c5baSBill Taylor  */
219e39c5baSBill Taylor 
229e39c5baSBill Taylor /*
2317a2b317SBill Taylor  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor  */
259e39c5baSBill Taylor 
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor  * hermon_cfg.c
289e39c5baSBill Taylor  *    Hermon Configuration Profile Routines
299e39c5baSBill Taylor  *
309e39c5baSBill Taylor  *    Implements the routines necessary for initializing and (later) tearing
319e39c5baSBill Taylor  *    down the list of Hermon configuration information.
329e39c5baSBill Taylor  */
339e39c5baSBill Taylor 
34*de710d24SJosef 'Jeff' Sipek #include <sys/sysmacros.h>
359e39c5baSBill Taylor #include <sys/types.h>
369e39c5baSBill Taylor #include <sys/conf.h>
379e39c5baSBill Taylor #include <sys/ddi.h>
389e39c5baSBill Taylor #include <sys/sunddi.h>
399e39c5baSBill Taylor #include <sys/modctl.h>
409e39c5baSBill Taylor #include <sys/bitmap.h>
419e39c5baSBill Taylor 
429e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
439e39c5baSBill Taylor 
449e39c5baSBill Taylor /*
459e39c5baSBill Taylor  * Below are the elements that make up the Hermon configuration profile.
469e39c5baSBill Taylor  * For advanced users who wish to alter these values, this can be done via
479e39c5baSBill Taylor  * the /etc/system file. By default, values are assigned to the number of
489e39c5baSBill Taylor  * supported resources, either from the HCA's reported capacities or by
499e39c5baSBill Taylor  * a by-design limit in the driver.
509e39c5baSBill Taylor  */
519e39c5baSBill Taylor 
529e39c5baSBill Taylor /* Number of supported QPs, CQs and SRQs */
539e39c5baSBill Taylor uint32_t hermon_log_num_qp		= HERMON_NUM_QP_SHIFT;
549e39c5baSBill Taylor uint32_t hermon_log_num_cq		= HERMON_NUM_CQ_SHIFT;
559e39c5baSBill Taylor uint32_t hermon_log_num_srq		= HERMON_NUM_SRQ_SHIFT;
569e39c5baSBill Taylor 
579e39c5baSBill Taylor /* Number of supported SGL per WQE for SQ/RQ, and for SRQ */
589e39c5baSBill Taylor /* XXX use the same for all queues if limitation in srq.h is resolved */
599e39c5baSBill Taylor uint32_t hermon_wqe_max_sgl		= HERMON_NUM_SGL_PER_WQE;
609e39c5baSBill Taylor uint32_t hermon_srq_max_sgl		= HERMON_SRQ_MAX_SGL;
619e39c5baSBill Taylor 
629e39c5baSBill Taylor /* Maximum "responder resources" (in) and "initiator depth" (out) per QP */
639e39c5baSBill Taylor uint32_t hermon_log_num_rdb_per_qp	= HERMON_LOG_NUM_RDB_PER_QP;
649e39c5baSBill Taylor 
659e39c5baSBill Taylor /*
669e39c5baSBill Taylor  * Number of multicast groups (MCGs), number of QP per MCG, and the number
679e39c5baSBill Taylor  * of entries (from the total number) in the multicast group "hash table"
689e39c5baSBill Taylor  */
699e39c5baSBill Taylor uint32_t hermon_log_num_mcg		= HERMON_NUM_MCG_SHIFT;
709e39c5baSBill Taylor uint32_t hermon_num_qp_per_mcg		= HERMON_NUM_QP_PER_MCG;
719fa01fafSagiri uint32_t hermon_log_num_mcg_hash	= HERMON_NUM_MCG_HASH_SHIFT;
729e39c5baSBill Taylor 
739e39c5baSBill Taylor /* Number of UD AVs */
749e39c5baSBill Taylor uint32_t hermon_log_num_ah		= HERMON_NUM_AH_SHIFT;
759e39c5baSBill Taylor 
769e39c5baSBill Taylor /* Number of EQs and their default size */
779e39c5baSBill Taylor uint32_t hermon_log_num_eq		= HERMON_NUM_EQ_SHIFT;
789e39c5baSBill Taylor uint32_t hermon_log_eq_sz		= HERMON_DEFAULT_EQ_SZ_SHIFT;
799e39c5baSBill Taylor 
809e39c5baSBill Taylor /*
819e39c5baSBill Taylor  * Number of supported MPTs, MTTs and also the maximum MPT size.
829e39c5baSBill Taylor  */
839e39c5baSBill Taylor uint32_t hermon_log_num_mtt		= HERMON_NUM_MTT_SHIFT;
849e39c5baSBill Taylor uint32_t hermon_log_num_dmpt		= HERMON_NUM_DMPT_SHIFT;
859e39c5baSBill Taylor uint32_t hermon_log_max_mrw_sz		= HERMON_MAX_MEM_MPT_SHIFT;
869e39c5baSBill Taylor 
879e39c5baSBill Taylor /*
889e39c5baSBill Taylor  * Number of supported UAR (User Access Regions) for this HCA.
899e39c5baSBill Taylor  * We could in the future read in uar_sz from devlim, and thus
909e39c5baSBill Taylor  * derive the number of UAR. Since this is derived from PAGESIZE,
919e39c5baSBill Taylor  * however, this means that x86 systems would have twice as many
929e39c5baSBill Taylor  * UARs as SPARC systems. Therefore for consistency's sake, we will
939e39c5baSBill Taylor  * just use 1024 pages, which is the maximum on SPARC systems.
949e39c5baSBill Taylor  */
959e39c5baSBill Taylor uint32_t hermon_log_num_uar		= HERMON_NUM_UAR_SHIFT;
969e39c5baSBill Taylor 
979e39c5baSBill Taylor /*
989e39c5baSBill Taylor  * Number of remaps allowed for FMR before a sync is required.  This value
999e39c5baSBill Taylor  * determines how many times we can fmr_deregister() before the underlying fmr
1009e39c5baSBill Taylor  * framework places the region to wait for an MTT_SYNC operation, cleaning up
1019e39c5baSBill Taylor  * the old mappings.
1029e39c5baSBill Taylor  */
1039e39c5baSBill Taylor uint32_t hermon_fmr_num_remaps		= HERMON_FMR_MAX_REMAPS;
1049e39c5baSBill Taylor 
1059e39c5baSBill Taylor /*
1069e39c5baSBill Taylor  * Number of supported Hermon mailboxes ("In" and "Out") and their maximum
1079e39c5baSBill Taylor  * sizes, respectively
1089e39c5baSBill Taylor  */
1099e39c5baSBill Taylor uint32_t hermon_log_num_inmbox		= HERMON_NUM_MAILBOXES_SHIFT;
1109e39c5baSBill Taylor uint32_t hermon_log_num_outmbox		= HERMON_NUM_MAILBOXES_SHIFT;
1119e39c5baSBill Taylor uint32_t hermon_log_inmbox_size		= HERMON_MBOX_SIZE_SHIFT;
1129e39c5baSBill Taylor uint32_t hermon_log_outmbox_size	= HERMON_MBOX_SIZE_SHIFT;
1139e39c5baSBill Taylor uint32_t hermon_log_num_intr_inmbox	= HERMON_NUM_INTR_MAILBOXES_SHIFT;
1149e39c5baSBill Taylor uint32_t hermon_log_num_intr_outmbox	= HERMON_NUM_INTR_MAILBOXES_SHIFT;
1159e39c5baSBill Taylor 
1169e39c5baSBill Taylor /* Number of supported Protection Domains (PD) */
1179e39c5baSBill Taylor uint32_t hermon_log_num_pd		= HERMON_NUM_PD_SHIFT;
1189e39c5baSBill Taylor 
1199e39c5baSBill Taylor /*
1209e39c5baSBill Taylor  * Number of total supported PKeys per PKey table (i.e.
1219e39c5baSBill Taylor  * per port).  Also the number of SGID per GID table.
1229e39c5baSBill Taylor  */
1239e39c5baSBill Taylor uint32_t hermon_log_max_pkeytbl		= HERMON_NUM_PKEYTBL_SHIFT;
1249e39c5baSBill Taylor uint32_t hermon_log_max_gidtbl		= HERMON_NUM_GIDTBL_SHIFT;
1259e39c5baSBill Taylor 
1269e39c5baSBill Taylor /* Maximum supported MTU and portwidth */
1279e39c5baSBill Taylor uint32_t hermon_max_mtu			= HERMON_MAX_MTU;
1289e39c5baSBill Taylor uint32_t hermon_max_port_width		= HERMON_MAX_PORT_WIDTH;
1299e39c5baSBill Taylor 
1309e39c5baSBill Taylor /* Number of supported Virtual Lanes (VL) */
1319e39c5baSBill Taylor uint32_t hermon_max_vlcap		= HERMON_MAX_VLCAP;
1329e39c5baSBill Taylor 
1339e39c5baSBill Taylor /*
1349e39c5baSBill Taylor  * Whether or not to use the built-in (i.e. in firmware) agents for QP0 and
1359e39c5baSBill Taylor  * QP1, respectively.
1369e39c5baSBill Taylor  */
13776c04273SRajkumar Sivaprakasam uint32_t hermon_qp0_agents_in_fw	= 0;
1389e39c5baSBill Taylor uint32_t hermon_qp1_agents_in_fw	= 0;
1399e39c5baSBill Taylor 
1409e39c5baSBill Taylor /*
1419e39c5baSBill Taylor  * Whether DMA mappings should bypass the PCI IOMMU or not.
1429e39c5baSBill Taylor  * hermon_iommu_bypass is a global setting for all memory addresses.
1439e39c5baSBill Taylor  */
1449e39c5baSBill Taylor uint32_t hermon_iommu_bypass		= 1;
1459e39c5baSBill Taylor 
1469e39c5baSBill Taylor /*
1479e39c5baSBill Taylor  * Whether *DATA* buffers should be bound w/ Relaxed Ordering (RO) turned on
1489e39c5baSBill Taylor  * via the SW workaround (HCAs don't support RO in HW).  Defaulted on,
1499e39c5baSBill Taylor  * though care must be taken w/ some Userland clients that *MAY* have
1509e39c5baSBill Taylor  * peeked in the data to understand when data xfer was done - MPI does
1519e39c5baSBill Taylor  * as an efficiency
1529e39c5baSBill Taylor  */
1539e39c5baSBill Taylor 
1549e39c5baSBill Taylor uint32_t hermon_kernel_data_ro		= HERMON_RO_ENABLED;	/* default */
1559e39c5baSBill Taylor uint32_t hermon_user_data_ro		= HERMON_RO_ENABLED;	/* default */
1569e39c5baSBill Taylor 
1579e39c5baSBill Taylor /*
1589e39c5baSBill Taylor  * Whether Hermon should use MSI (Message Signaled Interrupts), if available.
1599e39c5baSBill Taylor  * Note: 0 indicates 'legacy interrupt', 1 indicates MSI (if available)
1609e39c5baSBill Taylor  */
1619e39c5baSBill Taylor uint32_t hermon_use_msi_if_avail	= 1;
1629e39c5baSBill Taylor 
1639e39c5baSBill Taylor /*
1649e39c5baSBill Taylor  * This is a patchable variable that determines the time we will wait after
1659e39c5baSBill Taylor  * initiating SW reset before we do our first read from Hermon config space.
1669e39c5baSBill Taylor  * If this value is set too small (less than the default 100ms), it is
1679e39c5baSBill Taylor  * possible for Hermon hardware to be unready to respond to the config cycle
1689e39c5baSBill Taylor  * reads.  This could cause master abort on the PCI bridge.  Note: If
1699e39c5baSBill Taylor  * "hermon_sw_reset_delay" is set to zero, then no software reset of the Hermon
1709e39c5baSBill Taylor  * device will be attempted.
1719e39c5baSBill Taylor  */
1729e39c5baSBill Taylor uint32_t hermon_sw_reset_delay		= HERMON_SW_RESET_DELAY;
1739e39c5baSBill Taylor 
1749e39c5baSBill Taylor /*
1759e39c5baSBill Taylor  * These are patchable variables for hermon command polling. The poll_delay is
1769e39c5baSBill Taylor  * the number of usec to wait in-between calls to poll the 'go' bit.  The
1779e39c5baSBill Taylor  * poll_max is the total number of usec to loop in waiting for the 'go' bit to
1789e39c5baSBill Taylor  * clear.
1799e39c5baSBill Taylor  */
1809e39c5baSBill Taylor uint32_t hermon_cmd_poll_delay		= HERMON_CMD_POLL_DELAY;
1819e39c5baSBill Taylor uint32_t hermon_cmd_poll_max		= HERMON_CMD_POLL_MAX;
1829e39c5baSBill Taylor 
1839e39c5baSBill Taylor /*
1849e39c5baSBill Taylor  * This is a patchable variable that determines the frequency with which
1859e39c5baSBill Taylor  * the AckReq bit will be set in outgoing RC packets.  The AckReq bit will be
1869e39c5baSBill Taylor  * set in at least every 2^hermon_qp_ackreq_freq packets (but at least once
1879e39c5baSBill Taylor  * per message, i.e. in the last packet).  Tuning this value can increase
1889e39c5baSBill Taylor  * IB fabric utilization by cutting down on the number of unnecessary ACKs.
1899e39c5baSBill Taylor  */
1909e39c5baSBill Taylor uint32_t hermon_qp_ackreq_freq		= HERMON_QP_ACKREQ_FREQ;
1919e39c5baSBill Taylor 
1929e39c5baSBill Taylor static void hermon_cfg_wqe_sizes(hermon_state_t *state,
1939e39c5baSBill Taylor     hermon_cfg_profile_t *cp);
1949e39c5baSBill Taylor #ifdef __sparc
1959e39c5baSBill Taylor static void hermon_check_iommu_bypass(hermon_state_t *state,
1969e39c5baSBill Taylor     hermon_cfg_profile_t *cp);
1979e39c5baSBill Taylor #endif
1989e39c5baSBill Taylor 
1999e39c5baSBill Taylor /*
2009e39c5baSBill Taylor  * hermon_cfg_profile_init_phase1()
2019e39c5baSBill Taylor  *    Context: Only called from attach() path context
2029e39c5baSBill Taylor  */
2039e39c5baSBill Taylor int
hermon_cfg_profile_init_phase1(hermon_state_t * state)2049e39c5baSBill Taylor hermon_cfg_profile_init_phase1(hermon_state_t *state)
2059e39c5baSBill Taylor {
2069e39c5baSBill Taylor 	hermon_cfg_profile_t	*cp;
2079e39c5baSBill Taylor 
2089e39c5baSBill Taylor 	/*
2099e39c5baSBill Taylor 	 * Allocate space for the configuration profile structure
2109e39c5baSBill Taylor 	 */
2119e39c5baSBill Taylor 	cp = (hermon_cfg_profile_t *)kmem_zalloc(sizeof (hermon_cfg_profile_t),
2129e39c5baSBill Taylor 	    KM_SLEEP);
2139e39c5baSBill Taylor 
2149e39c5baSBill Taylor 	/*
2159e39c5baSBill Taylor 	 * Common to all profiles.
2169e39c5baSBill Taylor 	 */
2179e39c5baSBill Taylor 	cp->cp_qp0_agents_in_fw		= hermon_qp0_agents_in_fw;
2189e39c5baSBill Taylor 	cp->cp_qp1_agents_in_fw		= hermon_qp1_agents_in_fw;
2199e39c5baSBill Taylor 	cp->cp_sw_reset_delay		= hermon_sw_reset_delay;
2209e39c5baSBill Taylor 	cp->cp_cmd_poll_delay		= hermon_cmd_poll_delay;
2219e39c5baSBill Taylor 	cp->cp_cmd_poll_max		= hermon_cmd_poll_max;
2229e39c5baSBill Taylor 	cp->cp_ackreq_freq		= hermon_qp_ackreq_freq;
2239e39c5baSBill Taylor 	cp->cp_fmr_max_remaps		= hermon_fmr_num_remaps;
2249e39c5baSBill Taylor 
2259e39c5baSBill Taylor 	/*
2269e39c5baSBill Taylor 	 * Although most of the configuration is enabled in "phase2" of the
2279e39c5baSBill Taylor 	 * cfg_profile_init, we have to setup the OUT mailboxes soon, since
2289e39c5baSBill Taylor 	 * they are used immediately after this "phase1" completes, to run the
2299e39c5baSBill Taylor 	 * firmware and get the device limits, which we'll need for 'phase2'.
2309e39c5baSBill Taylor 	 * That's done in rsrc_init_phase1, called shortly after we do this
2319e39c5baSBill Taylor 	 * and the sw reset - see hermon.c
2329e39c5baSBill Taylor 	 */
2339e39c5baSBill Taylor 	if (state->hs_cfg_profile_setting == HERMON_CFG_MEMFREE) {
2349e39c5baSBill Taylor 		cp->cp_log_num_outmbox		= hermon_log_num_outmbox;
2359e39c5baSBill Taylor 		cp->cp_log_outmbox_size		= hermon_log_outmbox_size;
2369e39c5baSBill Taylor 		cp->cp_log_num_inmbox		= hermon_log_num_inmbox;
2379e39c5baSBill Taylor 		cp->cp_log_inmbox_size		= hermon_log_inmbox_size;
2389e39c5baSBill Taylor 		cp->cp_log_num_intr_inmbox	= hermon_log_num_intr_inmbox;
2399e39c5baSBill Taylor 		cp->cp_log_num_intr_outmbox	= hermon_log_num_intr_outmbox;
2409e39c5baSBill Taylor 
2419e39c5baSBill Taylor 	} else {
2429e39c5baSBill Taylor 		return (DDI_FAILURE);
2439e39c5baSBill Taylor 	}
2449e39c5baSBill Taylor 
245c7facc54SBill Taylor 	/*
246c7facc54SBill Taylor 	 * Set IOMMU bypass or not.  Ensure consistency of flags with
247c7facc54SBill Taylor 	 * architecture type.
248c7facc54SBill Taylor 	 */
249c7facc54SBill Taylor #ifdef __sparc
250c7facc54SBill Taylor 	if (hermon_iommu_bypass == 1) {
251c7facc54SBill Taylor 		hermon_check_iommu_bypass(state, cp);
252c7facc54SBill Taylor 	} else {
253c7facc54SBill Taylor 		cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL;
254c7facc54SBill Taylor 	}
255c7facc54SBill Taylor #else
256c7facc54SBill Taylor 	cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL;
257c7facc54SBill Taylor #endif
258c7facc54SBill Taylor 
2599e39c5baSBill Taylor 	/* Attach the configuration profile to Hermon softstate */
2609e39c5baSBill Taylor 	state->hs_cfg_profile = cp;
2619e39c5baSBill Taylor 
2629e39c5baSBill Taylor 	return (DDI_SUCCESS);
2639e39c5baSBill Taylor }
2649e39c5baSBill Taylor 
2659e39c5baSBill Taylor /*
2669e39c5baSBill Taylor  * hermon_cfg_profile_init_phase2()
2679e39c5baSBill Taylor  *    Context: Only called from attach() path context
2689e39c5baSBill Taylor  */
2699e39c5baSBill Taylor int
hermon_cfg_profile_init_phase2(hermon_state_t * state)2709e39c5baSBill Taylor hermon_cfg_profile_init_phase2(hermon_state_t *state)
2719e39c5baSBill Taylor {
2729e39c5baSBill Taylor 	hermon_cfg_profile_t	*cp;
2739e39c5baSBill Taylor 	hermon_hw_querydevlim_t	*devlim;
2749e39c5baSBill Taylor 	hermon_hw_query_port_t	*port;
2759e39c5baSBill Taylor 	uint32_t		num, size;
2769e39c5baSBill Taylor 	int			i;
2779e39c5baSBill Taylor 
2789e39c5baSBill Taylor 	/* Read in the device limits */
2799e39c5baSBill Taylor 	devlim = &state->hs_devlim;
2809e39c5baSBill Taylor 	/* and the port information */
2819e39c5baSBill Taylor 	port = &state->hs_queryport;
2829e39c5baSBill Taylor 
2839e39c5baSBill Taylor 	/* Read the configuration profile */
2849e39c5baSBill Taylor 	cp = state->hs_cfg_profile;
2859e39c5baSBill Taylor 
2869e39c5baSBill Taylor 	/*
2879e39c5baSBill Taylor 	 * We configure all Hermon HCAs with the same profile, which
2889e39c5baSBill Taylor 	 * is based upon the default value assignments above. If we want to
2899e39c5baSBill Taylor 	 * add additional profiles in the future, they can be added here.
2909e39c5baSBill Taylor 	 * Note the reference to "Memfree" is a holdover from Arbel/Sinai
2919e39c5baSBill Taylor 	 */
2929e39c5baSBill Taylor 	if (state->hs_cfg_profile_setting != HERMON_CFG_MEMFREE) {
2939e39c5baSBill Taylor 		return (DDI_FAILURE);
2949e39c5baSBill Taylor 	}
2959e39c5baSBill Taylor 
2969e39c5baSBill Taylor 	/*
2979e39c5baSBill Taylor 	 * Note for most configuration parameters, we use the lesser of our
2989e39c5baSBill Taylor 	 * desired configuration value or the device-defined maximum value.
2999e39c5baSBill Taylor 	 */
3009e39c5baSBill Taylor 	cp->cp_log_num_mtt	= min(hermon_log_num_mtt, devlim->log_max_mtt);
3019e39c5baSBill Taylor 	cp->cp_log_num_dmpt = min(hermon_log_num_dmpt, devlim->log_max_dmpt);
3029e39c5baSBill Taylor 	cp->cp_log_num_cmpt	= HERMON_LOG_CMPT_PER_TYPE + 2;	/* times 4, */
3039e39c5baSBill Taylor 								/* per PRM */
3049e39c5baSBill Taylor 	cp->cp_log_max_mrw_sz	= min(hermon_log_max_mrw_sz,
3059e39c5baSBill Taylor 	    devlim->log_max_mrw_sz);
3069e39c5baSBill Taylor 	cp->cp_log_num_pd	= min(hermon_log_num_pd, devlim->log_max_pd);
3079e39c5baSBill Taylor 	cp->cp_log_num_qp	= min(hermon_log_num_qp, devlim->log_max_qp);
3089e39c5baSBill Taylor 	cp->cp_log_num_cq	= min(hermon_log_num_cq, devlim->log_max_cq);
3099e39c5baSBill Taylor 	cp->cp_log_num_srq	= min(hermon_log_num_srq, devlim->log_max_srq);
3109e39c5baSBill Taylor 	cp->cp_log_num_eq	= min(hermon_log_num_eq, devlim->log_max_eq);
3119e39c5baSBill Taylor 	cp->cp_log_eq_sz	= min(hermon_log_eq_sz, devlim->log_max_eq_sz);
3129e39c5baSBill Taylor 	cp->cp_log_num_rdb	= cp->cp_log_num_qp +
3139e39c5baSBill Taylor 	    min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp);
3149e39c5baSBill Taylor 	cp->cp_hca_max_rdma_in_qp = cp->cp_hca_max_rdma_out_qp =
3159e39c5baSBill Taylor 	    1 << min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp);
3169fa01fafSagiri 	cp->cp_num_qp_per_mcg	= max(hermon_num_qp_per_mcg,
3179fa01fafSagiri 	    HERMON_NUM_QP_PER_MCG_MIN);
3189fa01fafSagiri 	cp->cp_num_qp_per_mcg	= min(cp->cp_num_qp_per_mcg,
3199fa01fafSagiri 	    (1 << devlim->log_max_qp_mcg) - 8);
3209fa01fafSagiri 	cp->cp_num_qp_per_mcg	= (1 << highbit(cp->cp_num_qp_per_mcg + 7)) - 8;
3219e39c5baSBill Taylor 	cp->cp_log_num_mcg 	= min(hermon_log_num_mcg, devlim->log_max_mcg);
3229e39c5baSBill Taylor 	cp->cp_log_num_mcg_hash	= hermon_log_num_mcg_hash;
3239e39c5baSBill Taylor 
3249e39c5baSBill Taylor 	/* until srq_resize is debugged, disable it */
3259e39c5baSBill Taylor 	cp->cp_srq_resize_enabled = 0;
3269e39c5baSBill Taylor 
3279e39c5baSBill Taylor 	/* cp->cp_log_num_uar	= hermon_log_num_uar; */
3289e39c5baSBill Taylor 	/*
3299e39c5baSBill Taylor 	 * now, we HAVE to calculate the number of UAR pages, so that we can
3309e39c5baSBill Taylor 	 * get the blueflame stuff correct as well
3319e39c5baSBill Taylor 	 */
3329e39c5baSBill Taylor 
3339e39c5baSBill Taylor 	size = devlim->log_max_uar_sz;
3349e39c5baSBill Taylor 	/* 1MB (2^^20) times size (2^^size) / sparc_pg (2^^13) */
3359e39c5baSBill Taylor 	num = (20 + size) - 13;		/* XXX - consider using PAGESHIFT */
3369e39c5baSBill Taylor 	if (devlim->blu_flm)
3379e39c5baSBill Taylor 		num -= 1;	/* if blueflame, only half the size for UARs */
3389e39c5baSBill Taylor 	cp->cp_log_num_uar	= min(hermon_log_num_uar, num);
3399e39c5baSBill Taylor 
3409e39c5baSBill Taylor 
3419e39c5baSBill Taylor 	/* while we're at it, calculate the index of the kernel uar page */
3429e39c5baSBill Taylor 	/* either the reserved uar's or 128, whichever is smaller */
3439e39c5baSBill Taylor 	state->hs_kernel_uar_index = (devlim->num_rsvd_uar > 128) ?
3449e39c5baSBill Taylor 	    devlim->num_rsvd_uar : 128;
3459e39c5baSBill Taylor 
3469e39c5baSBill Taylor 	cp->cp_log_max_pkeytbl	= port->log_max_pkey;
3479e39c5baSBill Taylor 
3489e39c5baSBill Taylor 	cp->cp_log_max_qp_sz	= devlim->log_max_qp_sz;
3499e39c5baSBill Taylor 	cp->cp_log_max_cq_sz	= devlim->log_max_cq_sz;
3509e39c5baSBill Taylor 	cp->cp_log_max_srq_sz	= devlim->log_max_srq_sz;
3519e39c5baSBill Taylor 	cp->cp_log_max_gidtbl	= port->log_max_gid;
3529e39c5baSBill Taylor 	cp->cp_max_mtu		= port->ib_mtu;	/* XXX now from query_port */
3539e39c5baSBill Taylor 	cp->cp_max_port_width	= port->ib_port_wid;  /* now from query_port */
3549e39c5baSBill Taylor 	cp->cp_max_vlcap	= port->max_vl;
3559e39c5baSBill Taylor 	cp->cp_log_num_ah	= hermon_log_num_ah;
3569e39c5baSBill Taylor 
35717a2b317SBill Taylor 	/* Paranoia, ensure no arrays indexed by port_num are out of bounds */
35817a2b317SBill Taylor 	cp->cp_num_ports	= devlim->num_ports;
35917a2b317SBill Taylor 	if (cp->cp_num_ports > HERMON_MAX_PORTS) {
36017a2b317SBill Taylor 		cmn_err(CE_CONT, "device has more ports (%d) than are "
36117a2b317SBill Taylor 		    "supported; Using %d ports\n",
36217a2b317SBill Taylor 		    cp->cp_num_ports, HERMON_MAX_PORTS);
36317a2b317SBill Taylor 		cp->cp_num_ports = HERMON_MAX_PORTS;
36417a2b317SBill Taylor 	};
36517a2b317SBill Taylor 
3669e39c5baSBill Taylor 	/* allocate variable sized arrays */
3679e39c5baSBill Taylor 	for (i = 0; i < HERMON_MAX_PORTS; i++) {
3689e39c5baSBill Taylor 		state->hs_pkey[i] = kmem_zalloc((1 << cp->cp_log_max_pkeytbl) *
3699e39c5baSBill Taylor 		    sizeof (ib_pkey_t), KM_SLEEP);
3709e39c5baSBill Taylor 		state->hs_guid[i] = kmem_zalloc((1 << cp->cp_log_max_gidtbl) *
3719e39c5baSBill Taylor 		    sizeof (ib_guid_t), KM_SLEEP);
3729e39c5baSBill Taylor 	}
3739e39c5baSBill Taylor 
3749e39c5baSBill Taylor 	/* Determine WQE sizes from requested max SGLs */
3759e39c5baSBill Taylor 	hermon_cfg_wqe_sizes(state, cp);
3769e39c5baSBill Taylor 
3779e39c5baSBill Taylor 	/* Set whether to use MSIs or not */
3789e39c5baSBill Taylor 	cp->cp_use_msi_if_avail = hermon_use_msi_if_avail;
3799e39c5baSBill Taylor 
380f35eb4e6SBill Taylor #if !defined(_ELF64)
381f35eb4e6SBill Taylor 	/*
382f35eb4e6SBill Taylor 	 * Need to reduce the hermon kernel virtual memory footprint
383f35eb4e6SBill Taylor 	 * on 32-bit kernels.
384f35eb4e6SBill Taylor 	 */
385f35eb4e6SBill Taylor 	cp->cp_log_num_mtt	-= 6;
386f35eb4e6SBill Taylor 	cp->cp_log_num_dmpt	-= 6;
387f35eb4e6SBill Taylor 	cp->cp_log_num_pd	-= 6;
388f35eb4e6SBill Taylor 	cp->cp_log_num_qp	-= 6;
389f35eb4e6SBill Taylor 	cp->cp_log_num_cq	-= 6;
390f35eb4e6SBill Taylor 	cp->cp_log_num_srq	-= 6;
391f35eb4e6SBill Taylor 	cp->cp_log_num_rdb	= cp->cp_log_num_qp +
392f35eb4e6SBill Taylor 	    min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp);
393f35eb4e6SBill Taylor 	cp->cp_hca_max_rdma_in_qp = cp->cp_hca_max_rdma_out_qp =
394f35eb4e6SBill Taylor 	    1 << min(hermon_log_num_rdb_per_qp, devlim->log_max_ra_req_qp);
395f35eb4e6SBill Taylor #endif
396f35eb4e6SBill Taylor 
3979e39c5baSBill Taylor 	return (DDI_SUCCESS);
3989e39c5baSBill Taylor }
3999e39c5baSBill Taylor 
4009e39c5baSBill Taylor 
4019e39c5baSBill Taylor /*
4029e39c5baSBill Taylor  * hermon_cfg_profile_fini()
4039e39c5baSBill Taylor  *    Context: Only called from attach() and/or detach() path contexts
4049e39c5baSBill Taylor  */
4059e39c5baSBill Taylor void
hermon_cfg_profile_fini(hermon_state_t * state)4069e39c5baSBill Taylor hermon_cfg_profile_fini(hermon_state_t *state)
4079e39c5baSBill Taylor {
4089e39c5baSBill Taylor 	/*
4099e39c5baSBill Taylor 	 * Free up the space for configuration profile
4109e39c5baSBill Taylor 	 */
4119e39c5baSBill Taylor 	kmem_free(state->hs_cfg_profile, sizeof (hermon_cfg_profile_t));
4129e39c5baSBill Taylor }
4139e39c5baSBill Taylor 
4149e39c5baSBill Taylor 
4159e39c5baSBill Taylor /*
4169e39c5baSBill Taylor  * hermon_cfg_wqe_sizes()
4179e39c5baSBill Taylor  *    Context: Only called from attach() path context
4189e39c5baSBill Taylor  */
4199e39c5baSBill Taylor static void
hermon_cfg_wqe_sizes(hermon_state_t * state,hermon_cfg_profile_t * cp)4209e39c5baSBill Taylor hermon_cfg_wqe_sizes(hermon_state_t *state, hermon_cfg_profile_t *cp)
4219e39c5baSBill Taylor {
4229e39c5baSBill Taylor 	uint_t	max_size, log2;
4239e39c5baSBill Taylor 	uint_t	max_sgl, real_max_sgl;
4249e39c5baSBill Taylor 
4259e39c5baSBill Taylor 	/*
4269e39c5baSBill Taylor 	 * Get the requested maximum number SGL per WQE from the Hermon
4279e39c5baSBill Taylor 	 * patchable variable
4289e39c5baSBill Taylor 	 */
4299e39c5baSBill Taylor 	max_sgl = hermon_wqe_max_sgl;
4309e39c5baSBill Taylor 
4319e39c5baSBill Taylor 	/*
4329e39c5baSBill Taylor 	 * Use requested maximum number of SGL to calculate the max descriptor
4339e39c5baSBill Taylor 	 * size (while guaranteeing that the descriptor size is a power-of-2
4349e39c5baSBill Taylor 	 * cachelines).  We have to use the calculation for QP1 MLX transport
4359e39c5baSBill Taylor 	 * because the possibility that we might need to inline a GRH, along
4369e39c5baSBill Taylor 	 * with all the other headers and alignment restrictions, sets the
4379e39c5baSBill Taylor 	 * maximum for the number of SGLs that we can advertise support for.
4389e39c5baSBill Taylor 	 */
4399e39c5baSBill Taylor 	max_size = (HERMON_QP_WQE_MLX_QP1_HDRS + (max_sgl << 4));
4409e39c5baSBill Taylor 	log2 = highbit(max_size);
441*de710d24SJosef 'Jeff' Sipek 	if (ISP2(max_size)) {
4429e39c5baSBill Taylor 		log2 = log2 - 1;
4439e39c5baSBill Taylor 	}
4449e39c5baSBill Taylor 	max_size = (1 << log2);
4459e39c5baSBill Taylor 
4469e39c5baSBill Taylor 	max_size = min(max_size, state->hs_devlim.max_desc_sz_sq);
4479e39c5baSBill Taylor 
4489e39c5baSBill Taylor 	/*
4499e39c5baSBill Taylor 	 * Then use the calculated max descriptor size to determine the "real"
4509e39c5baSBill Taylor 	 * maximum SGL (the number beyond which we would roll over to the next
4519e39c5baSBill Taylor 	 * power-of-2).
4529e39c5baSBill Taylor 	 */
4539e39c5baSBill Taylor 	real_max_sgl = (max_size - HERMON_QP_WQE_MLX_QP1_HDRS) >> 4;
4549e39c5baSBill Taylor 
4559e39c5baSBill Taylor 	/* Then save away this configuration information */
4569e39c5baSBill Taylor 	cp->cp_wqe_max_sgl	= max_sgl;
4579e39c5baSBill Taylor 	cp->cp_wqe_real_max_sgl = real_max_sgl;
4589e39c5baSBill Taylor 
4599e39c5baSBill Taylor 	/* SRQ SGL gets set to it's own patchable variable value */
4609e39c5baSBill Taylor 	cp->cp_srq_max_sgl		= hermon_srq_max_sgl;
4619e39c5baSBill Taylor }
4629e39c5baSBill Taylor 
4639e39c5baSBill Taylor #ifdef __sparc
4649e39c5baSBill Taylor /*
4659e39c5baSBill Taylor  * hermon_check_iommu_bypass()
4669e39c5baSBill Taylor  *    Context: Only called from attach() path context
4679e39c5baSBill Taylor  *    XXX This is a DMA allocation routine outside the normal
4689e39c5baSBill Taylor  *	  path. FMA hardening will not like this.
4699e39c5baSBill Taylor  */
4709e39c5baSBill Taylor static void
hermon_check_iommu_bypass(hermon_state_t * state,hermon_cfg_profile_t * cp)4719e39c5baSBill Taylor hermon_check_iommu_bypass(hermon_state_t *state, hermon_cfg_profile_t *cp)
4729e39c5baSBill Taylor {
4739e39c5baSBill Taylor 	ddi_dma_handle_t	dmahdl;
4749e39c5baSBill Taylor 	ddi_dma_attr_t		dma_attr;
4759e39c5baSBill Taylor 	int			status;
4769e39c5baSBill Taylor 	ddi_acc_handle_t	acc_hdl;
4779e39c5baSBill Taylor 	caddr_t			kaddr;
4789e39c5baSBill Taylor 	size_t			actual_len;
4799e39c5baSBill Taylor 	ddi_dma_cookie_t	cookie;
4809e39c5baSBill Taylor 	uint_t			cookiecnt;
4819e39c5baSBill Taylor 
4829e39c5baSBill Taylor 	hermon_dma_attr_init(state, &dma_attr);
4839e39c5baSBill Taylor 
4849e39c5baSBill Taylor 	/* Try mapping for IOMMU bypass (Force Physical) */
4859e39c5baSBill Taylor 	dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL |
4869e39c5baSBill Taylor 	    DDI_DMA_RELAXED_ORDERING;
4879e39c5baSBill Taylor 
4889e39c5baSBill Taylor 	/*
4899e39c5baSBill Taylor 	 * Call ddi_dma_alloc_handle().  If this returns DDI_DMA_BADATTR then
4909e39c5baSBill Taylor 	 * it is not possible to use IOMMU bypass with our PCI bridge parent.
4919e39c5baSBill Taylor 	 * Since the function we are in can only be called if iommu bypass was
4929e39c5baSBill Taylor 	 * requested in the config profile, we configure for bypass if the
4939e39c5baSBill Taylor 	 * ddi_dma_alloc_handle() was successful.  Otherwise, we configure
4949e39c5baSBill Taylor 	 * for non-bypass (ie: normal) mapping.
4959e39c5baSBill Taylor 	 */
4969e39c5baSBill Taylor 	status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
4979e39c5baSBill Taylor 	    DDI_DMA_SLEEP, NULL, &dmahdl);
4989e39c5baSBill Taylor 	if (status == DDI_DMA_BADATTR) {
4999e39c5baSBill Taylor 		cp->cp_iommu_bypass = HERMON_BINDMEM_NORMAL;
5009e39c5baSBill Taylor 		return;
5019e39c5baSBill Taylor 	} else if (status != DDI_SUCCESS) {	/* failed somehow */
5029e39c5baSBill Taylor 		hermon_kernel_data_ro = HERMON_RO_DISABLED;
5039e39c5baSBill Taylor 		hermon_user_data_ro = HERMON_RO_DISABLED;
5049e39c5baSBill Taylor 		cp->cp_iommu_bypass = HERMON_BINDMEM_BYPASS;
5059e39c5baSBill Taylor 		return;
5069e39c5baSBill Taylor 	} else {
5079e39c5baSBill Taylor 		cp->cp_iommu_bypass = HERMON_BINDMEM_BYPASS;
5089e39c5baSBill Taylor 	}
5099e39c5baSBill Taylor 
5109e39c5baSBill Taylor 	status = ddi_dma_mem_alloc(dmahdl, 256,
5119e39c5baSBill Taylor 	    &state->hs_reg_accattr, DDI_DMA_CONSISTENT,
5129e39c5baSBill Taylor 	    DDI_DMA_SLEEP, NULL, (caddr_t *)&kaddr, &actual_len, &acc_hdl);
5139e39c5baSBill Taylor 
5149e39c5baSBill Taylor 	if (status != DDI_SUCCESS) {		/* failed somehow */
5159e39c5baSBill Taylor 		hermon_kernel_data_ro = HERMON_RO_DISABLED;
5169e39c5baSBill Taylor 		hermon_user_data_ro = HERMON_RO_DISABLED;
5179e39c5baSBill Taylor 		ddi_dma_free_handle(&dmahdl);
5189e39c5baSBill Taylor 		return;
5199e39c5baSBill Taylor 	}
5209e39c5baSBill Taylor 
5219e39c5baSBill Taylor 	status = ddi_dma_addr_bind_handle(dmahdl, NULL, kaddr, actual_len,
5229e39c5baSBill Taylor 	    DDI_DMA_RDWR, DDI_DMA_SLEEP, NULL, &cookie, &cookiecnt);
5239e39c5baSBill Taylor 
5249e39c5baSBill Taylor 	if (status == DDI_DMA_MAPPED) {
5259e39c5baSBill Taylor 		(void) ddi_dma_unbind_handle(dmahdl);
5269e39c5baSBill Taylor 	} else {
5279e39c5baSBill Taylor 		hermon_kernel_data_ro = HERMON_RO_DISABLED;
5289e39c5baSBill Taylor 		hermon_user_data_ro = HERMON_RO_DISABLED;
5299e39c5baSBill Taylor 	}
5309e39c5baSBill Taylor 
5319e39c5baSBill Taylor 	ddi_dma_mem_free(&acc_hdl);
5329e39c5baSBill Taylor 	ddi_dma_free_handle(&dmahdl);
5339e39c5baSBill Taylor }
5349e39c5baSBill Taylor #endif