19e39c5baSBill Taylor /*
29e39c5baSBill Taylor * CDDL HEADER START
39e39c5baSBill Taylor *
49e39c5baSBill Taylor * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor *
89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor * See the License for the specific language governing permissions
119e39c5baSBill Taylor * and limitations under the License.
129e39c5baSBill Taylor *
139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor *
199e39c5baSBill Taylor * CDDL HEADER END
209e39c5baSBill Taylor */
219e39c5baSBill Taylor
229e39c5baSBill Taylor /*
23*17a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor */
259e39c5baSBill Taylor
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor * hermon_cq.c
289e39c5baSBill Taylor * Hermon Completion Queue Processing Routines
299e39c5baSBill Taylor *
309e39c5baSBill Taylor * Implements all the routines necessary for allocating, freeing, resizing,
319e39c5baSBill Taylor * and handling the completion type events that the Hermon hardware can
329e39c5baSBill Taylor * generate.
339e39c5baSBill Taylor */
349e39c5baSBill Taylor
359e39c5baSBill Taylor #include <sys/types.h>
369e39c5baSBill Taylor #include <sys/conf.h>
379e39c5baSBill Taylor #include <sys/ddi.h>
389e39c5baSBill Taylor #include <sys/sunddi.h>
399e39c5baSBill Taylor #include <sys/modctl.h>
409e39c5baSBill Taylor #include <sys/bitmap.h>
419e39c5baSBill Taylor #include <sys/sysmacros.h>
429e39c5baSBill Taylor
439e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h>
449e39c5baSBill Taylor
459e39c5baSBill Taylor int hermon_should_panic = 0; /* debugging aid */
469e39c5baSBill Taylor
479e39c5baSBill Taylor #define hermon_cq_update_ci_doorbell(cq) \
489e39c5baSBill Taylor /* Build the doorbell record data (low 24 bits only) */ \
499e39c5baSBill Taylor HERMON_UAR_DB_RECORD_WRITE(cq->cq_arm_ci_vdbr, \
509e39c5baSBill Taylor cq->cq_consindx & 0x00FFFFFF)
519e39c5baSBill Taylor
529e39c5baSBill Taylor static int hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq,
539e39c5baSBill Taylor uint_t cmd);
549e39c5baSBill Taylor #pragma inline(hermon_cq_arm_doorbell)
559e39c5baSBill Taylor static void hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr);
569e39c5baSBill Taylor #pragma inline(hermon_arm_cq_dbr_init)
579e39c5baSBill Taylor static void hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
589e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc);
599e39c5baSBill Taylor static void hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
609e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc);
619e39c5baSBill Taylor
629e39c5baSBill Taylor
639e39c5baSBill Taylor /*
649e39c5baSBill Taylor * hermon_cq_alloc()
659e39c5baSBill Taylor * Context: Can be called only from user or kernel context.
669e39c5baSBill Taylor */
679e39c5baSBill Taylor int
hermon_cq_alloc(hermon_state_t * state,ibt_cq_hdl_t ibt_cqhdl,ibt_cq_attr_t * cq_attr,uint_t * actual_size,hermon_cqhdl_t * cqhdl,uint_t sleepflag)689e39c5baSBill Taylor hermon_cq_alloc(hermon_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
699e39c5baSBill Taylor ibt_cq_attr_t *cq_attr, uint_t *actual_size, hermon_cqhdl_t *cqhdl,
709e39c5baSBill Taylor uint_t sleepflag)
719e39c5baSBill Taylor {
729e39c5baSBill Taylor hermon_rsrc_t *cqc, *rsrc;
739e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb;
749e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry;
759e39c5baSBill Taylor hermon_cqhdl_t cq;
769e39c5baSBill Taylor ibt_mr_attr_t mr_attr;
779e39c5baSBill Taylor hermon_mr_options_t op;
789e39c5baSBill Taylor hermon_pdhdl_t pd;
799e39c5baSBill Taylor hermon_mrhdl_t mr;
809e39c5baSBill Taylor hermon_hw_cqe_t *buf;
819e39c5baSBill Taylor uint64_t value;
829e39c5baSBill Taylor uint32_t log_cq_size, uarpg;
839e39c5baSBill Taylor uint_t cq_is_umap;
849e39c5baSBill Taylor uint32_t status, flag;
85*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp;
869e39c5baSBill Taylor
879e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq_attr))
889e39c5baSBill Taylor
899e39c5baSBill Taylor /*
909e39c5baSBill Taylor * Determine whether CQ is being allocated for userland access or
919e39c5baSBill Taylor * whether it is being allocated for kernel access. If the CQ is
929e39c5baSBill Taylor * being allocated for userland access, then lookup the UAR
939e39c5baSBill Taylor * page number for the current process. Note: If this is not found
949e39c5baSBill Taylor * (e.g. if the process has not previously open()'d the Hermon driver),
959e39c5baSBill Taylor * then an error is returned.
969e39c5baSBill Taylor */
979e39c5baSBill Taylor cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
989e39c5baSBill Taylor if (cq_is_umap) {
999e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
1009e39c5baSBill Taylor MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
1019e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1029e39c5baSBill Taylor status = IBT_INVALID_PARAM;
1039e39c5baSBill Taylor goto cqalloc_fail;
1049e39c5baSBill Taylor }
1059e39c5baSBill Taylor uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
1069e39c5baSBill Taylor } else {
1079e39c5baSBill Taylor uarpg = state->hs_kernel_uar_index;
1089e39c5baSBill Taylor }
1099e39c5baSBill Taylor
1109e39c5baSBill Taylor /* Use the internal protection domain (PD) for setting up CQs */
1119e39c5baSBill Taylor pd = state->hs_pdhdl_internal;
1129e39c5baSBill Taylor
1139e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */
1149e39c5baSBill Taylor hermon_pd_refcnt_inc(pd);
1159e39c5baSBill Taylor
1169e39c5baSBill Taylor /*
1179e39c5baSBill Taylor * Allocate an CQ context entry. This will be filled in with all
1189e39c5baSBill Taylor * the necessary parameters to define the Completion Queue. And then
1199e39c5baSBill Taylor * ownership will be passed to the hardware in the final step
1209e39c5baSBill Taylor * below. If we fail here, we must undo the protection domain
1219e39c5baSBill Taylor * reference count.
1229e39c5baSBill Taylor */
1239e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_CQC, 1, sleepflag, &cqc);
1249e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1259e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1269e39c5baSBill Taylor goto cqalloc_fail1;
1279e39c5baSBill Taylor }
1289e39c5baSBill Taylor
1299e39c5baSBill Taylor /*
1309e39c5baSBill Taylor * Allocate the software structure for tracking the completion queue
1319e39c5baSBill Taylor * (i.e. the Hermon Completion Queue handle). If we fail here, we must
1329e39c5baSBill Taylor * undo the protection domain reference count and the previous
1339e39c5baSBill Taylor * resource allocation.
1349e39c5baSBill Taylor */
1359e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_CQHDL, 1, sleepflag, &rsrc);
1369e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1379e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1389e39c5baSBill Taylor goto cqalloc_fail2;
1399e39c5baSBill Taylor }
1409e39c5baSBill Taylor cq = (hermon_cqhdl_t)rsrc->hr_addr;
1419e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
1429e39c5baSBill Taylor cq->cq_is_umap = cq_is_umap;
1439e39c5baSBill Taylor cq->cq_cqnum = cqc->hr_indx; /* just use index, implicit in Hermon */
144*17a2b317SBill Taylor cq->cq_intmod_count = 0;
145*17a2b317SBill Taylor cq->cq_intmod_usec = 0;
1469e39c5baSBill Taylor
1479e39c5baSBill Taylor /*
1489e39c5baSBill Taylor * If this will be a user-mappable CQ, then allocate an entry for
1499e39c5baSBill Taylor * the "userland resources database". This will later be added to
1509e39c5baSBill Taylor * the database (after all further CQ operations are successful).
1519e39c5baSBill Taylor * If we fail here, we must undo the reference counts and the
1529e39c5baSBill Taylor * previous resource allocation.
1539e39c5baSBill Taylor */
1549e39c5baSBill Taylor if (cq->cq_is_umap) {
1559e39c5baSBill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance, cq->cq_cqnum,
1569e39c5baSBill Taylor MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
1579e39c5baSBill Taylor if (umapdb == NULL) {
1589e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1599e39c5baSBill Taylor goto cqalloc_fail3;
1609e39c5baSBill Taylor }
1619e39c5baSBill Taylor }
1629e39c5baSBill Taylor
1639e39c5baSBill Taylor
1649e39c5baSBill Taylor /*
1659e39c5baSBill Taylor * Allocate the doorbell record. We'll need one for the CQ, handling
1669e39c5baSBill Taylor * both consumer index (SET CI) and the CQ state (CQ ARM).
1679e39c5baSBill Taylor */
1689e39c5baSBill Taylor
1699e39c5baSBill Taylor status = hermon_dbr_alloc(state, uarpg, &cq->cq_arm_ci_dbr_acchdl,
1709e39c5baSBill Taylor &cq->cq_arm_ci_vdbr, &cq->cq_arm_ci_pdbr, &cq->cq_dbr_mapoffset);
1719e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1729e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
1739e39c5baSBill Taylor goto cqalloc_fail4;
1749e39c5baSBill Taylor }
1759e39c5baSBill Taylor
1769e39c5baSBill Taylor /*
1779e39c5baSBill Taylor * Calculate the appropriate size for the completion queue.
1789e39c5baSBill Taylor * Note: All Hermon CQs must be a power-of-2 minus 1 in size. Also
1799e39c5baSBill Taylor * they may not be any smaller than HERMON_CQ_MIN_SIZE. This step is
1809e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2
1819e39c5baSBill Taylor */
1829e39c5baSBill Taylor cq_attr->cq_size = max(cq_attr->cq_size, HERMON_CQ_MIN_SIZE);
1839e39c5baSBill Taylor log_cq_size = highbit(cq_attr->cq_size);
1849e39c5baSBill Taylor
1859e39c5baSBill Taylor /*
1869e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
1879e39c5baSBill Taylor * with the device limits and/or software-configured limits)
1889e39c5baSBill Taylor */
1899e39c5baSBill Taylor if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
1909e39c5baSBill Taylor status = IBT_HCA_CQ_EXCEEDED;
1919e39c5baSBill Taylor goto cqalloc_fail4a;
1929e39c5baSBill Taylor }
1939e39c5baSBill Taylor
1949e39c5baSBill Taylor /*
1959e39c5baSBill Taylor * Allocate the memory for Completion Queue.
1969e39c5baSBill Taylor *
1979e39c5baSBill Taylor * Note: Although we use the common queue allocation routine, we
1989e39c5baSBill Taylor * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
1999e39c5baSBill Taylor * kernel system memory) for kernel CQs because it would be
2009e39c5baSBill Taylor * inefficient to have CQs located in DDR memory. This is primarily
2019e39c5baSBill Taylor * because CQs are read from (by software) more than they are written
2029e39c5baSBill Taylor * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
2039e39c5baSBill Taylor * user-mappable CQs for a similar reason.)
2049e39c5baSBill Taylor * It is also worth noting that, unlike Hermon QP work queues,
2059e39c5baSBill Taylor * completion queues do not have the same strict alignment
2069e39c5baSBill Taylor * requirements. It is sufficient for the CQ memory to be both
2079e39c5baSBill Taylor * aligned to and bound to addresses which are a multiple of CQE size.
2089e39c5baSBill Taylor */
2099e39c5baSBill Taylor cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
2109e39c5baSBill Taylor
2119e39c5baSBill Taylor cq->cq_cqinfo.qa_alloc_align = PAGESIZE;
2129e39c5baSBill Taylor cq->cq_cqinfo.qa_bind_align = PAGESIZE;
2139e39c5baSBill Taylor if (cq->cq_is_umap) {
2149e39c5baSBill Taylor cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
2159e39c5baSBill Taylor } else {
2169e39c5baSBill Taylor cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
2179e39c5baSBill Taylor hermon_arm_cq_dbr_init(cq->cq_arm_ci_vdbr);
2189e39c5baSBill Taylor }
2199e39c5baSBill Taylor status = hermon_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
2209e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2219e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
2229e39c5baSBill Taylor goto cqalloc_fail4;
2239e39c5baSBill Taylor }
2249e39c5baSBill Taylor buf = (hermon_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
2259e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
2269e39c5baSBill Taylor
2279e39c5baSBill Taylor /*
2289e39c5baSBill Taylor * The ownership bit of the CQE's is set by the HW during the process
2299e39c5baSBill Taylor * of transferrring ownership of the CQ (PRM 09.35c, 14.2.1, note D1
2309e39c5baSBill Taylor *
2319e39c5baSBill Taylor */
2329e39c5baSBill Taylor
2339e39c5baSBill Taylor /*
2349e39c5baSBill Taylor * Register the memory for the CQ. The memory for the CQ must
2359e39c5baSBill Taylor * be registered in the Hermon TPT tables. This gives us the LKey
2369e39c5baSBill Taylor * to specify in the CQ context below. Note: If this is a user-
2379e39c5baSBill Taylor * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
2389e39c5baSBill Taylor */
2399e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
2409e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
2419e39c5baSBill Taylor mr_attr.mr_len = cq->cq_cqinfo.qa_size;
2429e39c5baSBill Taylor mr_attr.mr_as = NULL;
2439e39c5baSBill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
2449e39c5baSBill Taylor op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
2459e39c5baSBill Taylor op.mro_bind_dmahdl = cq->cq_cqinfo.qa_dmahdl;
2469e39c5baSBill Taylor op.mro_bind_override_addr = 0;
2479e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
2489e39c5baSBill Taylor HERMON_CQ_CMPT);
2499e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2509e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
2519e39c5baSBill Taylor goto cqalloc_fail5;
2529e39c5baSBill Taylor }
2539e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2549e39c5baSBill Taylor
255*17a2b317SBill Taylor cq->cq_erreqnum = HERMON_CQ_ERREQNUM_GET(state);
256*17a2b317SBill Taylor if (cq_attr->cq_flags & IBT_CQ_HID) {
257*17a2b317SBill Taylor if (!HERMON_HID_VALID(state, cq_attr->cq_hid)) {
258*17a2b317SBill Taylor IBTF_DPRINTF_L2("CQalloc", "bad handler id 0x%x",
259*17a2b317SBill Taylor cq_attr->cq_hid);
260*17a2b317SBill Taylor status = IBT_INVALID_PARAM;
261*17a2b317SBill Taylor goto cqalloc_fail5;
262*17a2b317SBill Taylor }
263*17a2b317SBill Taylor cq->cq_eqnum = HERMON_HID_TO_EQNUM(state, cq_attr->cq_hid);
264*17a2b317SBill Taylor IBTF_DPRINTF_L2("cqalloc", "hid: eqn %d", cq->cq_eqnum);
265*17a2b317SBill Taylor } else {
266*17a2b317SBill Taylor cq_schedp = (hermon_cq_sched_t *)cq_attr->cq_sched;
267*17a2b317SBill Taylor if (cq_schedp == NULL) {
268*17a2b317SBill Taylor cq_schedp = &state->hs_cq_sched_default;
269*17a2b317SBill Taylor } else if (cq_schedp != &state->hs_cq_sched_default) {
270*17a2b317SBill Taylor int i;
271*17a2b317SBill Taylor hermon_cq_sched_t *tmp;
272*17a2b317SBill Taylor
273*17a2b317SBill Taylor tmp = state->hs_cq_sched_array;
274*17a2b317SBill Taylor for (i = 0; i < state->hs_cq_sched_array_size; i++)
275*17a2b317SBill Taylor if (cq_schedp == &tmp[i])
276*17a2b317SBill Taylor break; /* found it */
277*17a2b317SBill Taylor if (i >= state->hs_cq_sched_array_size) {
278*17a2b317SBill Taylor cmn_err(CE_CONT, "!Invalid cq_sched argument: "
279*17a2b317SBill Taylor "ignored\n");
280*17a2b317SBill Taylor cq_schedp = &state->hs_cq_sched_default;
281*17a2b317SBill Taylor }
282*17a2b317SBill Taylor }
283*17a2b317SBill Taylor cq->cq_eqnum = HERMON_HID_TO_EQNUM(state,
284*17a2b317SBill Taylor HERMON_CQSCHED_NEXT_HID(cq_schedp));
285*17a2b317SBill Taylor IBTF_DPRINTF_L2("cqalloc", "sched: first-1 %d, len %d, "
286*17a2b317SBill Taylor "eqn %d", cq_schedp->cqs_start_hid - 1,
287*17a2b317SBill Taylor cq_schedp->cqs_len, cq->cq_eqnum);
288*17a2b317SBill Taylor }
2899e39c5baSBill Taylor
2909e39c5baSBill Taylor /*
2919e39c5baSBill Taylor * Fill in the CQC entry. This is the final step before passing
2929e39c5baSBill Taylor * ownership of the CQC entry to the Hermon hardware. We use all of
2939e39c5baSBill Taylor * the information collected/calculated above to fill in the
2949e39c5baSBill Taylor * requisite portions of the CQC. Note: If this CQ is going to be
2959e39c5baSBill Taylor * used for userland access, then we need to set the UAR page number
2969e39c5baSBill Taylor * appropriately (otherwise it's a "don't care")
2979e39c5baSBill Taylor */
2989e39c5baSBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
2999e39c5baSBill Taylor
3009e39c5baSBill Taylor cqc_entry.state = HERMON_CQ_DISARMED;
3019e39c5baSBill Taylor cqc_entry.pg_offs = cq->cq_cqinfo.qa_pgoffs >> 5;
3029e39c5baSBill Taylor cqc_entry.log_cq_sz = log_cq_size;
3039e39c5baSBill Taylor cqc_entry.usr_page = uarpg;
3049e39c5baSBill Taylor cqc_entry.c_eqn = cq->cq_eqnum;
3059e39c5baSBill Taylor cqc_entry.log2_pgsz = mr->mr_log2_pgsz;
3069e39c5baSBill Taylor cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
3079e39c5baSBill Taylor cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;
3089e39c5baSBill Taylor cqc_entry.dbr_addrh = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 32);
3099e39c5baSBill Taylor cqc_entry.dbr_addrl = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 3);
3109e39c5baSBill Taylor
3119e39c5baSBill Taylor /*
3129e39c5baSBill Taylor * Write the CQC entry to hardware - we pass ownership of
3139e39c5baSBill Taylor * the entry to the hardware (using the Hermon SW2HW_CQ firmware
3149e39c5baSBill Taylor * command). Note: In general, this operation shouldn't fail. But
3159e39c5baSBill Taylor * if it does, we have to undo everything we've done above before
3169e39c5baSBill Taylor * returning error.
3179e39c5baSBill Taylor */
3189e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_CQ, &cqc_entry,
3199e39c5baSBill Taylor sizeof (hermon_hw_cqc_t), cq->cq_cqnum, sleepflag);
3209e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) {
3219e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_CQ command failed: %08x\n",
3229e39c5baSBill Taylor status);
3239e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
3249e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
3259e39c5baSBill Taylor }
3269e39c5baSBill Taylor status = ibc_get_ci_failure(0);
3279e39c5baSBill Taylor goto cqalloc_fail6;
3289e39c5baSBill Taylor }
3299e39c5baSBill Taylor
3309e39c5baSBill Taylor /*
3319e39c5baSBill Taylor * Fill in the rest of the Hermon Completion Queue handle. Having
3329e39c5baSBill Taylor * successfully transferred ownership of the CQC, we can update the
3339e39c5baSBill Taylor * following fields for use in further operations on the CQ.
3349e39c5baSBill Taylor */
3359e39c5baSBill Taylor cq->cq_resize_hdl = 0;
3369e39c5baSBill Taylor cq->cq_cqcrsrcp = cqc;
3379e39c5baSBill Taylor cq->cq_rsrcp = rsrc;
3389e39c5baSBill Taylor cq->cq_consindx = 0;
3399e39c5baSBill Taylor /* least restrictive */
3409e39c5baSBill Taylor cq->cq_buf = buf;
3419e39c5baSBill Taylor cq->cq_bufsz = (1 << log_cq_size);
3429e39c5baSBill Taylor cq->cq_log_cqsz = log_cq_size;
3439e39c5baSBill Taylor cq->cq_mrhdl = mr;
3449e39c5baSBill Taylor cq->cq_refcnt = 0;
3459e39c5baSBill Taylor cq->cq_is_special = 0;
3469e39c5baSBill Taylor cq->cq_uarpg = uarpg;
3479e39c5baSBill Taylor cq->cq_umap_dhp = (devmap_cookie_t)NULL;
3489e39c5baSBill Taylor avl_create(&cq->cq_wrid_wqhdr_avl_tree, hermon_wrid_workq_compare,
3499e39c5baSBill Taylor sizeof (struct hermon_workq_avl_s),
3509e39c5baSBill Taylor offsetof(struct hermon_workq_avl_s, wqa_link));
3519e39c5baSBill Taylor
3529e39c5baSBill Taylor cq->cq_hdlrarg = (void *)ibt_cqhdl;
3539e39c5baSBill Taylor
3549e39c5baSBill Taylor /*
3559e39c5baSBill Taylor * Put CQ handle in Hermon CQNum-to-CQHdl list. Then fill in the
3569e39c5baSBill Taylor * "actual_size" and "cqhdl" and return success
3579e39c5baSBill Taylor */
358*17a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, cq);
3599e39c5baSBill Taylor
3609e39c5baSBill Taylor /*
3619e39c5baSBill Taylor * If this is a user-mappable CQ, then we need to insert the previously
3629e39c5baSBill Taylor * allocated entry into the "userland resources database". This will
3639e39c5baSBill Taylor * allow for later lookup during devmap() (i.e. mmap()) calls.
3649e39c5baSBill Taylor */
3659e39c5baSBill Taylor if (cq->cq_is_umap) {
3669e39c5baSBill Taylor hermon_umap_db_add(umapdb);
3679e39c5baSBill Taylor }
3689e39c5baSBill Taylor
3699e39c5baSBill Taylor /*
3709e39c5baSBill Taylor * Fill in the return arguments (if necessary). This includes the
3719e39c5baSBill Taylor * real completion queue size.
3729e39c5baSBill Taylor */
3739e39c5baSBill Taylor if (actual_size != NULL) {
3749e39c5baSBill Taylor *actual_size = (1 << log_cq_size) - 1;
3759e39c5baSBill Taylor }
3769e39c5baSBill Taylor *cqhdl = cq;
3779e39c5baSBill Taylor
3789e39c5baSBill Taylor return (DDI_SUCCESS);
3799e39c5baSBill Taylor
3809e39c5baSBill Taylor /*
3819e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine
3829e39c5baSBill Taylor */
3839e39c5baSBill Taylor cqalloc_fail6:
3849e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
3859e39c5baSBill Taylor sleepflag) != DDI_SUCCESS) {
3869e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister CQ memory");
3879e39c5baSBill Taylor }
3889e39c5baSBill Taylor cqalloc_fail5:
3899e39c5baSBill Taylor hermon_queue_free(&cq->cq_cqinfo);
3909e39c5baSBill Taylor cqalloc_fail4a:
3919e39c5baSBill Taylor hermon_dbr_free(state, uarpg, cq->cq_arm_ci_vdbr);
3929e39c5baSBill Taylor cqalloc_fail4:
3939e39c5baSBill Taylor if (cq_is_umap) {
3949e39c5baSBill Taylor hermon_umap_db_free(umapdb);
3959e39c5baSBill Taylor }
3969e39c5baSBill Taylor cqalloc_fail3:
3979e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc);
3989e39c5baSBill Taylor cqalloc_fail2:
3999e39c5baSBill Taylor hermon_rsrc_free(state, &cqc);
4009e39c5baSBill Taylor cqalloc_fail1:
4019e39c5baSBill Taylor hermon_pd_refcnt_dec(pd);
4029e39c5baSBill Taylor cqalloc_fail:
4039e39c5baSBill Taylor return (status);
4049e39c5baSBill Taylor }
4059e39c5baSBill Taylor
4069e39c5baSBill Taylor
4079e39c5baSBill Taylor /*
4089e39c5baSBill Taylor * hermon_cq_free()
4099e39c5baSBill Taylor * Context: Can be called only from user or kernel context.
4109e39c5baSBill Taylor */
4119e39c5baSBill Taylor /* ARGSUSED */
4129e39c5baSBill Taylor int
hermon_cq_free(hermon_state_t * state,hermon_cqhdl_t * cqhdl,uint_t sleepflag)4139e39c5baSBill Taylor hermon_cq_free(hermon_state_t *state, hermon_cqhdl_t *cqhdl, uint_t sleepflag)
4149e39c5baSBill Taylor {
4159e39c5baSBill Taylor hermon_rsrc_t *cqc, *rsrc;
4169e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb;
4179e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry;
4189e39c5baSBill Taylor hermon_pdhdl_t pd;
4199e39c5baSBill Taylor hermon_mrhdl_t mr;
4209e39c5baSBill Taylor hermon_cqhdl_t cq, resize;
4219e39c5baSBill Taylor uint32_t cqnum;
4229e39c5baSBill Taylor uint64_t value;
4239e39c5baSBill Taylor uint_t maxprot;
4249e39c5baSBill Taylor int status;
4259e39c5baSBill Taylor
4269e39c5baSBill Taylor /*
4279e39c5baSBill Taylor * Pull all the necessary information from the Hermon Completion Queue
4289e39c5baSBill Taylor * handle. This is necessary here because the resource for the
4299e39c5baSBill Taylor * CQ handle is going to be freed up as part of this operation.
4309e39c5baSBill Taylor */
4319e39c5baSBill Taylor cq = *cqhdl;
4329e39c5baSBill Taylor mutex_enter(&cq->cq_lock);
4339e39c5baSBill Taylor cqc = cq->cq_cqcrsrcp;
4349e39c5baSBill Taylor rsrc = cq->cq_rsrcp;
4359e39c5baSBill Taylor pd = state->hs_pdhdl_internal;
4369e39c5baSBill Taylor mr = cq->cq_mrhdl;
4379e39c5baSBill Taylor cqnum = cq->cq_cqnum;
4389e39c5baSBill Taylor
4399e39c5baSBill Taylor resize = cq->cq_resize_hdl; /* save the handle for later */
4409e39c5baSBill Taylor
4419e39c5baSBill Taylor /*
4429e39c5baSBill Taylor * If there are work queues still associated with the CQ, then return
4439e39c5baSBill Taylor * an error. Otherwise, we will be holding the CQ lock.
4449e39c5baSBill Taylor */
4459e39c5baSBill Taylor if (cq->cq_refcnt != 0) {
4469e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
4479e39c5baSBill Taylor return (IBT_CQ_BUSY);
4489e39c5baSBill Taylor }
4499e39c5baSBill Taylor
4509e39c5baSBill Taylor /*
4519e39c5baSBill Taylor * If this was a user-mappable CQ, then we need to remove its entry
4529e39c5baSBill Taylor * from the "userland resources database". If it is also currently
4539e39c5baSBill Taylor * mmap()'d out to a user process, then we need to call
4549e39c5baSBill Taylor * devmap_devmem_remap() to remap the CQ memory to an invalid mapping.
4559e39c5baSBill Taylor * We also need to invalidate the CQ tracking information for the
4569e39c5baSBill Taylor * user mapping.
4579e39c5baSBill Taylor */
4589e39c5baSBill Taylor if (cq->cq_is_umap) {
4599e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, cqnum,
4609e39c5baSBill Taylor MLNX_UMAP_CQMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
4619e39c5baSBill Taylor &umapdb);
4629e39c5baSBill Taylor if (status != DDI_SUCCESS) {
4639e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
4649e39c5baSBill Taylor HERMON_WARNING(state, "failed to find in database");
4659e39c5baSBill Taylor return (ibc_get_ci_failure(0));
4669e39c5baSBill Taylor }
4679e39c5baSBill Taylor hermon_umap_db_free(umapdb);
4689e39c5baSBill Taylor if (cq->cq_umap_dhp != NULL) {
4699e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER);
4709e39c5baSBill Taylor status = devmap_devmem_remap(cq->cq_umap_dhp,
4719e39c5baSBill Taylor state->hs_dip, 0, 0, cq->cq_cqinfo.qa_size,
4729e39c5baSBill Taylor maxprot, DEVMAP_MAPPING_INVALID, NULL);
4739e39c5baSBill Taylor if (status != DDI_SUCCESS) {
4749e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
4759e39c5baSBill Taylor HERMON_WARNING(state, "failed in CQ memory "
4769e39c5baSBill Taylor "devmap_devmem_remap()");
4779e39c5baSBill Taylor return (ibc_get_ci_failure(0));
4789e39c5baSBill Taylor }
4799e39c5baSBill Taylor cq->cq_umap_dhp = (devmap_cookie_t)NULL;
4809e39c5baSBill Taylor }
4819e39c5baSBill Taylor }
4829e39c5baSBill Taylor
4839e39c5baSBill Taylor /*
4849e39c5baSBill Taylor * Put NULL into the Arbel CQNum-to-CQHdl list. This will allow any
4859e39c5baSBill Taylor * in-progress events to detect that the CQ corresponding to this
4869e39c5baSBill Taylor * number has been freed.
4879e39c5baSBill Taylor */
488*17a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, NULL);
4899e39c5baSBill Taylor
4909e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
4919e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq))
4929e39c5baSBill Taylor
4939e39c5baSBill Taylor /*
4949e39c5baSBill Taylor * Reclaim CQC entry from hardware (using the Hermon HW2SW_CQ
4959e39c5baSBill Taylor * firmware command). If the ownership transfer fails for any reason,
4969e39c5baSBill Taylor * then it is an indication that something (either in HW or SW) has
4979e39c5baSBill Taylor * gone seriously wrong.
4989e39c5baSBill Taylor */
4999e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
5009e39c5baSBill Taylor sizeof (hermon_hw_cqc_t), cqnum, sleepflag);
5019e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) {
5029e39c5baSBill Taylor HERMON_WARNING(state, "failed to reclaim CQC ownership");
5039e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: HW2SW_CQ command failed: %08x\n",
5049e39c5baSBill Taylor status);
5059e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
5069e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
5079e39c5baSBill Taylor }
5089e39c5baSBill Taylor return (ibc_get_ci_failure(0));
5099e39c5baSBill Taylor }
5109e39c5baSBill Taylor
5119e39c5baSBill Taylor /*
5129e39c5baSBill Taylor * From here on, we start reliquishing resources - but check to see
5139e39c5baSBill Taylor * if a resize was in progress - if so, we need to relinquish those
5149e39c5baSBill Taylor * resources as well
5159e39c5baSBill Taylor */
5169e39c5baSBill Taylor
5179e39c5baSBill Taylor
5189e39c5baSBill Taylor /*
5199e39c5baSBill Taylor * Deregister the memory for the Completion Queue. If this fails
5209e39c5baSBill Taylor * for any reason, then it is an indication that something (either
5219e39c5baSBill Taylor * in HW or SW) has gone seriously wrong. So we print a warning
5229e39c5baSBill Taylor * message and return.
5239e39c5baSBill Taylor */
5249e39c5baSBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
5259e39c5baSBill Taylor sleepflag);
5269e39c5baSBill Taylor if (status != DDI_SUCCESS) {
5279e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister CQ memory");
5289e39c5baSBill Taylor return (ibc_get_ci_failure(0));
5299e39c5baSBill Taylor }
5309e39c5baSBill Taylor
5319e39c5baSBill Taylor if (resize) { /* there was a pointer to a handle */
5329e39c5baSBill Taylor mr = resize->cq_mrhdl; /* reuse the pointer to the region */
5339e39c5baSBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
5349e39c5baSBill Taylor sleepflag);
5359e39c5baSBill Taylor if (status != DDI_SUCCESS) {
5369e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister resize CQ "
5379e39c5baSBill Taylor "memory");
5389e39c5baSBill Taylor return (ibc_get_ci_failure(0));
5399e39c5baSBill Taylor }
5409e39c5baSBill Taylor }
5419e39c5baSBill Taylor
5429e39c5baSBill Taylor /* Free the memory for the CQ */
5439e39c5baSBill Taylor hermon_queue_free(&cq->cq_cqinfo);
5449e39c5baSBill Taylor if (resize) {
5459e39c5baSBill Taylor hermon_queue_free(&resize->cq_cqinfo);
5469e39c5baSBill Taylor /* and the temporary handle */
5479e39c5baSBill Taylor kmem_free(resize, sizeof (struct hermon_sw_cq_s));
5489e39c5baSBill Taylor }
5499e39c5baSBill Taylor
5509e39c5baSBill Taylor /* everything else does not matter for the resize in progress */
5519e39c5baSBill Taylor
5529e39c5baSBill Taylor /* Free the dbr */
5539e39c5baSBill Taylor hermon_dbr_free(state, cq->cq_uarpg, cq->cq_arm_ci_vdbr);
5549e39c5baSBill Taylor
5559e39c5baSBill Taylor /* Free the Hermon Completion Queue handle */
5569e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc);
5579e39c5baSBill Taylor
5589e39c5baSBill Taylor /* Free up the CQC entry resource */
5599e39c5baSBill Taylor hermon_rsrc_free(state, &cqc);
5609e39c5baSBill Taylor
5619e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */
5629e39c5baSBill Taylor hermon_pd_refcnt_dec(pd);
5639e39c5baSBill Taylor
5649e39c5baSBill Taylor /* Set the cqhdl pointer to NULL and return success */
5659e39c5baSBill Taylor *cqhdl = NULL;
5669e39c5baSBill Taylor
5679e39c5baSBill Taylor return (DDI_SUCCESS);
5689e39c5baSBill Taylor }
5699e39c5baSBill Taylor
5709e39c5baSBill Taylor
5719e39c5baSBill Taylor /*
5729e39c5baSBill Taylor * hermon_cq_resize()
5739e39c5baSBill Taylor * Context: Can be called only from user or kernel context.
5749e39c5baSBill Taylor */
5759e39c5baSBill Taylor int
hermon_cq_resize(hermon_state_t * state,hermon_cqhdl_t cq,uint_t req_size,uint_t * actual_size,uint_t sleepflag)5769e39c5baSBill Taylor hermon_cq_resize(hermon_state_t *state, hermon_cqhdl_t cq, uint_t req_size,
5779e39c5baSBill Taylor uint_t *actual_size, uint_t sleepflag)
5789e39c5baSBill Taylor {
5799e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry;
5809e39c5baSBill Taylor hermon_cqhdl_t resize_hdl;
5819e39c5baSBill Taylor hermon_qalloc_info_t new_cqinfo;
5829e39c5baSBill Taylor ibt_mr_attr_t mr_attr;
5839e39c5baSBill Taylor hermon_mr_options_t op;
5849e39c5baSBill Taylor hermon_pdhdl_t pd;
5859e39c5baSBill Taylor hermon_mrhdl_t mr;
5869e39c5baSBill Taylor hermon_hw_cqe_t *buf;
5879e39c5baSBill Taylor uint32_t new_prod_indx;
5889e39c5baSBill Taylor uint_t log_cq_size;
5899e39c5baSBill Taylor int status, flag;
5909e39c5baSBill Taylor
5919e39c5baSBill Taylor if (cq->cq_resize_hdl != 0) { /* already in process */
5929e39c5baSBill Taylor status = IBT_CQ_BUSY;
5939e39c5baSBill Taylor goto cqresize_fail;
5949e39c5baSBill Taylor }
5959e39c5baSBill Taylor
5969e39c5baSBill Taylor
5979e39c5baSBill Taylor /* Use the internal protection domain (PD) for CQs */
5989e39c5baSBill Taylor pd = state->hs_pdhdl_internal;
5999e39c5baSBill Taylor
6009e39c5baSBill Taylor /*
6019e39c5baSBill Taylor * Calculate the appropriate size for the new resized completion queue.
6029e39c5baSBill Taylor * Note: All Hermon CQs must be a power-of-2 minus 1 in size. Also
6039e39c5baSBill Taylor * they may not be any smaller than HERMON_CQ_MIN_SIZE. This step is
6049e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2
6059e39c5baSBill Taylor */
6069e39c5baSBill Taylor req_size = max(req_size, HERMON_CQ_MIN_SIZE);
6079e39c5baSBill Taylor log_cq_size = highbit(req_size);
6089e39c5baSBill Taylor
6099e39c5baSBill Taylor /*
6109e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
6119e39c5baSBill Taylor * with the device limits and/or software-configured limits)
6129e39c5baSBill Taylor */
6139e39c5baSBill Taylor if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
6149e39c5baSBill Taylor status = IBT_HCA_CQ_EXCEEDED;
6159e39c5baSBill Taylor goto cqresize_fail;
6169e39c5baSBill Taylor }
6179e39c5baSBill Taylor
6189e39c5baSBill Taylor /*
6199e39c5baSBill Taylor * Allocate the memory for newly resized Completion Queue.
6209e39c5baSBill Taylor *
6219e39c5baSBill Taylor * Note: Although we use the common queue allocation routine, we
6229e39c5baSBill Taylor * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
6239e39c5baSBill Taylor * kernel system memory) for kernel CQs because it would be
6249e39c5baSBill Taylor * inefficient to have CQs located in DDR memory. This is the same
6259e39c5baSBill Taylor * as we do when we first allocate completion queues primarily
6269e39c5baSBill Taylor * because CQs are read from (by software) more than they are written
6279e39c5baSBill Taylor * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
6289e39c5baSBill Taylor * user-mappable CQs for a similar reason.)
6299e39c5baSBill Taylor * It is also worth noting that, unlike Hermon QP work queues,
6309e39c5baSBill Taylor * completion queues do not have the same strict alignment
6319e39c5baSBill Taylor * requirements. It is sufficient for the CQ memory to be both
6329e39c5baSBill Taylor * aligned to and bound to addresses which are a multiple of CQE size.
6339e39c5baSBill Taylor */
6349e39c5baSBill Taylor
6359e39c5baSBill Taylor /* first, alloc the resize_handle */
6369e39c5baSBill Taylor resize_hdl = kmem_zalloc(sizeof (struct hermon_sw_cq_s), KM_SLEEP);
6379e39c5baSBill Taylor
6389e39c5baSBill Taylor new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
6399e39c5baSBill Taylor new_cqinfo.qa_alloc_align = PAGESIZE;
6409e39c5baSBill Taylor new_cqinfo.qa_bind_align = PAGESIZE;
6419e39c5baSBill Taylor if (cq->cq_is_umap) {
6429e39c5baSBill Taylor new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
6439e39c5baSBill Taylor } else {
6449e39c5baSBill Taylor new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
6459e39c5baSBill Taylor }
6469e39c5baSBill Taylor status = hermon_queue_alloc(state, &new_cqinfo, sleepflag);
6479e39c5baSBill Taylor if (status != DDI_SUCCESS) {
6489e39c5baSBill Taylor /* free the resize handle */
6499e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
6509e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
6519e39c5baSBill Taylor goto cqresize_fail;
6529e39c5baSBill Taylor }
6539e39c5baSBill Taylor buf = (hermon_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
6549e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf))
6559e39c5baSBill Taylor
6569e39c5baSBill Taylor /*
6579e39c5baSBill Taylor * No initialization of the cq is needed - the command will do it
6589e39c5baSBill Taylor */
6599e39c5baSBill Taylor
6609e39c5baSBill Taylor /*
6619e39c5baSBill Taylor * Register the memory for the CQ. The memory for the CQ must
6629e39c5baSBill Taylor * be registered in the Hermon TPT tables. This gives us the LKey
6639e39c5baSBill Taylor * to specify in the CQ context below.
6649e39c5baSBill Taylor */
6659e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
6669e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
6679e39c5baSBill Taylor mr_attr.mr_len = new_cqinfo.qa_size;
6689e39c5baSBill Taylor mr_attr.mr_as = NULL;
6699e39c5baSBill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
6709e39c5baSBill Taylor op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
6719e39c5baSBill Taylor op.mro_bind_dmahdl = new_cqinfo.qa_dmahdl;
6729e39c5baSBill Taylor op.mro_bind_override_addr = 0;
6739e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
6749e39c5baSBill Taylor HERMON_CQ_CMPT);
6759e39c5baSBill Taylor if (status != DDI_SUCCESS) {
6769e39c5baSBill Taylor hermon_queue_free(&new_cqinfo);
6779e39c5baSBill Taylor /* free the resize handle */
6789e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
6799e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE;
6809e39c5baSBill Taylor goto cqresize_fail;
6819e39c5baSBill Taylor }
6829e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
6839e39c5baSBill Taylor
6849e39c5baSBill Taylor /*
6859e39c5baSBill Taylor * Now we grab the CQ lock. Since we will be updating the actual
6869e39c5baSBill Taylor * CQ location and the producer/consumer indexes, we should hold
6879e39c5baSBill Taylor * the lock.
6889e39c5baSBill Taylor *
6899e39c5baSBill Taylor * We do a ARBEL_NOSLEEP here (and below), though, because we are
6909e39c5baSBill Taylor * holding the "cq_lock" and if we got raised to interrupt level
6919e39c5baSBill Taylor * by priority inversion, we would not want to block in this routine
6929e39c5baSBill Taylor * waiting for success.
6939e39c5baSBill Taylor */
6949e39c5baSBill Taylor mutex_enter(&cq->cq_lock);
6959e39c5baSBill Taylor
6969e39c5baSBill Taylor /*
6979e39c5baSBill Taylor * Fill in the CQC entry. For the resize operation this is the
6989e39c5baSBill Taylor * final step before attempting the resize operation on the CQC entry.
6999e39c5baSBill Taylor * We use all of the information collected/calculated above to fill
7009e39c5baSBill Taylor * in the requisite portions of the CQC.
7019e39c5baSBill Taylor */
7029e39c5baSBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
7039e39c5baSBill Taylor cqc_entry.log_cq_sz = log_cq_size;
7049e39c5baSBill Taylor cqc_entry.pg_offs = new_cqinfo.qa_pgoffs >> 5;
7059e39c5baSBill Taylor cqc_entry.log2_pgsz = mr->mr_log2_pgsz;
7069e39c5baSBill Taylor cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
7079e39c5baSBill Taylor cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3;
7089e39c5baSBill Taylor
7099e39c5baSBill Taylor /*
7109e39c5baSBill Taylor * Write the CQC entry to hardware. Lastly, we pass ownership of
7119e39c5baSBill Taylor * the entry to the hardware (using the Hermon RESIZE_CQ firmware
7129e39c5baSBill Taylor * command). Note: In general, this operation shouldn't fail. But
7139e39c5baSBill Taylor * if it does, we have to undo everything we've done above before
7149e39c5baSBill Taylor * returning error. Also note that the status returned may indicate
7159e39c5baSBill Taylor * the code to return to the IBTF.
7169e39c5baSBill Taylor */
7179e39c5baSBill Taylor status = hermon_resize_cq_cmd_post(state, &cqc_entry, cq->cq_cqnum,
7189e39c5baSBill Taylor &new_prod_indx, HERMON_CMD_NOSLEEP_SPIN);
7199e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) {
7209e39c5baSBill Taylor /* Resize attempt has failed, drop CQ lock and cleanup */
7219e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
7229e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
7239e39c5baSBill Taylor sleepflag) != DDI_SUCCESS) {
7249e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister CQ memory");
7259e39c5baSBill Taylor }
7269e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
7279e39c5baSBill Taylor hermon_queue_free(&new_cqinfo);
7289e39c5baSBill Taylor if (status == HERMON_CMD_BAD_SIZE) {
7299e39c5baSBill Taylor return (IBT_CQ_SZ_INSUFFICIENT);
7309e39c5baSBill Taylor } else {
7319e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: RESIZE_CQ command failed: "
7329e39c5baSBill Taylor "%08x\n", status);
7339e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
7349e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR,
7359e39c5baSBill Taylor HCA_ERR_SRV_LOST);
7369e39c5baSBill Taylor }
7379e39c5baSBill Taylor return (ibc_get_ci_failure(0));
7389e39c5baSBill Taylor }
7399e39c5baSBill Taylor }
7409e39c5baSBill Taylor
7419e39c5baSBill Taylor /*
7429e39c5baSBill Taylor * For Hermon, we've alloc'd another handle structure and save off the
7439e39c5baSBill Taylor * important things in it. Then, in polling we check to see if there's
7449e39c5baSBill Taylor * a "resizing handle" and if so we look for the "special CQE", opcode
7459e39c5baSBill Taylor * 0x16, that indicates the transition to the new buffer.
7469e39c5baSBill Taylor *
7479e39c5baSBill Taylor * At that point, we'll adjust everything - including dereg and
7489e39c5baSBill Taylor * freeing of the original buffer, updating all the necessary fields
7499e39c5baSBill Taylor * in the cq_hdl, and setting up for the next cqe polling
7509e39c5baSBill Taylor */
7519e39c5baSBill Taylor
7529e39c5baSBill Taylor resize_hdl->cq_buf = buf;
7539e39c5baSBill Taylor resize_hdl->cq_bufsz = (1 << log_cq_size);
7549e39c5baSBill Taylor resize_hdl->cq_mrhdl = mr;
7559e39c5baSBill Taylor resize_hdl->cq_log_cqsz = log_cq_size;
7569e39c5baSBill Taylor
7579e39c5baSBill Taylor bcopy(&new_cqinfo, &(resize_hdl->cq_cqinfo),
7589e39c5baSBill Taylor sizeof (struct hermon_qalloc_info_s));
7599e39c5baSBill Taylor
7609e39c5baSBill Taylor /* now, save the address in the cq_handle */
7619e39c5baSBill Taylor cq->cq_resize_hdl = resize_hdl;
7629e39c5baSBill Taylor
7639e39c5baSBill Taylor /*
7649e39c5baSBill Taylor * Drop the CQ lock now.
7659e39c5baSBill Taylor */
7669e39c5baSBill Taylor
7679e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
7689e39c5baSBill Taylor /*
7699e39c5baSBill Taylor * Fill in the return arguments (if necessary). This includes the
7709e39c5baSBill Taylor * real new completion queue size.
7719e39c5baSBill Taylor */
7729e39c5baSBill Taylor if (actual_size != NULL) {
7739e39c5baSBill Taylor *actual_size = (1 << log_cq_size) - 1;
7749e39c5baSBill Taylor }
7759e39c5baSBill Taylor
7769e39c5baSBill Taylor return (DDI_SUCCESS);
7779e39c5baSBill Taylor
7789e39c5baSBill Taylor cqresize_fail:
7799e39c5baSBill Taylor return (status);
7809e39c5baSBill Taylor }
7819e39c5baSBill Taylor
7829e39c5baSBill Taylor
7839e39c5baSBill Taylor /*
7849e39c5baSBill Taylor * hermon_cq_modify()
7859e39c5baSBill Taylor * Context: Can be called base context.
7869e39c5baSBill Taylor */
7879e39c5baSBill Taylor /* ARGSUSED */
7889e39c5baSBill Taylor int
hermon_cq_modify(hermon_state_t * state,hermon_cqhdl_t cq,uint_t count,uint_t usec,ibt_cq_handler_id_t hid,uint_t sleepflag)7899e39c5baSBill Taylor hermon_cq_modify(hermon_state_t *state, hermon_cqhdl_t cq,
7909e39c5baSBill Taylor uint_t count, uint_t usec, ibt_cq_handler_id_t hid, uint_t sleepflag)
7919e39c5baSBill Taylor {
7929e39c5baSBill Taylor int status;
7939e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry;
7949e39c5baSBill Taylor
7959e39c5baSBill Taylor mutex_enter(&cq->cq_lock);
7969e39c5baSBill Taylor if (count != cq->cq_intmod_count ||
7979e39c5baSBill Taylor usec != cq->cq_intmod_usec) {
7989e39c5baSBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
7999e39c5baSBill Taylor cqc_entry.cq_max_cnt = count;
8009e39c5baSBill Taylor cqc_entry.cq_period = usec;
8019e39c5baSBill Taylor status = hermon_modify_cq_cmd_post(state, &cqc_entry,
8029e39c5baSBill Taylor cq->cq_cqnum, MODIFY_MODERATION_CQ, sleepflag);
8039e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) {
8049e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
805*17a2b317SBill Taylor cmn_err(CE_CONT, "Hermon: MODIFY_MODERATION_CQ "
806*17a2b317SBill Taylor "command failed: %08x\n", status);
8079e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
8089e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR,
8099e39c5baSBill Taylor HCA_ERR_SRV_LOST);
8109e39c5baSBill Taylor }
8119e39c5baSBill Taylor return (ibc_get_ci_failure(0));
8129e39c5baSBill Taylor }
8139e39c5baSBill Taylor cq->cq_intmod_count = count;
8149e39c5baSBill Taylor cq->cq_intmod_usec = usec;
8159e39c5baSBill Taylor }
816*17a2b317SBill Taylor if (hid && (hid - 1 != cq->cq_eqnum)) {
817*17a2b317SBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t));
818*17a2b317SBill Taylor cqc_entry.c_eqn = HERMON_HID_TO_EQNUM(state, hid);
819*17a2b317SBill Taylor status = hermon_modify_cq_cmd_post(state, &cqc_entry,
820*17a2b317SBill Taylor cq->cq_cqnum, MODIFY_EQN, sleepflag);
821*17a2b317SBill Taylor if (status != HERMON_CMD_SUCCESS) {
822*17a2b317SBill Taylor mutex_exit(&cq->cq_lock);
823*17a2b317SBill Taylor cmn_err(CE_CONT, "Hermon: MODIFY_EQN command failed: "
824*17a2b317SBill Taylor "%08x\n", status);
825*17a2b317SBill Taylor if (status == HERMON_CMD_INVALID_STATUS) {
826*17a2b317SBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR,
827*17a2b317SBill Taylor HCA_ERR_SRV_LOST);
828*17a2b317SBill Taylor }
829*17a2b317SBill Taylor return (ibc_get_ci_failure(0));
830*17a2b317SBill Taylor }
831*17a2b317SBill Taylor cq->cq_eqnum = hid - 1;
832*17a2b317SBill Taylor }
8339e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
8349e39c5baSBill Taylor return (DDI_SUCCESS);
8359e39c5baSBill Taylor }
8369e39c5baSBill Taylor
8379e39c5baSBill Taylor /*
8389e39c5baSBill Taylor * hermon_cq_notify()
8399e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
8409e39c5baSBill Taylor */
8419e39c5baSBill Taylor int
hermon_cq_notify(hermon_state_t * state,hermon_cqhdl_t cq,ibt_cq_notify_flags_t flags)8429e39c5baSBill Taylor hermon_cq_notify(hermon_state_t *state, hermon_cqhdl_t cq,
8439e39c5baSBill Taylor ibt_cq_notify_flags_t flags)
8449e39c5baSBill Taylor {
8459e39c5baSBill Taylor uint_t cmd;
8469e39c5baSBill Taylor ibt_status_t status;
8479e39c5baSBill Taylor
8489e39c5baSBill Taylor /* Validate IBT flags and call doorbell routine. */
8499e39c5baSBill Taylor if (flags == IBT_NEXT_COMPLETION) {
8509e39c5baSBill Taylor cmd = HERMON_CQDB_NOTIFY_CQ;
8519e39c5baSBill Taylor } else if (flags == IBT_NEXT_SOLICITED) {
8529e39c5baSBill Taylor cmd = HERMON_CQDB_NOTIFY_CQ_SOLICIT;
8539e39c5baSBill Taylor } else {
8549e39c5baSBill Taylor return (IBT_CQ_NOTIFY_TYPE_INVALID);
8559e39c5baSBill Taylor }
8569e39c5baSBill Taylor
8579e39c5baSBill Taylor status = hermon_cq_arm_doorbell(state, cq, cmd);
8589e39c5baSBill Taylor return (status);
8599e39c5baSBill Taylor }
8609e39c5baSBill Taylor
8619e39c5baSBill Taylor
8629e39c5baSBill Taylor /*
8639e39c5baSBill Taylor * hermon_cq_poll()
8649e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
8659e39c5baSBill Taylor */
8669e39c5baSBill Taylor int
hermon_cq_poll(hermon_state_t * state,hermon_cqhdl_t cq,ibt_wc_t * wc_p,uint_t num_wc,uint_t * num_polled)8679e39c5baSBill Taylor hermon_cq_poll(hermon_state_t *state, hermon_cqhdl_t cq, ibt_wc_t *wc_p,
8689e39c5baSBill Taylor uint_t num_wc, uint_t *num_polled)
8699e39c5baSBill Taylor {
8709e39c5baSBill Taylor hermon_hw_cqe_t *cqe;
8719e39c5baSBill Taylor uint_t opcode;
872*17a2b317SBill Taylor uint32_t cons_indx, wrap_around_mask, shift, mask;
8739e39c5baSBill Taylor uint32_t polled_cnt, spec_op = 0;
8749e39c5baSBill Taylor int status;
8759e39c5baSBill Taylor
8769e39c5baSBill Taylor /*
8779e39c5baSBill Taylor * Check for user-mappable CQ memory. Note: We do not allow kernel
8789e39c5baSBill Taylor * clients to poll CQ memory that is accessible directly by the user.
8799e39c5baSBill Taylor * If the CQ memory is user accessible, then return an error.
8809e39c5baSBill Taylor */
8819e39c5baSBill Taylor if (cq->cq_is_umap) {
8829e39c5baSBill Taylor return (IBT_CQ_HDL_INVALID);
8839e39c5baSBill Taylor }
8849e39c5baSBill Taylor
8859e39c5baSBill Taylor mutex_enter(&cq->cq_lock);
8869e39c5baSBill Taylor
8879e39c5baSBill Taylor /* Get the consumer index */
8889e39c5baSBill Taylor cons_indx = cq->cq_consindx;
889*17a2b317SBill Taylor shift = cq->cq_log_cqsz;
890*17a2b317SBill Taylor mask = cq->cq_bufsz;
8919e39c5baSBill Taylor
8929e39c5baSBill Taylor /*
8939e39c5baSBill Taylor * Calculate the wrap around mask. Note: This operation only works
8949e39c5baSBill Taylor * because all Hermon completion queues have power-of-2 sizes
8959e39c5baSBill Taylor */
8969e39c5baSBill Taylor wrap_around_mask = (cq->cq_bufsz - 1);
8979e39c5baSBill Taylor
8989e39c5baSBill Taylor /* Calculate the pointer to the first CQ entry */
8999e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9009e39c5baSBill Taylor
9019e39c5baSBill Taylor /*
9029e39c5baSBill Taylor * Keep pulling entries from the CQ until we find an entry owned by
9039e39c5baSBill Taylor * the hardware. As long as there the CQE's owned by SW, process
9049e39c5baSBill Taylor * each entry by calling hermon_cq_cqe_consume() and updating the CQ
9059e39c5baSBill Taylor * consumer index. Note: We only update the consumer index if
9069e39c5baSBill Taylor * hermon_cq_cqe_consume() returns HERMON_CQ_SYNC_AND_DB. Otherwise,
9079e39c5baSBill Taylor * it indicates that we are going to "recycle" the CQE (probably
9089e39c5baSBill Taylor * because it is a error CQE and corresponds to more than one
9099e39c5baSBill Taylor * completion).
9109e39c5baSBill Taylor */
9119e39c5baSBill Taylor polled_cnt = 0;
912*17a2b317SBill Taylor while (HERMON_CQE_OWNER_IS_SW(cq, cqe, cons_indx, shift, mask)) {
9139e39c5baSBill Taylor if (cq->cq_resize_hdl != 0) { /* in midst of resize */
9149e39c5baSBill Taylor /* peek at the opcode */
9159e39c5baSBill Taylor opcode = HERMON_CQE_OPCODE_GET(cq, cqe);
9169e39c5baSBill Taylor if (opcode == HERMON_CQE_RCV_RESIZE_CODE) {
9179e39c5baSBill Taylor hermon_cq_resize_helper(state, cq);
9189e39c5baSBill Taylor
9199e39c5baSBill Taylor /* Increment the consumer index */
9209e39c5baSBill Taylor cons_indx = (cons_indx + 1);
9219e39c5baSBill Taylor spec_op = 1; /* plus one for the limiting CQE */
9229e39c5baSBill Taylor
9239e39c5baSBill Taylor wrap_around_mask = (cq->cq_bufsz - 1);
9249e39c5baSBill Taylor
9259e39c5baSBill Taylor /* Update the pointer to the next CQ entry */
9269e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9279e39c5baSBill Taylor
9289e39c5baSBill Taylor continue;
9299e39c5baSBill Taylor }
9309e39c5baSBill Taylor } /* in resizing CQ */
9319e39c5baSBill Taylor
9329e39c5baSBill Taylor /*
9339e39c5baSBill Taylor * either resizing and not the special opcode, or
9349e39c5baSBill Taylor * not resizing at all
9359e39c5baSBill Taylor */
9369e39c5baSBill Taylor hermon_cq_cqe_consume(state, cq, cqe, &wc_p[polled_cnt++]);
9379e39c5baSBill Taylor
9389e39c5baSBill Taylor /* Increment the consumer index */
9399e39c5baSBill Taylor cons_indx = (cons_indx + 1);
9409e39c5baSBill Taylor
9419e39c5baSBill Taylor /* Update the pointer to the next CQ entry */
9429e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9439e39c5baSBill Taylor
9449e39c5baSBill Taylor /*
9459e39c5baSBill Taylor * If we have run out of space to store work completions,
9469e39c5baSBill Taylor * then stop and return the ones we have pulled of the CQ.
9479e39c5baSBill Taylor */
9489e39c5baSBill Taylor if (polled_cnt >= num_wc) {
9499e39c5baSBill Taylor break;
9509e39c5baSBill Taylor }
9519e39c5baSBill Taylor }
9529e39c5baSBill Taylor
9539e39c5baSBill Taylor /*
9549e39c5baSBill Taylor * Now we only ring the doorbell (to update the consumer index) if
955*17a2b317SBill Taylor * we've actually consumed a CQ entry.
9569e39c5baSBill Taylor */
9579e39c5baSBill Taylor if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) {
9589e39c5baSBill Taylor /*
9599e39c5baSBill Taylor * Update the consumer index in both the CQ handle and the
9609e39c5baSBill Taylor * doorbell record.
9619e39c5baSBill Taylor */
9629e39c5baSBill Taylor cq->cq_consindx = cons_indx;
9639e39c5baSBill Taylor hermon_cq_update_ci_doorbell(cq);
9649e39c5baSBill Taylor
9659e39c5baSBill Taylor } else if (polled_cnt == 0) {
9669e39c5baSBill Taylor if (spec_op != 0) {
9679e39c5baSBill Taylor /* if we got the special opcode, update the consindx */
9689e39c5baSBill Taylor cq->cq_consindx = cons_indx;
9699e39c5baSBill Taylor hermon_cq_update_ci_doorbell(cq);
9709e39c5baSBill Taylor }
9719e39c5baSBill Taylor }
9729e39c5baSBill Taylor
9739e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
9749e39c5baSBill Taylor
9759e39c5baSBill Taylor /* Set "num_polled" (if necessary) */
9769e39c5baSBill Taylor if (num_polled != NULL) {
9779e39c5baSBill Taylor *num_polled = polled_cnt;
9789e39c5baSBill Taylor }
9799e39c5baSBill Taylor
9809e39c5baSBill Taylor /* Set CQ_EMPTY condition if needed, otherwise return success */
9819e39c5baSBill Taylor if (polled_cnt == 0) {
9829e39c5baSBill Taylor status = IBT_CQ_EMPTY;
9839e39c5baSBill Taylor } else {
9849e39c5baSBill Taylor status = DDI_SUCCESS;
9859e39c5baSBill Taylor }
9869e39c5baSBill Taylor
9879e39c5baSBill Taylor /*
9889e39c5baSBill Taylor * Check if the system is currently panicking. If it is, then call
9899e39c5baSBill Taylor * the Hermon interrupt service routine. This step is necessary here
9909e39c5baSBill Taylor * because we might be in a polled I/O mode and without the call to
9919e39c5baSBill Taylor * hermon_isr() - and its subsequent calls to poll and rearm each
9929e39c5baSBill Taylor * event queue - we might overflow our EQs and render the system
9939e39c5baSBill Taylor * unable to sync/dump.
9949e39c5baSBill Taylor */
9959e39c5baSBill Taylor if (ddi_in_panic() != 0) {
9969e39c5baSBill Taylor (void) hermon_isr((caddr_t)state, (caddr_t)NULL);
9979e39c5baSBill Taylor }
9989e39c5baSBill Taylor return (status);
9999e39c5baSBill Taylor }
10009e39c5baSBill Taylor
10019e39c5baSBill Taylor /*
10029e39c5baSBill Taylor * cmd_sn must be initialized to 1 to enable proper reenabling
10039e39c5baSBill Taylor * by hermon_arm_cq_dbr_update().
10049e39c5baSBill Taylor */
10059e39c5baSBill Taylor static void
hermon_arm_cq_dbr_init(hermon_dbr_t * cq_arm_dbr)10069e39c5baSBill Taylor hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr)
10079e39c5baSBill Taylor {
10089e39c5baSBill Taylor uint32_t *target;
10099e39c5baSBill Taylor
10109e39c5baSBill Taylor target = (uint32_t *)cq_arm_dbr + 1;
10119e39c5baSBill Taylor *target = htonl(1 << HERMON_CQDB_CMDSN_SHIFT);
10129e39c5baSBill Taylor }
10139e39c5baSBill Taylor
10149e39c5baSBill Taylor
10159e39c5baSBill Taylor /*
10169e39c5baSBill Taylor * User cmd_sn needs help from this kernel function to know
10179e39c5baSBill Taylor * when it should be incremented (modulo 4). We do an atomic
10189e39c5baSBill Taylor * update of the arm_cq dbr to communicate this fact. We retry
10199e39c5baSBill Taylor * in the case that user library is racing with us. We zero
10209e39c5baSBill Taylor * out the cmd field so that the user library can use the cmd
10219e39c5baSBill Taylor * field to track the last command it issued (solicited verses any).
10229e39c5baSBill Taylor */
10239e39c5baSBill Taylor static void
hermon_arm_cq_dbr_update(hermon_dbr_t * cq_arm_dbr)10249e39c5baSBill Taylor hermon_arm_cq_dbr_update(hermon_dbr_t *cq_arm_dbr)
10259e39c5baSBill Taylor {
10269e39c5baSBill Taylor uint32_t tmp, cmp, new;
10279e39c5baSBill Taylor uint32_t old_cmd_sn, new_cmd_sn;
10289e39c5baSBill Taylor uint32_t *target;
10299e39c5baSBill Taylor int retries = 0;
10309e39c5baSBill Taylor
10319e39c5baSBill Taylor target = (uint32_t *)cq_arm_dbr + 1;
10329e39c5baSBill Taylor retry:
10339e39c5baSBill Taylor cmp = *target;
10349e39c5baSBill Taylor tmp = htonl(cmp);
10359e39c5baSBill Taylor old_cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT);
10369e39c5baSBill Taylor new_cmd_sn = (old_cmd_sn + (0x1 << HERMON_CQDB_CMDSN_SHIFT)) &
10379e39c5baSBill Taylor (0x3 << HERMON_CQDB_CMDSN_SHIFT);
10389e39c5baSBill Taylor new = htonl((tmp & ~(0x37 << HERMON_CQDB_CMD_SHIFT)) | new_cmd_sn);
10399e39c5baSBill Taylor tmp = atomic_cas_32(target, cmp, new);
10409e39c5baSBill Taylor if (tmp != cmp) { /* cas failed, so need to retry */
10419e39c5baSBill Taylor drv_usecwait(retries & 0xff); /* avoid race */
10429e39c5baSBill Taylor if (++retries > 100000) {
10439e39c5baSBill Taylor cmn_err(CE_CONT, "cas failed in hermon\n");
10449e39c5baSBill Taylor retries = 0;
10459e39c5baSBill Taylor }
10469e39c5baSBill Taylor goto retry;
10479e39c5baSBill Taylor }
10489e39c5baSBill Taylor }
10499e39c5baSBill Taylor
10509e39c5baSBill Taylor
10519e39c5baSBill Taylor /*
10529e39c5baSBill Taylor * hermon_cq_handler()
10539e39c5baSBill Taylor * Context: Only called from interrupt context
10549e39c5baSBill Taylor */
1055*17a2b317SBill Taylor /* ARGSUSED */
10569e39c5baSBill Taylor int
hermon_cq_handler(hermon_state_t * state,hermon_eqhdl_t eq,hermon_hw_eqe_t * eqe)10579e39c5baSBill Taylor hermon_cq_handler(hermon_state_t *state, hermon_eqhdl_t eq,
10589e39c5baSBill Taylor hermon_hw_eqe_t *eqe)
10599e39c5baSBill Taylor {
10609e39c5baSBill Taylor hermon_cqhdl_t cq;
10619e39c5baSBill Taylor uint_t cqnum;
10629e39c5baSBill Taylor
10639e39c5baSBill Taylor /* Get the CQ handle from CQ number in event descriptor */
10649e39c5baSBill Taylor cqnum = HERMON_EQE_CQNUM_GET(eq, eqe);
10659e39c5baSBill Taylor cq = hermon_cqhdl_from_cqnum(state, cqnum);
10669e39c5baSBill Taylor
10679e39c5baSBill Taylor /*
10689e39c5baSBill Taylor * If the CQ handle is NULL, this is probably an indication
10699e39c5baSBill Taylor * that the CQ has been freed already. In which case, we
10709e39c5baSBill Taylor * should not deliver this event.
10719e39c5baSBill Taylor *
10729e39c5baSBill Taylor * We also check that the CQ number in the handle is the
10739e39c5baSBill Taylor * same as the CQ number in the event queue entry. This
10749e39c5baSBill Taylor * extra check allows us to handle the case where a CQ was
10759e39c5baSBill Taylor * freed and then allocated again in the time it took to
10769e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
10779e39c5baSBill Taylor * the non-constrained portion of the CQ number every time
10789e39c5baSBill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance
10799e39c5baSBill Taylor * that a stale event could be passed to the client's CQ
10809e39c5baSBill Taylor * handler.
10819e39c5baSBill Taylor *
10829e39c5baSBill Taylor * Lastly, we check if "hs_ibtfpriv" is NULL. If it is then it
10839e39c5baSBill Taylor * means that we've have either received this event before we
10849e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
10859e39c5baSBill Taylor * are in the process of detaching.
10869e39c5baSBill Taylor */
10879e39c5baSBill Taylor if ((cq != NULL) && (cq->cq_cqnum == cqnum) &&
10889e39c5baSBill Taylor (state->hs_ibtfpriv != NULL)) {
10899e39c5baSBill Taylor hermon_arm_cq_dbr_update(cq->cq_arm_ci_vdbr);
10909e39c5baSBill Taylor HERMON_DO_IBTF_CQ_CALLB(state, cq);
10919e39c5baSBill Taylor }
10929e39c5baSBill Taylor
10939e39c5baSBill Taylor return (DDI_SUCCESS);
10949e39c5baSBill Taylor }
10959e39c5baSBill Taylor
10969e39c5baSBill Taylor
10979e39c5baSBill Taylor /*
10989e39c5baSBill Taylor * hermon_cq_err_handler()
10999e39c5baSBill Taylor * Context: Only called from interrupt context
11009e39c5baSBill Taylor */
1101*17a2b317SBill Taylor /* ARGSUSED */
11029e39c5baSBill Taylor int
hermon_cq_err_handler(hermon_state_t * state,hermon_eqhdl_t eq,hermon_hw_eqe_t * eqe)11039e39c5baSBill Taylor hermon_cq_err_handler(hermon_state_t *state, hermon_eqhdl_t eq,
11049e39c5baSBill Taylor hermon_hw_eqe_t *eqe)
11059e39c5baSBill Taylor {
11069e39c5baSBill Taylor hermon_cqhdl_t cq;
11079e39c5baSBill Taylor uint_t cqnum;
11089e39c5baSBill Taylor ibc_async_event_t event;
11099e39c5baSBill Taylor ibt_async_code_t type;
11109e39c5baSBill Taylor
11119e39c5baSBill Taylor HERMON_FMANOTE(state, HERMON_FMA_OVERRUN);
11129e39c5baSBill Taylor /* Get the CQ handle from CQ number in event descriptor */
11139e39c5baSBill Taylor cqnum = HERMON_EQE_CQNUM_GET(eq, eqe);
11149e39c5baSBill Taylor cq = hermon_cqhdl_from_cqnum(state, cqnum);
11159e39c5baSBill Taylor
11169e39c5baSBill Taylor /*
11179e39c5baSBill Taylor * If the CQ handle is NULL, this is probably an indication
11189e39c5baSBill Taylor * that the CQ has been freed already. In which case, we
11199e39c5baSBill Taylor * should not deliver this event.
11209e39c5baSBill Taylor *
11219e39c5baSBill Taylor * We also check that the CQ number in the handle is the
11229e39c5baSBill Taylor * same as the CQ number in the event queue entry. This
11239e39c5baSBill Taylor * extra check allows us to handle the case where a CQ was
11249e39c5baSBill Taylor * freed and then allocated again in the time it took to
11259e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing
11269e39c5baSBill Taylor * the non-constrained portion of the CQ number every time
11279e39c5baSBill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance
11289e39c5baSBill Taylor * that a stale event could be passed to the client's CQ
11299e39c5baSBill Taylor * handler.
11309e39c5baSBill Taylor *
11319e39c5baSBill Taylor * And then we check if "hs_ibtfpriv" is NULL. If it is then it
11329e39c5baSBill Taylor * means that we've have either received this event before we
11339e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we
11349e39c5baSBill Taylor * are in the process of detaching.
11359e39c5baSBill Taylor */
11369e39c5baSBill Taylor if ((cq != NULL) && (cq->cq_cqnum == cqnum) &&
11379e39c5baSBill Taylor (state->hs_ibtfpriv != NULL)) {
11389e39c5baSBill Taylor event.ev_cq_hdl = (ibt_cq_hdl_t)cq->cq_hdlrarg;
11399e39c5baSBill Taylor type = IBT_ERROR_CQ;
11409e39c5baSBill Taylor HERMON_DO_IBTF_ASYNC_CALLB(state, type, &event);
11419e39c5baSBill Taylor }
11429e39c5baSBill Taylor
11439e39c5baSBill Taylor return (DDI_SUCCESS);
11449e39c5baSBill Taylor }
11459e39c5baSBill Taylor
11469e39c5baSBill Taylor
11479e39c5baSBill Taylor /*
11489e39c5baSBill Taylor * hermon_cq_refcnt_inc()
11499e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
11509e39c5baSBill Taylor */
11519e39c5baSBill Taylor int
hermon_cq_refcnt_inc(hermon_cqhdl_t cq,uint_t is_special)11529e39c5baSBill Taylor hermon_cq_refcnt_inc(hermon_cqhdl_t cq, uint_t is_special)
11539e39c5baSBill Taylor {
11549e39c5baSBill Taylor /*
11559e39c5baSBill Taylor * Increment the completion queue's reference count. Note: In order
11569e39c5baSBill Taylor * to ensure compliance with IBA C11-15, we must ensure that a given
11579e39c5baSBill Taylor * CQ is not used for both special (SMI/GSI) QP and non-special QP.
11589e39c5baSBill Taylor * This is accomplished here by keeping track of how the referenced
11599e39c5baSBill Taylor * CQ is being used.
11609e39c5baSBill Taylor */
11619e39c5baSBill Taylor mutex_enter(&cq->cq_lock);
11629e39c5baSBill Taylor if (cq->cq_refcnt == 0) {
11639e39c5baSBill Taylor cq->cq_is_special = is_special;
11649e39c5baSBill Taylor } else {
11659e39c5baSBill Taylor if (cq->cq_is_special != is_special) {
11669e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
11679e39c5baSBill Taylor return (DDI_FAILURE);
11689e39c5baSBill Taylor }
11699e39c5baSBill Taylor }
11709e39c5baSBill Taylor cq->cq_refcnt++;
11719e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
11729e39c5baSBill Taylor return (DDI_SUCCESS);
11739e39c5baSBill Taylor }
11749e39c5baSBill Taylor
11759e39c5baSBill Taylor
11769e39c5baSBill Taylor /*
11779e39c5baSBill Taylor * hermon_cq_refcnt_dec()
11789e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
11799e39c5baSBill Taylor */
11809e39c5baSBill Taylor void
hermon_cq_refcnt_dec(hermon_cqhdl_t cq)11819e39c5baSBill Taylor hermon_cq_refcnt_dec(hermon_cqhdl_t cq)
11829e39c5baSBill Taylor {
11839e39c5baSBill Taylor /* Decrement the completion queue's reference count */
11849e39c5baSBill Taylor mutex_enter(&cq->cq_lock);
11859e39c5baSBill Taylor cq->cq_refcnt--;
11869e39c5baSBill Taylor mutex_exit(&cq->cq_lock);
11879e39c5baSBill Taylor }
11889e39c5baSBill Taylor
11899e39c5baSBill Taylor
11909e39c5baSBill Taylor /*
11919e39c5baSBill Taylor * hermon_cq_arm_doorbell()
11929e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
11939e39c5baSBill Taylor */
11949e39c5baSBill Taylor static int
hermon_cq_arm_doorbell(hermon_state_t * state,hermon_cqhdl_t cq,uint_t cq_cmd)11959e39c5baSBill Taylor hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq, uint_t cq_cmd)
11969e39c5baSBill Taylor {
11979e39c5baSBill Taylor uint32_t cq_num;
11989e39c5baSBill Taylor uint32_t *target;
11999e39c5baSBill Taylor uint32_t old_cmd, cmp, new, tmp, cmd_sn;
12009e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
12019e39c5baSBill Taylor
12029e39c5baSBill Taylor /* initialize the FMA retry loop */
12039e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
12049e39c5baSBill Taylor
12059e39c5baSBill Taylor cq_num = cq->cq_cqnum;
12069e39c5baSBill Taylor target = (uint32_t *)cq->cq_arm_ci_vdbr + 1;
12079e39c5baSBill Taylor
12089e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell register. */
12099e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt, fm_status,
12109e39c5baSBill Taylor fm_test_num);
12119e39c5baSBill Taylor retry:
12129e39c5baSBill Taylor cmp = *target;
12139e39c5baSBill Taylor tmp = htonl(cmp);
12149e39c5baSBill Taylor old_cmd = tmp & (0x7 << HERMON_CQDB_CMD_SHIFT);
12159e39c5baSBill Taylor cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT);
12169e39c5baSBill Taylor if (cq_cmd == HERMON_CQDB_NOTIFY_CQ) {
12179e39c5baSBill Taylor if (old_cmd != HERMON_CQDB_NOTIFY_CQ) {
12189e39c5baSBill Taylor cmd_sn |= (HERMON_CQDB_NOTIFY_CQ <<
12199e39c5baSBill Taylor HERMON_CQDB_CMD_SHIFT);
12209e39c5baSBill Taylor new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF));
12219e39c5baSBill Taylor tmp = atomic_cas_32(target, cmp, new);
12229e39c5baSBill Taylor if (tmp != cmp)
12239e39c5baSBill Taylor goto retry;
12249e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *)
12259e39c5baSBill Taylor &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) <<
12269e39c5baSBill Taylor 32) | (cq->cq_consindx & 0xFFFFFF));
12279e39c5baSBill Taylor } /* else it's already armed */
12289e39c5baSBill Taylor } else {
12299e39c5baSBill Taylor ASSERT(cq_cmd == HERMON_CQDB_NOTIFY_CQ_SOLICIT);
12309e39c5baSBill Taylor if (old_cmd != HERMON_CQDB_NOTIFY_CQ &&
12319e39c5baSBill Taylor old_cmd != HERMON_CQDB_NOTIFY_CQ_SOLICIT) {
12329e39c5baSBill Taylor cmd_sn |= (HERMON_CQDB_NOTIFY_CQ_SOLICIT <<
12339e39c5baSBill Taylor HERMON_CQDB_CMD_SHIFT);
12349e39c5baSBill Taylor new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF));
12359e39c5baSBill Taylor tmp = atomic_cas_32(target, cmp, new);
12369e39c5baSBill Taylor if (tmp != cmp)
12379e39c5baSBill Taylor goto retry;
12389e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *)
12399e39c5baSBill Taylor &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) <<
12409e39c5baSBill Taylor 32) | (cq->cq_consindx & 0xFFFFFF));
12419e39c5baSBill Taylor } /* else it's already armed */
12429e39c5baSBill Taylor }
12439e39c5baSBill Taylor
12449e39c5baSBill Taylor /* the FMA retry loop ends. */
12459e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt, fm_status,
12469e39c5baSBill Taylor fm_test_num);
12479e39c5baSBill Taylor
12489e39c5baSBill Taylor return (IBT_SUCCESS);
12499e39c5baSBill Taylor
12509e39c5baSBill Taylor pio_error:
12519e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
12529e39c5baSBill Taylor return (ibc_get_ci_failure(0));
12539e39c5baSBill Taylor }
12549e39c5baSBill Taylor
12559e39c5baSBill Taylor
12569e39c5baSBill Taylor /*
12579e39c5baSBill Taylor * hermon_cqhdl_from_cqnum()
12589e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
12599e39c5baSBill Taylor *
12609e39c5baSBill Taylor * This routine is important because changing the unconstrained
12619e39c5baSBill Taylor * portion of the CQ number is critical to the detection of a
12629e39c5baSBill Taylor * potential race condition in the CQ handler code (i.e. the case
12639e39c5baSBill Taylor * where a CQ is freed and alloc'd again before an event for the
12649e39c5baSBill Taylor * "old" CQ can be handled).
12659e39c5baSBill Taylor *
12669e39c5baSBill Taylor * While this is not a perfect solution (not sure that one exists)
12679e39c5baSBill Taylor * it does help to mitigate the chance that this race condition will
12689e39c5baSBill Taylor * cause us to deliver a "stale" event to the new CQ owner. Note:
12699e39c5baSBill Taylor * this solution does not scale well because the number of constrained
12709e39c5baSBill Taylor * bits increases (and, hence, the number of unconstrained bits
12719e39c5baSBill Taylor * decreases) as the number of supported CQs grows. For small and
12729e39c5baSBill Taylor * intermediate values, it should hopefully provide sufficient
12739e39c5baSBill Taylor * protection.
12749e39c5baSBill Taylor */
12759e39c5baSBill Taylor hermon_cqhdl_t
hermon_cqhdl_from_cqnum(hermon_state_t * state,uint_t cqnum)12769e39c5baSBill Taylor hermon_cqhdl_from_cqnum(hermon_state_t *state, uint_t cqnum)
12779e39c5baSBill Taylor {
12789e39c5baSBill Taylor uint_t cqindx, cqmask;
12799e39c5baSBill Taylor
12809e39c5baSBill Taylor /* Calculate the CQ table index from the cqnum */
12819e39c5baSBill Taylor cqmask = (1 << state->hs_cfg_profile->cp_log_num_cq) - 1;
12829e39c5baSBill Taylor cqindx = cqnum & cqmask;
1283*17a2b317SBill Taylor return (hermon_icm_num_to_hdl(state, HERMON_CQC, cqindx));
12849e39c5baSBill Taylor }
12859e39c5baSBill Taylor
12869e39c5baSBill Taylor /*
12879e39c5baSBill Taylor * hermon_cq_cqe_consume()
12889e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
12899e39c5baSBill Taylor */
12909e39c5baSBill Taylor static void
hermon_cq_cqe_consume(hermon_state_t * state,hermon_cqhdl_t cq,hermon_hw_cqe_t * cqe,ibt_wc_t * wc)12919e39c5baSBill Taylor hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
12929e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc)
12939e39c5baSBill Taylor {
12949e39c5baSBill Taylor uint_t opcode, qpnum, qp1_indx;
12959e39c5baSBill Taylor ibt_wc_flags_t flags;
12969e39c5baSBill Taylor ibt_wrc_opcode_t type;
12979e39c5baSBill Taylor
12989e39c5baSBill Taylor /*
12999e39c5baSBill Taylor * Determine if this is an "error" CQE by examining "opcode". If it
13009e39c5baSBill Taylor * is an error CQE, then call hermon_cq_errcqe_consume() and return
13019e39c5baSBill Taylor * whatever status it returns. Otherwise, this is a successful
13029e39c5baSBill Taylor * completion.
13039e39c5baSBill Taylor */
13049e39c5baSBill Taylor opcode = HERMON_CQE_OPCODE_GET(cq, cqe);
13059e39c5baSBill Taylor if ((opcode == HERMON_CQE_SEND_ERR_OPCODE) ||
13069e39c5baSBill Taylor (opcode == HERMON_CQE_RECV_ERR_OPCODE)) {
13079e39c5baSBill Taylor hermon_cq_errcqe_consume(state, cq, cqe, wc);
13089e39c5baSBill Taylor return;
13099e39c5baSBill Taylor }
13109e39c5baSBill Taylor
13119e39c5baSBill Taylor /*
13129e39c5baSBill Taylor * Fetch the Work Request ID using the information in the CQE.
13139e39c5baSBill Taylor * See hermon_wr.c for more details.
13149e39c5baSBill Taylor */
13159e39c5baSBill Taylor wc->wc_id = hermon_wrid_get_entry(cq, cqe);
13169e39c5baSBill Taylor
13179e39c5baSBill Taylor /*
13189e39c5baSBill Taylor * Parse the CQE opcode to determine completion type. This will set
13199e39c5baSBill Taylor * not only the type of the completion, but also any flags that might
13209e39c5baSBill Taylor * be associated with it (e.g. whether immediate data is present).
13219e39c5baSBill Taylor */
13229e39c5baSBill Taylor flags = IBT_WC_NO_FLAGS;
1323*17a2b317SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->hs_fcoib_may_be_running))
13249e39c5baSBill Taylor if (HERMON_CQE_SENDRECV_GET(cq, cqe) != HERMON_COMPLETION_RECV) {
13259e39c5baSBill Taylor
13269e39c5baSBill Taylor /* Send CQE */
13279e39c5baSBill Taylor switch (opcode) {
13289e39c5baSBill Taylor case HERMON_CQE_SND_RDMAWR_IMM:
13299e39c5baSBill Taylor case HERMON_CQE_SND_RDMAWR:
13309e39c5baSBill Taylor type = IBT_WRC_RDMAW;
13319e39c5baSBill Taylor break;
13329e39c5baSBill Taylor
1333*17a2b317SBill Taylor case HERMON_CQE_SND_SEND_INV:
13349e39c5baSBill Taylor case HERMON_CQE_SND_SEND_IMM:
13359e39c5baSBill Taylor case HERMON_CQE_SND_SEND:
13369e39c5baSBill Taylor type = IBT_WRC_SEND;
13379e39c5baSBill Taylor break;
13389e39c5baSBill Taylor
13399e39c5baSBill Taylor case HERMON_CQE_SND_LSO:
13409e39c5baSBill Taylor type = IBT_WRC_SEND_LSO;
13419e39c5baSBill Taylor break;
13429e39c5baSBill Taylor
13439e39c5baSBill Taylor case HERMON_CQE_SND_RDMARD:
13449e39c5baSBill Taylor type = IBT_WRC_RDMAR;
13459e39c5baSBill Taylor break;
13469e39c5baSBill Taylor
13479e39c5baSBill Taylor case HERMON_CQE_SND_ATOMIC_CS:
13489e39c5baSBill Taylor type = IBT_WRC_CSWAP;
13499e39c5baSBill Taylor break;
13509e39c5baSBill Taylor
13519e39c5baSBill Taylor case HERMON_CQE_SND_ATOMIC_FA:
13529e39c5baSBill Taylor type = IBT_WRC_FADD;
13539e39c5baSBill Taylor break;
13549e39c5baSBill Taylor
13559e39c5baSBill Taylor case HERMON_CQE_SND_BIND_MW:
13569e39c5baSBill Taylor type = IBT_WRC_BIND;
13579e39c5baSBill Taylor break;
13589e39c5baSBill Taylor
1359*17a2b317SBill Taylor case HERMON_CQE_SND_FRWR:
1360*17a2b317SBill Taylor type = IBT_WRC_FAST_REG_PMR;
1361*17a2b317SBill Taylor break;
1362*17a2b317SBill Taylor
1363*17a2b317SBill Taylor case HERMON_CQE_SND_LCL_INV:
1364*17a2b317SBill Taylor type = IBT_WRC_LOCAL_INVALIDATE;
1365*17a2b317SBill Taylor break;
1366*17a2b317SBill Taylor
13679e39c5baSBill Taylor default:
13689e39c5baSBill Taylor HERMON_WARNING(state, "unknown send CQE type");
13699e39c5baSBill Taylor wc->wc_status = IBT_WC_LOCAL_QP_OP_ERR;
13709e39c5baSBill Taylor return;
13719e39c5baSBill Taylor }
1372*17a2b317SBill Taylor } else if ((state->hs_fcoib_may_be_running == B_TRUE) &&
1373*17a2b317SBill Taylor hermon_fcoib_is_fexch_qpn(state, HERMON_CQE_QPNUM_GET(cq, cqe))) {
1374*17a2b317SBill Taylor type = IBT_WRC_RECV;
1375*17a2b317SBill Taylor if (HERMON_CQE_FEXCH_DIFE(cq, cqe))
1376*17a2b317SBill Taylor flags |= IBT_WC_DIF_ERROR;
1377*17a2b317SBill Taylor wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe);
1378*17a2b317SBill Taylor wc->wc_fexch_seq_cnt = HERMON_CQE_FEXCH_SEQ_CNT(cq, cqe);
1379*17a2b317SBill Taylor wc->wc_fexch_tx_bytes_xfer = HERMON_CQE_FEXCH_TX_BYTES(cq, cqe);
1380*17a2b317SBill Taylor wc->wc_fexch_rx_bytes_xfer = HERMON_CQE_FEXCH_RX_BYTES(cq, cqe);
1381*17a2b317SBill Taylor wc->wc_fexch_seq_id = HERMON_CQE_FEXCH_SEQ_ID(cq, cqe);
1382*17a2b317SBill Taylor wc->wc_detail = HERMON_CQE_FEXCH_DETAIL(cq, cqe) &
1383*17a2b317SBill Taylor IBT_WC_DETAIL_FC_MATCH_MASK;
1384*17a2b317SBill Taylor wc->wc_rkey = HERMON_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe);
1385*17a2b317SBill Taylor flags |= IBT_WC_FEXCH_FMT | IBT_WC_RKEY_INVALIDATED;
13869e39c5baSBill Taylor } else {
1387*17a2b317SBill Taylor /*
1388*17a2b317SBill Taylor * Parse the remaining contents of the CQE into the work
1389*17a2b317SBill Taylor * completion. This means filling in SL, QP number, SLID,
1390*17a2b317SBill Taylor * immediate data, etc.
1391*17a2b317SBill Taylor *
1392*17a2b317SBill Taylor * Note: Not all of these fields are valid in a given
1393*17a2b317SBill Taylor * completion. Many of them depend on the actual type of
1394*17a2b317SBill Taylor * completion. So we fill in all of the fields and leave
1395*17a2b317SBill Taylor * it up to the IBTF and consumer to sort out which are
1396*17a2b317SBill Taylor * valid based on their context.
1397*17a2b317SBill Taylor */
1398*17a2b317SBill Taylor wc->wc_sl = HERMON_CQE_SL_GET(cq, cqe);
1399*17a2b317SBill Taylor wc->wc_qpn = HERMON_CQE_DQPN_GET(cq, cqe);
1400*17a2b317SBill Taylor wc->wc_slid = HERMON_CQE_DLID_GET(cq, cqe);
1401*17a2b317SBill Taylor wc->wc_immed_data =
1402*17a2b317SBill Taylor HERMON_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe);
1403*17a2b317SBill Taylor wc->wc_ethertype = (wc->wc_immed_data & 0xFFFF);
1404*17a2b317SBill Taylor wc->wc_pkey_ix = (wc->wc_immed_data &
1405*17a2b317SBill Taylor ((1 << state->hs_queryport.log_max_pkey) - 1));
1406*17a2b317SBill Taylor /*
1407*17a2b317SBill Taylor * Fill in "bytes transferred" as appropriate. Also,
1408*17a2b317SBill Taylor * if necessary, fill in the "path bits" field.
1409*17a2b317SBill Taylor */
1410*17a2b317SBill Taylor wc->wc_path_bits = HERMON_CQE_PATHBITS_GET(cq, cqe);
1411*17a2b317SBill Taylor wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe);
1412*17a2b317SBill Taylor
1413*17a2b317SBill Taylor /*
1414*17a2b317SBill Taylor * Check for GRH, update the flags, then fill in "wc_flags"
1415*17a2b317SBill Taylor * field in the work completion
1416*17a2b317SBill Taylor */
1417*17a2b317SBill Taylor if (HERMON_CQE_GRH_GET(cq, cqe) != 0) {
1418*17a2b317SBill Taylor flags |= IBT_WC_GRH_PRESENT;
1419*17a2b317SBill Taylor }
14209e39c5baSBill Taylor
14219e39c5baSBill Taylor /* Receive CQE */
1422*17a2b317SBill Taylor switch (opcode) {
14239e39c5baSBill Taylor case HERMON_CQE_RCV_SEND_IMM:
14249e39c5baSBill Taylor /*
14259e39c5baSBill Taylor * Note: According to the PRM, all QP1 recv
14269e39c5baSBill Taylor * completions look like the result of a Send with
14279e39c5baSBill Taylor * Immediate. They are not, however, (MADs are Send
14289e39c5baSBill Taylor * Only) so we need to check the QP number and set
14299e39c5baSBill Taylor * the flag only if it is non-QP1.
14309e39c5baSBill Taylor */
14319e39c5baSBill Taylor qpnum = HERMON_CQE_QPNUM_GET(cq, cqe);
14329e39c5baSBill Taylor qp1_indx = state->hs_spec_qp1->hr_indx;
14339e39c5baSBill Taylor if ((qpnum < qp1_indx) || (qpnum > qp1_indx + 1)) {
14349e39c5baSBill Taylor flags |= IBT_WC_IMMED_DATA_PRESENT;
14359e39c5baSBill Taylor }
14369e39c5baSBill Taylor /* FALLTHROUGH */
1437*17a2b317SBill Taylor
14389e39c5baSBill Taylor case HERMON_CQE_RCV_SEND:
14399e39c5baSBill Taylor type = IBT_WRC_RECV;
14409e39c5baSBill Taylor if (HERMON_CQE_IS_IPOK(cq, cqe)) {
14419e39c5baSBill Taylor wc->wc_cksum = HERMON_CQE_CKSUM(cq, cqe);
14429e39c5baSBill Taylor flags |= IBT_WC_CKSUM_OK;
14439e39c5baSBill Taylor wc->wc_detail = IBT_WC_DETAIL_ALL_FLAGS_MASK &
14449e39c5baSBill Taylor HERMON_CQE_IPOIB_STATUS(cq, cqe);
14459e39c5baSBill Taylor }
14469e39c5baSBill Taylor break;
1447*17a2b317SBill Taylor
1448*17a2b317SBill Taylor case HERMON_CQE_RCV_SEND_INV:
1449*17a2b317SBill Taylor type = IBT_WRC_RECV;
1450*17a2b317SBill Taylor flags |= IBT_WC_RKEY_INVALIDATED;
1451*17a2b317SBill Taylor wc->wc_rkey = wc->wc_immed_data; /* same field in cqe */
1452*17a2b317SBill Taylor break;
1453*17a2b317SBill Taylor
14549e39c5baSBill Taylor case HERMON_CQE_RCV_RDMAWR_IMM:
14559e39c5baSBill Taylor flags |= IBT_WC_IMMED_DATA_PRESENT;
14569e39c5baSBill Taylor type = IBT_WRC_RECV_RDMAWI;
14579e39c5baSBill Taylor break;
14589e39c5baSBill Taylor
14599e39c5baSBill Taylor default:
14609e39c5baSBill Taylor
14619e39c5baSBill Taylor HERMON_WARNING(state, "unknown recv CQE type");
14629e39c5baSBill Taylor wc->wc_status = IBT_WC_LOCAL_QP_OP_ERR;
14639e39c5baSBill Taylor return;
14649e39c5baSBill Taylor }
14659e39c5baSBill Taylor }
14669e39c5baSBill Taylor wc->wc_type = type;
14679e39c5baSBill Taylor wc->wc_flags = flags;
14689e39c5baSBill Taylor wc->wc_status = IBT_WC_SUCCESS;
14699e39c5baSBill Taylor }
14709e39c5baSBill Taylor
14719e39c5baSBill Taylor /*
14729e39c5baSBill Taylor * hermon_cq_errcqe_consume()
14739e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
14749e39c5baSBill Taylor */
14759e39c5baSBill Taylor static void
hermon_cq_errcqe_consume(hermon_state_t * state,hermon_cqhdl_t cq,hermon_hw_cqe_t * cqe,ibt_wc_t * wc)14769e39c5baSBill Taylor hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
14779e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc)
14789e39c5baSBill Taylor {
14799e39c5baSBill Taylor uint32_t imm_eth_pkey_cred;
14809e39c5baSBill Taylor uint_t status;
14819e39c5baSBill Taylor ibt_wc_status_t ibt_status;
14829e39c5baSBill Taylor
14839e39c5baSBill Taylor /*
14849e39c5baSBill Taylor * Fetch the Work Request ID using the information in the CQE.
14859e39c5baSBill Taylor * See hermon_wr.c for more details.
14869e39c5baSBill Taylor */
14879e39c5baSBill Taylor wc->wc_id = hermon_wrid_get_entry(cq, cqe);
14889e39c5baSBill Taylor
14899e39c5baSBill Taylor /*
14909e39c5baSBill Taylor * Parse the CQE opcode to determine completion type. We know that
14919e39c5baSBill Taylor * the CQE is an error completion, so we extract only the completion
14929e39c5baSBill Taylor * status/syndrome here.
14939e39c5baSBill Taylor */
14949e39c5baSBill Taylor imm_eth_pkey_cred = HERMON_CQE_ERROR_SYNDROME_GET(cq, cqe);
14959e39c5baSBill Taylor status = imm_eth_pkey_cred;
1496c7facc54SBill Taylor if (status != HERMON_CQE_WR_FLUSHED_ERR)
1497*17a2b317SBill Taylor IBTF_DPRINTF_L2("CQE ERR", "cqe %p QPN %x indx %x status 0x%x "
1498*17a2b317SBill Taylor "vendor syndrome %x", cqe, HERMON_CQE_QPNUM_GET(cq, cqe),
1499*17a2b317SBill Taylor HERMON_CQE_WQECNTR_GET(cq, cqe), status,
1500*17a2b317SBill Taylor HERMON_CQE_ERROR_VENDOR_SYNDROME_GET(cq, cqe));
15019e39c5baSBill Taylor switch (status) {
15029e39c5baSBill Taylor case HERMON_CQE_LOC_LEN_ERR:
1503*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCLEN);
15049e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_LEN_ERR;
15059e39c5baSBill Taylor break;
15069e39c5baSBill Taylor
15079e39c5baSBill Taylor case HERMON_CQE_LOC_OP_ERR:
1508*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCQPOP);
15099e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_QP_OP_ERR;
15109e39c5baSBill Taylor break;
15119e39c5baSBill Taylor
15129e39c5baSBill Taylor case HERMON_CQE_LOC_PROT_ERR:
1513*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCPROT);
15149e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_PROTECT_ERR;
1515*17a2b317SBill Taylor IBTF_DPRINTF_L2("ERRCQE", "is at %p", cqe);
15169e39c5baSBill Taylor if (hermon_should_panic) {
15179e39c5baSBill Taylor cmn_err(CE_PANIC, "Hermon intentional PANIC - "
15189e39c5baSBill Taylor "Local Protection Error\n");
15199e39c5baSBill Taylor }
15209e39c5baSBill Taylor break;
15219e39c5baSBill Taylor
15229e39c5baSBill Taylor case HERMON_CQE_WR_FLUSHED_ERR:
15239e39c5baSBill Taylor ibt_status = IBT_WC_WR_FLUSHED_ERR;
15249e39c5baSBill Taylor break;
15259e39c5baSBill Taylor
15269e39c5baSBill Taylor case HERMON_CQE_MW_BIND_ERR:
1527*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_MWBIND);
15289e39c5baSBill Taylor ibt_status = IBT_WC_MEM_WIN_BIND_ERR;
15299e39c5baSBill Taylor break;
15309e39c5baSBill Taylor
15319e39c5baSBill Taylor case HERMON_CQE_BAD_RESPONSE_ERR:
1532*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_RESP);
15339e39c5baSBill Taylor ibt_status = IBT_WC_BAD_RESPONSE_ERR;
15349e39c5baSBill Taylor break;
15359e39c5baSBill Taylor
15369e39c5baSBill Taylor case HERMON_CQE_LOCAL_ACCESS_ERR:
1537*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCACC);
15389e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_ACCESS_ERR;
15399e39c5baSBill Taylor break;
15409e39c5baSBill Taylor
15419e39c5baSBill Taylor case HERMON_CQE_REM_INV_REQ_ERR:
1542*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_REMREQ);
15439e39c5baSBill Taylor ibt_status = IBT_WC_REMOTE_INVALID_REQ_ERR;
15449e39c5baSBill Taylor break;
15459e39c5baSBill Taylor
15469e39c5baSBill Taylor case HERMON_CQE_REM_ACC_ERR:
1547*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_REMACC);
15489e39c5baSBill Taylor ibt_status = IBT_WC_REMOTE_ACCESS_ERR;
15499e39c5baSBill Taylor break;
15509e39c5baSBill Taylor
15519e39c5baSBill Taylor case HERMON_CQE_REM_OP_ERR:
1552*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_REMOP);
15539e39c5baSBill Taylor ibt_status = IBT_WC_REMOTE_OP_ERR;
15549e39c5baSBill Taylor break;
15559e39c5baSBill Taylor
15569e39c5baSBill Taylor case HERMON_CQE_TRANS_TO_ERR:
1557*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_XPORTCNT);
15589e39c5baSBill Taylor ibt_status = IBT_WC_TRANS_TIMEOUT_ERR;
15599e39c5baSBill Taylor break;
15609e39c5baSBill Taylor
15619e39c5baSBill Taylor case HERMON_CQE_RNRNAK_TO_ERR:
1562*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_RNRCNT);
15639e39c5baSBill Taylor ibt_status = IBT_WC_RNR_NAK_TIMEOUT_ERR;
15649e39c5baSBill Taylor break;
15659e39c5baSBill Taylor
15669e39c5baSBill Taylor /*
15679e39c5baSBill Taylor * The following error codes are not supported in the Hermon driver
15689e39c5baSBill Taylor * as they relate only to Reliable Datagram completion statuses:
15699e39c5baSBill Taylor * case HERMON_CQE_LOCAL_RDD_VIO_ERR:
15709e39c5baSBill Taylor * case HERMON_CQE_REM_INV_RD_REQ_ERR:
15719e39c5baSBill Taylor * case HERMON_CQE_EEC_REM_ABORTED_ERR:
15729e39c5baSBill Taylor * case HERMON_CQE_INV_EEC_NUM_ERR:
15739e39c5baSBill Taylor * case HERMON_CQE_INV_EEC_STATE_ERR:
15749e39c5baSBill Taylor * case HERMON_CQE_LOC_EEC_ERR:
15759e39c5baSBill Taylor */
15769e39c5baSBill Taylor
15779e39c5baSBill Taylor default:
15789e39c5baSBill Taylor HERMON_WARNING(state, "unknown error CQE status");
15799e39c5baSBill Taylor HERMON_FMANOTE(state, HERMON_FMA_UNKN);
15809e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_QP_OP_ERR;
15819e39c5baSBill Taylor break;
15829e39c5baSBill Taylor }
15839e39c5baSBill Taylor
15849e39c5baSBill Taylor wc->wc_status = ibt_status;
15859e39c5baSBill Taylor }
15869e39c5baSBill Taylor
15879e39c5baSBill Taylor
15889e39c5baSBill Taylor /*
15899e39c5baSBill Taylor * hermon_cq_resize_helper()
15909e39c5baSBill Taylor * Context: Can be called only from user or kernel context.
15919e39c5baSBill Taylor */
15929e39c5baSBill Taylor void
hermon_cq_resize_helper(hermon_state_t * state,hermon_cqhdl_t cq)15939e39c5baSBill Taylor hermon_cq_resize_helper(hermon_state_t *state, hermon_cqhdl_t cq)
15949e39c5baSBill Taylor {
15959e39c5baSBill Taylor hermon_cqhdl_t resize_hdl;
15969e39c5baSBill Taylor int status;
15979e39c5baSBill Taylor
15989e39c5baSBill Taylor /*
15999e39c5baSBill Taylor * we're here because we found the special cqe opcode, so we have
16009e39c5baSBill Taylor * to update the cq_handle, release the old resources, clear the
16019e39c5baSBill Taylor * flag in the cq_hdl, and release the resize_hdl. When we return
16029e39c5baSBill Taylor * above, it will take care of the rest
16039e39c5baSBill Taylor */
16049e39c5baSBill Taylor ASSERT(MUTEX_HELD(&cq->cq_lock));
16059e39c5baSBill Taylor
16069e39c5baSBill Taylor resize_hdl = cq->cq_resize_hdl;
16079e39c5baSBill Taylor
16089e39c5baSBill Taylor /*
16099e39c5baSBill Taylor * Deregister the memory for the old Completion Queue. Note: We
16109e39c5baSBill Taylor * really can't return error here because we have no good way to
16119e39c5baSBill Taylor * cleanup. Plus, the deregistration really shouldn't ever happen.
16129e39c5baSBill Taylor * So, if it does, it is an indication that something has gone
16139e39c5baSBill Taylor * seriously wrong. So we print a warning message and return error
16149e39c5baSBill Taylor * (knowing, of course, that the "old" CQ memory will be leaked)
16159e39c5baSBill Taylor */
16169e39c5baSBill Taylor status = hermon_mr_deregister(state, &cq->cq_mrhdl, HERMON_MR_DEREG_ALL,
16179e39c5baSBill Taylor HERMON_SLEEP);
16189e39c5baSBill Taylor if (status != DDI_SUCCESS) {
16199e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister old CQ memory");
16209e39c5baSBill Taylor }
16219e39c5baSBill Taylor
16229e39c5baSBill Taylor /* Next, free the memory from the old CQ buffer */
16239e39c5baSBill Taylor hermon_queue_free(&cq->cq_cqinfo);
16249e39c5baSBill Taylor
16259e39c5baSBill Taylor /* now we can update the cq_hdl with the new things saved */
16269e39c5baSBill Taylor
16279e39c5baSBill Taylor cq->cq_buf = resize_hdl->cq_buf;
16289e39c5baSBill Taylor cq->cq_mrhdl = resize_hdl->cq_mrhdl;
16299e39c5baSBill Taylor cq->cq_bufsz = resize_hdl->cq_bufsz;
16309e39c5baSBill Taylor cq->cq_log_cqsz = resize_hdl->cq_log_cqsz;
16319e39c5baSBill Taylor cq->cq_umap_dhp = cq->cq_resize_hdl->cq_umap_dhp;
16329e39c5baSBill Taylor cq->cq_resize_hdl = 0;
16339e39c5baSBill Taylor bcopy(&resize_hdl->cq_cqinfo, &cq->cq_cqinfo,
16349e39c5baSBill Taylor sizeof (struct hermon_qalloc_info_s));
16359e39c5baSBill Taylor
16369e39c5baSBill Taylor /* finally, release the resizing handle */
16379e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
16389e39c5baSBill Taylor }
16399e39c5baSBill Taylor
16409e39c5baSBill Taylor
16419e39c5baSBill Taylor /*
16429e39c5baSBill Taylor * hermon_cq_entries_flush()
16439e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
16449e39c5baSBill Taylor */
16459e39c5baSBill Taylor /* ARGSUSED */
16469e39c5baSBill Taylor void
hermon_cq_entries_flush(hermon_state_t * state,hermon_qphdl_t qp)16479e39c5baSBill Taylor hermon_cq_entries_flush(hermon_state_t *state, hermon_qphdl_t qp)
16489e39c5baSBill Taylor {
16499e39c5baSBill Taylor hermon_cqhdl_t cq;
16509e39c5baSBill Taylor hermon_hw_cqe_t *cqe, *next_cqe;
16519e39c5baSBill Taylor hermon_srqhdl_t srq;
16529e39c5baSBill Taylor hermon_workq_hdr_t *wq;
16539e39c5baSBill Taylor uint32_t cons_indx, tail_cons_indx, wrap_around_mask;
16549e39c5baSBill Taylor uint32_t new_indx, check_indx, qpnum;
1655*17a2b317SBill Taylor uint32_t shift, mask;
16569e39c5baSBill Taylor int outstanding_cqes;
16579e39c5baSBill Taylor
16589e39c5baSBill Taylor qpnum = qp->qp_qpnum;
16599e39c5baSBill Taylor if ((srq = qp->qp_srqhdl) != NULL)
16609e39c5baSBill Taylor wq = qp->qp_srqhdl->srq_wq_wqhdr;
16619e39c5baSBill Taylor else
16629e39c5baSBill Taylor wq = NULL;
16639e39c5baSBill Taylor cq = qp->qp_rq_cqhdl;
16649e39c5baSBill Taylor
1665*17a2b317SBill Taylor if (cq == NULL) {
1666*17a2b317SBill Taylor cq = qp->qp_sq_cqhdl;
1667*17a2b317SBill Taylor }
1668*17a2b317SBill Taylor
16699e39c5baSBill Taylor do_send_cq: /* loop back to here if send_cq is not the same as recv_cq */
1670*17a2b317SBill Taylor if (cq == NULL)
1671*17a2b317SBill Taylor return;
16729e39c5baSBill Taylor
16739e39c5baSBill Taylor cons_indx = cq->cq_consindx;
1674*17a2b317SBill Taylor shift = cq->cq_log_cqsz;
1675*17a2b317SBill Taylor mask = cq->cq_bufsz;
1676*17a2b317SBill Taylor wrap_around_mask = mask - 1;
16779e39c5baSBill Taylor
16789e39c5baSBill Taylor /* Calculate the pointer to the first CQ entry */
16799e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
16809e39c5baSBill Taylor
16819e39c5baSBill Taylor /*
16829e39c5baSBill Taylor * Loop through the CQ looking for entries owned by software. If an
16839e39c5baSBill Taylor * entry is owned by software then we increment an 'outstanding_cqes'
16849e39c5baSBill Taylor * count to know how many entries total we have on our CQ. We use this
16859e39c5baSBill Taylor * value further down to know how many entries to loop through looking
16869e39c5baSBill Taylor * for our same QP number.
16879e39c5baSBill Taylor */
16889e39c5baSBill Taylor outstanding_cqes = 0;
16899e39c5baSBill Taylor tail_cons_indx = cons_indx;
1690*17a2b317SBill Taylor while (HERMON_CQE_OWNER_IS_SW(cq, cqe, tail_cons_indx, shift, mask)) {
16919e39c5baSBill Taylor /* increment total cqes count */
16929e39c5baSBill Taylor outstanding_cqes++;
16939e39c5baSBill Taylor
16949e39c5baSBill Taylor /* increment the consumer index */
16959e39c5baSBill Taylor tail_cons_indx++;
16969e39c5baSBill Taylor
16979e39c5baSBill Taylor /* update the pointer to the next cq entry */
16989e39c5baSBill Taylor cqe = &cq->cq_buf[tail_cons_indx & wrap_around_mask];
16999e39c5baSBill Taylor }
17009e39c5baSBill Taylor
17019e39c5baSBill Taylor /*
17029e39c5baSBill Taylor * Using the 'tail_cons_indx' that was just set, we now know how many
17039e39c5baSBill Taylor * total CQEs possible there are. Set the 'check_indx' and the
17049e39c5baSBill Taylor * 'new_indx' to the last entry identified by 'tail_cons_indx'
17059e39c5baSBill Taylor */
17069e39c5baSBill Taylor check_indx = new_indx = (tail_cons_indx - 1);
17079e39c5baSBill Taylor
17089e39c5baSBill Taylor while (--outstanding_cqes >= 0) {
17099e39c5baSBill Taylor cqe = &cq->cq_buf[check_indx & wrap_around_mask];
17109e39c5baSBill Taylor
17119e39c5baSBill Taylor /*
17129e39c5baSBill Taylor * If the QP number is the same in the CQE as the QP, then
17139e39c5baSBill Taylor * we must "consume" it. If it is for an SRQ wqe, then we
17149e39c5baSBill Taylor * also must free the wqe back onto the free list of the SRQ.
17159e39c5baSBill Taylor */
17169e39c5baSBill Taylor if (qpnum == HERMON_CQE_QPNUM_GET(cq, cqe)) {
17179e39c5baSBill Taylor if (srq && (HERMON_CQE_SENDRECV_GET(cq, cqe) ==
17189e39c5baSBill Taylor HERMON_COMPLETION_RECV)) {
17199e39c5baSBill Taylor uint64_t *desc;
17209e39c5baSBill Taylor int indx;
17219e39c5baSBill Taylor
17229e39c5baSBill Taylor /* Add wqe back to SRQ free list */
17239e39c5baSBill Taylor indx = HERMON_CQE_WQEADDRSZ_GET(cq, cqe) &
17249e39c5baSBill Taylor wq->wq_mask;
17259e39c5baSBill Taylor desc = HERMON_SRQ_WQE_ADDR(srq, wq->wq_tail);
17269e39c5baSBill Taylor ((uint16_t *)desc)[1] = htons(indx);
17279e39c5baSBill Taylor wq->wq_tail = indx;
17289e39c5baSBill Taylor }
17299e39c5baSBill Taylor } else { /* CQEs for other QPNs need to remain */
17309e39c5baSBill Taylor if (check_indx != new_indx) {
17319e39c5baSBill Taylor next_cqe =
17329e39c5baSBill Taylor &cq->cq_buf[new_indx & wrap_around_mask];
17339e39c5baSBill Taylor /* Copy the CQE into the "next_cqe" pointer. */
17349e39c5baSBill Taylor bcopy(cqe, next_cqe, sizeof (hermon_hw_cqe_t));
17359e39c5baSBill Taylor }
17369e39c5baSBill Taylor new_indx--; /* move index to next CQE to fill */
17379e39c5baSBill Taylor }
17389e39c5baSBill Taylor check_indx--; /* move index to next CQE to check */
17399e39c5baSBill Taylor }
17409e39c5baSBill Taylor
17419e39c5baSBill Taylor /*
17429e39c5baSBill Taylor * Update consumer index to be the 'new_indx'. This moves it past all
17439e39c5baSBill Taylor * removed entries. Because 'new_indx' is pointing to the last
17449e39c5baSBill Taylor * previously valid SW owned entry, we add 1 to point the cons_indx to
17459e39c5baSBill Taylor * the first HW owned entry.
17469e39c5baSBill Taylor */
17479e39c5baSBill Taylor cons_indx = (new_indx + 1);
17489e39c5baSBill Taylor
17499e39c5baSBill Taylor /*
17509e39c5baSBill Taylor * Now we only ring the doorbell (to update the consumer index) if
17519e39c5baSBill Taylor * we've actually consumed a CQ entry. If we found no QP number
17529e39c5baSBill Taylor * matches above, then we would not have removed anything. So only if
17539e39c5baSBill Taylor * something was removed do we ring the doorbell.
17549e39c5baSBill Taylor */
17559e39c5baSBill Taylor if (cq->cq_consindx != cons_indx) {
17569e39c5baSBill Taylor /*
17579e39c5baSBill Taylor * Update the consumer index in both the CQ handle and the
17589e39c5baSBill Taylor * doorbell record.
17599e39c5baSBill Taylor */
17609e39c5baSBill Taylor cq->cq_consindx = cons_indx;
17619e39c5baSBill Taylor
17629e39c5baSBill Taylor hermon_cq_update_ci_doorbell(cq);
17639e39c5baSBill Taylor
17649e39c5baSBill Taylor }
17659e39c5baSBill Taylor if (cq != qp->qp_sq_cqhdl) {
17669e39c5baSBill Taylor cq = qp->qp_sq_cqhdl;
17679e39c5baSBill Taylor goto do_send_cq;
17689e39c5baSBill Taylor }
17699e39c5baSBill Taylor }
1770*17a2b317SBill Taylor
1771*17a2b317SBill Taylor /*
1772*17a2b317SBill Taylor * hermon_get_cq_sched_list()
1773*17a2b317SBill Taylor * Context: Only called from attach() path context
1774*17a2b317SBill Taylor *
1775*17a2b317SBill Taylor * Read properties, creating entries in hs_cq_sched_list with
1776*17a2b317SBill Taylor * information about the requested "expected" and "minimum"
1777*17a2b317SBill Taylor * number of MSI-X interrupt vectors per list entry.
1778*17a2b317SBill Taylor */
1779*17a2b317SBill Taylor static int
hermon_get_cq_sched_list(hermon_state_t * state)1780*17a2b317SBill Taylor hermon_get_cq_sched_list(hermon_state_t *state)
1781*17a2b317SBill Taylor {
1782*17a2b317SBill Taylor char **listp, ulp_prop[HERMON_CQH_MAX + 4];
1783*17a2b317SBill Taylor uint_t nlist, i, j, ndata;
1784*17a2b317SBill Taylor int *data;
1785*17a2b317SBill Taylor size_t len;
1786*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp;
1787*17a2b317SBill Taylor
1788*17a2b317SBill Taylor if (ddi_prop_lookup_string_array(DDI_DEV_T_ANY, state->hs_dip,
1789*17a2b317SBill Taylor DDI_PROP_DONTPASS, "cqh-group-list", &listp, &nlist) !=
1790*17a2b317SBill Taylor DDI_PROP_SUCCESS)
1791*17a2b317SBill Taylor return (0);
1792*17a2b317SBill Taylor
1793*17a2b317SBill Taylor state->hs_cq_sched_array_size = nlist;
1794*17a2b317SBill Taylor state->hs_cq_sched_array = cq_schedp = kmem_zalloc(nlist *
1795*17a2b317SBill Taylor sizeof (hermon_cq_sched_t), KM_SLEEP);
1796*17a2b317SBill Taylor for (i = 0; i < nlist; i++) {
1797*17a2b317SBill Taylor if ((len = strlen(listp[i])) >= HERMON_CQH_MAX) {
1798*17a2b317SBill Taylor cmn_err(CE_CONT, "'cqh' property name too long\n");
1799*17a2b317SBill Taylor goto game_over;
1800*17a2b317SBill Taylor }
1801*17a2b317SBill Taylor for (j = 0; j < i; j++) {
1802*17a2b317SBill Taylor if (strcmp(listp[j], listp[i]) == 0) {
1803*17a2b317SBill Taylor cmn_err(CE_CONT, "Duplicate 'cqh' property\n");
1804*17a2b317SBill Taylor goto game_over;
1805*17a2b317SBill Taylor }
1806*17a2b317SBill Taylor }
1807*17a2b317SBill Taylor (void) strncpy(cq_schedp[i].cqs_name, listp[i], HERMON_CQH_MAX);
1808*17a2b317SBill Taylor ulp_prop[0] = 'c';
1809*17a2b317SBill Taylor ulp_prop[1] = 'q';
1810*17a2b317SBill Taylor ulp_prop[2] = 'h';
1811*17a2b317SBill Taylor ulp_prop[3] = '-';
1812*17a2b317SBill Taylor (void) strncpy(ulp_prop + 4, listp[i], len + 1);
1813*17a2b317SBill Taylor if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip,
1814*17a2b317SBill Taylor DDI_PROP_DONTPASS, ulp_prop, &data, &ndata) !=
1815*17a2b317SBill Taylor DDI_PROP_SUCCESS) {
1816*17a2b317SBill Taylor cmn_err(CE_CONT, "property '%s' not found\n", ulp_prop);
1817*17a2b317SBill Taylor goto game_over;
1818*17a2b317SBill Taylor }
1819*17a2b317SBill Taylor if (ndata != 2) {
1820*17a2b317SBill Taylor cmn_err(CE_CONT, "property '%s' does not "
1821*17a2b317SBill Taylor "have 2 integers\n", ulp_prop);
1822*17a2b317SBill Taylor goto game_over_free_data;
1823*17a2b317SBill Taylor }
1824*17a2b317SBill Taylor cq_schedp[i].cqs_desired = data[0];
1825*17a2b317SBill Taylor cq_schedp[i].cqs_minimum = data[1];
1826*17a2b317SBill Taylor cq_schedp[i].cqs_refcnt = 0;
1827*17a2b317SBill Taylor ddi_prop_free(data);
1828*17a2b317SBill Taylor }
1829*17a2b317SBill Taylor if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip,
1830*17a2b317SBill Taylor DDI_PROP_DONTPASS, "cqh-default", &data, &ndata) !=
1831*17a2b317SBill Taylor DDI_PROP_SUCCESS) {
1832*17a2b317SBill Taylor cmn_err(CE_CONT, "property 'cqh-default' not found\n");
1833*17a2b317SBill Taylor goto game_over;
1834*17a2b317SBill Taylor }
1835*17a2b317SBill Taylor if (ndata != 2) {
1836*17a2b317SBill Taylor cmn_err(CE_CONT, "property 'cqh-default' does not "
1837*17a2b317SBill Taylor "have 2 integers\n");
1838*17a2b317SBill Taylor goto game_over_free_data;
1839*17a2b317SBill Taylor }
1840*17a2b317SBill Taylor cq_schedp = &state->hs_cq_sched_default;
1841*17a2b317SBill Taylor cq_schedp->cqs_desired = data[0];
1842*17a2b317SBill Taylor cq_schedp->cqs_minimum = data[1];
1843*17a2b317SBill Taylor cq_schedp->cqs_refcnt = 0;
1844*17a2b317SBill Taylor ddi_prop_free(data);
1845*17a2b317SBill Taylor ddi_prop_free(listp);
1846*17a2b317SBill Taylor return (1); /* game on */
1847*17a2b317SBill Taylor
1848*17a2b317SBill Taylor game_over_free_data:
1849*17a2b317SBill Taylor ddi_prop_free(data);
1850*17a2b317SBill Taylor game_over:
1851*17a2b317SBill Taylor cmn_err(CE_CONT, "Error in 'cqh' properties in hermon.conf\n");
1852*17a2b317SBill Taylor cmn_err(CE_CONT, "completion handler groups not being used\n");
1853*17a2b317SBill Taylor kmem_free(cq_schedp, nlist * sizeof (hermon_cq_sched_t));
1854*17a2b317SBill Taylor state->hs_cq_sched_array_size = 0;
1855*17a2b317SBill Taylor ddi_prop_free(listp);
1856*17a2b317SBill Taylor return (0);
1857*17a2b317SBill Taylor }
1858*17a2b317SBill Taylor
1859*17a2b317SBill Taylor /*
1860*17a2b317SBill Taylor * hermon_cq_sched_init()
1861*17a2b317SBill Taylor * Context: Only called from attach() path context
1862*17a2b317SBill Taylor *
1863*17a2b317SBill Taylor * Read the hermon.conf properties looking for cq_sched info,
1864*17a2b317SBill Taylor * creating reserved pools of MSI-X interrupt ranges for the
1865*17a2b317SBill Taylor * specified ULPs.
1866*17a2b317SBill Taylor */
1867*17a2b317SBill Taylor int
hermon_cq_sched_init(hermon_state_t * state)1868*17a2b317SBill Taylor hermon_cq_sched_init(hermon_state_t *state)
1869*17a2b317SBill Taylor {
1870*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp, *defp;
1871*17a2b317SBill Taylor int i, desired, array_size;
1872*17a2b317SBill Taylor
1873*17a2b317SBill Taylor mutex_init(&state->hs_cq_sched_lock, NULL, MUTEX_DRIVER,
1874*17a2b317SBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri));
1875*17a2b317SBill Taylor
1876*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock);
1877*17a2b317SBill Taylor state->hs_cq_sched_array = NULL;
1878*17a2b317SBill Taylor
1879*17a2b317SBill Taylor /* initialize cq_sched_default */
1880*17a2b317SBill Taylor defp = &state->hs_cq_sched_default;
1881*17a2b317SBill Taylor defp->cqs_start_hid = 1;
1882*17a2b317SBill Taylor defp->cqs_len = state->hs_intrmsi_allocd;
1883*17a2b317SBill Taylor defp->cqs_next_alloc = defp->cqs_len - 1;
1884*17a2b317SBill Taylor (void) strncpy(defp->cqs_name, "default", 8);
1885*17a2b317SBill Taylor
1886*17a2b317SBill Taylor /* Read properties to determine which ULPs use cq_sched */
1887*17a2b317SBill Taylor if (hermon_get_cq_sched_list(state) == 0)
1888*17a2b317SBill Taylor goto done;
1889*17a2b317SBill Taylor
1890*17a2b317SBill Taylor /* Determine if we have enough vectors, or if we have to scale down */
1891*17a2b317SBill Taylor desired = defp->cqs_desired; /* default desired (from hermon.conf) */
1892*17a2b317SBill Taylor if (desired <= 0)
1893*17a2b317SBill Taylor goto done; /* all interrupts in the default pool */
1894*17a2b317SBill Taylor cq_schedp = state->hs_cq_sched_array;
1895*17a2b317SBill Taylor array_size = state->hs_cq_sched_array_size;
1896*17a2b317SBill Taylor for (i = 0; i < array_size; i++)
1897*17a2b317SBill Taylor desired += cq_schedp[i].cqs_desired;
1898*17a2b317SBill Taylor if (desired > state->hs_intrmsi_allocd) {
1899*17a2b317SBill Taylor cmn_err(CE_CONT, "#interrupts allocated (%d) is less than "
1900*17a2b317SBill Taylor "the #interrupts desired (%d)\n",
1901*17a2b317SBill Taylor state->hs_intrmsi_allocd, desired);
1902*17a2b317SBill Taylor cmn_err(CE_CONT, "completion handler groups not being used\n");
1903*17a2b317SBill Taylor goto done; /* all interrupts in the default pool */
1904*17a2b317SBill Taylor }
1905*17a2b317SBill Taylor /* Game on. For each cq_sched group, reserve the MSI-X range */
1906*17a2b317SBill Taylor for (i = 0; i < array_size; i++) {
1907*17a2b317SBill Taylor desired = cq_schedp[i].cqs_desired;
1908*17a2b317SBill Taylor cq_schedp[i].cqs_start_hid = defp->cqs_start_hid;
1909*17a2b317SBill Taylor cq_schedp[i].cqs_len = desired;
1910*17a2b317SBill Taylor cq_schedp[i].cqs_next_alloc = desired - 1;
1911*17a2b317SBill Taylor defp->cqs_len -= desired;
1912*17a2b317SBill Taylor defp->cqs_start_hid += desired;
1913*17a2b317SBill Taylor }
1914*17a2b317SBill Taylor /* reset default's start allocation seed */
1915*17a2b317SBill Taylor state->hs_cq_sched_default.cqs_next_alloc =
1916*17a2b317SBill Taylor state->hs_cq_sched_default.cqs_len - 1;
1917*17a2b317SBill Taylor
1918*17a2b317SBill Taylor done:
1919*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock);
1920*17a2b317SBill Taylor return (IBT_SUCCESS);
1921*17a2b317SBill Taylor }
1922*17a2b317SBill Taylor
1923*17a2b317SBill Taylor void
hermon_cq_sched_fini(hermon_state_t * state)1924*17a2b317SBill Taylor hermon_cq_sched_fini(hermon_state_t *state)
1925*17a2b317SBill Taylor {
1926*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock);
1927*17a2b317SBill Taylor if (state->hs_cq_sched_array_size) {
1928*17a2b317SBill Taylor kmem_free(state->hs_cq_sched_array, sizeof (hermon_cq_sched_t) *
1929*17a2b317SBill Taylor state->hs_cq_sched_array_size);
1930*17a2b317SBill Taylor state->hs_cq_sched_array_size = 0;
1931*17a2b317SBill Taylor state->hs_cq_sched_array = NULL;
1932*17a2b317SBill Taylor }
1933*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock);
1934*17a2b317SBill Taylor mutex_destroy(&state->hs_cq_sched_lock);
1935*17a2b317SBill Taylor }
1936*17a2b317SBill Taylor
1937*17a2b317SBill Taylor int
hermon_cq_sched_alloc(hermon_state_t * state,ibt_cq_sched_attr_t * attr,hermon_cq_sched_t ** cq_sched_pp)1938*17a2b317SBill Taylor hermon_cq_sched_alloc(hermon_state_t *state, ibt_cq_sched_attr_t *attr,
1939*17a2b317SBill Taylor hermon_cq_sched_t **cq_sched_pp)
1940*17a2b317SBill Taylor {
1941*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp;
1942*17a2b317SBill Taylor int i;
1943*17a2b317SBill Taylor char *name;
1944*17a2b317SBill Taylor ibt_cq_sched_flags_t flags;
1945*17a2b317SBill Taylor
1946*17a2b317SBill Taylor flags = attr->cqs_flags;
1947*17a2b317SBill Taylor if ((flags & (IBT_CQS_SCHED_GROUP | IBT_CQS_EXACT_SCHED_GROUP)) == 0) {
1948*17a2b317SBill Taylor *cq_sched_pp = NULL;
1949*17a2b317SBill Taylor return (IBT_SUCCESS);
1950*17a2b317SBill Taylor }
1951*17a2b317SBill Taylor name = attr->cqs_pool_name;
1952*17a2b317SBill Taylor
1953*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock);
1954*17a2b317SBill Taylor cq_schedp = state->hs_cq_sched_array;
1955*17a2b317SBill Taylor for (i = 0; i < state->hs_cq_sched_array_size; i++, cq_schedp++) {
1956*17a2b317SBill Taylor if (strcmp(name, cq_schedp->cqs_name) == 0) {
1957*17a2b317SBill Taylor if (cq_schedp->cqs_len != 0)
1958*17a2b317SBill Taylor cq_schedp->cqs_refcnt++;
1959*17a2b317SBill Taylor break; /* found it */
1960*17a2b317SBill Taylor }
1961*17a2b317SBill Taylor }
1962*17a2b317SBill Taylor if ((i == state->hs_cq_sched_array_size) || /* not found, or */
1963*17a2b317SBill Taylor (cq_schedp->cqs_len == 0)) /* defined, but no dedicated intr's */
1964*17a2b317SBill Taylor cq_schedp = NULL;
1965*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock);
1966*17a2b317SBill Taylor
1967*17a2b317SBill Taylor *cq_sched_pp = cq_schedp; /* set to valid hdl, or to NULL */
1968*17a2b317SBill Taylor if ((cq_schedp == NULL) &&
1969*17a2b317SBill Taylor (attr->cqs_flags & IBT_CQS_EXACT_SCHED_GROUP))
1970*17a2b317SBill Taylor return (IBT_CQ_NO_SCHED_GROUP);
1971*17a2b317SBill Taylor else
1972*17a2b317SBill Taylor return (IBT_SUCCESS);
1973*17a2b317SBill Taylor }
1974*17a2b317SBill Taylor
1975*17a2b317SBill Taylor int
hermon_cq_sched_free(hermon_state_t * state,hermon_cq_sched_t * cq_schedp)1976*17a2b317SBill Taylor hermon_cq_sched_free(hermon_state_t *state, hermon_cq_sched_t *cq_schedp)
1977*17a2b317SBill Taylor {
1978*17a2b317SBill Taylor if (cq_schedp != NULL) {
1979*17a2b317SBill Taylor /* Just decrement refcnt */
1980*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock);
1981*17a2b317SBill Taylor if (cq_schedp->cqs_refcnt == 0)
1982*17a2b317SBill Taylor HERMON_WARNING(state, "cq_sched free underflow\n");
1983*17a2b317SBill Taylor else
1984*17a2b317SBill Taylor cq_schedp->cqs_refcnt--;
1985*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock);
1986*17a2b317SBill Taylor }
1987*17a2b317SBill Taylor return (IBT_SUCCESS);
1988*17a2b317SBill Taylor }
1989