19e39c5baSBill Taylor /* 29e39c5baSBill Taylor * CDDL HEADER START 39e39c5baSBill Taylor * 49e39c5baSBill Taylor * The contents of this file are subject to the terms of the 59e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 69e39c5baSBill Taylor * You may not use this file except in compliance with the License. 79e39c5baSBill Taylor * 89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 109e39c5baSBill Taylor * See the License for the specific language governing permissions 119e39c5baSBill Taylor * and limitations under the License. 129e39c5baSBill Taylor * 139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 189e39c5baSBill Taylor * 199e39c5baSBill Taylor * CDDL HEADER END 209e39c5baSBill Taylor */ 219e39c5baSBill Taylor 229e39c5baSBill Taylor /* 23*17a2b317SBill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 249e39c5baSBill Taylor */ 259e39c5baSBill Taylor 269e39c5baSBill Taylor /* 279e39c5baSBill Taylor * hermon_cq.c 289e39c5baSBill Taylor * Hermon Completion Queue Processing Routines 299e39c5baSBill Taylor * 309e39c5baSBill Taylor * Implements all the routines necessary for allocating, freeing, resizing, 319e39c5baSBill Taylor * and handling the completion type events that the Hermon hardware can 329e39c5baSBill Taylor * generate. 339e39c5baSBill Taylor */ 349e39c5baSBill Taylor 359e39c5baSBill Taylor #include <sys/types.h> 369e39c5baSBill Taylor #include <sys/conf.h> 379e39c5baSBill Taylor #include <sys/ddi.h> 389e39c5baSBill Taylor #include <sys/sunddi.h> 399e39c5baSBill Taylor #include <sys/modctl.h> 409e39c5baSBill Taylor #include <sys/bitmap.h> 419e39c5baSBill Taylor #include <sys/sysmacros.h> 429e39c5baSBill Taylor 439e39c5baSBill Taylor #include <sys/ib/adapters/hermon/hermon.h> 449e39c5baSBill Taylor 459e39c5baSBill Taylor int hermon_should_panic = 0; /* debugging aid */ 469e39c5baSBill Taylor 479e39c5baSBill Taylor #define hermon_cq_update_ci_doorbell(cq) \ 489e39c5baSBill Taylor /* Build the doorbell record data (low 24 bits only) */ \ 499e39c5baSBill Taylor HERMON_UAR_DB_RECORD_WRITE(cq->cq_arm_ci_vdbr, \ 509e39c5baSBill Taylor cq->cq_consindx & 0x00FFFFFF) 519e39c5baSBill Taylor 529e39c5baSBill Taylor static int hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq, 539e39c5baSBill Taylor uint_t cmd); 549e39c5baSBill Taylor #pragma inline(hermon_cq_arm_doorbell) 559e39c5baSBill Taylor static void hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr); 569e39c5baSBill Taylor #pragma inline(hermon_arm_cq_dbr_init) 579e39c5baSBill Taylor static void hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq, 589e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc); 599e39c5baSBill Taylor static void hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq, 609e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc); 619e39c5baSBill Taylor 629e39c5baSBill Taylor 639e39c5baSBill Taylor /* 649e39c5baSBill Taylor * hermon_cq_alloc() 659e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 669e39c5baSBill Taylor */ 679e39c5baSBill Taylor int 689e39c5baSBill Taylor hermon_cq_alloc(hermon_state_t *state, ibt_cq_hdl_t ibt_cqhdl, 699e39c5baSBill Taylor ibt_cq_attr_t *cq_attr, uint_t *actual_size, hermon_cqhdl_t *cqhdl, 709e39c5baSBill Taylor uint_t sleepflag) 719e39c5baSBill Taylor { 729e39c5baSBill Taylor hermon_rsrc_t *cqc, *rsrc; 739e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 749e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry; 759e39c5baSBill Taylor hermon_cqhdl_t cq; 769e39c5baSBill Taylor ibt_mr_attr_t mr_attr; 779e39c5baSBill Taylor hermon_mr_options_t op; 789e39c5baSBill Taylor hermon_pdhdl_t pd; 799e39c5baSBill Taylor hermon_mrhdl_t mr; 809e39c5baSBill Taylor hermon_hw_cqe_t *buf; 819e39c5baSBill Taylor uint64_t value; 829e39c5baSBill Taylor uint32_t log_cq_size, uarpg; 839e39c5baSBill Taylor uint_t cq_is_umap; 849e39c5baSBill Taylor uint32_t status, flag; 85*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp; 869e39c5baSBill Taylor 879e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq_attr)) 889e39c5baSBill Taylor 899e39c5baSBill Taylor /* 909e39c5baSBill Taylor * Determine whether CQ is being allocated for userland access or 919e39c5baSBill Taylor * whether it is being allocated for kernel access. If the CQ is 929e39c5baSBill Taylor * being allocated for userland access, then lookup the UAR 939e39c5baSBill Taylor * page number for the current process. Note: If this is not found 949e39c5baSBill Taylor * (e.g. if the process has not previously open()'d the Hermon driver), 959e39c5baSBill Taylor * then an error is returned. 969e39c5baSBill Taylor */ 979e39c5baSBill Taylor cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0; 989e39c5baSBill Taylor if (cq_is_umap) { 999e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(), 1009e39c5baSBill Taylor MLNX_UMAP_UARPG_RSRC, &value, 0, NULL); 1019e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1029e39c5baSBill Taylor status = IBT_INVALID_PARAM; 1039e39c5baSBill Taylor goto cqalloc_fail; 1049e39c5baSBill Taylor } 1059e39c5baSBill Taylor uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx; 1069e39c5baSBill Taylor } else { 1079e39c5baSBill Taylor uarpg = state->hs_kernel_uar_index; 1089e39c5baSBill Taylor } 1099e39c5baSBill Taylor 1109e39c5baSBill Taylor /* Use the internal protection domain (PD) for setting up CQs */ 1119e39c5baSBill Taylor pd = state->hs_pdhdl_internal; 1129e39c5baSBill Taylor 1139e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */ 1149e39c5baSBill Taylor hermon_pd_refcnt_inc(pd); 1159e39c5baSBill Taylor 1169e39c5baSBill Taylor /* 1179e39c5baSBill Taylor * Allocate an CQ context entry. This will be filled in with all 1189e39c5baSBill Taylor * the necessary parameters to define the Completion Queue. And then 1199e39c5baSBill Taylor * ownership will be passed to the hardware in the final step 1209e39c5baSBill Taylor * below. If we fail here, we must undo the protection domain 1219e39c5baSBill Taylor * reference count. 1229e39c5baSBill Taylor */ 1239e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_CQC, 1, sleepflag, &cqc); 1249e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1259e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 1269e39c5baSBill Taylor goto cqalloc_fail1; 1279e39c5baSBill Taylor } 1289e39c5baSBill Taylor 1299e39c5baSBill Taylor /* 1309e39c5baSBill Taylor * Allocate the software structure for tracking the completion queue 1319e39c5baSBill Taylor * (i.e. the Hermon Completion Queue handle). If we fail here, we must 1329e39c5baSBill Taylor * undo the protection domain reference count and the previous 1339e39c5baSBill Taylor * resource allocation. 1349e39c5baSBill Taylor */ 1359e39c5baSBill Taylor status = hermon_rsrc_alloc(state, HERMON_CQHDL, 1, sleepflag, &rsrc); 1369e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1379e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 1389e39c5baSBill Taylor goto cqalloc_fail2; 1399e39c5baSBill Taylor } 1409e39c5baSBill Taylor cq = (hermon_cqhdl_t)rsrc->hr_addr; 1419e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq)) 1429e39c5baSBill Taylor cq->cq_is_umap = cq_is_umap; 1439e39c5baSBill Taylor cq->cq_cqnum = cqc->hr_indx; /* just use index, implicit in Hermon */ 144*17a2b317SBill Taylor cq->cq_intmod_count = 0; 145*17a2b317SBill Taylor cq->cq_intmod_usec = 0; 1469e39c5baSBill Taylor 1479e39c5baSBill Taylor /* 1489e39c5baSBill Taylor * If this will be a user-mappable CQ, then allocate an entry for 1499e39c5baSBill Taylor * the "userland resources database". This will later be added to 1509e39c5baSBill Taylor * the database (after all further CQ operations are successful). 1519e39c5baSBill Taylor * If we fail here, we must undo the reference counts and the 1529e39c5baSBill Taylor * previous resource allocation. 1539e39c5baSBill Taylor */ 1549e39c5baSBill Taylor if (cq->cq_is_umap) { 1559e39c5baSBill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance, cq->cq_cqnum, 1569e39c5baSBill Taylor MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc); 1579e39c5baSBill Taylor if (umapdb == NULL) { 1589e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 1599e39c5baSBill Taylor goto cqalloc_fail3; 1609e39c5baSBill Taylor } 1619e39c5baSBill Taylor } 1629e39c5baSBill Taylor 1639e39c5baSBill Taylor 1649e39c5baSBill Taylor /* 1659e39c5baSBill Taylor * Allocate the doorbell record. We'll need one for the CQ, handling 1669e39c5baSBill Taylor * both consumer index (SET CI) and the CQ state (CQ ARM). 1679e39c5baSBill Taylor */ 1689e39c5baSBill Taylor 1699e39c5baSBill Taylor status = hermon_dbr_alloc(state, uarpg, &cq->cq_arm_ci_dbr_acchdl, 1709e39c5baSBill Taylor &cq->cq_arm_ci_vdbr, &cq->cq_arm_ci_pdbr, &cq->cq_dbr_mapoffset); 1719e39c5baSBill Taylor if (status != DDI_SUCCESS) { 1729e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 1739e39c5baSBill Taylor goto cqalloc_fail4; 1749e39c5baSBill Taylor } 1759e39c5baSBill Taylor 1769e39c5baSBill Taylor /* 1779e39c5baSBill Taylor * Calculate the appropriate size for the completion queue. 1789e39c5baSBill Taylor * Note: All Hermon CQs must be a power-of-2 minus 1 in size. Also 1799e39c5baSBill Taylor * they may not be any smaller than HERMON_CQ_MIN_SIZE. This step is 1809e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2 1819e39c5baSBill Taylor */ 1829e39c5baSBill Taylor cq_attr->cq_size = max(cq_attr->cq_size, HERMON_CQ_MIN_SIZE); 1839e39c5baSBill Taylor log_cq_size = highbit(cq_attr->cq_size); 1849e39c5baSBill Taylor 1859e39c5baSBill Taylor /* 1869e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent 1879e39c5baSBill Taylor * with the device limits and/or software-configured limits) 1889e39c5baSBill Taylor */ 1899e39c5baSBill Taylor if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) { 1909e39c5baSBill Taylor status = IBT_HCA_CQ_EXCEEDED; 1919e39c5baSBill Taylor goto cqalloc_fail4a; 1929e39c5baSBill Taylor } 1939e39c5baSBill Taylor 1949e39c5baSBill Taylor /* 1959e39c5baSBill Taylor * Allocate the memory for Completion Queue. 1969e39c5baSBill Taylor * 1979e39c5baSBill Taylor * Note: Although we use the common queue allocation routine, we 1989e39c5baSBill Taylor * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in 1999e39c5baSBill Taylor * kernel system memory) for kernel CQs because it would be 2009e39c5baSBill Taylor * inefficient to have CQs located in DDR memory. This is primarily 2019e39c5baSBill Taylor * because CQs are read from (by software) more than they are written 2029e39c5baSBill Taylor * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all 2039e39c5baSBill Taylor * user-mappable CQs for a similar reason.) 2049e39c5baSBill Taylor * It is also worth noting that, unlike Hermon QP work queues, 2059e39c5baSBill Taylor * completion queues do not have the same strict alignment 2069e39c5baSBill Taylor * requirements. It is sufficient for the CQ memory to be both 2079e39c5baSBill Taylor * aligned to and bound to addresses which are a multiple of CQE size. 2089e39c5baSBill Taylor */ 2099e39c5baSBill Taylor cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t); 2109e39c5baSBill Taylor 2119e39c5baSBill Taylor cq->cq_cqinfo.qa_alloc_align = PAGESIZE; 2129e39c5baSBill Taylor cq->cq_cqinfo.qa_bind_align = PAGESIZE; 2139e39c5baSBill Taylor if (cq->cq_is_umap) { 2149e39c5baSBill Taylor cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND; 2159e39c5baSBill Taylor } else { 2169e39c5baSBill Taylor cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL; 2179e39c5baSBill Taylor hermon_arm_cq_dbr_init(cq->cq_arm_ci_vdbr); 2189e39c5baSBill Taylor } 2199e39c5baSBill Taylor status = hermon_queue_alloc(state, &cq->cq_cqinfo, sleepflag); 2209e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2219e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 2229e39c5baSBill Taylor goto cqalloc_fail4; 2239e39c5baSBill Taylor } 2249e39c5baSBill Taylor buf = (hermon_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned; 2259e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf)) 2269e39c5baSBill Taylor 2279e39c5baSBill Taylor /* 2289e39c5baSBill Taylor * The ownership bit of the CQE's is set by the HW during the process 2299e39c5baSBill Taylor * of transferrring ownership of the CQ (PRM 09.35c, 14.2.1, note D1 2309e39c5baSBill Taylor * 2319e39c5baSBill Taylor */ 2329e39c5baSBill Taylor 2339e39c5baSBill Taylor /* 2349e39c5baSBill Taylor * Register the memory for the CQ. The memory for the CQ must 2359e39c5baSBill Taylor * be registered in the Hermon TPT tables. This gives us the LKey 2369e39c5baSBill Taylor * to specify in the CQ context below. Note: If this is a user- 2379e39c5baSBill Taylor * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping. 2389e39c5baSBill Taylor */ 2399e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP; 2409e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf; 2419e39c5baSBill Taylor mr_attr.mr_len = cq->cq_cqinfo.qa_size; 2429e39c5baSBill Taylor mr_attr.mr_as = NULL; 2439e39c5baSBill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE; 2449e39c5baSBill Taylor op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass; 2459e39c5baSBill Taylor op.mro_bind_dmahdl = cq->cq_cqinfo.qa_dmahdl; 2469e39c5baSBill Taylor op.mro_bind_override_addr = 0; 2479e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &op, 2489e39c5baSBill Taylor HERMON_CQ_CMPT); 2499e39c5baSBill Taylor if (status != DDI_SUCCESS) { 2509e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 2519e39c5baSBill Taylor goto cqalloc_fail5; 2529e39c5baSBill Taylor } 2539e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) 2549e39c5baSBill Taylor 255*17a2b317SBill Taylor cq->cq_erreqnum = HERMON_CQ_ERREQNUM_GET(state); 256*17a2b317SBill Taylor if (cq_attr->cq_flags & IBT_CQ_HID) { 257*17a2b317SBill Taylor if (!HERMON_HID_VALID(state, cq_attr->cq_hid)) { 258*17a2b317SBill Taylor IBTF_DPRINTF_L2("CQalloc", "bad handler id 0x%x", 259*17a2b317SBill Taylor cq_attr->cq_hid); 260*17a2b317SBill Taylor status = IBT_INVALID_PARAM; 261*17a2b317SBill Taylor goto cqalloc_fail5; 262*17a2b317SBill Taylor } 263*17a2b317SBill Taylor cq->cq_eqnum = HERMON_HID_TO_EQNUM(state, cq_attr->cq_hid); 264*17a2b317SBill Taylor IBTF_DPRINTF_L2("cqalloc", "hid: eqn %d", cq->cq_eqnum); 265*17a2b317SBill Taylor } else { 266*17a2b317SBill Taylor cq_schedp = (hermon_cq_sched_t *)cq_attr->cq_sched; 267*17a2b317SBill Taylor if (cq_schedp == NULL) { 268*17a2b317SBill Taylor cq_schedp = &state->hs_cq_sched_default; 269*17a2b317SBill Taylor } else if (cq_schedp != &state->hs_cq_sched_default) { 270*17a2b317SBill Taylor int i; 271*17a2b317SBill Taylor hermon_cq_sched_t *tmp; 272*17a2b317SBill Taylor 273*17a2b317SBill Taylor tmp = state->hs_cq_sched_array; 274*17a2b317SBill Taylor for (i = 0; i < state->hs_cq_sched_array_size; i++) 275*17a2b317SBill Taylor if (cq_schedp == &tmp[i]) 276*17a2b317SBill Taylor break; /* found it */ 277*17a2b317SBill Taylor if (i >= state->hs_cq_sched_array_size) { 278*17a2b317SBill Taylor cmn_err(CE_CONT, "!Invalid cq_sched argument: " 279*17a2b317SBill Taylor "ignored\n"); 280*17a2b317SBill Taylor cq_schedp = &state->hs_cq_sched_default; 281*17a2b317SBill Taylor } 282*17a2b317SBill Taylor } 283*17a2b317SBill Taylor cq->cq_eqnum = HERMON_HID_TO_EQNUM(state, 284*17a2b317SBill Taylor HERMON_CQSCHED_NEXT_HID(cq_schedp)); 285*17a2b317SBill Taylor IBTF_DPRINTF_L2("cqalloc", "sched: first-1 %d, len %d, " 286*17a2b317SBill Taylor "eqn %d", cq_schedp->cqs_start_hid - 1, 287*17a2b317SBill Taylor cq_schedp->cqs_len, cq->cq_eqnum); 288*17a2b317SBill Taylor } 2899e39c5baSBill Taylor 2909e39c5baSBill Taylor /* 2919e39c5baSBill Taylor * Fill in the CQC entry. This is the final step before passing 2929e39c5baSBill Taylor * ownership of the CQC entry to the Hermon hardware. We use all of 2939e39c5baSBill Taylor * the information collected/calculated above to fill in the 2949e39c5baSBill Taylor * requisite portions of the CQC. Note: If this CQ is going to be 2959e39c5baSBill Taylor * used for userland access, then we need to set the UAR page number 2969e39c5baSBill Taylor * appropriately (otherwise it's a "don't care") 2979e39c5baSBill Taylor */ 2989e39c5baSBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t)); 2999e39c5baSBill Taylor 3009e39c5baSBill Taylor cqc_entry.state = HERMON_CQ_DISARMED; 3019e39c5baSBill Taylor cqc_entry.pg_offs = cq->cq_cqinfo.qa_pgoffs >> 5; 3029e39c5baSBill Taylor cqc_entry.log_cq_sz = log_cq_size; 3039e39c5baSBill Taylor cqc_entry.usr_page = uarpg; 3049e39c5baSBill Taylor cqc_entry.c_eqn = cq->cq_eqnum; 3059e39c5baSBill Taylor cqc_entry.log2_pgsz = mr->mr_log2_pgsz; 3069e39c5baSBill Taylor cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF); 3079e39c5baSBill Taylor cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3; 3089e39c5baSBill Taylor cqc_entry.dbr_addrh = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 32); 3099e39c5baSBill Taylor cqc_entry.dbr_addrl = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 3); 3109e39c5baSBill Taylor 3119e39c5baSBill Taylor /* 3129e39c5baSBill Taylor * Write the CQC entry to hardware - we pass ownership of 3139e39c5baSBill Taylor * the entry to the hardware (using the Hermon SW2HW_CQ firmware 3149e39c5baSBill Taylor * command). Note: In general, this operation shouldn't fail. But 3159e39c5baSBill Taylor * if it does, we have to undo everything we've done above before 3169e39c5baSBill Taylor * returning error. 3179e39c5baSBill Taylor */ 3189e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_CQ, &cqc_entry, 3199e39c5baSBill Taylor sizeof (hermon_hw_cqc_t), cq->cq_cqnum, sleepflag); 3209e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 3219e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_CQ command failed: %08x\n", 3229e39c5baSBill Taylor status); 3239e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 3249e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 3259e39c5baSBill Taylor } 3269e39c5baSBill Taylor status = ibc_get_ci_failure(0); 3279e39c5baSBill Taylor goto cqalloc_fail6; 3289e39c5baSBill Taylor } 3299e39c5baSBill Taylor 3309e39c5baSBill Taylor /* 3319e39c5baSBill Taylor * Fill in the rest of the Hermon Completion Queue handle. Having 3329e39c5baSBill Taylor * successfully transferred ownership of the CQC, we can update the 3339e39c5baSBill Taylor * following fields for use in further operations on the CQ. 3349e39c5baSBill Taylor */ 3359e39c5baSBill Taylor cq->cq_resize_hdl = 0; 3369e39c5baSBill Taylor cq->cq_cqcrsrcp = cqc; 3379e39c5baSBill Taylor cq->cq_rsrcp = rsrc; 3389e39c5baSBill Taylor cq->cq_consindx = 0; 3399e39c5baSBill Taylor /* least restrictive */ 3409e39c5baSBill Taylor cq->cq_buf = buf; 3419e39c5baSBill Taylor cq->cq_bufsz = (1 << log_cq_size); 3429e39c5baSBill Taylor cq->cq_log_cqsz = log_cq_size; 3439e39c5baSBill Taylor cq->cq_mrhdl = mr; 3449e39c5baSBill Taylor cq->cq_refcnt = 0; 3459e39c5baSBill Taylor cq->cq_is_special = 0; 3469e39c5baSBill Taylor cq->cq_uarpg = uarpg; 3479e39c5baSBill Taylor cq->cq_umap_dhp = (devmap_cookie_t)NULL; 3489e39c5baSBill Taylor avl_create(&cq->cq_wrid_wqhdr_avl_tree, hermon_wrid_workq_compare, 3499e39c5baSBill Taylor sizeof (struct hermon_workq_avl_s), 3509e39c5baSBill Taylor offsetof(struct hermon_workq_avl_s, wqa_link)); 3519e39c5baSBill Taylor 3529e39c5baSBill Taylor cq->cq_hdlrarg = (void *)ibt_cqhdl; 3539e39c5baSBill Taylor 3549e39c5baSBill Taylor /* 3559e39c5baSBill Taylor * Put CQ handle in Hermon CQNum-to-CQHdl list. Then fill in the 3569e39c5baSBill Taylor * "actual_size" and "cqhdl" and return success 3579e39c5baSBill Taylor */ 358*17a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, cq); 3599e39c5baSBill Taylor 3609e39c5baSBill Taylor /* 3619e39c5baSBill Taylor * If this is a user-mappable CQ, then we need to insert the previously 3629e39c5baSBill Taylor * allocated entry into the "userland resources database". This will 3639e39c5baSBill Taylor * allow for later lookup during devmap() (i.e. mmap()) calls. 3649e39c5baSBill Taylor */ 3659e39c5baSBill Taylor if (cq->cq_is_umap) { 3669e39c5baSBill Taylor hermon_umap_db_add(umapdb); 3679e39c5baSBill Taylor } 3689e39c5baSBill Taylor 3699e39c5baSBill Taylor /* 3709e39c5baSBill Taylor * Fill in the return arguments (if necessary). This includes the 3719e39c5baSBill Taylor * real completion queue size. 3729e39c5baSBill Taylor */ 3739e39c5baSBill Taylor if (actual_size != NULL) { 3749e39c5baSBill Taylor *actual_size = (1 << log_cq_size) - 1; 3759e39c5baSBill Taylor } 3769e39c5baSBill Taylor *cqhdl = cq; 3779e39c5baSBill Taylor 3789e39c5baSBill Taylor return (DDI_SUCCESS); 3799e39c5baSBill Taylor 3809e39c5baSBill Taylor /* 3819e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine 3829e39c5baSBill Taylor */ 3839e39c5baSBill Taylor cqalloc_fail6: 3849e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL, 3859e39c5baSBill Taylor sleepflag) != DDI_SUCCESS) { 3869e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister CQ memory"); 3879e39c5baSBill Taylor } 3889e39c5baSBill Taylor cqalloc_fail5: 3899e39c5baSBill Taylor hermon_queue_free(&cq->cq_cqinfo); 3909e39c5baSBill Taylor cqalloc_fail4a: 3919e39c5baSBill Taylor hermon_dbr_free(state, uarpg, cq->cq_arm_ci_vdbr); 3929e39c5baSBill Taylor cqalloc_fail4: 3939e39c5baSBill Taylor if (cq_is_umap) { 3949e39c5baSBill Taylor hermon_umap_db_free(umapdb); 3959e39c5baSBill Taylor } 3969e39c5baSBill Taylor cqalloc_fail3: 3979e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 3989e39c5baSBill Taylor cqalloc_fail2: 3999e39c5baSBill Taylor hermon_rsrc_free(state, &cqc); 4009e39c5baSBill Taylor cqalloc_fail1: 4019e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 4029e39c5baSBill Taylor cqalloc_fail: 4039e39c5baSBill Taylor return (status); 4049e39c5baSBill Taylor } 4059e39c5baSBill Taylor 4069e39c5baSBill Taylor 4079e39c5baSBill Taylor /* 4089e39c5baSBill Taylor * hermon_cq_free() 4099e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 4109e39c5baSBill Taylor */ 4119e39c5baSBill Taylor /* ARGSUSED */ 4129e39c5baSBill Taylor int 4139e39c5baSBill Taylor hermon_cq_free(hermon_state_t *state, hermon_cqhdl_t *cqhdl, uint_t sleepflag) 4149e39c5baSBill Taylor { 4159e39c5baSBill Taylor hermon_rsrc_t *cqc, *rsrc; 4169e39c5baSBill Taylor hermon_umap_db_entry_t *umapdb; 4179e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry; 4189e39c5baSBill Taylor hermon_pdhdl_t pd; 4199e39c5baSBill Taylor hermon_mrhdl_t mr; 4209e39c5baSBill Taylor hermon_cqhdl_t cq, resize; 4219e39c5baSBill Taylor uint32_t cqnum; 4229e39c5baSBill Taylor uint64_t value; 4239e39c5baSBill Taylor uint_t maxprot; 4249e39c5baSBill Taylor int status; 4259e39c5baSBill Taylor 4269e39c5baSBill Taylor /* 4279e39c5baSBill Taylor * Pull all the necessary information from the Hermon Completion Queue 4289e39c5baSBill Taylor * handle. This is necessary here because the resource for the 4299e39c5baSBill Taylor * CQ handle is going to be freed up as part of this operation. 4309e39c5baSBill Taylor */ 4319e39c5baSBill Taylor cq = *cqhdl; 4329e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 4339e39c5baSBill Taylor cqc = cq->cq_cqcrsrcp; 4349e39c5baSBill Taylor rsrc = cq->cq_rsrcp; 4359e39c5baSBill Taylor pd = state->hs_pdhdl_internal; 4369e39c5baSBill Taylor mr = cq->cq_mrhdl; 4379e39c5baSBill Taylor cqnum = cq->cq_cqnum; 4389e39c5baSBill Taylor 4399e39c5baSBill Taylor resize = cq->cq_resize_hdl; /* save the handle for later */ 4409e39c5baSBill Taylor 4419e39c5baSBill Taylor /* 4429e39c5baSBill Taylor * If there are work queues still associated with the CQ, then return 4439e39c5baSBill Taylor * an error. Otherwise, we will be holding the CQ lock. 4449e39c5baSBill Taylor */ 4459e39c5baSBill Taylor if (cq->cq_refcnt != 0) { 4469e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 4479e39c5baSBill Taylor return (IBT_CQ_BUSY); 4489e39c5baSBill Taylor } 4499e39c5baSBill Taylor 4509e39c5baSBill Taylor /* 4519e39c5baSBill Taylor * If this was a user-mappable CQ, then we need to remove its entry 4529e39c5baSBill Taylor * from the "userland resources database". If it is also currently 4539e39c5baSBill Taylor * mmap()'d out to a user process, then we need to call 4549e39c5baSBill Taylor * devmap_devmem_remap() to remap the CQ memory to an invalid mapping. 4559e39c5baSBill Taylor * We also need to invalidate the CQ tracking information for the 4569e39c5baSBill Taylor * user mapping. 4579e39c5baSBill Taylor */ 4589e39c5baSBill Taylor if (cq->cq_is_umap) { 4599e39c5baSBill Taylor status = hermon_umap_db_find(state->hs_instance, cqnum, 4609e39c5baSBill Taylor MLNX_UMAP_CQMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE, 4619e39c5baSBill Taylor &umapdb); 4629e39c5baSBill Taylor if (status != DDI_SUCCESS) { 4639e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 4649e39c5baSBill Taylor HERMON_WARNING(state, "failed to find in database"); 4659e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 4669e39c5baSBill Taylor } 4679e39c5baSBill Taylor hermon_umap_db_free(umapdb); 4689e39c5baSBill Taylor if (cq->cq_umap_dhp != NULL) { 4699e39c5baSBill Taylor maxprot = (PROT_READ | PROT_WRITE | PROT_USER); 4709e39c5baSBill Taylor status = devmap_devmem_remap(cq->cq_umap_dhp, 4719e39c5baSBill Taylor state->hs_dip, 0, 0, cq->cq_cqinfo.qa_size, 4729e39c5baSBill Taylor maxprot, DEVMAP_MAPPING_INVALID, NULL); 4739e39c5baSBill Taylor if (status != DDI_SUCCESS) { 4749e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 4759e39c5baSBill Taylor HERMON_WARNING(state, "failed in CQ memory " 4769e39c5baSBill Taylor "devmap_devmem_remap()"); 4779e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 4789e39c5baSBill Taylor } 4799e39c5baSBill Taylor cq->cq_umap_dhp = (devmap_cookie_t)NULL; 4809e39c5baSBill Taylor } 4819e39c5baSBill Taylor } 4829e39c5baSBill Taylor 4839e39c5baSBill Taylor /* 4849e39c5baSBill Taylor * Put NULL into the Arbel CQNum-to-CQHdl list. This will allow any 4859e39c5baSBill Taylor * in-progress events to detect that the CQ corresponding to this 4869e39c5baSBill Taylor * number has been freed. 4879e39c5baSBill Taylor */ 488*17a2b317SBill Taylor hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, NULL); 4899e39c5baSBill Taylor 4909e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 4919e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cq)) 4929e39c5baSBill Taylor 4939e39c5baSBill Taylor /* 4949e39c5baSBill Taylor * Reclaim CQC entry from hardware (using the Hermon HW2SW_CQ 4959e39c5baSBill Taylor * firmware command). If the ownership transfer fails for any reason, 4969e39c5baSBill Taylor * then it is an indication that something (either in HW or SW) has 4979e39c5baSBill Taylor * gone seriously wrong. 4989e39c5baSBill Taylor */ 4999e39c5baSBill Taylor status = hermon_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry, 5009e39c5baSBill Taylor sizeof (hermon_hw_cqc_t), cqnum, sleepflag); 5019e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 5029e39c5baSBill Taylor HERMON_WARNING(state, "failed to reclaim CQC ownership"); 5039e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: HW2SW_CQ command failed: %08x\n", 5049e39c5baSBill Taylor status); 5059e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 5069e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 5079e39c5baSBill Taylor } 5089e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 5099e39c5baSBill Taylor } 5109e39c5baSBill Taylor 5119e39c5baSBill Taylor /* 5129e39c5baSBill Taylor * From here on, we start reliquishing resources - but check to see 5139e39c5baSBill Taylor * if a resize was in progress - if so, we need to relinquish those 5149e39c5baSBill Taylor * resources as well 5159e39c5baSBill Taylor */ 5169e39c5baSBill Taylor 5179e39c5baSBill Taylor 5189e39c5baSBill Taylor /* 5199e39c5baSBill Taylor * Deregister the memory for the Completion Queue. If this fails 5209e39c5baSBill Taylor * for any reason, then it is an indication that something (either 5219e39c5baSBill Taylor * in HW or SW) has gone seriously wrong. So we print a warning 5229e39c5baSBill Taylor * message and return. 5239e39c5baSBill Taylor */ 5249e39c5baSBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL, 5259e39c5baSBill Taylor sleepflag); 5269e39c5baSBill Taylor if (status != DDI_SUCCESS) { 5279e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister CQ memory"); 5289e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 5299e39c5baSBill Taylor } 5309e39c5baSBill Taylor 5319e39c5baSBill Taylor if (resize) { /* there was a pointer to a handle */ 5329e39c5baSBill Taylor mr = resize->cq_mrhdl; /* reuse the pointer to the region */ 5339e39c5baSBill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL, 5349e39c5baSBill Taylor sleepflag); 5359e39c5baSBill Taylor if (status != DDI_SUCCESS) { 5369e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister resize CQ " 5379e39c5baSBill Taylor "memory"); 5389e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 5399e39c5baSBill Taylor } 5409e39c5baSBill Taylor } 5419e39c5baSBill Taylor 5429e39c5baSBill Taylor /* Free the memory for the CQ */ 5439e39c5baSBill Taylor hermon_queue_free(&cq->cq_cqinfo); 5449e39c5baSBill Taylor if (resize) { 5459e39c5baSBill Taylor hermon_queue_free(&resize->cq_cqinfo); 5469e39c5baSBill Taylor /* and the temporary handle */ 5479e39c5baSBill Taylor kmem_free(resize, sizeof (struct hermon_sw_cq_s)); 5489e39c5baSBill Taylor } 5499e39c5baSBill Taylor 5509e39c5baSBill Taylor /* everything else does not matter for the resize in progress */ 5519e39c5baSBill Taylor 5529e39c5baSBill Taylor /* Free the dbr */ 5539e39c5baSBill Taylor hermon_dbr_free(state, cq->cq_uarpg, cq->cq_arm_ci_vdbr); 5549e39c5baSBill Taylor 5559e39c5baSBill Taylor /* Free the Hermon Completion Queue handle */ 5569e39c5baSBill Taylor hermon_rsrc_free(state, &rsrc); 5579e39c5baSBill Taylor 5589e39c5baSBill Taylor /* Free up the CQC entry resource */ 5599e39c5baSBill Taylor hermon_rsrc_free(state, &cqc); 5609e39c5baSBill Taylor 5619e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */ 5629e39c5baSBill Taylor hermon_pd_refcnt_dec(pd); 5639e39c5baSBill Taylor 5649e39c5baSBill Taylor /* Set the cqhdl pointer to NULL and return success */ 5659e39c5baSBill Taylor *cqhdl = NULL; 5669e39c5baSBill Taylor 5679e39c5baSBill Taylor return (DDI_SUCCESS); 5689e39c5baSBill Taylor } 5699e39c5baSBill Taylor 5709e39c5baSBill Taylor 5719e39c5baSBill Taylor /* 5729e39c5baSBill Taylor * hermon_cq_resize() 5739e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 5749e39c5baSBill Taylor */ 5759e39c5baSBill Taylor int 5769e39c5baSBill Taylor hermon_cq_resize(hermon_state_t *state, hermon_cqhdl_t cq, uint_t req_size, 5779e39c5baSBill Taylor uint_t *actual_size, uint_t sleepflag) 5789e39c5baSBill Taylor { 5799e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry; 5809e39c5baSBill Taylor hermon_cqhdl_t resize_hdl; 5819e39c5baSBill Taylor hermon_qalloc_info_t new_cqinfo; 5829e39c5baSBill Taylor ibt_mr_attr_t mr_attr; 5839e39c5baSBill Taylor hermon_mr_options_t op; 5849e39c5baSBill Taylor hermon_pdhdl_t pd; 5859e39c5baSBill Taylor hermon_mrhdl_t mr; 5869e39c5baSBill Taylor hermon_hw_cqe_t *buf; 5879e39c5baSBill Taylor uint32_t new_prod_indx; 5889e39c5baSBill Taylor uint_t log_cq_size; 5899e39c5baSBill Taylor int status, flag; 5909e39c5baSBill Taylor 5919e39c5baSBill Taylor if (cq->cq_resize_hdl != 0) { /* already in process */ 5929e39c5baSBill Taylor status = IBT_CQ_BUSY; 5939e39c5baSBill Taylor goto cqresize_fail; 5949e39c5baSBill Taylor } 5959e39c5baSBill Taylor 5969e39c5baSBill Taylor 5979e39c5baSBill Taylor /* Use the internal protection domain (PD) for CQs */ 5989e39c5baSBill Taylor pd = state->hs_pdhdl_internal; 5999e39c5baSBill Taylor 6009e39c5baSBill Taylor /* 6019e39c5baSBill Taylor * Calculate the appropriate size for the new resized completion queue. 6029e39c5baSBill Taylor * Note: All Hermon CQs must be a power-of-2 minus 1 in size. Also 6039e39c5baSBill Taylor * they may not be any smaller than HERMON_CQ_MIN_SIZE. This step is 6049e39c5baSBill Taylor * to round the requested size up to the next highest power-of-2 6059e39c5baSBill Taylor */ 6069e39c5baSBill Taylor req_size = max(req_size, HERMON_CQ_MIN_SIZE); 6079e39c5baSBill Taylor log_cq_size = highbit(req_size); 6089e39c5baSBill Taylor 6099e39c5baSBill Taylor /* 6109e39c5baSBill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent 6119e39c5baSBill Taylor * with the device limits and/or software-configured limits) 6129e39c5baSBill Taylor */ 6139e39c5baSBill Taylor if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) { 6149e39c5baSBill Taylor status = IBT_HCA_CQ_EXCEEDED; 6159e39c5baSBill Taylor goto cqresize_fail; 6169e39c5baSBill Taylor } 6179e39c5baSBill Taylor 6189e39c5baSBill Taylor /* 6199e39c5baSBill Taylor * Allocate the memory for newly resized Completion Queue. 6209e39c5baSBill Taylor * 6219e39c5baSBill Taylor * Note: Although we use the common queue allocation routine, we 6229e39c5baSBill Taylor * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in 6239e39c5baSBill Taylor * kernel system memory) for kernel CQs because it would be 6249e39c5baSBill Taylor * inefficient to have CQs located in DDR memory. This is the same 6259e39c5baSBill Taylor * as we do when we first allocate completion queues primarily 6269e39c5baSBill Taylor * because CQs are read from (by software) more than they are written 6279e39c5baSBill Taylor * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all 6289e39c5baSBill Taylor * user-mappable CQs for a similar reason.) 6299e39c5baSBill Taylor * It is also worth noting that, unlike Hermon QP work queues, 6309e39c5baSBill Taylor * completion queues do not have the same strict alignment 6319e39c5baSBill Taylor * requirements. It is sufficient for the CQ memory to be both 6329e39c5baSBill Taylor * aligned to and bound to addresses which are a multiple of CQE size. 6339e39c5baSBill Taylor */ 6349e39c5baSBill Taylor 6359e39c5baSBill Taylor /* first, alloc the resize_handle */ 6369e39c5baSBill Taylor resize_hdl = kmem_zalloc(sizeof (struct hermon_sw_cq_s), KM_SLEEP); 6379e39c5baSBill Taylor 6389e39c5baSBill Taylor new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t); 6399e39c5baSBill Taylor new_cqinfo.qa_alloc_align = PAGESIZE; 6409e39c5baSBill Taylor new_cqinfo.qa_bind_align = PAGESIZE; 6419e39c5baSBill Taylor if (cq->cq_is_umap) { 6429e39c5baSBill Taylor new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND; 6439e39c5baSBill Taylor } else { 6449e39c5baSBill Taylor new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL; 6459e39c5baSBill Taylor } 6469e39c5baSBill Taylor status = hermon_queue_alloc(state, &new_cqinfo, sleepflag); 6479e39c5baSBill Taylor if (status != DDI_SUCCESS) { 6489e39c5baSBill Taylor /* free the resize handle */ 6499e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s)); 6509e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 6519e39c5baSBill Taylor goto cqresize_fail; 6529e39c5baSBill Taylor } 6539e39c5baSBill Taylor buf = (hermon_hw_cqe_t *)new_cqinfo.qa_buf_aligned; 6549e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*buf)) 6559e39c5baSBill Taylor 6569e39c5baSBill Taylor /* 6579e39c5baSBill Taylor * No initialization of the cq is needed - the command will do it 6589e39c5baSBill Taylor */ 6599e39c5baSBill Taylor 6609e39c5baSBill Taylor /* 6619e39c5baSBill Taylor * Register the memory for the CQ. The memory for the CQ must 6629e39c5baSBill Taylor * be registered in the Hermon TPT tables. This gives us the LKey 6639e39c5baSBill Taylor * to specify in the CQ context below. 6649e39c5baSBill Taylor */ 6659e39c5baSBill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP; 6669e39c5baSBill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf; 6679e39c5baSBill Taylor mr_attr.mr_len = new_cqinfo.qa_size; 6689e39c5baSBill Taylor mr_attr.mr_as = NULL; 6699e39c5baSBill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE; 6709e39c5baSBill Taylor op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass; 6719e39c5baSBill Taylor op.mro_bind_dmahdl = new_cqinfo.qa_dmahdl; 6729e39c5baSBill Taylor op.mro_bind_override_addr = 0; 6739e39c5baSBill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &op, 6749e39c5baSBill Taylor HERMON_CQ_CMPT); 6759e39c5baSBill Taylor if (status != DDI_SUCCESS) { 6769e39c5baSBill Taylor hermon_queue_free(&new_cqinfo); 6779e39c5baSBill Taylor /* free the resize handle */ 6789e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s)); 6799e39c5baSBill Taylor status = IBT_INSUFF_RESOURCE; 6809e39c5baSBill Taylor goto cqresize_fail; 6819e39c5baSBill Taylor } 6829e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) 6839e39c5baSBill Taylor 6849e39c5baSBill Taylor /* 6859e39c5baSBill Taylor * Now we grab the CQ lock. Since we will be updating the actual 6869e39c5baSBill Taylor * CQ location and the producer/consumer indexes, we should hold 6879e39c5baSBill Taylor * the lock. 6889e39c5baSBill Taylor * 6899e39c5baSBill Taylor * We do a ARBEL_NOSLEEP here (and below), though, because we are 6909e39c5baSBill Taylor * holding the "cq_lock" and if we got raised to interrupt level 6919e39c5baSBill Taylor * by priority inversion, we would not want to block in this routine 6929e39c5baSBill Taylor * waiting for success. 6939e39c5baSBill Taylor */ 6949e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 6959e39c5baSBill Taylor 6969e39c5baSBill Taylor /* 6979e39c5baSBill Taylor * Fill in the CQC entry. For the resize operation this is the 6989e39c5baSBill Taylor * final step before attempting the resize operation on the CQC entry. 6999e39c5baSBill Taylor * We use all of the information collected/calculated above to fill 7009e39c5baSBill Taylor * in the requisite portions of the CQC. 7019e39c5baSBill Taylor */ 7029e39c5baSBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t)); 7039e39c5baSBill Taylor cqc_entry.log_cq_sz = log_cq_size; 7049e39c5baSBill Taylor cqc_entry.pg_offs = new_cqinfo.qa_pgoffs >> 5; 7059e39c5baSBill Taylor cqc_entry.log2_pgsz = mr->mr_log2_pgsz; 7069e39c5baSBill Taylor cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF); 7079e39c5baSBill Taylor cqc_entry.mtt_base_addl = mr->mr_mttaddr >> 3; 7089e39c5baSBill Taylor 7099e39c5baSBill Taylor /* 7109e39c5baSBill Taylor * Write the CQC entry to hardware. Lastly, we pass ownership of 7119e39c5baSBill Taylor * the entry to the hardware (using the Hermon RESIZE_CQ firmware 7129e39c5baSBill Taylor * command). Note: In general, this operation shouldn't fail. But 7139e39c5baSBill Taylor * if it does, we have to undo everything we've done above before 7149e39c5baSBill Taylor * returning error. Also note that the status returned may indicate 7159e39c5baSBill Taylor * the code to return to the IBTF. 7169e39c5baSBill Taylor */ 7179e39c5baSBill Taylor status = hermon_resize_cq_cmd_post(state, &cqc_entry, cq->cq_cqnum, 7189e39c5baSBill Taylor &new_prod_indx, HERMON_CMD_NOSLEEP_SPIN); 7199e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 7209e39c5baSBill Taylor /* Resize attempt has failed, drop CQ lock and cleanup */ 7219e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 7229e39c5baSBill Taylor if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL, 7239e39c5baSBill Taylor sleepflag) != DDI_SUCCESS) { 7249e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister CQ memory"); 7259e39c5baSBill Taylor } 7269e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s)); 7279e39c5baSBill Taylor hermon_queue_free(&new_cqinfo); 7289e39c5baSBill Taylor if (status == HERMON_CMD_BAD_SIZE) { 7299e39c5baSBill Taylor return (IBT_CQ_SZ_INSUFFICIENT); 7309e39c5baSBill Taylor } else { 7319e39c5baSBill Taylor cmn_err(CE_CONT, "Hermon: RESIZE_CQ command failed: " 7329e39c5baSBill Taylor "%08x\n", status); 7339e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 7349e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 7359e39c5baSBill Taylor HCA_ERR_SRV_LOST); 7369e39c5baSBill Taylor } 7379e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 7389e39c5baSBill Taylor } 7399e39c5baSBill Taylor } 7409e39c5baSBill Taylor 7419e39c5baSBill Taylor /* 7429e39c5baSBill Taylor * For Hermon, we've alloc'd another handle structure and save off the 7439e39c5baSBill Taylor * important things in it. Then, in polling we check to see if there's 7449e39c5baSBill Taylor * a "resizing handle" and if so we look for the "special CQE", opcode 7459e39c5baSBill Taylor * 0x16, that indicates the transition to the new buffer. 7469e39c5baSBill Taylor * 7479e39c5baSBill Taylor * At that point, we'll adjust everything - including dereg and 7489e39c5baSBill Taylor * freeing of the original buffer, updating all the necessary fields 7499e39c5baSBill Taylor * in the cq_hdl, and setting up for the next cqe polling 7509e39c5baSBill Taylor */ 7519e39c5baSBill Taylor 7529e39c5baSBill Taylor resize_hdl->cq_buf = buf; 7539e39c5baSBill Taylor resize_hdl->cq_bufsz = (1 << log_cq_size); 7549e39c5baSBill Taylor resize_hdl->cq_mrhdl = mr; 7559e39c5baSBill Taylor resize_hdl->cq_log_cqsz = log_cq_size; 7569e39c5baSBill Taylor 7579e39c5baSBill Taylor bcopy(&new_cqinfo, &(resize_hdl->cq_cqinfo), 7589e39c5baSBill Taylor sizeof (struct hermon_qalloc_info_s)); 7599e39c5baSBill Taylor 7609e39c5baSBill Taylor /* now, save the address in the cq_handle */ 7619e39c5baSBill Taylor cq->cq_resize_hdl = resize_hdl; 7629e39c5baSBill Taylor 7639e39c5baSBill Taylor /* 7649e39c5baSBill Taylor * Drop the CQ lock now. 7659e39c5baSBill Taylor */ 7669e39c5baSBill Taylor 7679e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 7689e39c5baSBill Taylor /* 7699e39c5baSBill Taylor * Fill in the return arguments (if necessary). This includes the 7709e39c5baSBill Taylor * real new completion queue size. 7719e39c5baSBill Taylor */ 7729e39c5baSBill Taylor if (actual_size != NULL) { 7739e39c5baSBill Taylor *actual_size = (1 << log_cq_size) - 1; 7749e39c5baSBill Taylor } 7759e39c5baSBill Taylor 7769e39c5baSBill Taylor return (DDI_SUCCESS); 7779e39c5baSBill Taylor 7789e39c5baSBill Taylor cqresize_fail: 7799e39c5baSBill Taylor return (status); 7809e39c5baSBill Taylor } 7819e39c5baSBill Taylor 7829e39c5baSBill Taylor 7839e39c5baSBill Taylor /* 7849e39c5baSBill Taylor * hermon_cq_modify() 7859e39c5baSBill Taylor * Context: Can be called base context. 7869e39c5baSBill Taylor */ 7879e39c5baSBill Taylor /* ARGSUSED */ 7889e39c5baSBill Taylor int 7899e39c5baSBill Taylor hermon_cq_modify(hermon_state_t *state, hermon_cqhdl_t cq, 7909e39c5baSBill Taylor uint_t count, uint_t usec, ibt_cq_handler_id_t hid, uint_t sleepflag) 7919e39c5baSBill Taylor { 7929e39c5baSBill Taylor int status; 7939e39c5baSBill Taylor hermon_hw_cqc_t cqc_entry; 7949e39c5baSBill Taylor 7959e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 7969e39c5baSBill Taylor if (count != cq->cq_intmod_count || 7979e39c5baSBill Taylor usec != cq->cq_intmod_usec) { 7989e39c5baSBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t)); 7999e39c5baSBill Taylor cqc_entry.cq_max_cnt = count; 8009e39c5baSBill Taylor cqc_entry.cq_period = usec; 8019e39c5baSBill Taylor status = hermon_modify_cq_cmd_post(state, &cqc_entry, 8029e39c5baSBill Taylor cq->cq_cqnum, MODIFY_MODERATION_CQ, sleepflag); 8039e39c5baSBill Taylor if (status != HERMON_CMD_SUCCESS) { 8049e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 805*17a2b317SBill Taylor cmn_err(CE_CONT, "Hermon: MODIFY_MODERATION_CQ " 806*17a2b317SBill Taylor "command failed: %08x\n", status); 8079e39c5baSBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 8089e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 8099e39c5baSBill Taylor HCA_ERR_SRV_LOST); 8109e39c5baSBill Taylor } 8119e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 8129e39c5baSBill Taylor } 8139e39c5baSBill Taylor cq->cq_intmod_count = count; 8149e39c5baSBill Taylor cq->cq_intmod_usec = usec; 8159e39c5baSBill Taylor } 816*17a2b317SBill Taylor if (hid && (hid - 1 != cq->cq_eqnum)) { 817*17a2b317SBill Taylor bzero(&cqc_entry, sizeof (hermon_hw_cqc_t)); 818*17a2b317SBill Taylor cqc_entry.c_eqn = HERMON_HID_TO_EQNUM(state, hid); 819*17a2b317SBill Taylor status = hermon_modify_cq_cmd_post(state, &cqc_entry, 820*17a2b317SBill Taylor cq->cq_cqnum, MODIFY_EQN, sleepflag); 821*17a2b317SBill Taylor if (status != HERMON_CMD_SUCCESS) { 822*17a2b317SBill Taylor mutex_exit(&cq->cq_lock); 823*17a2b317SBill Taylor cmn_err(CE_CONT, "Hermon: MODIFY_EQN command failed: " 824*17a2b317SBill Taylor "%08x\n", status); 825*17a2b317SBill Taylor if (status == HERMON_CMD_INVALID_STATUS) { 826*17a2b317SBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, 827*17a2b317SBill Taylor HCA_ERR_SRV_LOST); 828*17a2b317SBill Taylor } 829*17a2b317SBill Taylor return (ibc_get_ci_failure(0)); 830*17a2b317SBill Taylor } 831*17a2b317SBill Taylor cq->cq_eqnum = hid - 1; 832*17a2b317SBill Taylor } 8339e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 8349e39c5baSBill Taylor return (DDI_SUCCESS); 8359e39c5baSBill Taylor } 8369e39c5baSBill Taylor 8379e39c5baSBill Taylor /* 8389e39c5baSBill Taylor * hermon_cq_notify() 8399e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 8409e39c5baSBill Taylor */ 8419e39c5baSBill Taylor int 8429e39c5baSBill Taylor hermon_cq_notify(hermon_state_t *state, hermon_cqhdl_t cq, 8439e39c5baSBill Taylor ibt_cq_notify_flags_t flags) 8449e39c5baSBill Taylor { 8459e39c5baSBill Taylor uint_t cmd; 8469e39c5baSBill Taylor ibt_status_t status; 8479e39c5baSBill Taylor 8489e39c5baSBill Taylor /* Validate IBT flags and call doorbell routine. */ 8499e39c5baSBill Taylor if (flags == IBT_NEXT_COMPLETION) { 8509e39c5baSBill Taylor cmd = HERMON_CQDB_NOTIFY_CQ; 8519e39c5baSBill Taylor } else if (flags == IBT_NEXT_SOLICITED) { 8529e39c5baSBill Taylor cmd = HERMON_CQDB_NOTIFY_CQ_SOLICIT; 8539e39c5baSBill Taylor } else { 8549e39c5baSBill Taylor return (IBT_CQ_NOTIFY_TYPE_INVALID); 8559e39c5baSBill Taylor } 8569e39c5baSBill Taylor 8579e39c5baSBill Taylor status = hermon_cq_arm_doorbell(state, cq, cmd); 8589e39c5baSBill Taylor return (status); 8599e39c5baSBill Taylor } 8609e39c5baSBill Taylor 8619e39c5baSBill Taylor 8629e39c5baSBill Taylor /* 8639e39c5baSBill Taylor * hermon_cq_poll() 8649e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 8659e39c5baSBill Taylor */ 8669e39c5baSBill Taylor int 8679e39c5baSBill Taylor hermon_cq_poll(hermon_state_t *state, hermon_cqhdl_t cq, ibt_wc_t *wc_p, 8689e39c5baSBill Taylor uint_t num_wc, uint_t *num_polled) 8699e39c5baSBill Taylor { 8709e39c5baSBill Taylor hermon_hw_cqe_t *cqe; 8719e39c5baSBill Taylor uint_t opcode; 872*17a2b317SBill Taylor uint32_t cons_indx, wrap_around_mask, shift, mask; 8739e39c5baSBill Taylor uint32_t polled_cnt, spec_op = 0; 8749e39c5baSBill Taylor int status; 8759e39c5baSBill Taylor 8769e39c5baSBill Taylor /* 8779e39c5baSBill Taylor * Check for user-mappable CQ memory. Note: We do not allow kernel 8789e39c5baSBill Taylor * clients to poll CQ memory that is accessible directly by the user. 8799e39c5baSBill Taylor * If the CQ memory is user accessible, then return an error. 8809e39c5baSBill Taylor */ 8819e39c5baSBill Taylor if (cq->cq_is_umap) { 8829e39c5baSBill Taylor return (IBT_CQ_HDL_INVALID); 8839e39c5baSBill Taylor } 8849e39c5baSBill Taylor 8859e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 8869e39c5baSBill Taylor 8879e39c5baSBill Taylor /* Get the consumer index */ 8889e39c5baSBill Taylor cons_indx = cq->cq_consindx; 889*17a2b317SBill Taylor shift = cq->cq_log_cqsz; 890*17a2b317SBill Taylor mask = cq->cq_bufsz; 8919e39c5baSBill Taylor 8929e39c5baSBill Taylor /* 8939e39c5baSBill Taylor * Calculate the wrap around mask. Note: This operation only works 8949e39c5baSBill Taylor * because all Hermon completion queues have power-of-2 sizes 8959e39c5baSBill Taylor */ 8969e39c5baSBill Taylor wrap_around_mask = (cq->cq_bufsz - 1); 8979e39c5baSBill Taylor 8989e39c5baSBill Taylor /* Calculate the pointer to the first CQ entry */ 8999e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask]; 9009e39c5baSBill Taylor 9019e39c5baSBill Taylor /* 9029e39c5baSBill Taylor * Keep pulling entries from the CQ until we find an entry owned by 9039e39c5baSBill Taylor * the hardware. As long as there the CQE's owned by SW, process 9049e39c5baSBill Taylor * each entry by calling hermon_cq_cqe_consume() and updating the CQ 9059e39c5baSBill Taylor * consumer index. Note: We only update the consumer index if 9069e39c5baSBill Taylor * hermon_cq_cqe_consume() returns HERMON_CQ_SYNC_AND_DB. Otherwise, 9079e39c5baSBill Taylor * it indicates that we are going to "recycle" the CQE (probably 9089e39c5baSBill Taylor * because it is a error CQE and corresponds to more than one 9099e39c5baSBill Taylor * completion). 9109e39c5baSBill Taylor */ 9119e39c5baSBill Taylor polled_cnt = 0; 912*17a2b317SBill Taylor while (HERMON_CQE_OWNER_IS_SW(cq, cqe, cons_indx, shift, mask)) { 9139e39c5baSBill Taylor if (cq->cq_resize_hdl != 0) { /* in midst of resize */ 9149e39c5baSBill Taylor /* peek at the opcode */ 9159e39c5baSBill Taylor opcode = HERMON_CQE_OPCODE_GET(cq, cqe); 9169e39c5baSBill Taylor if (opcode == HERMON_CQE_RCV_RESIZE_CODE) { 9179e39c5baSBill Taylor hermon_cq_resize_helper(state, cq); 9189e39c5baSBill Taylor 9199e39c5baSBill Taylor /* Increment the consumer index */ 9209e39c5baSBill Taylor cons_indx = (cons_indx + 1); 9219e39c5baSBill Taylor spec_op = 1; /* plus one for the limiting CQE */ 9229e39c5baSBill Taylor 9239e39c5baSBill Taylor wrap_around_mask = (cq->cq_bufsz - 1); 9249e39c5baSBill Taylor 9259e39c5baSBill Taylor /* Update the pointer to the next CQ entry */ 9269e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask]; 9279e39c5baSBill Taylor 9289e39c5baSBill Taylor continue; 9299e39c5baSBill Taylor } 9309e39c5baSBill Taylor } /* in resizing CQ */ 9319e39c5baSBill Taylor 9329e39c5baSBill Taylor /* 9339e39c5baSBill Taylor * either resizing and not the special opcode, or 9349e39c5baSBill Taylor * not resizing at all 9359e39c5baSBill Taylor */ 9369e39c5baSBill Taylor hermon_cq_cqe_consume(state, cq, cqe, &wc_p[polled_cnt++]); 9379e39c5baSBill Taylor 9389e39c5baSBill Taylor /* Increment the consumer index */ 9399e39c5baSBill Taylor cons_indx = (cons_indx + 1); 9409e39c5baSBill Taylor 9419e39c5baSBill Taylor /* Update the pointer to the next CQ entry */ 9429e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask]; 9439e39c5baSBill Taylor 9449e39c5baSBill Taylor /* 9459e39c5baSBill Taylor * If we have run out of space to store work completions, 9469e39c5baSBill Taylor * then stop and return the ones we have pulled of the CQ. 9479e39c5baSBill Taylor */ 9489e39c5baSBill Taylor if (polled_cnt >= num_wc) { 9499e39c5baSBill Taylor break; 9509e39c5baSBill Taylor } 9519e39c5baSBill Taylor } 9529e39c5baSBill Taylor 9539e39c5baSBill Taylor /* 9549e39c5baSBill Taylor * Now we only ring the doorbell (to update the consumer index) if 955*17a2b317SBill Taylor * we've actually consumed a CQ entry. 9569e39c5baSBill Taylor */ 9579e39c5baSBill Taylor if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) { 9589e39c5baSBill Taylor /* 9599e39c5baSBill Taylor * Update the consumer index in both the CQ handle and the 9609e39c5baSBill Taylor * doorbell record. 9619e39c5baSBill Taylor */ 9629e39c5baSBill Taylor cq->cq_consindx = cons_indx; 9639e39c5baSBill Taylor hermon_cq_update_ci_doorbell(cq); 9649e39c5baSBill Taylor 9659e39c5baSBill Taylor } else if (polled_cnt == 0) { 9669e39c5baSBill Taylor if (spec_op != 0) { 9679e39c5baSBill Taylor /* if we got the special opcode, update the consindx */ 9689e39c5baSBill Taylor cq->cq_consindx = cons_indx; 9699e39c5baSBill Taylor hermon_cq_update_ci_doorbell(cq); 9709e39c5baSBill Taylor } 9719e39c5baSBill Taylor } 9729e39c5baSBill Taylor 9739e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 9749e39c5baSBill Taylor 9759e39c5baSBill Taylor /* Set "num_polled" (if necessary) */ 9769e39c5baSBill Taylor if (num_polled != NULL) { 9779e39c5baSBill Taylor *num_polled = polled_cnt; 9789e39c5baSBill Taylor } 9799e39c5baSBill Taylor 9809e39c5baSBill Taylor /* Set CQ_EMPTY condition if needed, otherwise return success */ 9819e39c5baSBill Taylor if (polled_cnt == 0) { 9829e39c5baSBill Taylor status = IBT_CQ_EMPTY; 9839e39c5baSBill Taylor } else { 9849e39c5baSBill Taylor status = DDI_SUCCESS; 9859e39c5baSBill Taylor } 9869e39c5baSBill Taylor 9879e39c5baSBill Taylor /* 9889e39c5baSBill Taylor * Check if the system is currently panicking. If it is, then call 9899e39c5baSBill Taylor * the Hermon interrupt service routine. This step is necessary here 9909e39c5baSBill Taylor * because we might be in a polled I/O mode and without the call to 9919e39c5baSBill Taylor * hermon_isr() - and its subsequent calls to poll and rearm each 9929e39c5baSBill Taylor * event queue - we might overflow our EQs and render the system 9939e39c5baSBill Taylor * unable to sync/dump. 9949e39c5baSBill Taylor */ 9959e39c5baSBill Taylor if (ddi_in_panic() != 0) { 9969e39c5baSBill Taylor (void) hermon_isr((caddr_t)state, (caddr_t)NULL); 9979e39c5baSBill Taylor } 9989e39c5baSBill Taylor return (status); 9999e39c5baSBill Taylor } 10009e39c5baSBill Taylor 10019e39c5baSBill Taylor /* 10029e39c5baSBill Taylor * cmd_sn must be initialized to 1 to enable proper reenabling 10039e39c5baSBill Taylor * by hermon_arm_cq_dbr_update(). 10049e39c5baSBill Taylor */ 10059e39c5baSBill Taylor static void 10069e39c5baSBill Taylor hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr) 10079e39c5baSBill Taylor { 10089e39c5baSBill Taylor uint32_t *target; 10099e39c5baSBill Taylor 10109e39c5baSBill Taylor target = (uint32_t *)cq_arm_dbr + 1; 10119e39c5baSBill Taylor *target = htonl(1 << HERMON_CQDB_CMDSN_SHIFT); 10129e39c5baSBill Taylor } 10139e39c5baSBill Taylor 10149e39c5baSBill Taylor 10159e39c5baSBill Taylor /* 10169e39c5baSBill Taylor * User cmd_sn needs help from this kernel function to know 10179e39c5baSBill Taylor * when it should be incremented (modulo 4). We do an atomic 10189e39c5baSBill Taylor * update of the arm_cq dbr to communicate this fact. We retry 10199e39c5baSBill Taylor * in the case that user library is racing with us. We zero 10209e39c5baSBill Taylor * out the cmd field so that the user library can use the cmd 10219e39c5baSBill Taylor * field to track the last command it issued (solicited verses any). 10229e39c5baSBill Taylor */ 10239e39c5baSBill Taylor static void 10249e39c5baSBill Taylor hermon_arm_cq_dbr_update(hermon_dbr_t *cq_arm_dbr) 10259e39c5baSBill Taylor { 10269e39c5baSBill Taylor uint32_t tmp, cmp, new; 10279e39c5baSBill Taylor uint32_t old_cmd_sn, new_cmd_sn; 10289e39c5baSBill Taylor uint32_t *target; 10299e39c5baSBill Taylor int retries = 0; 10309e39c5baSBill Taylor 10319e39c5baSBill Taylor target = (uint32_t *)cq_arm_dbr + 1; 10329e39c5baSBill Taylor retry: 10339e39c5baSBill Taylor cmp = *target; 10349e39c5baSBill Taylor tmp = htonl(cmp); 10359e39c5baSBill Taylor old_cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT); 10369e39c5baSBill Taylor new_cmd_sn = (old_cmd_sn + (0x1 << HERMON_CQDB_CMDSN_SHIFT)) & 10379e39c5baSBill Taylor (0x3 << HERMON_CQDB_CMDSN_SHIFT); 10389e39c5baSBill Taylor new = htonl((tmp & ~(0x37 << HERMON_CQDB_CMD_SHIFT)) | new_cmd_sn); 10399e39c5baSBill Taylor tmp = atomic_cas_32(target, cmp, new); 10409e39c5baSBill Taylor if (tmp != cmp) { /* cas failed, so need to retry */ 10419e39c5baSBill Taylor drv_usecwait(retries & 0xff); /* avoid race */ 10429e39c5baSBill Taylor if (++retries > 100000) { 10439e39c5baSBill Taylor cmn_err(CE_CONT, "cas failed in hermon\n"); 10449e39c5baSBill Taylor retries = 0; 10459e39c5baSBill Taylor } 10469e39c5baSBill Taylor goto retry; 10479e39c5baSBill Taylor } 10489e39c5baSBill Taylor } 10499e39c5baSBill Taylor 10509e39c5baSBill Taylor 10519e39c5baSBill Taylor /* 10529e39c5baSBill Taylor * hermon_cq_handler() 10539e39c5baSBill Taylor * Context: Only called from interrupt context 10549e39c5baSBill Taylor */ 1055*17a2b317SBill Taylor /* ARGSUSED */ 10569e39c5baSBill Taylor int 10579e39c5baSBill Taylor hermon_cq_handler(hermon_state_t *state, hermon_eqhdl_t eq, 10589e39c5baSBill Taylor hermon_hw_eqe_t *eqe) 10599e39c5baSBill Taylor { 10609e39c5baSBill Taylor hermon_cqhdl_t cq; 10619e39c5baSBill Taylor uint_t cqnum; 10629e39c5baSBill Taylor 10639e39c5baSBill Taylor /* Get the CQ handle from CQ number in event descriptor */ 10649e39c5baSBill Taylor cqnum = HERMON_EQE_CQNUM_GET(eq, eqe); 10659e39c5baSBill Taylor cq = hermon_cqhdl_from_cqnum(state, cqnum); 10669e39c5baSBill Taylor 10679e39c5baSBill Taylor /* 10689e39c5baSBill Taylor * If the CQ handle is NULL, this is probably an indication 10699e39c5baSBill Taylor * that the CQ has been freed already. In which case, we 10709e39c5baSBill Taylor * should not deliver this event. 10719e39c5baSBill Taylor * 10729e39c5baSBill Taylor * We also check that the CQ number in the handle is the 10739e39c5baSBill Taylor * same as the CQ number in the event queue entry. This 10749e39c5baSBill Taylor * extra check allows us to handle the case where a CQ was 10759e39c5baSBill Taylor * freed and then allocated again in the time it took to 10769e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing 10779e39c5baSBill Taylor * the non-constrained portion of the CQ number every time 10789e39c5baSBill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance 10799e39c5baSBill Taylor * that a stale event could be passed to the client's CQ 10809e39c5baSBill Taylor * handler. 10819e39c5baSBill Taylor * 10829e39c5baSBill Taylor * Lastly, we check if "hs_ibtfpriv" is NULL. If it is then it 10839e39c5baSBill Taylor * means that we've have either received this event before we 10849e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we 10859e39c5baSBill Taylor * are in the process of detaching. 10869e39c5baSBill Taylor */ 10879e39c5baSBill Taylor if ((cq != NULL) && (cq->cq_cqnum == cqnum) && 10889e39c5baSBill Taylor (state->hs_ibtfpriv != NULL)) { 10899e39c5baSBill Taylor hermon_arm_cq_dbr_update(cq->cq_arm_ci_vdbr); 10909e39c5baSBill Taylor HERMON_DO_IBTF_CQ_CALLB(state, cq); 10919e39c5baSBill Taylor } 10929e39c5baSBill Taylor 10939e39c5baSBill Taylor return (DDI_SUCCESS); 10949e39c5baSBill Taylor } 10959e39c5baSBill Taylor 10969e39c5baSBill Taylor 10979e39c5baSBill Taylor /* 10989e39c5baSBill Taylor * hermon_cq_err_handler() 10999e39c5baSBill Taylor * Context: Only called from interrupt context 11009e39c5baSBill Taylor */ 1101*17a2b317SBill Taylor /* ARGSUSED */ 11029e39c5baSBill Taylor int 11039e39c5baSBill Taylor hermon_cq_err_handler(hermon_state_t *state, hermon_eqhdl_t eq, 11049e39c5baSBill Taylor hermon_hw_eqe_t *eqe) 11059e39c5baSBill Taylor { 11069e39c5baSBill Taylor hermon_cqhdl_t cq; 11079e39c5baSBill Taylor uint_t cqnum; 11089e39c5baSBill Taylor ibc_async_event_t event; 11099e39c5baSBill Taylor ibt_async_code_t type; 11109e39c5baSBill Taylor 11119e39c5baSBill Taylor HERMON_FMANOTE(state, HERMON_FMA_OVERRUN); 11129e39c5baSBill Taylor /* Get the CQ handle from CQ number in event descriptor */ 11139e39c5baSBill Taylor cqnum = HERMON_EQE_CQNUM_GET(eq, eqe); 11149e39c5baSBill Taylor cq = hermon_cqhdl_from_cqnum(state, cqnum); 11159e39c5baSBill Taylor 11169e39c5baSBill Taylor /* 11179e39c5baSBill Taylor * If the CQ handle is NULL, this is probably an indication 11189e39c5baSBill Taylor * that the CQ has been freed already. In which case, we 11199e39c5baSBill Taylor * should not deliver this event. 11209e39c5baSBill Taylor * 11219e39c5baSBill Taylor * We also check that the CQ number in the handle is the 11229e39c5baSBill Taylor * same as the CQ number in the event queue entry. This 11239e39c5baSBill Taylor * extra check allows us to handle the case where a CQ was 11249e39c5baSBill Taylor * freed and then allocated again in the time it took to 11259e39c5baSBill Taylor * handle the event queue processing. By constantly incrementing 11269e39c5baSBill Taylor * the non-constrained portion of the CQ number every time 11279e39c5baSBill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance 11289e39c5baSBill Taylor * that a stale event could be passed to the client's CQ 11299e39c5baSBill Taylor * handler. 11309e39c5baSBill Taylor * 11319e39c5baSBill Taylor * And then we check if "hs_ibtfpriv" is NULL. If it is then it 11329e39c5baSBill Taylor * means that we've have either received this event before we 11339e39c5baSBill Taylor * finished attaching to the IBTF or we've received it while we 11349e39c5baSBill Taylor * are in the process of detaching. 11359e39c5baSBill Taylor */ 11369e39c5baSBill Taylor if ((cq != NULL) && (cq->cq_cqnum == cqnum) && 11379e39c5baSBill Taylor (state->hs_ibtfpriv != NULL)) { 11389e39c5baSBill Taylor event.ev_cq_hdl = (ibt_cq_hdl_t)cq->cq_hdlrarg; 11399e39c5baSBill Taylor type = IBT_ERROR_CQ; 11409e39c5baSBill Taylor HERMON_DO_IBTF_ASYNC_CALLB(state, type, &event); 11419e39c5baSBill Taylor } 11429e39c5baSBill Taylor 11439e39c5baSBill Taylor return (DDI_SUCCESS); 11449e39c5baSBill Taylor } 11459e39c5baSBill Taylor 11469e39c5baSBill Taylor 11479e39c5baSBill Taylor /* 11489e39c5baSBill Taylor * hermon_cq_refcnt_inc() 11499e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 11509e39c5baSBill Taylor */ 11519e39c5baSBill Taylor int 11529e39c5baSBill Taylor hermon_cq_refcnt_inc(hermon_cqhdl_t cq, uint_t is_special) 11539e39c5baSBill Taylor { 11549e39c5baSBill Taylor /* 11559e39c5baSBill Taylor * Increment the completion queue's reference count. Note: In order 11569e39c5baSBill Taylor * to ensure compliance with IBA C11-15, we must ensure that a given 11579e39c5baSBill Taylor * CQ is not used for both special (SMI/GSI) QP and non-special QP. 11589e39c5baSBill Taylor * This is accomplished here by keeping track of how the referenced 11599e39c5baSBill Taylor * CQ is being used. 11609e39c5baSBill Taylor */ 11619e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 11629e39c5baSBill Taylor if (cq->cq_refcnt == 0) { 11639e39c5baSBill Taylor cq->cq_is_special = is_special; 11649e39c5baSBill Taylor } else { 11659e39c5baSBill Taylor if (cq->cq_is_special != is_special) { 11669e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 11679e39c5baSBill Taylor return (DDI_FAILURE); 11689e39c5baSBill Taylor } 11699e39c5baSBill Taylor } 11709e39c5baSBill Taylor cq->cq_refcnt++; 11719e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 11729e39c5baSBill Taylor return (DDI_SUCCESS); 11739e39c5baSBill Taylor } 11749e39c5baSBill Taylor 11759e39c5baSBill Taylor 11769e39c5baSBill Taylor /* 11779e39c5baSBill Taylor * hermon_cq_refcnt_dec() 11789e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 11799e39c5baSBill Taylor */ 11809e39c5baSBill Taylor void 11819e39c5baSBill Taylor hermon_cq_refcnt_dec(hermon_cqhdl_t cq) 11829e39c5baSBill Taylor { 11839e39c5baSBill Taylor /* Decrement the completion queue's reference count */ 11849e39c5baSBill Taylor mutex_enter(&cq->cq_lock); 11859e39c5baSBill Taylor cq->cq_refcnt--; 11869e39c5baSBill Taylor mutex_exit(&cq->cq_lock); 11879e39c5baSBill Taylor } 11889e39c5baSBill Taylor 11899e39c5baSBill Taylor 11909e39c5baSBill Taylor /* 11919e39c5baSBill Taylor * hermon_cq_arm_doorbell() 11929e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 11939e39c5baSBill Taylor */ 11949e39c5baSBill Taylor static int 11959e39c5baSBill Taylor hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq, uint_t cq_cmd) 11969e39c5baSBill Taylor { 11979e39c5baSBill Taylor uint32_t cq_num; 11989e39c5baSBill Taylor uint32_t *target; 11999e39c5baSBill Taylor uint32_t old_cmd, cmp, new, tmp, cmd_sn; 12009e39c5baSBill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state); 12019e39c5baSBill Taylor 12029e39c5baSBill Taylor /* initialize the FMA retry loop */ 12039e39c5baSBill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num); 12049e39c5baSBill Taylor 12059e39c5baSBill Taylor cq_num = cq->cq_cqnum; 12069e39c5baSBill Taylor target = (uint32_t *)cq->cq_arm_ci_vdbr + 1; 12079e39c5baSBill Taylor 12089e39c5baSBill Taylor /* the FMA retry loop starts for Hermon doorbell register. */ 12099e39c5baSBill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt, fm_status, 12109e39c5baSBill Taylor fm_test_num); 12119e39c5baSBill Taylor retry: 12129e39c5baSBill Taylor cmp = *target; 12139e39c5baSBill Taylor tmp = htonl(cmp); 12149e39c5baSBill Taylor old_cmd = tmp & (0x7 << HERMON_CQDB_CMD_SHIFT); 12159e39c5baSBill Taylor cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT); 12169e39c5baSBill Taylor if (cq_cmd == HERMON_CQDB_NOTIFY_CQ) { 12179e39c5baSBill Taylor if (old_cmd != HERMON_CQDB_NOTIFY_CQ) { 12189e39c5baSBill Taylor cmd_sn |= (HERMON_CQDB_NOTIFY_CQ << 12199e39c5baSBill Taylor HERMON_CQDB_CMD_SHIFT); 12209e39c5baSBill Taylor new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF)); 12219e39c5baSBill Taylor tmp = atomic_cas_32(target, cmp, new); 12229e39c5baSBill Taylor if (tmp != cmp) 12239e39c5baSBill Taylor goto retry; 12249e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *) 12259e39c5baSBill Taylor &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) << 12269e39c5baSBill Taylor 32) | (cq->cq_consindx & 0xFFFFFF)); 12279e39c5baSBill Taylor } /* else it's already armed */ 12289e39c5baSBill Taylor } else { 12299e39c5baSBill Taylor ASSERT(cq_cmd == HERMON_CQDB_NOTIFY_CQ_SOLICIT); 12309e39c5baSBill Taylor if (old_cmd != HERMON_CQDB_NOTIFY_CQ && 12319e39c5baSBill Taylor old_cmd != HERMON_CQDB_NOTIFY_CQ_SOLICIT) { 12329e39c5baSBill Taylor cmd_sn |= (HERMON_CQDB_NOTIFY_CQ_SOLICIT << 12339e39c5baSBill Taylor HERMON_CQDB_CMD_SHIFT); 12349e39c5baSBill Taylor new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF)); 12359e39c5baSBill Taylor tmp = atomic_cas_32(target, cmp, new); 12369e39c5baSBill Taylor if (tmp != cmp) 12379e39c5baSBill Taylor goto retry; 12389e39c5baSBill Taylor HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *) 12399e39c5baSBill Taylor &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) << 12409e39c5baSBill Taylor 32) | (cq->cq_consindx & 0xFFFFFF)); 12419e39c5baSBill Taylor } /* else it's already armed */ 12429e39c5baSBill Taylor } 12439e39c5baSBill Taylor 12449e39c5baSBill Taylor /* the FMA retry loop ends. */ 12459e39c5baSBill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt, fm_status, 12469e39c5baSBill Taylor fm_test_num); 12479e39c5baSBill Taylor 12489e39c5baSBill Taylor return (IBT_SUCCESS); 12499e39c5baSBill Taylor 12509e39c5baSBill Taylor pio_error: 12519e39c5baSBill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST); 12529e39c5baSBill Taylor return (ibc_get_ci_failure(0)); 12539e39c5baSBill Taylor } 12549e39c5baSBill Taylor 12559e39c5baSBill Taylor 12569e39c5baSBill Taylor /* 12579e39c5baSBill Taylor * hermon_cqhdl_from_cqnum() 12589e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 12599e39c5baSBill Taylor * 12609e39c5baSBill Taylor * This routine is important because changing the unconstrained 12619e39c5baSBill Taylor * portion of the CQ number is critical to the detection of a 12629e39c5baSBill Taylor * potential race condition in the CQ handler code (i.e. the case 12639e39c5baSBill Taylor * where a CQ is freed and alloc'd again before an event for the 12649e39c5baSBill Taylor * "old" CQ can be handled). 12659e39c5baSBill Taylor * 12669e39c5baSBill Taylor * While this is not a perfect solution (not sure that one exists) 12679e39c5baSBill Taylor * it does help to mitigate the chance that this race condition will 12689e39c5baSBill Taylor * cause us to deliver a "stale" event to the new CQ owner. Note: 12699e39c5baSBill Taylor * this solution does not scale well because the number of constrained 12709e39c5baSBill Taylor * bits increases (and, hence, the number of unconstrained bits 12719e39c5baSBill Taylor * decreases) as the number of supported CQs grows. For small and 12729e39c5baSBill Taylor * intermediate values, it should hopefully provide sufficient 12739e39c5baSBill Taylor * protection. 12749e39c5baSBill Taylor */ 12759e39c5baSBill Taylor hermon_cqhdl_t 12769e39c5baSBill Taylor hermon_cqhdl_from_cqnum(hermon_state_t *state, uint_t cqnum) 12779e39c5baSBill Taylor { 12789e39c5baSBill Taylor uint_t cqindx, cqmask; 12799e39c5baSBill Taylor 12809e39c5baSBill Taylor /* Calculate the CQ table index from the cqnum */ 12819e39c5baSBill Taylor cqmask = (1 << state->hs_cfg_profile->cp_log_num_cq) - 1; 12829e39c5baSBill Taylor cqindx = cqnum & cqmask; 1283*17a2b317SBill Taylor return (hermon_icm_num_to_hdl(state, HERMON_CQC, cqindx)); 12849e39c5baSBill Taylor } 12859e39c5baSBill Taylor 12869e39c5baSBill Taylor /* 12879e39c5baSBill Taylor * hermon_cq_cqe_consume() 12889e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 12899e39c5baSBill Taylor */ 12909e39c5baSBill Taylor static void 12919e39c5baSBill Taylor hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq, 12929e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc) 12939e39c5baSBill Taylor { 12949e39c5baSBill Taylor uint_t opcode, qpnum, qp1_indx; 12959e39c5baSBill Taylor ibt_wc_flags_t flags; 12969e39c5baSBill Taylor ibt_wrc_opcode_t type; 12979e39c5baSBill Taylor 12989e39c5baSBill Taylor /* 12999e39c5baSBill Taylor * Determine if this is an "error" CQE by examining "opcode". If it 13009e39c5baSBill Taylor * is an error CQE, then call hermon_cq_errcqe_consume() and return 13019e39c5baSBill Taylor * whatever status it returns. Otherwise, this is a successful 13029e39c5baSBill Taylor * completion. 13039e39c5baSBill Taylor */ 13049e39c5baSBill Taylor opcode = HERMON_CQE_OPCODE_GET(cq, cqe); 13059e39c5baSBill Taylor if ((opcode == HERMON_CQE_SEND_ERR_OPCODE) || 13069e39c5baSBill Taylor (opcode == HERMON_CQE_RECV_ERR_OPCODE)) { 13079e39c5baSBill Taylor hermon_cq_errcqe_consume(state, cq, cqe, wc); 13089e39c5baSBill Taylor return; 13099e39c5baSBill Taylor } 13109e39c5baSBill Taylor 13119e39c5baSBill Taylor /* 13129e39c5baSBill Taylor * Fetch the Work Request ID using the information in the CQE. 13139e39c5baSBill Taylor * See hermon_wr.c for more details. 13149e39c5baSBill Taylor */ 13159e39c5baSBill Taylor wc->wc_id = hermon_wrid_get_entry(cq, cqe); 13169e39c5baSBill Taylor 13179e39c5baSBill Taylor /* 13189e39c5baSBill Taylor * Parse the CQE opcode to determine completion type. This will set 13199e39c5baSBill Taylor * not only the type of the completion, but also any flags that might 13209e39c5baSBill Taylor * be associated with it (e.g. whether immediate data is present). 13219e39c5baSBill Taylor */ 13229e39c5baSBill Taylor flags = IBT_WC_NO_FLAGS; 1323*17a2b317SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->hs_fcoib_may_be_running)) 13249e39c5baSBill Taylor if (HERMON_CQE_SENDRECV_GET(cq, cqe) != HERMON_COMPLETION_RECV) { 13259e39c5baSBill Taylor 13269e39c5baSBill Taylor /* Send CQE */ 13279e39c5baSBill Taylor switch (opcode) { 13289e39c5baSBill Taylor case HERMON_CQE_SND_RDMAWR_IMM: 13299e39c5baSBill Taylor case HERMON_CQE_SND_RDMAWR: 13309e39c5baSBill Taylor type = IBT_WRC_RDMAW; 13319e39c5baSBill Taylor break; 13329e39c5baSBill Taylor 1333*17a2b317SBill Taylor case HERMON_CQE_SND_SEND_INV: 13349e39c5baSBill Taylor case HERMON_CQE_SND_SEND_IMM: 13359e39c5baSBill Taylor case HERMON_CQE_SND_SEND: 13369e39c5baSBill Taylor type = IBT_WRC_SEND; 13379e39c5baSBill Taylor break; 13389e39c5baSBill Taylor 13399e39c5baSBill Taylor case HERMON_CQE_SND_LSO: 13409e39c5baSBill Taylor type = IBT_WRC_SEND_LSO; 13419e39c5baSBill Taylor break; 13429e39c5baSBill Taylor 13439e39c5baSBill Taylor case HERMON_CQE_SND_RDMARD: 13449e39c5baSBill Taylor type = IBT_WRC_RDMAR; 13459e39c5baSBill Taylor break; 13469e39c5baSBill Taylor 13479e39c5baSBill Taylor case HERMON_CQE_SND_ATOMIC_CS: 13489e39c5baSBill Taylor type = IBT_WRC_CSWAP; 13499e39c5baSBill Taylor break; 13509e39c5baSBill Taylor 13519e39c5baSBill Taylor case HERMON_CQE_SND_ATOMIC_FA: 13529e39c5baSBill Taylor type = IBT_WRC_FADD; 13539e39c5baSBill Taylor break; 13549e39c5baSBill Taylor 13559e39c5baSBill Taylor case HERMON_CQE_SND_BIND_MW: 13569e39c5baSBill Taylor type = IBT_WRC_BIND; 13579e39c5baSBill Taylor break; 13589e39c5baSBill Taylor 1359*17a2b317SBill Taylor case HERMON_CQE_SND_FRWR: 1360*17a2b317SBill Taylor type = IBT_WRC_FAST_REG_PMR; 1361*17a2b317SBill Taylor break; 1362*17a2b317SBill Taylor 1363*17a2b317SBill Taylor case HERMON_CQE_SND_LCL_INV: 1364*17a2b317SBill Taylor type = IBT_WRC_LOCAL_INVALIDATE; 1365*17a2b317SBill Taylor break; 1366*17a2b317SBill Taylor 13679e39c5baSBill Taylor default: 13689e39c5baSBill Taylor HERMON_WARNING(state, "unknown send CQE type"); 13699e39c5baSBill Taylor wc->wc_status = IBT_WC_LOCAL_QP_OP_ERR; 13709e39c5baSBill Taylor return; 13719e39c5baSBill Taylor } 1372*17a2b317SBill Taylor } else if ((state->hs_fcoib_may_be_running == B_TRUE) && 1373*17a2b317SBill Taylor hermon_fcoib_is_fexch_qpn(state, HERMON_CQE_QPNUM_GET(cq, cqe))) { 1374*17a2b317SBill Taylor type = IBT_WRC_RECV; 1375*17a2b317SBill Taylor if (HERMON_CQE_FEXCH_DIFE(cq, cqe)) 1376*17a2b317SBill Taylor flags |= IBT_WC_DIF_ERROR; 1377*17a2b317SBill Taylor wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe); 1378*17a2b317SBill Taylor wc->wc_fexch_seq_cnt = HERMON_CQE_FEXCH_SEQ_CNT(cq, cqe); 1379*17a2b317SBill Taylor wc->wc_fexch_tx_bytes_xfer = HERMON_CQE_FEXCH_TX_BYTES(cq, cqe); 1380*17a2b317SBill Taylor wc->wc_fexch_rx_bytes_xfer = HERMON_CQE_FEXCH_RX_BYTES(cq, cqe); 1381*17a2b317SBill Taylor wc->wc_fexch_seq_id = HERMON_CQE_FEXCH_SEQ_ID(cq, cqe); 1382*17a2b317SBill Taylor wc->wc_detail = HERMON_CQE_FEXCH_DETAIL(cq, cqe) & 1383*17a2b317SBill Taylor IBT_WC_DETAIL_FC_MATCH_MASK; 1384*17a2b317SBill Taylor wc->wc_rkey = HERMON_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe); 1385*17a2b317SBill Taylor flags |= IBT_WC_FEXCH_FMT | IBT_WC_RKEY_INVALIDATED; 13869e39c5baSBill Taylor } else { 1387*17a2b317SBill Taylor /* 1388*17a2b317SBill Taylor * Parse the remaining contents of the CQE into the work 1389*17a2b317SBill Taylor * completion. This means filling in SL, QP number, SLID, 1390*17a2b317SBill Taylor * immediate data, etc. 1391*17a2b317SBill Taylor * 1392*17a2b317SBill Taylor * Note: Not all of these fields are valid in a given 1393*17a2b317SBill Taylor * completion. Many of them depend on the actual type of 1394*17a2b317SBill Taylor * completion. So we fill in all of the fields and leave 1395*17a2b317SBill Taylor * it up to the IBTF and consumer to sort out which are 1396*17a2b317SBill Taylor * valid based on their context. 1397*17a2b317SBill Taylor */ 1398*17a2b317SBill Taylor wc->wc_sl = HERMON_CQE_SL_GET(cq, cqe); 1399*17a2b317SBill Taylor wc->wc_qpn = HERMON_CQE_DQPN_GET(cq, cqe); 1400*17a2b317SBill Taylor wc->wc_slid = HERMON_CQE_DLID_GET(cq, cqe); 1401*17a2b317SBill Taylor wc->wc_immed_data = 1402*17a2b317SBill Taylor HERMON_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe); 1403*17a2b317SBill Taylor wc->wc_ethertype = (wc->wc_immed_data & 0xFFFF); 1404*17a2b317SBill Taylor wc->wc_pkey_ix = (wc->wc_immed_data & 1405*17a2b317SBill Taylor ((1 << state->hs_queryport.log_max_pkey) - 1)); 1406*17a2b317SBill Taylor /* 1407*17a2b317SBill Taylor * Fill in "bytes transferred" as appropriate. Also, 1408*17a2b317SBill Taylor * if necessary, fill in the "path bits" field. 1409*17a2b317SBill Taylor */ 1410*17a2b317SBill Taylor wc->wc_path_bits = HERMON_CQE_PATHBITS_GET(cq, cqe); 1411*17a2b317SBill Taylor wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe); 1412*17a2b317SBill Taylor 1413*17a2b317SBill Taylor /* 1414*17a2b317SBill Taylor * Check for GRH, update the flags, then fill in "wc_flags" 1415*17a2b317SBill Taylor * field in the work completion 1416*17a2b317SBill Taylor */ 1417*17a2b317SBill Taylor if (HERMON_CQE_GRH_GET(cq, cqe) != 0) { 1418*17a2b317SBill Taylor flags |= IBT_WC_GRH_PRESENT; 1419*17a2b317SBill Taylor } 14209e39c5baSBill Taylor 14219e39c5baSBill Taylor /* Receive CQE */ 1422*17a2b317SBill Taylor switch (opcode) { 14239e39c5baSBill Taylor case HERMON_CQE_RCV_SEND_IMM: 14249e39c5baSBill Taylor /* 14259e39c5baSBill Taylor * Note: According to the PRM, all QP1 recv 14269e39c5baSBill Taylor * completions look like the result of a Send with 14279e39c5baSBill Taylor * Immediate. They are not, however, (MADs are Send 14289e39c5baSBill Taylor * Only) so we need to check the QP number and set 14299e39c5baSBill Taylor * the flag only if it is non-QP1. 14309e39c5baSBill Taylor */ 14319e39c5baSBill Taylor qpnum = HERMON_CQE_QPNUM_GET(cq, cqe); 14329e39c5baSBill Taylor qp1_indx = state->hs_spec_qp1->hr_indx; 14339e39c5baSBill Taylor if ((qpnum < qp1_indx) || (qpnum > qp1_indx + 1)) { 14349e39c5baSBill Taylor flags |= IBT_WC_IMMED_DATA_PRESENT; 14359e39c5baSBill Taylor } 14369e39c5baSBill Taylor /* FALLTHROUGH */ 1437*17a2b317SBill Taylor 14389e39c5baSBill Taylor case HERMON_CQE_RCV_SEND: 14399e39c5baSBill Taylor type = IBT_WRC_RECV; 14409e39c5baSBill Taylor if (HERMON_CQE_IS_IPOK(cq, cqe)) { 14419e39c5baSBill Taylor wc->wc_cksum = HERMON_CQE_CKSUM(cq, cqe); 14429e39c5baSBill Taylor flags |= IBT_WC_CKSUM_OK; 14439e39c5baSBill Taylor wc->wc_detail = IBT_WC_DETAIL_ALL_FLAGS_MASK & 14449e39c5baSBill Taylor HERMON_CQE_IPOIB_STATUS(cq, cqe); 14459e39c5baSBill Taylor } 14469e39c5baSBill Taylor break; 1447*17a2b317SBill Taylor 1448*17a2b317SBill Taylor case HERMON_CQE_RCV_SEND_INV: 1449*17a2b317SBill Taylor type = IBT_WRC_RECV; 1450*17a2b317SBill Taylor flags |= IBT_WC_RKEY_INVALIDATED; 1451*17a2b317SBill Taylor wc->wc_rkey = wc->wc_immed_data; /* same field in cqe */ 1452*17a2b317SBill Taylor break; 1453*17a2b317SBill Taylor 14549e39c5baSBill Taylor case HERMON_CQE_RCV_RDMAWR_IMM: 14559e39c5baSBill Taylor flags |= IBT_WC_IMMED_DATA_PRESENT; 14569e39c5baSBill Taylor type = IBT_WRC_RECV_RDMAWI; 14579e39c5baSBill Taylor break; 14589e39c5baSBill Taylor 14599e39c5baSBill Taylor default: 14609e39c5baSBill Taylor 14619e39c5baSBill Taylor HERMON_WARNING(state, "unknown recv CQE type"); 14629e39c5baSBill Taylor wc->wc_status = IBT_WC_LOCAL_QP_OP_ERR; 14639e39c5baSBill Taylor return; 14649e39c5baSBill Taylor } 14659e39c5baSBill Taylor } 14669e39c5baSBill Taylor wc->wc_type = type; 14679e39c5baSBill Taylor wc->wc_flags = flags; 14689e39c5baSBill Taylor wc->wc_status = IBT_WC_SUCCESS; 14699e39c5baSBill Taylor } 14709e39c5baSBill Taylor 14719e39c5baSBill Taylor /* 14729e39c5baSBill Taylor * hermon_cq_errcqe_consume() 14739e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 14749e39c5baSBill Taylor */ 14759e39c5baSBill Taylor static void 14769e39c5baSBill Taylor hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq, 14779e39c5baSBill Taylor hermon_hw_cqe_t *cqe, ibt_wc_t *wc) 14789e39c5baSBill Taylor { 14799e39c5baSBill Taylor uint32_t imm_eth_pkey_cred; 14809e39c5baSBill Taylor uint_t status; 14819e39c5baSBill Taylor ibt_wc_status_t ibt_status; 14829e39c5baSBill Taylor 14839e39c5baSBill Taylor /* 14849e39c5baSBill Taylor * Fetch the Work Request ID using the information in the CQE. 14859e39c5baSBill Taylor * See hermon_wr.c for more details. 14869e39c5baSBill Taylor */ 14879e39c5baSBill Taylor wc->wc_id = hermon_wrid_get_entry(cq, cqe); 14889e39c5baSBill Taylor 14899e39c5baSBill Taylor /* 14909e39c5baSBill Taylor * Parse the CQE opcode to determine completion type. We know that 14919e39c5baSBill Taylor * the CQE is an error completion, so we extract only the completion 14929e39c5baSBill Taylor * status/syndrome here. 14939e39c5baSBill Taylor */ 14949e39c5baSBill Taylor imm_eth_pkey_cred = HERMON_CQE_ERROR_SYNDROME_GET(cq, cqe); 14959e39c5baSBill Taylor status = imm_eth_pkey_cred; 1496c7facc54SBill Taylor if (status != HERMON_CQE_WR_FLUSHED_ERR) 1497*17a2b317SBill Taylor IBTF_DPRINTF_L2("CQE ERR", "cqe %p QPN %x indx %x status 0x%x " 1498*17a2b317SBill Taylor "vendor syndrome %x", cqe, HERMON_CQE_QPNUM_GET(cq, cqe), 1499*17a2b317SBill Taylor HERMON_CQE_WQECNTR_GET(cq, cqe), status, 1500*17a2b317SBill Taylor HERMON_CQE_ERROR_VENDOR_SYNDROME_GET(cq, cqe)); 15019e39c5baSBill Taylor switch (status) { 15029e39c5baSBill Taylor case HERMON_CQE_LOC_LEN_ERR: 1503*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCLEN); 15049e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_LEN_ERR; 15059e39c5baSBill Taylor break; 15069e39c5baSBill Taylor 15079e39c5baSBill Taylor case HERMON_CQE_LOC_OP_ERR: 1508*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCQPOP); 15099e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_QP_OP_ERR; 15109e39c5baSBill Taylor break; 15119e39c5baSBill Taylor 15129e39c5baSBill Taylor case HERMON_CQE_LOC_PROT_ERR: 1513*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCPROT); 15149e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_PROTECT_ERR; 1515*17a2b317SBill Taylor IBTF_DPRINTF_L2("ERRCQE", "is at %p", cqe); 15169e39c5baSBill Taylor if (hermon_should_panic) { 15179e39c5baSBill Taylor cmn_err(CE_PANIC, "Hermon intentional PANIC - " 15189e39c5baSBill Taylor "Local Protection Error\n"); 15199e39c5baSBill Taylor } 15209e39c5baSBill Taylor break; 15219e39c5baSBill Taylor 15229e39c5baSBill Taylor case HERMON_CQE_WR_FLUSHED_ERR: 15239e39c5baSBill Taylor ibt_status = IBT_WC_WR_FLUSHED_ERR; 15249e39c5baSBill Taylor break; 15259e39c5baSBill Taylor 15269e39c5baSBill Taylor case HERMON_CQE_MW_BIND_ERR: 1527*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_MWBIND); 15289e39c5baSBill Taylor ibt_status = IBT_WC_MEM_WIN_BIND_ERR; 15299e39c5baSBill Taylor break; 15309e39c5baSBill Taylor 15319e39c5baSBill Taylor case HERMON_CQE_BAD_RESPONSE_ERR: 1532*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_RESP); 15339e39c5baSBill Taylor ibt_status = IBT_WC_BAD_RESPONSE_ERR; 15349e39c5baSBill Taylor break; 15359e39c5baSBill Taylor 15369e39c5baSBill Taylor case HERMON_CQE_LOCAL_ACCESS_ERR: 1537*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_LOCACC); 15389e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_ACCESS_ERR; 15399e39c5baSBill Taylor break; 15409e39c5baSBill Taylor 15419e39c5baSBill Taylor case HERMON_CQE_REM_INV_REQ_ERR: 1542*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_REMREQ); 15439e39c5baSBill Taylor ibt_status = IBT_WC_REMOTE_INVALID_REQ_ERR; 15449e39c5baSBill Taylor break; 15459e39c5baSBill Taylor 15469e39c5baSBill Taylor case HERMON_CQE_REM_ACC_ERR: 1547*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_REMACC); 15489e39c5baSBill Taylor ibt_status = IBT_WC_REMOTE_ACCESS_ERR; 15499e39c5baSBill Taylor break; 15509e39c5baSBill Taylor 15519e39c5baSBill Taylor case HERMON_CQE_REM_OP_ERR: 1552*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_REMOP); 15539e39c5baSBill Taylor ibt_status = IBT_WC_REMOTE_OP_ERR; 15549e39c5baSBill Taylor break; 15559e39c5baSBill Taylor 15569e39c5baSBill Taylor case HERMON_CQE_TRANS_TO_ERR: 1557*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_XPORTCNT); 15589e39c5baSBill Taylor ibt_status = IBT_WC_TRANS_TIMEOUT_ERR; 15599e39c5baSBill Taylor break; 15609e39c5baSBill Taylor 15619e39c5baSBill Taylor case HERMON_CQE_RNRNAK_TO_ERR: 1562*17a2b317SBill Taylor HERMON_WARNING(state, HERMON_FMA_RNRCNT); 15639e39c5baSBill Taylor ibt_status = IBT_WC_RNR_NAK_TIMEOUT_ERR; 15649e39c5baSBill Taylor break; 15659e39c5baSBill Taylor 15669e39c5baSBill Taylor /* 15679e39c5baSBill Taylor * The following error codes are not supported in the Hermon driver 15689e39c5baSBill Taylor * as they relate only to Reliable Datagram completion statuses: 15699e39c5baSBill Taylor * case HERMON_CQE_LOCAL_RDD_VIO_ERR: 15709e39c5baSBill Taylor * case HERMON_CQE_REM_INV_RD_REQ_ERR: 15719e39c5baSBill Taylor * case HERMON_CQE_EEC_REM_ABORTED_ERR: 15729e39c5baSBill Taylor * case HERMON_CQE_INV_EEC_NUM_ERR: 15739e39c5baSBill Taylor * case HERMON_CQE_INV_EEC_STATE_ERR: 15749e39c5baSBill Taylor * case HERMON_CQE_LOC_EEC_ERR: 15759e39c5baSBill Taylor */ 15769e39c5baSBill Taylor 15779e39c5baSBill Taylor default: 15789e39c5baSBill Taylor HERMON_WARNING(state, "unknown error CQE status"); 15799e39c5baSBill Taylor HERMON_FMANOTE(state, HERMON_FMA_UNKN); 15809e39c5baSBill Taylor ibt_status = IBT_WC_LOCAL_QP_OP_ERR; 15819e39c5baSBill Taylor break; 15829e39c5baSBill Taylor } 15839e39c5baSBill Taylor 15849e39c5baSBill Taylor wc->wc_status = ibt_status; 15859e39c5baSBill Taylor } 15869e39c5baSBill Taylor 15879e39c5baSBill Taylor 15889e39c5baSBill Taylor /* 15899e39c5baSBill Taylor * hermon_cq_resize_helper() 15909e39c5baSBill Taylor * Context: Can be called only from user or kernel context. 15919e39c5baSBill Taylor */ 15929e39c5baSBill Taylor void 15939e39c5baSBill Taylor hermon_cq_resize_helper(hermon_state_t *state, hermon_cqhdl_t cq) 15949e39c5baSBill Taylor { 15959e39c5baSBill Taylor hermon_cqhdl_t resize_hdl; 15969e39c5baSBill Taylor int status; 15979e39c5baSBill Taylor 15989e39c5baSBill Taylor /* 15999e39c5baSBill Taylor * we're here because we found the special cqe opcode, so we have 16009e39c5baSBill Taylor * to update the cq_handle, release the old resources, clear the 16019e39c5baSBill Taylor * flag in the cq_hdl, and release the resize_hdl. When we return 16029e39c5baSBill Taylor * above, it will take care of the rest 16039e39c5baSBill Taylor */ 16049e39c5baSBill Taylor ASSERT(MUTEX_HELD(&cq->cq_lock)); 16059e39c5baSBill Taylor 16069e39c5baSBill Taylor resize_hdl = cq->cq_resize_hdl; 16079e39c5baSBill Taylor 16089e39c5baSBill Taylor /* 16099e39c5baSBill Taylor * Deregister the memory for the old Completion Queue. Note: We 16109e39c5baSBill Taylor * really can't return error here because we have no good way to 16119e39c5baSBill Taylor * cleanup. Plus, the deregistration really shouldn't ever happen. 16129e39c5baSBill Taylor * So, if it does, it is an indication that something has gone 16139e39c5baSBill Taylor * seriously wrong. So we print a warning message and return error 16149e39c5baSBill Taylor * (knowing, of course, that the "old" CQ memory will be leaked) 16159e39c5baSBill Taylor */ 16169e39c5baSBill Taylor status = hermon_mr_deregister(state, &cq->cq_mrhdl, HERMON_MR_DEREG_ALL, 16179e39c5baSBill Taylor HERMON_SLEEP); 16189e39c5baSBill Taylor if (status != DDI_SUCCESS) { 16199e39c5baSBill Taylor HERMON_WARNING(state, "failed to deregister old CQ memory"); 16209e39c5baSBill Taylor } 16219e39c5baSBill Taylor 16229e39c5baSBill Taylor /* Next, free the memory from the old CQ buffer */ 16239e39c5baSBill Taylor hermon_queue_free(&cq->cq_cqinfo); 16249e39c5baSBill Taylor 16259e39c5baSBill Taylor /* now we can update the cq_hdl with the new things saved */ 16269e39c5baSBill Taylor 16279e39c5baSBill Taylor cq->cq_buf = resize_hdl->cq_buf; 16289e39c5baSBill Taylor cq->cq_mrhdl = resize_hdl->cq_mrhdl; 16299e39c5baSBill Taylor cq->cq_bufsz = resize_hdl->cq_bufsz; 16309e39c5baSBill Taylor cq->cq_log_cqsz = resize_hdl->cq_log_cqsz; 16319e39c5baSBill Taylor cq->cq_umap_dhp = cq->cq_resize_hdl->cq_umap_dhp; 16329e39c5baSBill Taylor cq->cq_resize_hdl = 0; 16339e39c5baSBill Taylor bcopy(&resize_hdl->cq_cqinfo, &cq->cq_cqinfo, 16349e39c5baSBill Taylor sizeof (struct hermon_qalloc_info_s)); 16359e39c5baSBill Taylor 16369e39c5baSBill Taylor /* finally, release the resizing handle */ 16379e39c5baSBill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s)); 16389e39c5baSBill Taylor } 16399e39c5baSBill Taylor 16409e39c5baSBill Taylor 16419e39c5baSBill Taylor /* 16429e39c5baSBill Taylor * hermon_cq_entries_flush() 16439e39c5baSBill Taylor * Context: Can be called from interrupt or base context. 16449e39c5baSBill Taylor */ 16459e39c5baSBill Taylor /* ARGSUSED */ 16469e39c5baSBill Taylor void 16479e39c5baSBill Taylor hermon_cq_entries_flush(hermon_state_t *state, hermon_qphdl_t qp) 16489e39c5baSBill Taylor { 16499e39c5baSBill Taylor hermon_cqhdl_t cq; 16509e39c5baSBill Taylor hermon_hw_cqe_t *cqe, *next_cqe; 16519e39c5baSBill Taylor hermon_srqhdl_t srq; 16529e39c5baSBill Taylor hermon_workq_hdr_t *wq; 16539e39c5baSBill Taylor uint32_t cons_indx, tail_cons_indx, wrap_around_mask; 16549e39c5baSBill Taylor uint32_t new_indx, check_indx, qpnum; 1655*17a2b317SBill Taylor uint32_t shift, mask; 16569e39c5baSBill Taylor int outstanding_cqes; 16579e39c5baSBill Taylor 16589e39c5baSBill Taylor qpnum = qp->qp_qpnum; 16599e39c5baSBill Taylor if ((srq = qp->qp_srqhdl) != NULL) 16609e39c5baSBill Taylor wq = qp->qp_srqhdl->srq_wq_wqhdr; 16619e39c5baSBill Taylor else 16629e39c5baSBill Taylor wq = NULL; 16639e39c5baSBill Taylor cq = qp->qp_rq_cqhdl; 16649e39c5baSBill Taylor 1665*17a2b317SBill Taylor if (cq == NULL) { 1666*17a2b317SBill Taylor cq = qp->qp_sq_cqhdl; 1667*17a2b317SBill Taylor } 1668*17a2b317SBill Taylor 16699e39c5baSBill Taylor do_send_cq: /* loop back to here if send_cq is not the same as recv_cq */ 1670*17a2b317SBill Taylor if (cq == NULL) 1671*17a2b317SBill Taylor return; 16729e39c5baSBill Taylor 16739e39c5baSBill Taylor cons_indx = cq->cq_consindx; 1674*17a2b317SBill Taylor shift = cq->cq_log_cqsz; 1675*17a2b317SBill Taylor mask = cq->cq_bufsz; 1676*17a2b317SBill Taylor wrap_around_mask = mask - 1; 16779e39c5baSBill Taylor 16789e39c5baSBill Taylor /* Calculate the pointer to the first CQ entry */ 16799e39c5baSBill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask]; 16809e39c5baSBill Taylor 16819e39c5baSBill Taylor /* 16829e39c5baSBill Taylor * Loop through the CQ looking for entries owned by software. If an 16839e39c5baSBill Taylor * entry is owned by software then we increment an 'outstanding_cqes' 16849e39c5baSBill Taylor * count to know how many entries total we have on our CQ. We use this 16859e39c5baSBill Taylor * value further down to know how many entries to loop through looking 16869e39c5baSBill Taylor * for our same QP number. 16879e39c5baSBill Taylor */ 16889e39c5baSBill Taylor outstanding_cqes = 0; 16899e39c5baSBill Taylor tail_cons_indx = cons_indx; 1690*17a2b317SBill Taylor while (HERMON_CQE_OWNER_IS_SW(cq, cqe, tail_cons_indx, shift, mask)) { 16919e39c5baSBill Taylor /* increment total cqes count */ 16929e39c5baSBill Taylor outstanding_cqes++; 16939e39c5baSBill Taylor 16949e39c5baSBill Taylor /* increment the consumer index */ 16959e39c5baSBill Taylor tail_cons_indx++; 16969e39c5baSBill Taylor 16979e39c5baSBill Taylor /* update the pointer to the next cq entry */ 16989e39c5baSBill Taylor cqe = &cq->cq_buf[tail_cons_indx & wrap_around_mask]; 16999e39c5baSBill Taylor } 17009e39c5baSBill Taylor 17019e39c5baSBill Taylor /* 17029e39c5baSBill Taylor * Using the 'tail_cons_indx' that was just set, we now know how many 17039e39c5baSBill Taylor * total CQEs possible there are. Set the 'check_indx' and the 17049e39c5baSBill Taylor * 'new_indx' to the last entry identified by 'tail_cons_indx' 17059e39c5baSBill Taylor */ 17069e39c5baSBill Taylor check_indx = new_indx = (tail_cons_indx - 1); 17079e39c5baSBill Taylor 17089e39c5baSBill Taylor while (--outstanding_cqes >= 0) { 17099e39c5baSBill Taylor cqe = &cq->cq_buf[check_indx & wrap_around_mask]; 17109e39c5baSBill Taylor 17119e39c5baSBill Taylor /* 17129e39c5baSBill Taylor * If the QP number is the same in the CQE as the QP, then 17139e39c5baSBill Taylor * we must "consume" it. If it is for an SRQ wqe, then we 17149e39c5baSBill Taylor * also must free the wqe back onto the free list of the SRQ. 17159e39c5baSBill Taylor */ 17169e39c5baSBill Taylor if (qpnum == HERMON_CQE_QPNUM_GET(cq, cqe)) { 17179e39c5baSBill Taylor if (srq && (HERMON_CQE_SENDRECV_GET(cq, cqe) == 17189e39c5baSBill Taylor HERMON_COMPLETION_RECV)) { 17199e39c5baSBill Taylor uint64_t *desc; 17209e39c5baSBill Taylor int indx; 17219e39c5baSBill Taylor 17229e39c5baSBill Taylor /* Add wqe back to SRQ free list */ 17239e39c5baSBill Taylor indx = HERMON_CQE_WQEADDRSZ_GET(cq, cqe) & 17249e39c5baSBill Taylor wq->wq_mask; 17259e39c5baSBill Taylor desc = HERMON_SRQ_WQE_ADDR(srq, wq->wq_tail); 17269e39c5baSBill Taylor ((uint16_t *)desc)[1] = htons(indx); 17279e39c5baSBill Taylor wq->wq_tail = indx; 17289e39c5baSBill Taylor } 17299e39c5baSBill Taylor } else { /* CQEs for other QPNs need to remain */ 17309e39c5baSBill Taylor if (check_indx != new_indx) { 17319e39c5baSBill Taylor next_cqe = 17329e39c5baSBill Taylor &cq->cq_buf[new_indx & wrap_around_mask]; 17339e39c5baSBill Taylor /* Copy the CQE into the "next_cqe" pointer. */ 17349e39c5baSBill Taylor bcopy(cqe, next_cqe, sizeof (hermon_hw_cqe_t)); 17359e39c5baSBill Taylor } 17369e39c5baSBill Taylor new_indx--; /* move index to next CQE to fill */ 17379e39c5baSBill Taylor } 17389e39c5baSBill Taylor check_indx--; /* move index to next CQE to check */ 17399e39c5baSBill Taylor } 17409e39c5baSBill Taylor 17419e39c5baSBill Taylor /* 17429e39c5baSBill Taylor * Update consumer index to be the 'new_indx'. This moves it past all 17439e39c5baSBill Taylor * removed entries. Because 'new_indx' is pointing to the last 17449e39c5baSBill Taylor * previously valid SW owned entry, we add 1 to point the cons_indx to 17459e39c5baSBill Taylor * the first HW owned entry. 17469e39c5baSBill Taylor */ 17479e39c5baSBill Taylor cons_indx = (new_indx + 1); 17489e39c5baSBill Taylor 17499e39c5baSBill Taylor /* 17509e39c5baSBill Taylor * Now we only ring the doorbell (to update the consumer index) if 17519e39c5baSBill Taylor * we've actually consumed a CQ entry. If we found no QP number 17529e39c5baSBill Taylor * matches above, then we would not have removed anything. So only if 17539e39c5baSBill Taylor * something was removed do we ring the doorbell. 17549e39c5baSBill Taylor */ 17559e39c5baSBill Taylor if (cq->cq_consindx != cons_indx) { 17569e39c5baSBill Taylor /* 17579e39c5baSBill Taylor * Update the consumer index in both the CQ handle and the 17589e39c5baSBill Taylor * doorbell record. 17599e39c5baSBill Taylor */ 17609e39c5baSBill Taylor cq->cq_consindx = cons_indx; 17619e39c5baSBill Taylor 17629e39c5baSBill Taylor hermon_cq_update_ci_doorbell(cq); 17639e39c5baSBill Taylor 17649e39c5baSBill Taylor } 17659e39c5baSBill Taylor if (cq != qp->qp_sq_cqhdl) { 17669e39c5baSBill Taylor cq = qp->qp_sq_cqhdl; 17679e39c5baSBill Taylor goto do_send_cq; 17689e39c5baSBill Taylor } 17699e39c5baSBill Taylor } 1770*17a2b317SBill Taylor 1771*17a2b317SBill Taylor /* 1772*17a2b317SBill Taylor * hermon_get_cq_sched_list() 1773*17a2b317SBill Taylor * Context: Only called from attach() path context 1774*17a2b317SBill Taylor * 1775*17a2b317SBill Taylor * Read properties, creating entries in hs_cq_sched_list with 1776*17a2b317SBill Taylor * information about the requested "expected" and "minimum" 1777*17a2b317SBill Taylor * number of MSI-X interrupt vectors per list entry. 1778*17a2b317SBill Taylor */ 1779*17a2b317SBill Taylor static int 1780*17a2b317SBill Taylor hermon_get_cq_sched_list(hermon_state_t *state) 1781*17a2b317SBill Taylor { 1782*17a2b317SBill Taylor char **listp, ulp_prop[HERMON_CQH_MAX + 4]; 1783*17a2b317SBill Taylor uint_t nlist, i, j, ndata; 1784*17a2b317SBill Taylor int *data; 1785*17a2b317SBill Taylor size_t len; 1786*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp; 1787*17a2b317SBill Taylor 1788*17a2b317SBill Taylor if (ddi_prop_lookup_string_array(DDI_DEV_T_ANY, state->hs_dip, 1789*17a2b317SBill Taylor DDI_PROP_DONTPASS, "cqh-group-list", &listp, &nlist) != 1790*17a2b317SBill Taylor DDI_PROP_SUCCESS) 1791*17a2b317SBill Taylor return (0); 1792*17a2b317SBill Taylor 1793*17a2b317SBill Taylor state->hs_cq_sched_array_size = nlist; 1794*17a2b317SBill Taylor state->hs_cq_sched_array = cq_schedp = kmem_zalloc(nlist * 1795*17a2b317SBill Taylor sizeof (hermon_cq_sched_t), KM_SLEEP); 1796*17a2b317SBill Taylor for (i = 0; i < nlist; i++) { 1797*17a2b317SBill Taylor if ((len = strlen(listp[i])) >= HERMON_CQH_MAX) { 1798*17a2b317SBill Taylor cmn_err(CE_CONT, "'cqh' property name too long\n"); 1799*17a2b317SBill Taylor goto game_over; 1800*17a2b317SBill Taylor } 1801*17a2b317SBill Taylor for (j = 0; j < i; j++) { 1802*17a2b317SBill Taylor if (strcmp(listp[j], listp[i]) == 0) { 1803*17a2b317SBill Taylor cmn_err(CE_CONT, "Duplicate 'cqh' property\n"); 1804*17a2b317SBill Taylor goto game_over; 1805*17a2b317SBill Taylor } 1806*17a2b317SBill Taylor } 1807*17a2b317SBill Taylor (void) strncpy(cq_schedp[i].cqs_name, listp[i], HERMON_CQH_MAX); 1808*17a2b317SBill Taylor ulp_prop[0] = 'c'; 1809*17a2b317SBill Taylor ulp_prop[1] = 'q'; 1810*17a2b317SBill Taylor ulp_prop[2] = 'h'; 1811*17a2b317SBill Taylor ulp_prop[3] = '-'; 1812*17a2b317SBill Taylor (void) strncpy(ulp_prop + 4, listp[i], len + 1); 1813*17a2b317SBill Taylor if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip, 1814*17a2b317SBill Taylor DDI_PROP_DONTPASS, ulp_prop, &data, &ndata) != 1815*17a2b317SBill Taylor DDI_PROP_SUCCESS) { 1816*17a2b317SBill Taylor cmn_err(CE_CONT, "property '%s' not found\n", ulp_prop); 1817*17a2b317SBill Taylor goto game_over; 1818*17a2b317SBill Taylor } 1819*17a2b317SBill Taylor if (ndata != 2) { 1820*17a2b317SBill Taylor cmn_err(CE_CONT, "property '%s' does not " 1821*17a2b317SBill Taylor "have 2 integers\n", ulp_prop); 1822*17a2b317SBill Taylor goto game_over_free_data; 1823*17a2b317SBill Taylor } 1824*17a2b317SBill Taylor cq_schedp[i].cqs_desired = data[0]; 1825*17a2b317SBill Taylor cq_schedp[i].cqs_minimum = data[1]; 1826*17a2b317SBill Taylor cq_schedp[i].cqs_refcnt = 0; 1827*17a2b317SBill Taylor ddi_prop_free(data); 1828*17a2b317SBill Taylor } 1829*17a2b317SBill Taylor if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip, 1830*17a2b317SBill Taylor DDI_PROP_DONTPASS, "cqh-default", &data, &ndata) != 1831*17a2b317SBill Taylor DDI_PROP_SUCCESS) { 1832*17a2b317SBill Taylor cmn_err(CE_CONT, "property 'cqh-default' not found\n"); 1833*17a2b317SBill Taylor goto game_over; 1834*17a2b317SBill Taylor } 1835*17a2b317SBill Taylor if (ndata != 2) { 1836*17a2b317SBill Taylor cmn_err(CE_CONT, "property 'cqh-default' does not " 1837*17a2b317SBill Taylor "have 2 integers\n"); 1838*17a2b317SBill Taylor goto game_over_free_data; 1839*17a2b317SBill Taylor } 1840*17a2b317SBill Taylor cq_schedp = &state->hs_cq_sched_default; 1841*17a2b317SBill Taylor cq_schedp->cqs_desired = data[0]; 1842*17a2b317SBill Taylor cq_schedp->cqs_minimum = data[1]; 1843*17a2b317SBill Taylor cq_schedp->cqs_refcnt = 0; 1844*17a2b317SBill Taylor ddi_prop_free(data); 1845*17a2b317SBill Taylor ddi_prop_free(listp); 1846*17a2b317SBill Taylor return (1); /* game on */ 1847*17a2b317SBill Taylor 1848*17a2b317SBill Taylor game_over_free_data: 1849*17a2b317SBill Taylor ddi_prop_free(data); 1850*17a2b317SBill Taylor game_over: 1851*17a2b317SBill Taylor cmn_err(CE_CONT, "Error in 'cqh' properties in hermon.conf\n"); 1852*17a2b317SBill Taylor cmn_err(CE_CONT, "completion handler groups not being used\n"); 1853*17a2b317SBill Taylor kmem_free(cq_schedp, nlist * sizeof (hermon_cq_sched_t)); 1854*17a2b317SBill Taylor state->hs_cq_sched_array_size = 0; 1855*17a2b317SBill Taylor ddi_prop_free(listp); 1856*17a2b317SBill Taylor return (0); 1857*17a2b317SBill Taylor } 1858*17a2b317SBill Taylor 1859*17a2b317SBill Taylor /* 1860*17a2b317SBill Taylor * hermon_cq_sched_init() 1861*17a2b317SBill Taylor * Context: Only called from attach() path context 1862*17a2b317SBill Taylor * 1863*17a2b317SBill Taylor * Read the hermon.conf properties looking for cq_sched info, 1864*17a2b317SBill Taylor * creating reserved pools of MSI-X interrupt ranges for the 1865*17a2b317SBill Taylor * specified ULPs. 1866*17a2b317SBill Taylor */ 1867*17a2b317SBill Taylor int 1868*17a2b317SBill Taylor hermon_cq_sched_init(hermon_state_t *state) 1869*17a2b317SBill Taylor { 1870*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp, *defp; 1871*17a2b317SBill Taylor int i, desired, array_size; 1872*17a2b317SBill Taylor 1873*17a2b317SBill Taylor mutex_init(&state->hs_cq_sched_lock, NULL, MUTEX_DRIVER, 1874*17a2b317SBill Taylor DDI_INTR_PRI(state->hs_intrmsi_pri)); 1875*17a2b317SBill Taylor 1876*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock); 1877*17a2b317SBill Taylor state->hs_cq_sched_array = NULL; 1878*17a2b317SBill Taylor 1879*17a2b317SBill Taylor /* initialize cq_sched_default */ 1880*17a2b317SBill Taylor defp = &state->hs_cq_sched_default; 1881*17a2b317SBill Taylor defp->cqs_start_hid = 1; 1882*17a2b317SBill Taylor defp->cqs_len = state->hs_intrmsi_allocd; 1883*17a2b317SBill Taylor defp->cqs_next_alloc = defp->cqs_len - 1; 1884*17a2b317SBill Taylor (void) strncpy(defp->cqs_name, "default", 8); 1885*17a2b317SBill Taylor 1886*17a2b317SBill Taylor /* Read properties to determine which ULPs use cq_sched */ 1887*17a2b317SBill Taylor if (hermon_get_cq_sched_list(state) == 0) 1888*17a2b317SBill Taylor goto done; 1889*17a2b317SBill Taylor 1890*17a2b317SBill Taylor /* Determine if we have enough vectors, or if we have to scale down */ 1891*17a2b317SBill Taylor desired = defp->cqs_desired; /* default desired (from hermon.conf) */ 1892*17a2b317SBill Taylor if (desired <= 0) 1893*17a2b317SBill Taylor goto done; /* all interrupts in the default pool */ 1894*17a2b317SBill Taylor cq_schedp = state->hs_cq_sched_array; 1895*17a2b317SBill Taylor array_size = state->hs_cq_sched_array_size; 1896*17a2b317SBill Taylor for (i = 0; i < array_size; i++) 1897*17a2b317SBill Taylor desired += cq_schedp[i].cqs_desired; 1898*17a2b317SBill Taylor if (desired > state->hs_intrmsi_allocd) { 1899*17a2b317SBill Taylor cmn_err(CE_CONT, "#interrupts allocated (%d) is less than " 1900*17a2b317SBill Taylor "the #interrupts desired (%d)\n", 1901*17a2b317SBill Taylor state->hs_intrmsi_allocd, desired); 1902*17a2b317SBill Taylor cmn_err(CE_CONT, "completion handler groups not being used\n"); 1903*17a2b317SBill Taylor goto done; /* all interrupts in the default pool */ 1904*17a2b317SBill Taylor } 1905*17a2b317SBill Taylor /* Game on. For each cq_sched group, reserve the MSI-X range */ 1906*17a2b317SBill Taylor for (i = 0; i < array_size; i++) { 1907*17a2b317SBill Taylor desired = cq_schedp[i].cqs_desired; 1908*17a2b317SBill Taylor cq_schedp[i].cqs_start_hid = defp->cqs_start_hid; 1909*17a2b317SBill Taylor cq_schedp[i].cqs_len = desired; 1910*17a2b317SBill Taylor cq_schedp[i].cqs_next_alloc = desired - 1; 1911*17a2b317SBill Taylor defp->cqs_len -= desired; 1912*17a2b317SBill Taylor defp->cqs_start_hid += desired; 1913*17a2b317SBill Taylor } 1914*17a2b317SBill Taylor /* reset default's start allocation seed */ 1915*17a2b317SBill Taylor state->hs_cq_sched_default.cqs_next_alloc = 1916*17a2b317SBill Taylor state->hs_cq_sched_default.cqs_len - 1; 1917*17a2b317SBill Taylor 1918*17a2b317SBill Taylor done: 1919*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock); 1920*17a2b317SBill Taylor return (IBT_SUCCESS); 1921*17a2b317SBill Taylor } 1922*17a2b317SBill Taylor 1923*17a2b317SBill Taylor void 1924*17a2b317SBill Taylor hermon_cq_sched_fini(hermon_state_t *state) 1925*17a2b317SBill Taylor { 1926*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock); 1927*17a2b317SBill Taylor if (state->hs_cq_sched_array_size) { 1928*17a2b317SBill Taylor kmem_free(state->hs_cq_sched_array, sizeof (hermon_cq_sched_t) * 1929*17a2b317SBill Taylor state->hs_cq_sched_array_size); 1930*17a2b317SBill Taylor state->hs_cq_sched_array_size = 0; 1931*17a2b317SBill Taylor state->hs_cq_sched_array = NULL; 1932*17a2b317SBill Taylor } 1933*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock); 1934*17a2b317SBill Taylor mutex_destroy(&state->hs_cq_sched_lock); 1935*17a2b317SBill Taylor } 1936*17a2b317SBill Taylor 1937*17a2b317SBill Taylor int 1938*17a2b317SBill Taylor hermon_cq_sched_alloc(hermon_state_t *state, ibt_cq_sched_attr_t *attr, 1939*17a2b317SBill Taylor hermon_cq_sched_t **cq_sched_pp) 1940*17a2b317SBill Taylor { 1941*17a2b317SBill Taylor hermon_cq_sched_t *cq_schedp; 1942*17a2b317SBill Taylor int i; 1943*17a2b317SBill Taylor char *name; 1944*17a2b317SBill Taylor ibt_cq_sched_flags_t flags; 1945*17a2b317SBill Taylor 1946*17a2b317SBill Taylor flags = attr->cqs_flags; 1947*17a2b317SBill Taylor if ((flags & (IBT_CQS_SCHED_GROUP | IBT_CQS_EXACT_SCHED_GROUP)) == 0) { 1948*17a2b317SBill Taylor *cq_sched_pp = NULL; 1949*17a2b317SBill Taylor return (IBT_SUCCESS); 1950*17a2b317SBill Taylor } 1951*17a2b317SBill Taylor name = attr->cqs_pool_name; 1952*17a2b317SBill Taylor 1953*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock); 1954*17a2b317SBill Taylor cq_schedp = state->hs_cq_sched_array; 1955*17a2b317SBill Taylor for (i = 0; i < state->hs_cq_sched_array_size; i++, cq_schedp++) { 1956*17a2b317SBill Taylor if (strcmp(name, cq_schedp->cqs_name) == 0) { 1957*17a2b317SBill Taylor if (cq_schedp->cqs_len != 0) 1958*17a2b317SBill Taylor cq_schedp->cqs_refcnt++; 1959*17a2b317SBill Taylor break; /* found it */ 1960*17a2b317SBill Taylor } 1961*17a2b317SBill Taylor } 1962*17a2b317SBill Taylor if ((i == state->hs_cq_sched_array_size) || /* not found, or */ 1963*17a2b317SBill Taylor (cq_schedp->cqs_len == 0)) /* defined, but no dedicated intr's */ 1964*17a2b317SBill Taylor cq_schedp = NULL; 1965*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock); 1966*17a2b317SBill Taylor 1967*17a2b317SBill Taylor *cq_sched_pp = cq_schedp; /* set to valid hdl, or to NULL */ 1968*17a2b317SBill Taylor if ((cq_schedp == NULL) && 1969*17a2b317SBill Taylor (attr->cqs_flags & IBT_CQS_EXACT_SCHED_GROUP)) 1970*17a2b317SBill Taylor return (IBT_CQ_NO_SCHED_GROUP); 1971*17a2b317SBill Taylor else 1972*17a2b317SBill Taylor return (IBT_SUCCESS); 1973*17a2b317SBill Taylor } 1974*17a2b317SBill Taylor 1975*17a2b317SBill Taylor int 1976*17a2b317SBill Taylor hermon_cq_sched_free(hermon_state_t *state, hermon_cq_sched_t *cq_schedp) 1977*17a2b317SBill Taylor { 1978*17a2b317SBill Taylor if (cq_schedp != NULL) { 1979*17a2b317SBill Taylor /* Just decrement refcnt */ 1980*17a2b317SBill Taylor mutex_enter(&state->hs_cq_sched_lock); 1981*17a2b317SBill Taylor if (cq_schedp->cqs_refcnt == 0) 1982*17a2b317SBill Taylor HERMON_WARNING(state, "cq_sched free underflow\n"); 1983*17a2b317SBill Taylor else 1984*17a2b317SBill Taylor cq_schedp->cqs_refcnt--; 1985*17a2b317SBill Taylor mutex_exit(&state->hs_cq_sched_lock); 1986*17a2b317SBill Taylor } 1987*17a2b317SBill Taylor return (IBT_SUCCESS); 1988*17a2b317SBill Taylor } 1989