1*9e39c5baSBill Taylor /* 2*9e39c5baSBill Taylor * CDDL HEADER START 3*9e39c5baSBill Taylor * 4*9e39c5baSBill Taylor * The contents of this file are subject to the terms of the 5*9e39c5baSBill Taylor * Common Development and Distribution License (the "License"). 6*9e39c5baSBill Taylor * You may not use this file except in compliance with the License. 7*9e39c5baSBill Taylor * 8*9e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9*9e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing. 10*9e39c5baSBill Taylor * See the License for the specific language governing permissions 11*9e39c5baSBill Taylor * and limitations under the License. 12*9e39c5baSBill Taylor * 13*9e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each 14*9e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15*9e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the 16*9e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying 17*9e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner] 18*9e39c5baSBill Taylor * 19*9e39c5baSBill Taylor * CDDL HEADER END 20*9e39c5baSBill Taylor */ 21*9e39c5baSBill Taylor 22*9e39c5baSBill Taylor /* 23*9e39c5baSBill Taylor * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24*9e39c5baSBill Taylor * Use is subject to license terms. 25*9e39c5baSBill Taylor */ 26*9e39c5baSBill Taylor 27*9e39c5baSBill Taylor #ifndef _DAPL_TAVOR_HW_H 28*9e39c5baSBill Taylor #define _DAPL_TAVOR_HW_H 29*9e39c5baSBill Taylor 30*9e39c5baSBill Taylor /* 31*9e39c5baSBill Taylor * dapl_tavor_hw.h 32*9e39c5baSBill Taylor * Contains all the structure definitions and #defines for all Tavor 33*9e39c5baSBill Taylor * hardware resources and registers. 34*9e39c5baSBill Taylor * Most of these definitions have been replicated from the tavor_hw.h 35*9e39c5baSBill Taylor * header file used by the tavor device driver. 36*9e39c5baSBill Taylor */ 37*9e39c5baSBill Taylor 38*9e39c5baSBill Taylor #ifdef __cplusplus 39*9e39c5baSBill Taylor extern "C" { 40*9e39c5baSBill Taylor #endif 41*9e39c5baSBill Taylor 42*9e39c5baSBill Taylor #include "dapl.h" 43*9e39c5baSBill Taylor #include "dapl_tavor_ibtf.h" 44*9e39c5baSBill Taylor 45*9e39c5baSBill Taylor 46*9e39c5baSBill Taylor /* 47*9e39c5baSBill Taylor * Ownership flags used to define hardware or software ownership for 48*9e39c5baSBill Taylor * various Tavor resources 49*9e39c5baSBill Taylor */ 50*9e39c5baSBill Taylor #define TAVOR_HW_OWNER 0x1U 51*9e39c5baSBill Taylor #define TAVOR_SW_OWNER 0x0 52*9e39c5baSBill Taylor 53*9e39c5baSBill Taylor /* 54*9e39c5baSBill Taylor * Tavor Completion Queue Entries (CQE) 55*9e39c5baSBill Taylor * Each CQE contains enough information for the software to associate the 56*9e39c5baSBill Taylor * completion with the Work Queue Element (WQE) to which it corresponds. 57*9e39c5baSBill Taylor * 58*9e39c5baSBill Taylor * Note: The following structure is not #define'd with both little-endian 59*9e39c5baSBill Taylor * and big-endian definitions. This is because each CQE's individual 60*9e39c5baSBill Taylor * fields are not directly accessed except through the macros defined below. 61*9e39c5baSBill Taylor */ 62*9e39c5baSBill Taylor 63*9e39c5baSBill Taylor /* 64*9e39c5baSBill Taylor * The following defines are used for Tavor CQ error handling. Note: For 65*9e39c5baSBill Taylor * CQEs which correspond to error events, the Tavor device requires some 66*9e39c5baSBill Taylor * special handling by software. These defines are used to identify and 67*9e39c5baSBill Taylor * extract the necessary information from each error CQE, including status 68*9e39c5baSBill Taylor * code (above), doorbell count, and whether a error completion is for a 69*9e39c5baSBill Taylor * send or receive work request. 70*9e39c5baSBill Taylor */ 71*9e39c5baSBill Taylor #define TAVOR_CQE_ERR_STATUS_SHIFT 24 72*9e39c5baSBill Taylor #define TAVOR_CQE_ERR_STATUS_MASK 0xFF 73*9e39c5baSBill Taylor #define TAVOR_CQE_ERR_DBDCNT_MASK 0xFFFF 74*9e39c5baSBill Taylor #define TAVOR_CQE_SEND_ERR_OPCODE 0xFF 75*9e39c5baSBill Taylor #define TAVOR_CQE_RECV_ERR_OPCODE 0xFE 76*9e39c5baSBill Taylor #define TAVOR_CQ_SYNC_AND_DB 0 77*9e39c5baSBill Taylor #define TAVOR_CQ_RECYCLE_ENTRY 1 78*9e39c5baSBill Taylor 79*9e39c5baSBill Taylor /* 80*9e39c5baSBill Taylor * These are the defines for the Tavor CQ entry types. They are also 81*9e39c5baSBill Taylor * specified by the Tavor register specification. They indicate what type 82*9e39c5baSBill Taylor * of work request is completing (for successful completions). Note: The 83*9e39c5baSBill Taylor * "SND" or "RCV" in each define is used to indicate whether the completion 84*9e39c5baSBill Taylor * work request was from the Send work queue or the Receive work queue on 85*9e39c5baSBill Taylor * the associated QP. 86*9e39c5baSBill Taylor */ 87*9e39c5baSBill Taylor #define TAVOR_CQE_SND_RDMAWR 0x8 88*9e39c5baSBill Taylor #define TAVOR_CQE_SND_RDMAWR_IMM 0x9 89*9e39c5baSBill Taylor #define TAVOR_CQE_SND_SEND 0xA 90*9e39c5baSBill Taylor #define TAVOR_CQE_SND_SEND_IMM 0xB 91*9e39c5baSBill Taylor #define TAVOR_CQE_SND_RDMARD 0x10 92*9e39c5baSBill Taylor #define TAVOR_CQE_SND_ATOMIC_CS 0x11 93*9e39c5baSBill Taylor #define TAVOR_CQE_SND_ATOMIC_FA 0x12 94*9e39c5baSBill Taylor #define TAVOR_CQE_SND_BIND_MW 0x18 95*9e39c5baSBill Taylor #define TAVOR_CQE_RCV_RECV_IMM 0x3 96*9e39c5baSBill Taylor #define TAVOR_CQE_RCV_RECV_IMM2 0x5 97*9e39c5baSBill Taylor #define TAVOR_CQE_RCV_RECV 0x2 98*9e39c5baSBill Taylor #define TAVOR_CQE_RCV_RECV2 0x4 99*9e39c5baSBill Taylor #define TAVOR_CQE_RCV_RDMAWR_IMM 0x9 100*9e39c5baSBill Taylor #define TAVOR_CQE_RCV_RDMAWR_IMM2 0xB 101*9e39c5baSBill Taylor 102*9e39c5baSBill Taylor /* 103*9e39c5baSBill Taylor * These are the defines for the Tavor CQ completion statuses. They are 104*9e39c5baSBill Taylor * specified by the Tavor register specification. 105*9e39c5baSBill Taylor */ 106*9e39c5baSBill Taylor #define TAVOR_CQE_SUCCESS 0x0 107*9e39c5baSBill Taylor #define TAVOR_CQE_LOC_LEN_ERR 0x1 108*9e39c5baSBill Taylor #define TAVOR_CQE_LOC_OP_ERR 0x2 109*9e39c5baSBill Taylor #define TAVOR_CQE_LOC_EEC_ERR 0x3 /* unsupported: RD */ 110*9e39c5baSBill Taylor #define TAVOR_CQE_LOC_PROT_ERR 0x4 111*9e39c5baSBill Taylor #define TAVOR_CQE_WR_FLUSHED_ERR 0x5 112*9e39c5baSBill Taylor #define TAVOR_CQE_MW_BIND_ERR 0x6 113*9e39c5baSBill Taylor #define TAVOR_CQE_BAD_RESPONSE_ERR 0x10 114*9e39c5baSBill Taylor #define TAVOR_CQE_LOCAL_ACCESS_ERR 0x11 115*9e39c5baSBill Taylor #define TAVOR_CQE_REM_INV_REQ_ERR 0x12 116*9e39c5baSBill Taylor #define TAVOR_CQE_REM_ACC_ERR 0x13 117*9e39c5baSBill Taylor #define TAVOR_CQE_REM_OP_ERR 0x14 118*9e39c5baSBill Taylor #define TAVOR_CQE_TRANS_TO_ERR 0x15 119*9e39c5baSBill Taylor #define TAVOR_CQE_RNRNAK_TO_ERR 0x16 120*9e39c5baSBill Taylor #define TAVOR_CQE_LOCAL_RDD_VIO_ERR 0x20 /* unsupported: RD */ 121*9e39c5baSBill Taylor #define TAVOR_CQE_REM_INV_RD_REQ_ERR 0x21 /* unsupported: RD */ 122*9e39c5baSBill Taylor #define TAVOR_CQE_EEC_REM_ABORTED_ERR 0x22 /* unsupported: RD */ 123*9e39c5baSBill Taylor #define TAVOR_CQE_INV_EEC_NUM_ERR 0x23 /* unsupported: RD */ 124*9e39c5baSBill Taylor #define TAVOR_CQE_INV_EEC_STATE_ERR 0x24 /* unsupported: RD */ 125*9e39c5baSBill Taylor 126*9e39c5baSBill Taylor typedef struct tavor_hw_cqe_s { 127*9e39c5baSBill Taylor uint32_t ver :4; 128*9e39c5baSBill Taylor uint32_t :4; 129*9e39c5baSBill Taylor uint32_t my_qpn :24; 130*9e39c5baSBill Taylor uint32_t :8; 131*9e39c5baSBill Taylor uint32_t my_ee :24; 132*9e39c5baSBill Taylor uint32_t :8; 133*9e39c5baSBill Taylor uint32_t rqpn :24; 134*9e39c5baSBill Taylor uint32_t sl :4; 135*9e39c5baSBill Taylor uint32_t :4; 136*9e39c5baSBill Taylor uint32_t grh :1; 137*9e39c5baSBill Taylor uint32_t ml_path :7; 138*9e39c5baSBill Taylor uint32_t rlid :16; 139*9e39c5baSBill Taylor uint32_t imm_eth_pkey_cred; 140*9e39c5baSBill Taylor uint32_t byte_cnt; 141*9e39c5baSBill Taylor uint32_t wqe_addr :26; 142*9e39c5baSBill Taylor uint32_t wqe_sz :6; 143*9e39c5baSBill Taylor uint32_t opcode :8; 144*9e39c5baSBill Taylor uint32_t send_or_recv :1; 145*9e39c5baSBill Taylor uint32_t :15; 146*9e39c5baSBill Taylor uint32_t owner :1; 147*9e39c5baSBill Taylor uint32_t status :7; 148*9e39c5baSBill Taylor } tavor_hw_cqe_t; 149*9e39c5baSBill Taylor #define TAVOR_COMPLETION_RECV 0x0 150*9e39c5baSBill Taylor #define TAVOR_COMPLETION_SEND 0x1 151*9e39c5baSBill Taylor 152*9e39c5baSBill Taylor #define TAVOR_CQE_DEFAULT_VERSION 0x0 153*9e39c5baSBill Taylor 154*9e39c5baSBill Taylor /* 155*9e39c5baSBill Taylor * The following macros are used for extracting (and in some cases filling in) 156*9e39c5baSBill Taylor * information from CQEs 157*9e39c5baSBill Taylor */ 158*9e39c5baSBill Taylor #define TAVOR_CQE_QPNUM_MASK 0x00FFFFFF 159*9e39c5baSBill Taylor #define TAVOR_CQE_QPNUM_SHIFT 0 160*9e39c5baSBill Taylor #define TAVOR_CQE_DQPN_MASK 0x00FFFFFF 161*9e39c5baSBill Taylor #define TAVOR_CQE_DQPN_SHIFT 0 162*9e39c5baSBill Taylor #define TAVOR_CQE_SL_MASK 0xF0000000 163*9e39c5baSBill Taylor #define TAVOR_CQE_SL_SHIFT 28 164*9e39c5baSBill Taylor #define TAVOR_CQE_GRH_MASK 0x00800000 165*9e39c5baSBill Taylor #define TAVOR_CQE_GRH_SHIFT 23 166*9e39c5baSBill Taylor #define TAVOR_CQE_PATHBITS_MASK 0x007F0000 167*9e39c5baSBill Taylor #define TAVOR_CQE_PATHBITS_SHIFT 16 168*9e39c5baSBill Taylor #define TAVOR_CQE_DLID_MASK 0x0000FFFF 169*9e39c5baSBill Taylor #define TAVOR_CQE_DLID_SHIFT 0 170*9e39c5baSBill Taylor #define TAVOR_CQE_OPCODE_MASK 0xFF000000 171*9e39c5baSBill Taylor #define TAVOR_CQE_OPCODE_SHIFT 24 172*9e39c5baSBill Taylor #define TAVOR_CQE_SENDRECV_MASK 0x00800000 173*9e39c5baSBill Taylor #define TAVOR_CQE_SENDRECV_SHIFT 23 174*9e39c5baSBill Taylor #define TAVOR_CQE_OWNER_MASK 0x00000080 175*9e39c5baSBill Taylor #define TAVOR_CQE_OWNER_SHIFT 7 176*9e39c5baSBill Taylor 177*9e39c5baSBill Taylor #define TAVOR_CQE_QPNUM_GET(cqe) \ 178*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[0]) & TAVOR_CQE_QPNUM_MASK) >> \ 179*9e39c5baSBill Taylor TAVOR_CQE_QPNUM_SHIFT) 180*9e39c5baSBill Taylor #define TAVOR_CQE_DQPN_GET(cqe) \ 181*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[2]) & TAVOR_CQE_DQPN_MASK) >> \ 182*9e39c5baSBill Taylor TAVOR_CQE_DQPN_SHIFT) 183*9e39c5baSBill Taylor #define TAVOR_CQE_SL_GET(cqe) \ 184*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_SL_MASK) >> \ 185*9e39c5baSBill Taylor TAVOR_CQE_SL_SHIFT) 186*9e39c5baSBill Taylor #define TAVOR_CQE_GRH_GET(cqe) \ 187*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_GRH_MASK) >> \ 188*9e39c5baSBill Taylor TAVOR_CQE_GRH_SHIFT) 189*9e39c5baSBill Taylor #define TAVOR_CQE_PATHBITS_GET(cqe) \ 190*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_PATHBITS_MASK) >>\ 191*9e39c5baSBill Taylor TAVOR_CQE_PATHBITS_SHIFT) 192*9e39c5baSBill Taylor #define TAVOR_CQE_DLID_GET(cqe) \ 193*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[3]) & TAVOR_CQE_DLID_MASK) >> \ 194*9e39c5baSBill Taylor TAVOR_CQE_DLID_SHIFT) 195*9e39c5baSBill Taylor #define TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cqe) \ 196*9e39c5baSBill Taylor (BETOH_32(((uint32_t *)(cqe))[4])) 197*9e39c5baSBill Taylor #define TAVOR_CQE_IMM_ETH_PKEY_CRED_SET(cqe, arg) \ 198*9e39c5baSBill Taylor (((uint32_t *)(cqe))[4] = HTOBE_32((arg))) 199*9e39c5baSBill Taylor #define TAVOR_CQE_BYTECNT_GET(cqe) \ 200*9e39c5baSBill Taylor (BETOH_32(((uint32_t *)(cqe))[5])) 201*9e39c5baSBill Taylor #define TAVOR_CQE_WQEADDRSZ_GET(cqe) \ 202*9e39c5baSBill Taylor (BETOH_32(((uint32_t *)(cqe))[6])) 203*9e39c5baSBill Taylor #define TAVOR_CQE_WQEADDRSZ_SET(cqe, arg) \ 204*9e39c5baSBill Taylor (((uint32_t *)(cqe))[6] = HTOBE_32((arg))) 205*9e39c5baSBill Taylor #define TAVOR_CQE_OPCODE_GET(cqe) \ 206*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[7]) & TAVOR_CQE_OPCODE_MASK) >> \ 207*9e39c5baSBill Taylor TAVOR_CQE_OPCODE_SHIFT) 208*9e39c5baSBill Taylor #define TAVOR_CQE_SENDRECV_GET(cqe) \ 209*9e39c5baSBill Taylor ((BETOH_32(((uint32_t *)(cqe))[7]) & TAVOR_CQE_SENDRECV_MASK) >>\ 210*9e39c5baSBill Taylor TAVOR_CQE_SENDRECV_SHIFT) 211*9e39c5baSBill Taylor #define TAVOR_CQE_OWNER_IS_SW(cqe) \ 212*9e39c5baSBill Taylor (((BETOH_32(((uint32_t *)(cqe))[7]) & TAVOR_CQE_OWNER_MASK) >> \ 213*9e39c5baSBill Taylor TAVOR_CQE_OWNER_SHIFT) == TAVOR_SW_OWNER) 214*9e39c5baSBill Taylor #define TAVOR_CQE_OWNER_SET_HW(cqe) \ 215*9e39c5baSBill Taylor (((uint32_t *)(cqe))[7] = \ 216*9e39c5baSBill Taylor BETOH_32((TAVOR_HW_OWNER << TAVOR_CQE_OWNER_SHIFT) & \ 217*9e39c5baSBill Taylor TAVOR_CQE_OWNER_MASK)) 218*9e39c5baSBill Taylor 219*9e39c5baSBill Taylor /* 220*9e39c5baSBill Taylor * Tavor User Access Region (UAR) 221*9e39c5baSBill Taylor * Tavor doorbells are each rung by writing to the doorbell registers that 222*9e39c5baSBill Taylor * form a User Access Region (UAR). A doorbell is a write-only hardware 223*9e39c5baSBill Taylor * register which enables passing information from software to hardware 224*9e39c5baSBill Taylor * with minimum software latency. A write operation from the host software 225*9e39c5baSBill Taylor * to these doorbell registers passes information about the HCA resources 226*9e39c5baSBill Taylor * and initiates processing of the doorbell data. There are 6 types of 227*9e39c5baSBill Taylor * doorbells in Tavor. 228*9e39c5baSBill Taylor * 229*9e39c5baSBill Taylor * "Send Doorbell" for synchronizing the attachment of a WQE (or a chain 230*9e39c5baSBill Taylor * of WQEs) to the send queue. 231*9e39c5baSBill Taylor * "RD Send Doorbell" (Same as above, except for RD QPs) is not supported. 232*9e39c5baSBill Taylor * "Receive Doorbell" for synchronizing the attachment of a WQE (or a chain 233*9e39c5baSBill Taylor * of WQEs) to the receive queue. 234*9e39c5baSBill Taylor * "CQ Doorbell" for updating the CQ consumer index and requesting 235*9e39c5baSBill Taylor * completion notifications. 236*9e39c5baSBill Taylor * "EQ Doorbell" for updating the EQ consumer index, arming interrupt 237*9e39c5baSBill Taylor * triggering, and disarming CQ notification requests. 238*9e39c5baSBill Taylor * "InfiniBlast" (which would have enabled access to the "InfiniBlast 239*9e39c5baSBill Taylor * buffer") is not supported. 240*9e39c5baSBill Taylor * 241*9e39c5baSBill Taylor * Note: The tavor_hw_uar_t below is the container for all of the various 242*9e39c5baSBill Taylor * doorbell types. Below we first define several structures which make up 243*9e39c5baSBill Taylor * the contents of those doorbell types. 244*9e39c5baSBill Taylor * 245*9e39c5baSBill Taylor * Note also: The following structures are not #define'd with both little- 246*9e39c5baSBill Taylor * endian and big-endian definitions. This is because each doorbell type 247*9e39c5baSBill Taylor * is not directly accessed except through a single ddi_put64() operation 248*9e39c5baSBill Taylor * (see tavor_qp_send_doorbell, tavor_qp_recv_doorbell, tavor_cq_doorbell, 249*9e39c5baSBill Taylor * or tavor_eq_doorbell) 250*9e39c5baSBill Taylor */ 251*9e39c5baSBill Taylor typedef struct tavor_hw_uar_send_s { 252*9e39c5baSBill Taylor uint32_t nda :26; 253*9e39c5baSBill Taylor uint32_t fence :1; 254*9e39c5baSBill Taylor uint32_t nopcode :5; 255*9e39c5baSBill Taylor uint32_t qpn :24; 256*9e39c5baSBill Taylor uint32_t :2; 257*9e39c5baSBill Taylor uint32_t nds :6; 258*9e39c5baSBill Taylor } tavor_hw_uar_send_t; 259*9e39c5baSBill Taylor #define TAVOR_QPSNDDB_NDA_MASK 0xFFFFFFC0 260*9e39c5baSBill Taylor #define TAVOR_QPSNDDB_NDA_SHIFT 0x20 261*9e39c5baSBill Taylor #define TAVOR_QPSNDDB_F_SHIFT 0x25 262*9e39c5baSBill Taylor #define TAVOR_QPSNDDB_NOPCODE_SHIFT 0x20 263*9e39c5baSBill Taylor #define TAVOR_QPSNDDB_QPN_SHIFT 0x8 264*9e39c5baSBill Taylor 265*9e39c5baSBill Taylor typedef struct tavor_hw_uar_recv_s { 266*9e39c5baSBill Taylor uint32_t nda :26; 267*9e39c5baSBill Taylor uint32_t nds :6; 268*9e39c5baSBill Taylor uint32_t qpn :24; 269*9e39c5baSBill Taylor uint32_t credits :8; 270*9e39c5baSBill Taylor } tavor_hw_uar_recv_t; 271*9e39c5baSBill Taylor #define TAVOR_QPRCVDB_NDA_MASK 0xFFFFFFC0 272*9e39c5baSBill Taylor #define TAVOR_QPRCVDB_NDA_SHIFT 0x20 273*9e39c5baSBill Taylor #define TAVOR_QPRCVDB_NDS_SHIFT 0x20 274*9e39c5baSBill Taylor #define TAVOR_QPRCVDB_QPN_SHIFT 0x8 275*9e39c5baSBill Taylor /* Max descriptors per Tavor doorbell */ 276*9e39c5baSBill Taylor #define TAVOR_QP_MAXDESC_PER_DB 256 277*9e39c5baSBill Taylor 278*9e39c5baSBill Taylor typedef struct tavor_hw_uar_cq_s { 279*9e39c5baSBill Taylor uint32_t cmd :8; 280*9e39c5baSBill Taylor uint32_t cqn :24; 281*9e39c5baSBill Taylor uint32_t param; 282*9e39c5baSBill Taylor } tavor_hw_uar_cq_t; 283*9e39c5baSBill Taylor #define TAVOR_CQDB_CMD_SHIFT 0x38 284*9e39c5baSBill Taylor #define TAVOR_CQDB_CQN_SHIFT 0x20 285*9e39c5baSBill Taylor 286*9e39c5baSBill Taylor #define TAVOR_CQDB_INCR_CONSINDX 0x01 287*9e39c5baSBill Taylor #define TAVOR_CQDB_NOTIFY_CQ 0x02 288*9e39c5baSBill Taylor #define TAVOR_CQDB_NOTIFY_CQ_SOLICIT 0x03 289*9e39c5baSBill Taylor #define TAVOR_CQDB_SET_CONSINDX 0x04 290*9e39c5baSBill Taylor #define TAVOR_CQDB_NOTIFY_NCQ 0x05 291*9e39c5baSBill Taylor /* Default value for use in NOTIFY_CQ doorbell */ 292*9e39c5baSBill Taylor #define TAVOR_CQDB_DEFAULT_PARAM 0xFFFFFFFF 293*9e39c5baSBill Taylor 294*9e39c5baSBill Taylor typedef struct tavor_hw_uar_eq_s { 295*9e39c5baSBill Taylor uint32_t cmd :8; 296*9e39c5baSBill Taylor uint32_t :18; 297*9e39c5baSBill Taylor uint32_t eqn :6; 298*9e39c5baSBill Taylor uint32_t param; 299*9e39c5baSBill Taylor } tavor_hw_uar_eq_t; 300*9e39c5baSBill Taylor 301*9e39c5baSBill Taylor typedef struct tavor_hw_uar_s { 302*9e39c5baSBill Taylor uint32_t rsrv0[4]; /* "RD Send" unsupported */ 303*9e39c5baSBill Taylor uint64_t send; /* tavor_hw_uar_send_t */ 304*9e39c5baSBill Taylor uint64_t recv; /* tavor_hw_uar_recv_t */ 305*9e39c5baSBill Taylor uint64_t cq; /* tavor_hw_uar_cq_t */ 306*9e39c5baSBill Taylor uint64_t eq; /* tavor_hw_uar_eq_t */ 307*9e39c5baSBill Taylor uint32_t rsrv1[244]; 308*9e39c5baSBill Taylor uint32_t iblast[256]; /* "InfiniBlast" unsupported */ 309*9e39c5baSBill Taylor } tavor_hw_uar_t; 310*9e39c5baSBill Taylor 311*9e39c5baSBill Taylor typedef struct tavor_hw_uar32_s { 312*9e39c5baSBill Taylor uint32_t rsrv0[4]; /* "RD Send" unsupported */ 313*9e39c5baSBill Taylor uint32_t send[2]; /* tavor_hw_uar_send_t */ 314*9e39c5baSBill Taylor uint32_t recv[2]; /* tavor_hw_uar_recv_t */ 315*9e39c5baSBill Taylor uint32_t cq[2]; /* tavor_hw_uar_cq_t */ 316*9e39c5baSBill Taylor uint32_t eq[2]; /* tavor_hw_uar_eq_t */ 317*9e39c5baSBill Taylor uint32_t rsrv1[244]; 318*9e39c5baSBill Taylor uint32_t iblast[256]; /* "InfiniBlast" unsupported */ 319*9e39c5baSBill Taylor } tavor_hw_uar32_t; 320*9e39c5baSBill Taylor 321*9e39c5baSBill Taylor 322*9e39c5baSBill Taylor /* 323*9e39c5baSBill Taylor * Tavor Send Work Queue Element (WQE) 324*9e39c5baSBill Taylor * A Tavor Send WQE is built of the following segments, each of which is a 325*9e39c5baSBill Taylor * multiple of 16 bytes. Note: Each individual WQE may contain only a 326*9e39c5baSBill Taylor * subset of these segments described below (according to the operation type 327*9e39c5baSBill Taylor * and transport type of the QP). 328*9e39c5baSBill Taylor * 329*9e39c5baSBill Taylor * The first 16 bytes of ever WQE are formed from the "Next/Ctrl" segment. 330*9e39c5baSBill Taylor * This segment contains the address of the next WQE to be executed and the 331*9e39c5baSBill Taylor * information required in order to allocate the resources to execute the 332*9e39c5baSBill Taylor * next WQE. The "Ctrl" part of this segment contains the control 333*9e39c5baSBill Taylor * information required to execute the WQE, including the opcode and other 334*9e39c5baSBill Taylor * control information. 335*9e39c5baSBill Taylor * The "Datagram" segment contains address information required in order to 336*9e39c5baSBill Taylor * form a UD message. 337*9e39c5baSBill Taylor * The "Bind" segment contains the parameters required for a Bind Memory 338*9e39c5baSBill Taylor * Window operation. 339*9e39c5baSBill Taylor * The "Remote Address" segment is present only in RDMA or Atomic WQEs and 340*9e39c5baSBill Taylor * specifies remote virtual addresses and RKey, respectively. Length of 341*9e39c5baSBill Taylor * the remote access is calculated from the scatter/gather list (for 342*9e39c5baSBill Taylor * RDMA-write/RDMA-read) or set to eight (for Atomic). 343*9e39c5baSBill Taylor * The "Atomic" segment is present only in Atomic WQEs and specifies 344*9e39c5baSBill Taylor * Swap/Add and Compare data. 345*9e39c5baSBill Taylor * 346*9e39c5baSBill Taylor * Note: The following structures are not #define'd with both little-endian 347*9e39c5baSBill Taylor * and big-endian definitions. This is because their individual fields are 348*9e39c5baSBill Taylor * not directly accessed except through macros defined below. 349*9e39c5baSBill Taylor */ 350*9e39c5baSBill Taylor typedef struct tavor_hw_snd_wqe_nextctrl_s { 351*9e39c5baSBill Taylor uint32_t next_wqe_addr :26; 352*9e39c5baSBill Taylor uint32_t :1; 353*9e39c5baSBill Taylor uint32_t nopcode :5; 354*9e39c5baSBill Taylor uint32_t next_eec :24; 355*9e39c5baSBill Taylor uint32_t dbd :1; 356*9e39c5baSBill Taylor uint32_t fence :1; 357*9e39c5baSBill Taylor uint32_t nds :6; 358*9e39c5baSBill Taylor 359*9e39c5baSBill Taylor uint32_t :28; 360*9e39c5baSBill Taylor uint32_t c :1; 361*9e39c5baSBill Taylor uint32_t e :1; 362*9e39c5baSBill Taylor uint32_t s :1; 363*9e39c5baSBill Taylor uint32_t i :1; 364*9e39c5baSBill Taylor uint32_t immediate :32; 365*9e39c5baSBill Taylor } tavor_hw_snd_wqe_nextctrl_t; 366*9e39c5baSBill Taylor 367*9e39c5baSBill Taylor #define TAVOR_WQE_NDA_MASK 0x00000000FFFFFFC0 368*9e39c5baSBill Taylor #define TAVOR_WQE_NDS_MASK 0x3F 369*9e39c5baSBill Taylor #define TAVOR_WQE_DBD_MASK 0x80 370*9e39c5baSBill Taylor 371*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_FENCE_MASK 0x40 372*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_RDMAW 0x8 373*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_RDMAWI 0x9 374*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_SEND 0xA 375*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_SENDI 0xB 376*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_RDMAR 0x10 377*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_ATMCS 0x11 378*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_ATMFA 0x12 379*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_NOPCODE_BIND 0x18 380*9e39c5baSBill Taylor 381*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_SIGNALED_MASK 0x800000000ULL 382*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_EVENT_MASK 0x400000000ULL 383*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_SOLICIT_MASK 0x200000000ULL 384*9e39c5baSBill Taylor #define TAVOR_WQE_SEND_IMMEDIATE_MASK 0x100000000ULL 385*9e39c5baSBill Taylor 386*9e39c5baSBill Taylor #define TAVOR_WQE_SENDHDR_UD_AV_MASK 0xFFFFFFFFFFFFFFE0 387*9e39c5baSBill Taylor #define TAVOR_WQE_SENDHDR_UD_DQPN_MASK 0xFFFFFF 388*9e39c5baSBill Taylor 389*9e39c5baSBill Taylor typedef struct tavor_hw_snd_wqe_bind_s { 390*9e39c5baSBill Taylor uint32_t ae :1; 391*9e39c5baSBill Taylor uint32_t rw :1; 392*9e39c5baSBill Taylor uint32_t rr :1; 393*9e39c5baSBill Taylor uint32_t :29; 394*9e39c5baSBill Taylor uint32_t :32; 395*9e39c5baSBill Taylor uint32_t new_rkey; 396*9e39c5baSBill Taylor uint32_t reg_lkey; 397*9e39c5baSBill Taylor uint64_t addr; 398*9e39c5baSBill Taylor uint64_t len; 399*9e39c5baSBill Taylor } tavor_hw_snd_wqe_bind_t; 400*9e39c5baSBill Taylor #define TAVOR_WQE_SENDHDR_BIND_ATOM 0x8000000000000000ULL 401*9e39c5baSBill Taylor #define TAVOR_WQE_SENDHDR_BIND_WR 0x4000000000000000ULL 402*9e39c5baSBill Taylor #define TAVOR_WQE_SENDHDR_BIND_RD 0x2000000000000000ULL 403*9e39c5baSBill Taylor 404*9e39c5baSBill Taylor typedef struct tavor_hw_snd_wqe_remaddr_s { 405*9e39c5baSBill Taylor uint64_t vaddr; 406*9e39c5baSBill Taylor uint32_t rkey; 407*9e39c5baSBill Taylor uint32_t :32; 408*9e39c5baSBill Taylor } tavor_hw_snd_wqe_remaddr_t; 409*9e39c5baSBill Taylor 410*9e39c5baSBill Taylor /* 411*9e39c5baSBill Taylor * Tavor Receive Work Queue Element (WQE) 412*9e39c5baSBill Taylor * Like the Send WQE, the Receive WQE is built of 16-byte segments. The 413*9e39c5baSBill Taylor * segment is the "Next/Ctrl" segment (defined below). It is followed by 414*9e39c5baSBill Taylor * some number of scatter list entries for the incoming message. 415*9e39c5baSBill Taylor * 416*9e39c5baSBill Taylor * The format of the scatter-gather list entries is also shown below. For 417*9e39c5baSBill Taylor * Receive WQEs the "inline_data" field must be cleared (i.e. data segments 418*9e39c5baSBill Taylor * cannot contain inline data). 419*9e39c5baSBill Taylor */ 420*9e39c5baSBill Taylor typedef struct tavor_hw_rcv_wqe_nextctrl_s { 421*9e39c5baSBill Taylor uint32_t next_wqe_addr :26; 422*9e39c5baSBill Taylor uint32_t :5; 423*9e39c5baSBill Taylor uint32_t one :1; 424*9e39c5baSBill Taylor uint32_t :24; 425*9e39c5baSBill Taylor uint32_t dbd :1; 426*9e39c5baSBill Taylor uint32_t :1; 427*9e39c5baSBill Taylor uint32_t nds :6; 428*9e39c5baSBill Taylor 429*9e39c5baSBill Taylor uint32_t :28; 430*9e39c5baSBill Taylor uint32_t c :1; 431*9e39c5baSBill Taylor uint32_t e :1; 432*9e39c5baSBill Taylor uint32_t :2; 433*9e39c5baSBill Taylor uint32_t :32; 434*9e39c5baSBill Taylor } tavor_hw_rcv_wqe_nextctrl_t; 435*9e39c5baSBill Taylor 436*9e39c5baSBill Taylor /* 437*9e39c5baSBill Taylor * This bit must be set in the next/ctrl field of all Receive WQEs 438*9e39c5baSBill Taylor * as a workaround to a Tavor hardware erratum related to having 439*9e39c5baSBill Taylor * the first 32-bits in the WQE set to zero. 440*9e39c5baSBill Taylor */ 441*9e39c5baSBill Taylor #define TAVOR_RCV_WQE_NDA0_WA_MASK 0x0000000100000000ULL 442*9e39c5baSBill Taylor #define TAVOR_WQE_RCV_SIGNALED_MASK 0x800000000ULL 443*9e39c5baSBill Taylor #define TAVOR_WQE_RCV_EVENT_MASK 0x400000000ULL 444*9e39c5baSBill Taylor 445*9e39c5baSBill Taylor typedef struct tavor_hw_wqe_sgl_s { 446*9e39c5baSBill Taylor uint32_t inline_data :1; 447*9e39c5baSBill Taylor uint32_t byte_cnt :31; 448*9e39c5baSBill Taylor uint32_t lkey; 449*9e39c5baSBill Taylor uint64_t addr; 450*9e39c5baSBill Taylor } tavor_hw_wqe_sgl_t; 451*9e39c5baSBill Taylor #define TAVOR_WQE_SGL_BYTE_CNT_MASK 0x7FFFFFFF 452*9e39c5baSBill Taylor #define TAVOR_WQE_SGL_INLINE_MASK 0x80000000 453*9e39c5baSBill Taylor /* 454*9e39c5baSBill Taylor * The tavor_sw_wqe_dbinfo_t structure is used internally by the Tavor 455*9e39c5baSBill Taylor * driver to return information (from the tavor_wqe_mlx_build_nextctl() and 456*9e39c5baSBill Taylor * tavor_wqe_send_build_nextctl() routines) regarding the type of Tavor 457*9e39c5baSBill Taylor * doorbell necessary. 458*9e39c5baSBill Taylor */ 459*9e39c5baSBill Taylor typedef struct tavor_sw_wqe_dbinfo_s { 460*9e39c5baSBill Taylor uint_t db_nopcode; 461*9e39c5baSBill Taylor uint_t db_fence; 462*9e39c5baSBill Taylor } tavor_sw_wqe_dbinfo_t; 463*9e39c5baSBill Taylor 464*9e39c5baSBill Taylor 465*9e39c5baSBill Taylor /* 466*9e39c5baSBill Taylor * The following macros are used for building each of the individual 467*9e39c5baSBill Taylor * segments that can make up a Tavor WQE. Note: We try not to use the 468*9e39c5baSBill Taylor * structures (with their associated bitfields) here, instead opting to 469*9e39c5baSBill Taylor * build and put 64-bit or 32-bit chunks to the WQEs as appropriate, 470*9e39c5baSBill Taylor * primarily because using the bitfields appears to force more read-modify- 471*9e39c5baSBill Taylor * write operations. 472*9e39c5baSBill Taylor * 473*9e39c5baSBill Taylor * TAVOR_WQE_BUILD_REMADDR - Builds Remote Address Segment using 474*9e39c5baSBill Taylor * RDMA info from the work request 475*9e39c5baSBill Taylor * TAVOR_WQE_BUILD_BIND - Builds the Bind Memory Window 476*9e39c5baSBill Taylor * Segment using bind info from the 477*9e39c5baSBill Taylor * work request 478*9e39c5baSBill Taylor * TAVOR_WQE_LINKNEXT - Links the current WQE to the 479*9e39c5baSBill Taylor * previous one 480*9e39c5baSBill Taylor * TAVOR_WQE_LINKFIRST - Links the first WQE on the current 481*9e39c5baSBill Taylor * chain to the previous WQE 482*9e39c5baSBill Taylor */ 483*9e39c5baSBill Taylor 484*9e39c5baSBill Taylor #define TAVOR_WQE_BUILD_REMADDR(ra, wr_rdma) \ 485*9e39c5baSBill Taylor { \ 486*9e39c5baSBill Taylor uint64_t *tmp; \ 487*9e39c5baSBill Taylor \ 488*9e39c5baSBill Taylor tmp = (uint64_t *)(ra); \ 489*9e39c5baSBill Taylor tmp[0] = HTOBE_64((wr_rdma)->rdma_raddr); \ 490*9e39c5baSBill Taylor tmp[1] = HTOBE_64((uint64_t)(wr_rdma)->rdma_rkey << 32); \ 491*9e39c5baSBill Taylor } 492*9e39c5baSBill Taylor #define TAVOR_WQE_BUILD_BIND(bn, wr_bind) \ 493*9e39c5baSBill Taylor { \ 494*9e39c5baSBill Taylor uint64_t *tmp; \ 495*9e39c5baSBill Taylor uint64_t bn0_tmp; \ 496*9e39c5baSBill Taylor ibt_bind_flags_t bind_flags; \ 497*9e39c5baSBill Taylor \ 498*9e39c5baSBill Taylor tmp = (uint64_t *)(bn); \ 499*9e39c5baSBill Taylor bind_flags = (wr_bind)->bind_flags; \ 500*9e39c5baSBill Taylor bn0_tmp = (bind_flags & IBT_WR_BIND_ATOMIC) ? \ 501*9e39c5baSBill Taylor TAVOR_WQE_SENDHDR_BIND_ATOM : 0; \ 502*9e39c5baSBill Taylor bn0_tmp |= (bind_flags & IBT_WR_BIND_WRITE) ? \ 503*9e39c5baSBill Taylor TAVOR_WQE_SENDHDR_BIND_WR : 0; \ 504*9e39c5baSBill Taylor bn0_tmp |= (bind_flags & IBT_WR_BIND_READ) ? \ 505*9e39c5baSBill Taylor TAVOR_WQE_SENDHDR_BIND_RD : 0; \ 506*9e39c5baSBill Taylor tmp[0] = HTOBE_64(bn0_tmp); \ 507*9e39c5baSBill Taylor tmp[1] = HTOBE_64(((uint64_t)(wr_bind)->bind_rkey_out << 32) | \ 508*9e39c5baSBill Taylor (wr_bind)->bind_lkey); \ 509*9e39c5baSBill Taylor tmp[2] = HTOBE_64((wr_bind)->bind_va); \ 510*9e39c5baSBill Taylor tmp[3] = HTOBE_64((wr_bind)->bind_len); \ 511*9e39c5baSBill Taylor } 512*9e39c5baSBill Taylor 513*9e39c5baSBill Taylor #define TAVOR_WQE_BUILD_DATA_SEG(ds, sgl) \ 514*9e39c5baSBill Taylor { \ 515*9e39c5baSBill Taylor uint64_t *tmp; \ 516*9e39c5baSBill Taylor \ 517*9e39c5baSBill Taylor tmp = (uint64_t *)(ds); \ 518*9e39c5baSBill Taylor tmp[0] = HTOBE_64(((uint64_t)((sgl)->ds_len & \ 519*9e39c5baSBill Taylor TAVOR_WQE_SGL_BYTE_CNT_MASK) << 32) | (sgl)->ds_key); \ 520*9e39c5baSBill Taylor tmp[1] = HTOBE_64((sgl)->ds_va); \ 521*9e39c5baSBill Taylor } 522*9e39c5baSBill Taylor 523*9e39c5baSBill Taylor #define TAVOR_WQE_LINKNEXT(prev, ctrl, next) \ 524*9e39c5baSBill Taylor { \ 525*9e39c5baSBill Taylor ((uint64_t *)(prev))[1] = HTOBE_64((ctrl)); \ 526*9e39c5baSBill Taylor ((uint64_t *)(prev))[0] = HTOBE_64((next)); \ 527*9e39c5baSBill Taylor } 528*9e39c5baSBill Taylor 529*9e39c5baSBill Taylor #define TAVOR_WQE_LINKFIRST(prev, next) \ 530*9e39c5baSBill Taylor { \ 531*9e39c5baSBill Taylor ((uint64_t *)(prev))[0] = HTOBE_64((next)); \ 532*9e39c5baSBill Taylor } 533*9e39c5baSBill Taylor 534*9e39c5baSBill Taylor /* 535*9e39c5baSBill Taylor * The following macro is used to convert WQE address and size into the 536*9e39c5baSBill Taylor * "wqeaddrsz" value needed in the tavor_wrid_entry_t (see below). 537*9e39c5baSBill Taylor */ 538*9e39c5baSBill Taylor #define TAVOR_QP_WQEADDRSZ(addr, size) \ 539*9e39c5baSBill Taylor ((((uintptr_t)(addr)) & ~TAVOR_WQE_NDS_MASK) | \ 540*9e39c5baSBill Taylor ((size) & TAVOR_WQE_NDS_MASK)) 541*9e39c5baSBill Taylor 542*9e39c5baSBill Taylor /* 543*9e39c5baSBill Taylor * The following macros are used to calculate pointers to the Send or Receive 544*9e39c5baSBill Taylor * WQEs on a given QP, respectively 545*9e39c5baSBill Taylor */ 546*9e39c5baSBill Taylor #define TAVOR_QP_SQ_ENTRY(qp, tail) \ 547*9e39c5baSBill Taylor ((uint64_t *)((uintptr_t)((qp)->qp_sq_buf) + \ 548*9e39c5baSBill Taylor ((tail) * (qp)->qp_sq_wqesz))) 549*9e39c5baSBill Taylor #define TAVOR_QP_SQ_DESC(qp, tail) \ 550*9e39c5baSBill Taylor ((uint32_t)((qp)->qp_sq_desc_addr + \ 551*9e39c5baSBill Taylor ((tail) * (qp)->qp_sq_wqesz))) 552*9e39c5baSBill Taylor #define TAVOR_QP_RQ_ENTRY(qp, tail) \ 553*9e39c5baSBill Taylor ((uint64_t *)((uintptr_t)((qp)->qp_rq_buf) + \ 554*9e39c5baSBill Taylor ((tail) * (qp)->qp_rq_wqesz))) 555*9e39c5baSBill Taylor #define TAVOR_QP_RQ_DESC(qp, tail) \ 556*9e39c5baSBill Taylor ((uint32_t)((qp)->qp_rq_desc_addr + \ 557*9e39c5baSBill Taylor ((tail) * (qp)->qp_rq_wqesz))) 558*9e39c5baSBill Taylor #define TAVOR_SRQ_RQ_ENTRY(srq, tail) \ 559*9e39c5baSBill Taylor ((uint64_t *)((uintptr_t)((srq)->srq_wq_buf) + \ 560*9e39c5baSBill Taylor ((tail) * (srq)->srq_wq_wqesz))) 561*9e39c5baSBill Taylor #define TAVOR_SRQ_RQ_DESC(srq, tail) \ 562*9e39c5baSBill Taylor ((uint32_t)((srq)->srq_wq_desc_addr + \ 563*9e39c5baSBill Taylor ((tail) * (srq)->srq_wq_wqesz))) 564*9e39c5baSBill Taylor #define TAVOR_SRQ_WQ_INDEX(srq_wq_desc_addr, desc_addr, wqesz) \ 565*9e39c5baSBill Taylor ((uint32_t)(((desc_addr) - (srq_wq_desc_addr)) / (wqesz))) 566*9e39c5baSBill Taylor #define TAVOR_SRQ_WQ_ENTRY(srq, index) \ 567*9e39c5baSBill Taylor ((uint64_t *)(((uintptr_t)(srq)->srq_addr) + \ 568*9e39c5baSBill Taylor ((index) * (srq)->srq_wq_wqesz))) 569*9e39c5baSBill Taylor 570*9e39c5baSBill Taylor /* 571*9e39c5baSBill Taylor * Maximum header before the data bytes when inlining data. 572*9e39c5baSBill Taylor * "Header" includes the link (nextctrl) struct, a remote address struct 573*9e39c5baSBill Taylor * (only for RDMA Write, not for Send) and the 32-bit byte count field. 574*9e39c5baSBill Taylor */ 575*9e39c5baSBill Taylor #define TAVOR_INLINE_HEADER_SIZE_MAX 0x40 /* from tavor driver */ 576*9e39c5baSBill Taylor #define TAVOR_INLINE_HEADER_SIZE_RDMAW \ 577*9e39c5baSBill Taylor (sizeof (tavor_hw_snd_wqe_nextctrl_t) + \ 578*9e39c5baSBill Taylor sizeof (tavor_hw_snd_wqe_remaddr_t) + \ 579*9e39c5baSBill Taylor sizeof (uint32_t)) 580*9e39c5baSBill Taylor #define TAVOR_INLINE_HEADER_SIZE_SEND \ 581*9e39c5baSBill Taylor (sizeof (tavor_hw_snd_wqe_nextctrl_t) + \ 582*9e39c5baSBill Taylor sizeof (uint32_t)) 583*9e39c5baSBill Taylor 584*9e39c5baSBill Taylor /* 585*9e39c5baSBill Taylor * Function signatures 586*9e39c5baSBill Taylor */ 587*9e39c5baSBill Taylor extern int dapls_tavor_max_inline(void); 588*9e39c5baSBill Taylor 589*9e39c5baSBill Taylor #ifdef __cplusplus 590*9e39c5baSBill Taylor } 591*9e39c5baSBill Taylor #endif 592*9e39c5baSBill Taylor 593*9e39c5baSBill Taylor #endif /* _DAPL_TAVOR_HW_H */ 594