11bdd6c0eSSue Gleeson /* 21bdd6c0eSSue Gleeson * CDDL HEADER START 31bdd6c0eSSue Gleeson * 41bdd6c0eSSue Gleeson * The contents of this file are subject to the terms of the 51bdd6c0eSSue Gleeson * Common Development and Distribution License (the "License"). 61bdd6c0eSSue Gleeson * You may not use this file except in compliance with the License. 71bdd6c0eSSue Gleeson * 81bdd6c0eSSue Gleeson * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 91bdd6c0eSSue Gleeson * or http://www.opensolaris.org/os/licensing. 101bdd6c0eSSue Gleeson * See the License for the specific language governing permissions 111bdd6c0eSSue Gleeson * and limitations under the License. 121bdd6c0eSSue Gleeson * 131bdd6c0eSSue Gleeson * When distributing Covered Code, include this CDDL HEADER in each 141bdd6c0eSSue Gleeson * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 151bdd6c0eSSue Gleeson * If applicable, add the following below this CDDL HEADER, with the 161bdd6c0eSSue Gleeson * fields enclosed by brackets "[]" replaced with your own identifying 171bdd6c0eSSue Gleeson * information: Portions Copyright [yyyy] [name of copyright owner] 181bdd6c0eSSue Gleeson * 191bdd6c0eSSue Gleeson * CDDL HEADER END 201bdd6c0eSSue Gleeson */ 211bdd6c0eSSue Gleeson 221bdd6c0eSSue Gleeson /* 23c3a558e7SSue Gleeson * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 241bdd6c0eSSue Gleeson */ 251bdd6c0eSSue Gleeson 26fd826efaSJohn Levon /* 27fd826efaSJohn Levon * Copyright 2019, Joyent, Inc. 28fd826efaSJohn Levon */ 29fd826efaSJohn Levon 301bdd6c0eSSue Gleeson #ifndef _SRPT_IMPL_H_ 311bdd6c0eSSue Gleeson #define _SRPT_IMPL_H_ 321bdd6c0eSSue Gleeson 331bdd6c0eSSue Gleeson /* 341bdd6c0eSSue Gleeson * Prototypes and data structures for the SRP Target Port Provider. 351bdd6c0eSSue Gleeson */ 361bdd6c0eSSue Gleeson 371bdd6c0eSSue Gleeson #include <sys/types.h> 381bdd6c0eSSue Gleeson #include <sys/ddi.h> 391bdd6c0eSSue Gleeson #include <sys/ib/ibtl/ibti.h> 401bdd6c0eSSue Gleeson #include <sys/modctl.h> 411bdd6c0eSSue Gleeson 424558d122SViswanathan Kannappan #include <sys/stmf.h> 434558d122SViswanathan Kannappan #include <sys/stmf_ioctl.h> 444558d122SViswanathan Kannappan #include <sys/portif.h> 451bdd6c0eSSue Gleeson 461bdd6c0eSSue Gleeson #include <sys/ib/mgt/ibdma/ibdma.h> 471bdd6c0eSSue Gleeson 481bdd6c0eSSue Gleeson #ifdef __cplusplus 491bdd6c0eSSue Gleeson extern "C" { 501bdd6c0eSSue Gleeson #endif 511bdd6c0eSSue Gleeson 52191c289bSCharles Ting /* Format the session identifier */ 53191c289bSCharles Ting #define ALIAS_STR(s, a, b) \ 54191c289bSCharles Ting ((void) snprintf((s), sizeof ((s)), "%016llx:%016llx", \ 55191c289bSCharles Ting (u_longlong_t)(a), (u_longlong_t)(b))) 56191c289bSCharles Ting 57191c289bSCharles Ting /* Format the EUI name */ 58191c289bSCharles Ting #define EUI_STR(s, a) \ 59191c289bSCharles Ting ((void) snprintf((s), sizeof ((s)), "eui.%016llX", (u_longlong_t)(a))) 60191c289bSCharles Ting 611bdd6c0eSSue Gleeson /* 621bdd6c0eSSue Gleeson * We should/could consider making some of these values tunables. 631bdd6c0eSSue Gleeson * Specifically, SEND_MSG_SIZE and SEND_MSG_DEPTH. 641bdd6c0eSSue Gleeson */ 651bdd6c0eSSue Gleeson enum { 661bdd6c0eSSue Gleeson SRPT_DEFAULT_IOC_SRQ_SIZE = 4096, 671bdd6c0eSSue Gleeson SRPT_DEFAULT_SEND_MSG_DEPTH = 128, 68989ca640SSue Gleeson /* 69989ca640SSue Gleeson * SEND_MSG_SIZE must be a multiple of 64 as it is registered 70989ca640SSue Gleeson * as memory regions with IB. To support a scatter/gather table 71989ca640SSue Gleeson * size of 32, the size must be at not less than 960. To support 72989ca640SSue Gleeson * the maximum scatter/gather table size of 255, the IU must 73989ca640SSue Gleeson * be at least 4160 bytes. 74989ca640SSue Gleeson */ 75989ca640SSue Gleeson SRPT_DEFAULT_SEND_MSG_SIZE = 4160, 761bdd6c0eSSue Gleeson SRPT_DEFAULT_MAX_RDMA_SIZE = 65536, 771bdd6c0eSSue Gleeson SRPT_MIN_T_I_IU_LEN = 52, 781bdd6c0eSSue Gleeson SRPT_EUI_ID_LEN = 20, 791bdd6c0eSSue Gleeson SRPT_RECV_WC_POLL_SIZE = 16, 801bdd6c0eSSue Gleeson SRPT_SEND_WC_POLL_SIZE = 16, 811bdd6c0eSSue Gleeson SRPT_MAX_OUT_IO_PER_CMD = 16, 821bdd6c0eSSue Gleeson SRPT_FENCE_SEND = 1, 831bdd6c0eSSue Gleeson SRPT_NO_FENCE_SEND = 0 841bdd6c0eSSue Gleeson }; 851bdd6c0eSSue Gleeson 861bdd6c0eSSue Gleeson struct srpt_target_port_s; 871bdd6c0eSSue Gleeson 88191c289bSCharles Ting #define SRPT_ALIAS_LEN (SRP_PORT_ID_LEN * 2 + 2) 89191c289bSCharles Ting 901bdd6c0eSSue Gleeson /* 911bdd6c0eSSue Gleeson * SRP Session - represents a SCSI I_T_Nexus. 921bdd6c0eSSue Gleeson * 931bdd6c0eSSue Gleeson * Sessions map 1 or more initiator logins to a specific I/O 941bdd6c0eSSue Gleeson * Controller SCSI Target Port. Targets create sessions 951bdd6c0eSSue Gleeson * at initiator login and release when no longer referenced 961bdd6c0eSSue Gleeson * by a login. 971bdd6c0eSSue Gleeson */ 981bdd6c0eSSue Gleeson typedef struct srpt_session_s { 99fd826efaSJohn Levon krwlock_t ss_rwlock; 1001bdd6c0eSSue Gleeson list_node_t ss_node; 1011bdd6c0eSSue Gleeson 1021bdd6c0eSSue Gleeson /* 1031bdd6c0eSSue Gleeson * ADVANCED FEATURE, NOT YET SUPPORTED. 1041bdd6c0eSSue Gleeson * In multi-channel mode, multiple RDMA communication 1051bdd6c0eSSue Gleeson * channels may reference the same SCSI session. When 1061bdd6c0eSSue Gleeson * a channel releases its reference to the SCSI session, 1071bdd6c0eSSue Gleeson * it should have no tasks associated with the session. 1081bdd6c0eSSue Gleeson * 1091bdd6c0eSSue Gleeson * If multi-channel is implemented, add a channel list 1101bdd6c0eSSue Gleeson * to this object instead of tracking it on the target. 1111bdd6c0eSSue Gleeson * 1121bdd6c0eSSue Gleeson * Will also need a session state & mode. Mode is to 1131bdd6c0eSSue Gleeson * track if the session is MULTI or SINGLE channel. 1141bdd6c0eSSue Gleeson */ 1151bdd6c0eSSue Gleeson 1161bdd6c0eSSue Gleeson stmf_scsi_session_t *ss_ss; 1171bdd6c0eSSue Gleeson struct srpt_target_port_s *ss_tgt; 1181bdd6c0eSSue Gleeson list_t ss_task_list; 1191bdd6c0eSSue Gleeson 1201bdd6c0eSSue Gleeson /* 1211bdd6c0eSSue Gleeson * SRP Initiator and target identifiers are 128-bit. 1221bdd6c0eSSue Gleeson * 1231bdd6c0eSSue Gleeson * The specification defines the initiator to be 64-bits of 1241bdd6c0eSSue Gleeson * ID extension and 64 bits of GUID, but these are really 1251bdd6c0eSSue Gleeson * just a recommendation. Generally the extension is used 1261bdd6c0eSSue Gleeson * to create unique I_T_Nexus from the same initiator and 1271bdd6c0eSSue Gleeson * target. Initiators are inconsistent on the GUID they 1281bdd6c0eSSue Gleeson * use, some use the HCA Node, some the HCA port. 1291bdd6c0eSSue Gleeson * 1301bdd6c0eSSue Gleeson * The specification defines the target to be 64-bits of 1311bdd6c0eSSue Gleeson * service ID followed by 64-bits of I/O Controller GUID. 1321bdd6c0eSSue Gleeson * In the case where there is a single default target 1331bdd6c0eSSue Gleeson * service, they will be the same (our default). 1341bdd6c0eSSue Gleeson */ 1351bdd6c0eSSue Gleeson uint8_t ss_i_id[SRP_PORT_ID_LEN]; 1361bdd6c0eSSue Gleeson uint8_t ss_t_id[SRP_PORT_ID_LEN]; 1371bdd6c0eSSue Gleeson 1381bdd6c0eSSue Gleeson /* So we can see the full 128-bit initiator login from stmfadm */ 139191c289bSCharles Ting char ss_i_alias[SRPT_ALIAS_LEN]; 1401bdd6c0eSSue Gleeson uint8_t ss_hw_port; 141191c289bSCharles Ting 142191c289bSCharles Ting char ss_t_alias[SRPT_ALIAS_LEN]; 143191c289bSCharles Ting char ss_i_name[SRPT_EUI_ID_LEN + 1]; 144191c289bSCharles Ting char ss_t_name[SRPT_EUI_ID_LEN + 1]; 145191c289bSCharles Ting char ss_i_gid[SRPT_ALIAS_LEN]; 146191c289bSCharles Ting char ss_t_gid[SRPT_ALIAS_LEN]; 1471bdd6c0eSSue Gleeson } srpt_session_t; 1481bdd6c0eSSue Gleeson 1491bdd6c0eSSue Gleeson /* 1501bdd6c0eSSue Gleeson * Send work request types. 1511bdd6c0eSSue Gleeson */ 1521bdd6c0eSSue Gleeson typedef enum srpt_swqe_type_e { 1531bdd6c0eSSue Gleeson SRPT_SWQE_TYPE_DATA = 1, 1541bdd6c0eSSue Gleeson SRPT_SWQE_TYPE_RESP 1551bdd6c0eSSue Gleeson } srpt_swqe_type_t; 1561bdd6c0eSSue Gleeson 1571bdd6c0eSSue Gleeson typedef struct srpt_swqe_s { 1581bdd6c0eSSue Gleeson srpt_swqe_type_t sw_type; 1591bdd6c0eSSue Gleeson void *sw_addr; 1601bdd6c0eSSue Gleeson ibt_wrid_t sw_next; 1611bdd6c0eSSue Gleeson } srpt_swqe_t; 1621bdd6c0eSSue Gleeson 1631bdd6c0eSSue Gleeson /* 1641bdd6c0eSSue Gleeson * SRP Channel - the RDMA communications channel associated with 1651bdd6c0eSSue Gleeson * a specific SRP login. 1661bdd6c0eSSue Gleeson */ 1671bdd6c0eSSue Gleeson typedef enum srpt_channel_state_e { 1681bdd6c0eSSue Gleeson SRPT_CHANNEL_CONNECTING = 0, 1691bdd6c0eSSue Gleeson SRPT_CHANNEL_CONNECTED, 1701bdd6c0eSSue Gleeson SRPT_CHANNEL_DISCONNECTING 1711bdd6c0eSSue Gleeson } srpt_channel_state_t; 1721bdd6c0eSSue Gleeson 1731bdd6c0eSSue Gleeson typedef struct srpt_channel_s { 174fd826efaSJohn Levon krwlock_t ch_rwlock; 1751bdd6c0eSSue Gleeson 1761bdd6c0eSSue Gleeson kmutex_t ch_reflock; 1771bdd6c0eSSue Gleeson uint_t ch_refcnt; 1781bdd6c0eSSue Gleeson kcondvar_t ch_cv_complete; 1791bdd6c0eSSue Gleeson uint_t ch_cv_waiters; 1801bdd6c0eSSue Gleeson 1811bdd6c0eSSue Gleeson list_node_t ch_stp_node; 1821bdd6c0eSSue Gleeson srpt_channel_state_t ch_state; 1831bdd6c0eSSue Gleeson ibt_cq_hdl_t ch_scq_hdl; 1841bdd6c0eSSue Gleeson ibt_cq_hdl_t ch_rcq_hdl; 1851bdd6c0eSSue Gleeson ibt_channel_hdl_t ch_chan_hdl; 1861bdd6c0eSSue Gleeson ibt_chan_sizes_t ch_sizes; 1871bdd6c0eSSue Gleeson 1881bdd6c0eSSue Gleeson uint32_t ch_req_lim_delta; 1891bdd6c0eSSue Gleeson uint32_t ch_ti_iu_len; 1901bdd6c0eSSue Gleeson struct srpt_target_port_s *ch_tgt; 1911bdd6c0eSSue Gleeson srpt_session_t *ch_session; 1921bdd6c0eSSue Gleeson 1931bdd6c0eSSue Gleeson /* 1941bdd6c0eSSue Gleeson * Map IB send WQE request IDs to the 1951bdd6c0eSSue Gleeson * apporpriate operation type (for errors). 1961bdd6c0eSSue Gleeson */ 1971bdd6c0eSSue Gleeson kmutex_t ch_swqe_lock; 1981bdd6c0eSSue Gleeson srpt_swqe_t *ch_swqe; 1991bdd6c0eSSue Gleeson uint32_t ch_num_swqe; 2001bdd6c0eSSue Gleeson uint32_t ch_head; 2011bdd6c0eSSue Gleeson uint32_t ch_tail; 2021bdd6c0eSSue Gleeson uint32_t ch_swqe_posted; 2031bdd6c0eSSue Gleeson } srpt_channel_t; 2041bdd6c0eSSue Gleeson 2051bdd6c0eSSue Gleeson /* 2061bdd6c0eSSue Gleeson * SRP Information Unit (IU). Each IU structure contains 2071bdd6c0eSSue Gleeson * the buffer for the IU itself (received over the RC 2081bdd6c0eSSue Gleeson * channel), and all of the context required by the target 2091bdd6c0eSSue Gleeson * to process this request represented by the IU. 2101bdd6c0eSSue Gleeson * Available IU structures are managed on the I/O Controller 2111bdd6c0eSSue Gleeson * shared receive queue. 2121bdd6c0eSSue Gleeson */ 2131bdd6c0eSSue Gleeson enum { 2141bdd6c0eSSue Gleeson SRPT_IU_STMF_ABORTING = 1 << 0, /* STMF called abort */ 2151bdd6c0eSSue Gleeson SRPT_IU_SRP_ABORTING = 1 << 1, /* SRP initiator aborting */ 2161bdd6c0eSSue Gleeson SRPT_IU_ABORTED = 1 << 2, /* Task has been aborted */ 2171bdd6c0eSSue Gleeson SRPT_IU_RESP_SENT = 1 << 3 /* Response queued */ 2181bdd6c0eSSue Gleeson }; 2191bdd6c0eSSue Gleeson 2201bdd6c0eSSue Gleeson typedef struct srpt_iu_s { 2211bdd6c0eSSue Gleeson /* 2221bdd6c0eSSue Gleeson * The buffer for the IU itself. When unused (a 2231bdd6c0eSSue Gleeson * reference count of zero), this buffer is posted 2241bdd6c0eSSue Gleeson * on the I/O Controllers SRPT SRQ. 2251bdd6c0eSSue Gleeson */ 2261bdd6c0eSSue Gleeson void *iu_buf; 2271bdd6c0eSSue Gleeson ibt_wr_ds_t iu_sge; 2281bdd6c0eSSue Gleeson struct srpt_ioc_s *iu_ioc; 2291bdd6c0eSSue Gleeson uint_t iu_pool_ndx; 2301bdd6c0eSSue Gleeson kmutex_t iu_lock; 2311bdd6c0eSSue Gleeson 2321bdd6c0eSSue Gleeson /* 2331bdd6c0eSSue Gleeson * The following are reset for each IU request 2341bdd6c0eSSue Gleeson * processed by this buffer. 2351bdd6c0eSSue Gleeson */ 2361bdd6c0eSSue Gleeson list_node_t iu_ss_task_node; 2371bdd6c0eSSue Gleeson srpt_channel_t *iu_ch; 2381bdd6c0eSSue Gleeson 2391bdd6c0eSSue Gleeson uint_t iu_num_rdescs; 2401bdd6c0eSSue Gleeson srp_direct_desc_t *iu_rdescs; 2411bdd6c0eSSue Gleeson uint_t iu_tot_xfer_len; 2421bdd6c0eSSue Gleeson 2431bdd6c0eSSue Gleeson uint64_t iu_tag; 2441bdd6c0eSSue Gleeson uint_t iu_flags; 2451bdd6c0eSSue Gleeson uint32_t iu_sq_posted_cnt; 2461bdd6c0eSSue Gleeson scsi_task_t *iu_stmf_task; 2471bdd6c0eSSue Gleeson } srpt_iu_t; 2481bdd6c0eSSue Gleeson 2491bdd6c0eSSue Gleeson /* 2501bdd6c0eSSue Gleeson * SRP SCSI Target Port. By default each HCA creates a single 2511bdd6c0eSSue Gleeson * SCSI Target Port based on the associated I/O Controller 2521bdd6c0eSSue Gleeson * (HCA) node GUID and made available through each physical 2531bdd6c0eSSue Gleeson * hardware port of the I/O Controller. 2541bdd6c0eSSue Gleeson */ 2551bdd6c0eSSue Gleeson typedef enum srpt_target_state_e { 2561bdd6c0eSSue Gleeson SRPT_TGT_STATE_OFFLINE = 0, 2571bdd6c0eSSue Gleeson SRPT_TGT_STATE_ONLINING, 2581bdd6c0eSSue Gleeson SRPT_TGT_STATE_ONLINE, 2591bdd6c0eSSue Gleeson SRPT_TGT_STATE_OFFLINING 2601bdd6c0eSSue Gleeson } srpt_target_state_t; 2611bdd6c0eSSue Gleeson 2621bdd6c0eSSue Gleeson typedef struct srpt_hw_port_s { 2631bdd6c0eSSue Gleeson ibt_sbind_hdl_t hwp_bind_hdl; 2641bdd6c0eSSue Gleeson ib_gid_t hwp_gid; 2651bdd6c0eSSue Gleeson } srpt_hw_port_t; 2661bdd6c0eSSue Gleeson 2671bdd6c0eSSue Gleeson typedef struct srpt_target_port_s { 2681bdd6c0eSSue Gleeson stmf_local_port_t *tp_lport; 2691bdd6c0eSSue Gleeson struct srpt_ioc_s *tp_ioc; 2701bdd6c0eSSue Gleeson 2711bdd6c0eSSue Gleeson kmutex_t tp_lock; 2721bdd6c0eSSue Gleeson srpt_target_state_t tp_state; 2731bdd6c0eSSue Gleeson kcondvar_t tp_offline_complete; 2741bdd6c0eSSue Gleeson uint_t tp_drv_disabled; 2751bdd6c0eSSue Gleeson 2761bdd6c0eSSue Gleeson /* 2771bdd6c0eSSue Gleeson * We are using a simple list for channels right now, we 2781bdd6c0eSSue Gleeson * probably should switch this over to the AVL 2791bdd6c0eSSue Gleeson * implementation eventually (but lookups are not done 2801bdd6c0eSSue Gleeson * in the data path so this is not urgent). 2811bdd6c0eSSue Gleeson */ 2821bdd6c0eSSue Gleeson kmutex_t tp_ch_list_lock; 2831bdd6c0eSSue Gleeson list_t tp_ch_list; 2841bdd6c0eSSue Gleeson 2851bdd6c0eSSue Gleeson /* 2861bdd6c0eSSue Gleeson * A list of active sessions. Session lifetime is 2871bdd6c0eSSue Gleeson * determined by having active channels, but track 2881bdd6c0eSSue Gleeson * them here for easier determination to when a 2891bdd6c0eSSue Gleeson * target can truly be offlined, and as a step toward 2901bdd6c0eSSue Gleeson * being session-focused rather than channel-focused. 2911bdd6c0eSSue Gleeson * If we ever truly support multi-channel, move the 2921bdd6c0eSSue Gleeson * channels to be part of the session object. 2931bdd6c0eSSue Gleeson * 2941bdd6c0eSSue Gleeson * Sessions should remain on this list until they 2951bdd6c0eSSue Gleeson * are deregistered from STMF. This allows the target 2961bdd6c0eSSue Gleeson * to properly track when it can consider itself 'offline'. 2971bdd6c0eSSue Gleeson */ 2981bdd6c0eSSue Gleeson kmutex_t tp_sess_list_lock; 2991bdd6c0eSSue Gleeson kcondvar_t tp_sess_complete; 3001bdd6c0eSSue Gleeson list_t tp_sess_list; 3011bdd6c0eSSue Gleeson 3021bdd6c0eSSue Gleeson uint_t tp_srp_enabled; 3031bdd6c0eSSue Gleeson ibt_srv_hdl_t tp_ibt_svc_hdl; 3041bdd6c0eSSue Gleeson ibt_srv_desc_t tp_ibt_svc_desc; 3051bdd6c0eSSue Gleeson ib_svc_id_t tp_ibt_svc_id; 3061bdd6c0eSSue Gleeson scsi_devid_desc_t *tp_scsi_devid; 3071bdd6c0eSSue Gleeson uint8_t tp_srp_port_id[SRP_PORT_ID_LEN]; 3081bdd6c0eSSue Gleeson 3091bdd6c0eSSue Gleeson uint_t tp_nports; 3101bdd6c0eSSue Gleeson srpt_hw_port_t *tp_hw_port; 3113baa0db4SSue Gleeson /* 3123baa0db4SSue Gleeson * track the number of active ports so we can offline the target if 3133baa0db4SSue Gleeson * none 3143baa0db4SSue Gleeson */ 3153baa0db4SSue Gleeson uint32_t tp_num_active_ports; 3163baa0db4SSue Gleeson /* state STMF wants the target in. We may be offline due to no ports */ 3173baa0db4SSue Gleeson srpt_target_state_t tp_requested_state; 3181bdd6c0eSSue Gleeson } srpt_target_port_t; 3191bdd6c0eSSue Gleeson 3201bdd6c0eSSue Gleeson /* 3211bdd6c0eSSue Gleeson * SRP Target hardware device. A SRP Target hardware device 3221bdd6c0eSSue Gleeson * is an IB HCA. All ports of the HCA comprise a single 3231bdd6c0eSSue Gleeson * I/O Controller that is registered with the IB Device 3241bdd6c0eSSue Gleeson * Managment Agent. 3251bdd6c0eSSue Gleeson */ 3261bdd6c0eSSue Gleeson typedef struct srpt_ioc_s { 3271bdd6c0eSSue Gleeson list_node_t ioc_node; 3281bdd6c0eSSue Gleeson 329fd826efaSJohn Levon krwlock_t ioc_rwlock; 3301bdd6c0eSSue Gleeson ibt_hca_hdl_t ioc_ibt_hdl; 3311bdd6c0eSSue Gleeson ibt_hca_attr_t ioc_attr; 3321bdd6c0eSSue Gleeson ib_guid_t ioc_guid; 3331bdd6c0eSSue Gleeson 3341bdd6c0eSSue Gleeson /* 3351bdd6c0eSSue Gleeson * By default each HCA is a single SRP.T10 service based on 3361bdd6c0eSSue Gleeson * the HCA GUID. We have implemented the target here as a 3371bdd6c0eSSue Gleeson * pointer to facilitate moving to a list of targets if 3381bdd6c0eSSue Gleeson * appropriate down the road. 3391bdd6c0eSSue Gleeson */ 3401bdd6c0eSSue Gleeson srpt_target_port_t *ioc_tgt_port; 3411bdd6c0eSSue Gleeson 3421bdd6c0eSSue Gleeson 3431bdd6c0eSSue Gleeson /* 3441bdd6c0eSSue Gleeson * Each HCA registers a single I/O Controller with the 3451bdd6c0eSSue Gleeson * IB Device Management Agent. 3461bdd6c0eSSue Gleeson */ 3471bdd6c0eSSue Gleeson ibdma_hdl_t ioc_ibdma_hdl; 3481bdd6c0eSSue Gleeson ib_dm_ioc_ctrl_profile_t ioc_profile; 3491bdd6c0eSSue Gleeson ib_dm_srv_t ioc_svc; 3501bdd6c0eSSue Gleeson 3511bdd6c0eSSue Gleeson ibt_pd_hdl_t ioc_pd_hdl; 3521bdd6c0eSSue Gleeson ibt_srq_sizes_t ioc_srq_attr; 3531bdd6c0eSSue Gleeson ibt_srq_hdl_t ioc_srq_hdl; 3541bdd6c0eSSue Gleeson 3551bdd6c0eSSue Gleeson /* 3561bdd6c0eSSue Gleeson * The I/O Controller pool of IU resources allocated 3571bdd6c0eSSue Gleeson * at controller creation. 3581bdd6c0eSSue Gleeson */ 3591bdd6c0eSSue Gleeson uint32_t ioc_num_iu_entries; 3601bdd6c0eSSue Gleeson srpt_iu_t *ioc_iu_pool; 3611bdd6c0eSSue Gleeson ibt_mr_hdl_t ioc_iu_mr_hdl; 3621bdd6c0eSSue Gleeson void *ioc_iu_bufs; /* iu buffer space */ 3631bdd6c0eSSue Gleeson 3641bdd6c0eSSue Gleeson /* 3651bdd6c0eSSue Gleeson * Each I/O Controller has it's own data buffer 3661bdd6c0eSSue Gleeson * vmem arena. Pool is created at controller creation, 3671bdd6c0eSSue Gleeson * and expanded as required. This keeps IB memory 3681bdd6c0eSSue Gleeson * registrations to a minimum in the data path. 3691bdd6c0eSSue Gleeson */ 3701bdd6c0eSSue Gleeson struct srpt_vmem_pool_s *ioc_dbuf_pool; 3711bdd6c0eSSue Gleeson stmf_dbuf_store_t *ioc_stmf_ds; 3721bdd6c0eSSue Gleeson } srpt_ioc_t; 3731bdd6c0eSSue Gleeson 3741bdd6c0eSSue Gleeson /* 3751bdd6c0eSSue Gleeson * Memory regions 3761bdd6c0eSSue Gleeson */ 3771bdd6c0eSSue Gleeson typedef struct srpt_mr_s { 3781bdd6c0eSSue Gleeson ibt_mr_hdl_t mr_hdl; 3791bdd6c0eSSue Gleeson ib_vaddr_t mr_va; 3801bdd6c0eSSue Gleeson ib_memlen_t mr_len; 3811bdd6c0eSSue Gleeson ibt_lkey_t mr_lkey; 3821bdd6c0eSSue Gleeson ibt_rkey_t mr_rkey; 3831bdd6c0eSSue Gleeson avl_node_t mr_avl; 3841bdd6c0eSSue Gleeson } srpt_mr_t; 3851bdd6c0eSSue Gleeson 3861bdd6c0eSSue Gleeson /* 3871bdd6c0eSSue Gleeson * SRP Target vmem arena definition 3881bdd6c0eSSue Gleeson */ 3891bdd6c0eSSue Gleeson typedef struct srpt_vmem_pool_s { 3901bdd6c0eSSue Gleeson srpt_ioc_t *svp_ioc; 3911bdd6c0eSSue Gleeson ib_memlen_t svp_chunksize; 3921bdd6c0eSSue Gleeson vmem_t *svp_vmem; 3931bdd6c0eSSue Gleeson uint64_t svp_total_size; 3941bdd6c0eSSue Gleeson uint64_t svp_max_size; 3951bdd6c0eSSue Gleeson avl_tree_t svp_mr_list; 3961bdd6c0eSSue Gleeson krwlock_t svp_lock; 3971bdd6c0eSSue Gleeson ibt_mr_flags_t svp_flags; 3981bdd6c0eSSue Gleeson } srpt_vmem_pool_t; 3991bdd6c0eSSue Gleeson 4001bdd6c0eSSue Gleeson /* 4011bdd6c0eSSue Gleeson * SRP port provider data buffer, allocated and freed 4021bdd6c0eSSue Gleeson * via calls to the IOC datastore. 4031bdd6c0eSSue Gleeson */ 4041bdd6c0eSSue Gleeson typedef struct srpt_ds_dbuf_s { 4051bdd6c0eSSue Gleeson stmf_data_buf_t *db_stmf_buf; 4061bdd6c0eSSue Gleeson srpt_ioc_t *db_ioc; 4071bdd6c0eSSue Gleeson ibt_mr_hdl_t db_mr_hdl; 4081bdd6c0eSSue Gleeson ibt_wr_ds_t db_sge; 4091bdd6c0eSSue Gleeson srpt_iu_t *db_iu; 4101bdd6c0eSSue Gleeson } srpt_ds_dbuf_t; 4111bdd6c0eSSue Gleeson 4121bdd6c0eSSue Gleeson /* 4131bdd6c0eSSue Gleeson * SRP Target service state 4141bdd6c0eSSue Gleeson */ 4151bdd6c0eSSue Gleeson typedef enum { 4161bdd6c0eSSue Gleeson SRPT_SVC_DISABLED, 4171bdd6c0eSSue Gleeson SRPT_SVC_ENABLED 4181bdd6c0eSSue Gleeson } srpt_svc_state_t; 4191bdd6c0eSSue Gleeson 4201bdd6c0eSSue Gleeson typedef struct { 4211bdd6c0eSSue Gleeson ddi_modhandle_t ibdmah; 4221bdd6c0eSSue Gleeson ibdma_hdl_t (*ibdma_register)(ib_guid_t, 4231bdd6c0eSSue Gleeson ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *); 4241bdd6c0eSSue Gleeson ibdma_status_t (*ibdma_unregister)(ibdma_hdl_t); 4251bdd6c0eSSue Gleeson ibdma_status_t (*ibdma_update)(ibdma_hdl_t, 4261bdd6c0eSSue Gleeson ib_dm_ioc_ctrl_profile_t *, ib_dm_srv_t *); 4271bdd6c0eSSue Gleeson } srpt_ibdma_ops_t; 4281bdd6c0eSSue Gleeson 4291bdd6c0eSSue Gleeson /* 4301bdd6c0eSSue Gleeson * SRP Target protocol driver context data structure, maintaining 4311bdd6c0eSSue Gleeson * the global state of the protocol. 4321bdd6c0eSSue Gleeson */ 4331bdd6c0eSSue Gleeson typedef struct srpt_ctxt_s { 4341bdd6c0eSSue Gleeson dev_info_t *sc_dip; 4351bdd6c0eSSue Gleeson krwlock_t sc_rwlock; 4361bdd6c0eSSue Gleeson srpt_svc_state_t sc_svc_state; 4371bdd6c0eSSue Gleeson 4381bdd6c0eSSue Gleeson ibt_clnt_hdl_t sc_ibt_hdl; 4391bdd6c0eSSue Gleeson 4401bdd6c0eSSue Gleeson /* 4411bdd6c0eSSue Gleeson * SRP Target I/O Controllers. Each IBT HCA represents an 4421bdd6c0eSSue Gleeson * I/O Controller. Must hold rwlock as a writer to update. 4431bdd6c0eSSue Gleeson */ 4441bdd6c0eSSue Gleeson list_t sc_ioc_list; 4451bdd6c0eSSue Gleeson uint_t sc_num_iocs; 4461bdd6c0eSSue Gleeson 4471bdd6c0eSSue Gleeson /* Back-end COMSTAR port provider interface. */ 4481bdd6c0eSSue Gleeson stmf_port_provider_t *sc_pp; 4491bdd6c0eSSue Gleeson 4501bdd6c0eSSue Gleeson /* IBDMA entry points */ 4511bdd6c0eSSue Gleeson srpt_ibdma_ops_t sc_ibdma_ops; 452c3a558e7SSue Gleeson 453c3a558e7SSue Gleeson /* 454c3a558e7SSue Gleeson * List of explicitly configured HCAs and their configurable 455c3a558e7SSue Gleeson * attributes. 456c3a558e7SSue Gleeson */ 457c3a558e7SSue Gleeson nvlist_t *sc_cfg_hca_nv; 4581bdd6c0eSSue Gleeson } srpt_ctxt_t; 4591bdd6c0eSSue Gleeson 4601bdd6c0eSSue Gleeson typedef struct srpt_iu_data_s { 4611bdd6c0eSSue Gleeson union { 4621bdd6c0eSSue Gleeson uint8_t srp_op; 4631bdd6c0eSSue Gleeson srp_cmd_req_t srp_cmd; 4641bdd6c0eSSue Gleeson srp_tsk_mgmt_t srp_tsk_mgmt; 4651bdd6c0eSSue Gleeson srp_i_logout_t srp_i_logout; 4661bdd6c0eSSue Gleeson srp_rsp_t srp_rsp; 4671bdd6c0eSSue Gleeson } rx_iu; 4681bdd6c0eSSue Gleeson } srpt_iu_data_t; 4691bdd6c0eSSue Gleeson 4701bdd6c0eSSue Gleeson extern srpt_ctxt_t *srpt_ctxt; 4711bdd6c0eSSue Gleeson 4721bdd6c0eSSue Gleeson /* 4731bdd6c0eSSue Gleeson * For Non recoverable or Major Errors 4741bdd6c0eSSue Gleeson */ 4751bdd6c0eSSue Gleeson #define SRPT_LOG_L0 0 4761bdd6c0eSSue Gleeson 4771bdd6c0eSSue Gleeson /* 4781bdd6c0eSSue Gleeson * For additional information on Non recoverable errors and 4791bdd6c0eSSue Gleeson * warnings/informational message for sys-admin types. 4801bdd6c0eSSue Gleeson */ 4811bdd6c0eSSue Gleeson #define SRPT_LOG_L1 1 4821bdd6c0eSSue Gleeson 4831bdd6c0eSSue Gleeson /* 4841bdd6c0eSSue Gleeson * debug only 4851bdd6c0eSSue Gleeson * for more verbose trace than L1, for e.g. recoverable errors, 4861bdd6c0eSSue Gleeson * or intersting trace 4871bdd6c0eSSue Gleeson */ 4881bdd6c0eSSue Gleeson #define SRPT_LOG_L2 2 4891bdd6c0eSSue Gleeson 4901bdd6c0eSSue Gleeson /* 4911bdd6c0eSSue Gleeson * debug only 4921bdd6c0eSSue Gleeson * for more verbose trace than L2, for e.g. printing function entries.... 4931bdd6c0eSSue Gleeson */ 4941bdd6c0eSSue Gleeson #define SRPT_LOG_L3 3 4951bdd6c0eSSue Gleeson 4961bdd6c0eSSue Gleeson /* 4971bdd6c0eSSue Gleeson * debug only 4981bdd6c0eSSue Gleeson * for more verbose trace than L3, for e.g. printing minor function entries... 4991bdd6c0eSSue Gleeson */ 5001bdd6c0eSSue Gleeson #define SRPT_LOG_L4 4 5011bdd6c0eSSue Gleeson 5021bdd6c0eSSue Gleeson /* 5031bdd6c0eSSue Gleeson * srpt_errlevel can be set in the debugger to enable additional logging. 5041bdd6c0eSSue Gleeson * You can also add set srpt:srpt_errlevel={0,1,2,3,4} in /etc/system. 505bea257dfSSue Gleeson * The default log level is L1. 5061bdd6c0eSSue Gleeson */ 5071bdd6c0eSSue Gleeson #define SRPT_LOG_DEFAULT_LEVEL SRPT_LOG_L1 5081bdd6c0eSSue Gleeson 5091bdd6c0eSSue Gleeson extern uint_t srpt_errlevel; 5101bdd6c0eSSue Gleeson 5111bdd6c0eSSue Gleeson 5121bdd6c0eSSue Gleeson #define SRPT_DPRINTF_L0(...) cmn_err(CE_WARN, __VA_ARGS__) 5131bdd6c0eSSue Gleeson #define SRPT_DPRINTF_L1(...) cmn_err(CE_NOTE, __VA_ARGS__) 5141bdd6c0eSSue Gleeson #define SRPT_DPRINTF_L2(...) if (srpt_errlevel >= SRPT_LOG_L2) { \ 5151bdd6c0eSSue Gleeson cmn_err(CE_NOTE, __VA_ARGS__);\ 5161bdd6c0eSSue Gleeson } 5171bdd6c0eSSue Gleeson #ifdef DEBUG 5181bdd6c0eSSue Gleeson #define SRPT_DPRINTF_L3(...) if (srpt_errlevel >= SRPT_LOG_L3) { \ 5191bdd6c0eSSue Gleeson cmn_err(CE_NOTE, __VA_ARGS__);\ 5201bdd6c0eSSue Gleeson } 5211bdd6c0eSSue Gleeson #define SRPT_DPRINTF_L4(...) if (srpt_errlevel >= SRPT_LOG_L4) { \ 5221bdd6c0eSSue Gleeson cmn_err(CE_NOTE, __VA_ARGS__);\ 5231bdd6c0eSSue Gleeson } 5241bdd6c0eSSue Gleeson #else 525*679c9deaSJohn Levon #define SRPT_DPRINTF_L3(...) (void)(0) 526*679c9deaSJohn Levon #define SRPT_DPRINTF_L4(...) (void)(0) 5271bdd6c0eSSue Gleeson #endif 5281bdd6c0eSSue Gleeson 5291bdd6c0eSSue Gleeson #ifdef __cplusplus 5301bdd6c0eSSue Gleeson } 5311bdd6c0eSSue Gleeson #endif 5321bdd6c0eSSue Gleeson 5331bdd6c0eSSue Gleeson #endif /* _SRPT_IMPL_H_ */ 534